diff --git a/builder/core/pom.xml b/builder/core/pom.xml index 6171fabf..cf580383 100644 --- a/builder/core/pom.xml +++ b/builder/core/pom.xml @@ -23,6 +23,14 @@ builder-core + + com.antgroup.openspg.reasoner + reasoner-local-runner + + + com.antgroup.openspg.reasoner + reasoner-cloudext-warehouse + com.antgroup.openspg.server api-http-client @@ -35,10 +43,6 @@ com.antgroup.openspg.cloudext cloudext-impl-graph-store-tugraph - - com.antgroup.openspg.cloudext - cloudext-interface-table-store - com.antgroup.openspg.cloudext cloudext-interface-search-engine @@ -47,13 +51,6 @@ com.antgroup.openspg.cloudext cloudext-impl-search-engine-elasticsearch - - com.antgroup.kg.reasoner - local - ${kgreasoner-version} - system - ${project.basedir}/../../server/lib/spgreasoner-local-0.0.1.jar - com.antgroup.openspg.server core-schema-model diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/RelationMappingProcessor.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/RelationMappingProcessor.java index bc71daa6..52347dd9 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/RelationMappingProcessor.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/RelationMappingProcessor.java @@ -9,7 +9,7 @@ import com.antgroup.openspg.builder.model.pipeline.config.RelationMappingNodeCon import com.antgroup.openspg.builder.model.record.BaseRecord; import com.antgroup.openspg.builder.model.record.BuilderRecord; import com.antgroup.openspg.builder.model.record.RelationRecord; -import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor.EdgeRecordConvertor; +import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util.EdgeRecordConvertor; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.identifier.RelationIdentifier; import com.antgroup.openspg.core.schema.model.predicate.Relation; diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/SPGTypeMappingProcessor.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/SPGTypeMappingProcessor.java index 9f599301..a7ba647c 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/SPGTypeMappingProcessor.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/SPGTypeMappingProcessor.java @@ -22,7 +22,7 @@ import com.antgroup.openspg.builder.model.pipeline.config.SPGTypeMappingNodeConf import com.antgroup.openspg.builder.model.record.BaseAdvancedRecord; import com.antgroup.openspg.builder.model.record.BaseRecord; import com.antgroup.openspg.builder.model.record.BuilderRecord; -import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor.VertexRecordConvertor; +import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util.VertexRecordConvertor; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.type.BaseSPGType; diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/ReasonProcessor.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/ReasonProcessor.java index 9b51f126..0c41dd15 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/ReasonProcessor.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/ReasonProcessor.java @@ -1,10 +1,5 @@ package com.antgroup.openspg.builder.core.reason; -import com.antgroup.kg.reasoner.catalog.impl.OpenKgCatalog; -import com.antgroup.kg.reasoner.common.graph.vertex.IVertexId; -import com.antgroup.kg.reasoner.graphstate.GraphState; -import com.antgroup.kg.reasoner.graphstate.impl.CloudExtGraphState; -import com.antgroup.kg.reasoner.lube.catalog.Catalog; import com.antgroup.openspg.builder.core.physical.process.BaseProcessor; import com.antgroup.openspg.builder.core.reason.impl.CausalConceptReasoner; import com.antgroup.openspg.builder.core.reason.impl.InductiveConceptReasoner; @@ -16,6 +11,11 @@ import com.antgroup.openspg.builder.model.record.BaseSPGRecord; import com.antgroup.openspg.core.schema.model.semantic.DynamicTaxonomySemantic; import com.antgroup.openspg.core.schema.model.semantic.LogicalCausationSemantic; import com.antgroup.openspg.core.schema.model.type.ConceptList; +import com.antgroup.openspg.reasoner.catalog.impl.OpenSPGCatalog; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.warehouse.cloudext.CloudExtGraphState; import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; import com.google.common.collect.Lists; import java.util.*; @@ -85,8 +85,8 @@ public class ReasonProcessor extends BaseProcessorcore model runner/local + testdata diff --git a/builder/testdata/pom.xml b/builder/testdata/pom.xml new file mode 100644 index 00000000..7df3ebbd --- /dev/null +++ b/builder/testdata/pom.xml @@ -0,0 +1,23 @@ + + + 4.0.0 + + com.antgroup.openspg.builder + builder-parent + 0.0.1-SNAPSHOT + + + builder-testdata + + + + com.antgroup.openspg.builder + builder-model + + + com.antgroup.openspg.server + testdata + + + diff --git a/builder/testdata/src/main/java/com/antgroup/openspg/builder/test/RiskMiningRecord.java b/builder/testdata/src/main/java/com/antgroup/openspg/builder/test/RiskMiningRecord.java new file mode 100644 index 00000000..b9bc8ce4 --- /dev/null +++ b/builder/testdata/src/main/java/com/antgroup/openspg/builder/test/RiskMiningRecord.java @@ -0,0 +1,39 @@ +package com.antgroup.openspg.builder.test; + +import com.antgroup.openspg.builder.model.record.EntityRecord; +import com.antgroup.openspg.builder.model.record.property.SPGPropertyRecord; +import com.antgroup.openspg.builder.model.record.property.SPGPropertyValue; +import com.antgroup.openspg.server.testdata.RiskMiningSchema; +import com.google.common.collect.Lists; + +public class RiskMiningRecord { + + public static final EntityRecord PERSON_RECORD1 = + new EntityRecord( + RiskMiningSchema.PERSON, + "0", + Lists.newArrayList( + new SPGPropertyRecord( + RiskMiningSchema.PERSON.getPropertyMap().get("name"), + new SPGPropertyValue("裘**")), + new SPGPropertyRecord( + RiskMiningSchema.PERSON.getPropertyMap().get("age"), new SPGPropertyValue("58")), + new SPGPropertyRecord( + RiskMiningSchema.PERSON.getPropertyMap().get("hasPhone"), + new SPGPropertyValue("154****7458")))); + + public static final EntityRecord PERSON_RECORD1_NORMALIZED = + new EntityRecord( + RiskMiningSchema.PERSON, + "0", + Lists.newArrayList( + new SPGPropertyRecord( + RiskMiningSchema.PERSON.getPropertyMap().get("name"), + new SPGPropertyValue("裘**").setStds(Lists.newArrayList("裘**"))), + new SPGPropertyRecord( + RiskMiningSchema.PERSON.getPropertyMap().get("age"), + new SPGPropertyValue("58").setStds(Lists.newArrayList(58L))), + new SPGPropertyRecord( + RiskMiningSchema.PERSON.getPropertyMap().get("hasPhone"), + new SPGPropertyValue("154****7458").setStds(Lists.newArrayList("154****7458"))))); +} diff --git a/cloudext/impl/object-store/local/pom.xml b/cloudext/impl/object-store/local/pom.xml deleted file mode 100644 index a20049d3..00000000 --- a/cloudext/impl/object-store/local/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.cloudext - cloudext-parent - 0.0.1-SNAPSHOT - ../../../pom.xml - - - cloudext-impl-object-store-local - - - - com.antgroup.openspg.cloudext - cloudext-interface-object-store - - - commons-io - commons-io - - - diff --git a/cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClient.java b/cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClient.java deleted file mode 100644 index ab665712..00000000 --- a/cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClient.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.objectstore.local; - -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient; -import com.antgroup.openspg.cloudext.interfaces.objectstore.cmd.ObjectStoreSaveCmd; -import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath; -import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo; -import java.io.File; -import java.io.IOException; -import lombok.Getter; -import org.apache.commons.io.FileUtils; - -public class LocalObjectStoreClient implements ObjectStoreClient { - - @Getter private final ObjectStoreConnectionInfo connInfo; - private final String localRootDir; - - public LocalObjectStoreClient(ObjectStoreConnectionInfo connInfo) { - this.connInfo = connInfo; - this.localRootDir = (String) connInfo.getNotNullParam("localDir"); - } - - @Override - public ObjectStorePath save(ObjectStoreSaveCmd cmd) { - ObjectStorePath path = cmd.getPath(); - File file = new File("./" + localRootDir + "/" + path.getRelativePath()); - try { - FileUtils.copyInputStreamToFile(cmd.getInputStream(), file); - } catch (IOException e) { - throw new RuntimeException(e); - } - - path.setAbsolutePath(file.getAbsolutePath()); - return path; - } -} diff --git a/cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClientDriver.java b/cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClientDriver.java deleted file mode 100644 index 89c6f46f..00000000 --- a/cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClientDriver.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.objectstore.local; - -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient; -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriver; -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriverManager; -import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo; - -public class LocalObjectStoreClientDriver implements ObjectStoreClientDriver { - - static { - ObjectStoreClientDriverManager.registerDriver(new LocalObjectStoreClientDriver()); - } - - @Override - public String driverScheme() { - return "local"; - } - - @Override - public ObjectStoreClient connect(ObjectStoreConnectionInfo connInfo) { - return new LocalObjectStoreClient(connInfo); - } -} diff --git a/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java b/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java index e7fb2c72..868bbd7e 100644 --- a/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java +++ b/cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java @@ -32,7 +32,7 @@ import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.QueryGroup; import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.TermQuery; import com.antgroup.openspg.common.util.StringUtils; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.dtflys.forest.http.ForestResponse; import com.google.gson.reflect.TypeToken; import java.util.ArrayList; @@ -93,7 +93,8 @@ public class ElasticSearchRecordUtils { throw new RuntimeException("mGetIdxRecords error, errorMsg=" + response.getContent()); } - EsDocs esDocs = JSON.deserialize(response.getContent(), new TypeToken() {}.getType()); + EsDocs esDocs = + SchemaJsonUtils.deserialize(response.getContent(), new TypeToken() {}.getType()); return esDocs.toIdxRecord(); } @@ -119,7 +120,7 @@ public class ElasticSearchRecordUtils { if (StringUtils.isBlank(hits)) { return Collections.emptyList(); } - EsHits esHits = JSON.deserialize(hits, new TypeToken() {}.getType()); + EsHits esHits = SchemaJsonUtils.deserialize(hits, new TypeToken() {}.getType()); return esHits.toIdxRecord(); } diff --git a/cloudext/impl/table-store/local/pom.xml b/cloudext/impl/table-store/local/pom.xml deleted file mode 100644 index 88b3a7dd..00000000 --- a/cloudext/impl/table-store/local/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.cloudext - cloudext-parent - 0.0.1-SNAPSHOT - ../../../pom.xml - - - cloudext-impl-table-store-local - - - - com.antgroup.openspg.cloudext - cloudext-interface-table-store - - - com.opencsv - opencsv - - - diff --git a/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableFileHandler.java b/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableFileHandler.java deleted file mode 100644 index cf47173f..00000000 --- a/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableFileHandler.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.tablestore.local; - -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableFileHandler; -import com.antgroup.openspg.cloudext.interfaces.tablestore.model.TableRecord; -import com.opencsv.CSVWriter; -import java.util.Arrays; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ArrayBlockingQueue; - -public class LocalTableFileHandler implements TableFileHandler { - - private final CSVWriter csvWriter; - private final String tableName; - private static final int DEFAULT_BUFFER_SIZE = 1000; - private final Queue buffer; - - public LocalTableFileHandler(CSVWriter csvWriter, String tableName) { - this.csvWriter = csvWriter; - this.tableName = tableName; - buffer = new ArrayBlockingQueue<>(DEFAULT_BUFFER_SIZE); - } - - @Override - public String getTableName() { - return tableName; - } - - @Override - public int write(TableRecord record) { - if (record.getValues() != null && record.getValues().length > 0) { - buffer.add(Arrays.stream(record.getValues()).map(Object::toString).toArray(String[]::new)); - if (buffer.size() == DEFAULT_BUFFER_SIZE) { - flushBuffer(); - } - return 1; - } - return 0; - } - - @Override - public int batchWrite(List records) { - for (TableRecord record : records) { - write(record); - } - return records.size(); - } - - private synchronized void flushBuffer() { - while (!buffer.isEmpty()) { - csvWriter.writeNext(buffer.poll()); - } - } - - @Override - public void close() throws Exception { - if (csvWriter != null) { - flushBuffer(); - csvWriter.close(); - } - } -} diff --git a/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClient.java b/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClient.java deleted file mode 100644 index 58130292..00000000 --- a/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClient.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.tablestore.local; - -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableFileHandler; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient; -import com.antgroup.openspg.cloudext.interfaces.tablestore.cmd.TableFileCreateCmd; -import com.antgroup.openspg.cloudext.interfaces.tablestore.model.ColumnMeta; -import com.antgroup.openspg.common.util.tuple.Tuple2; -import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo; -import com.opencsv.CSVWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Arrays; -import lombok.Getter; - -public class LocalTableStoreClient implements TableStoreClient { - - @Getter private final TableStoreConnectionInfo connInfo; - private final String localRootDir; - - public LocalTableStoreClient(TableStoreConnectionInfo connInfo) { - this.connInfo = connInfo; - this.localRootDir = (String) connInfo.getNotNullParam("localDir"); - new File("./" + localRootDir).mkdirs(); - } - - @Override - public TableFileHandler create(TableFileCreateCmd cmd) { - try { - Tuple2 tuple2 = buildCsvWriter(cmd); - return new LocalTableFileHandler(tuple2.first, tuple2.second); - } catch (Throwable e) { - throw new RuntimeException(e); - } - } - - private Tuple2 buildCsvWriter(TableFileCreateCmd cmd) throws IOException { - String filePath = "./" + localRootDir + "/" + cmd.getName() + ".csv"; - CSVWriter csvWriter = new CSVWriter(new FileWriter(filePath)); - - String[] columns = - Arrays.stream(cmd.getColumns()).map(ColumnMeta::getName).toArray(String[]::new); - csvWriter.writeNext(columns); - return Tuple2.of(csvWriter, filePath); - } -} diff --git a/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClientDriver.java b/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClientDriver.java deleted file mode 100644 index 81a9b65d..00000000 --- a/cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClientDriver.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.tablestore.local; - -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriver; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriverManager; -import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo; - -public class LocalTableStoreClientDriver implements TableStoreClientDriver { - - static { - TableStoreClientDriverManager.registerDriver(new LocalTableStoreClientDriver()); - } - - @Override - public String driverScheme() { - return "local"; - } - - @Override - public TableStoreClient connect(TableStoreConnectionInfo connInfo) { - return new LocalTableStoreClient(connInfo); - } -} diff --git a/cloudext/interface/graph-store/pom.xml b/cloudext/interface/graph-store/pom.xml index fb00db04..a45c60e0 100644 --- a/cloudext/interface/graph-store/pom.xml +++ b/cloudext/interface/graph-store/pom.xml @@ -32,5 +32,9 @@ com.antgroup.openspg.builder builder-model + + com.antgroup.openspg.builder + builder-testdata + diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/BaseSPGRecord2LPGStrategy.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/BaseSPGRecord2LPGStrategy.java index fe080562..93a8b5d3 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/BaseSPGRecord2LPGStrategy.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/BaseSPGRecord2LPGStrategy.java @@ -18,8 +18,8 @@ import com.antgroup.openspg.builder.model.record.RecordAlterOperationEnum; import com.antgroup.openspg.builder.model.record.SPGRecordAlterItem; import com.antgroup.openspg.builder.model.record.property.SPGPropertyRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.BaseLPGGraphStoreClient; -import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor.EdgeRecordConvertor; -import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor.VertexRecordConvertor; +import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util.EdgeRecordConvertor; +import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util.VertexRecordConvertor; import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.OneHopLPGRecordQuery; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.Direction; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGRecordAlterItem; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/RelationRecord2LPGStrategy.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/RelationRecord2LPGStrategy.java index c1020474..38b710a1 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/RelationRecord2LPGStrategy.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/RelationRecord2LPGStrategy.java @@ -15,7 +15,7 @@ package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl. import com.antgroup.openspg.builder.model.record.RelationRecord; import com.antgroup.openspg.builder.model.record.SPGRecordAlterItem; -import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor.EdgeRecordConvertor; +import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util.EdgeRecordConvertor; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGRecordAlterItem; import com.google.common.collect.Lists; import java.util.List; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/SPGTypeRecord2LPGStrategy.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/SPGTypeRecord2LPGStrategy.java index f968c539..9a71f663 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/SPGTypeRecord2LPGStrategy.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/strategy/SPGTypeRecord2LPGStrategy.java @@ -15,7 +15,7 @@ package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl. import com.antgroup.openspg.builder.model.record.BaseAdvancedRecord; import com.antgroup.openspg.builder.model.record.SPGRecordAlterItem; -import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor.VertexRecordConvertor; +import com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util.VertexRecordConvertor; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGRecordAlterItem; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord; import java.util.List; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/EdgeRecordConvertor.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/EdgeRecordConvertor.java similarity index 74% rename from cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/EdgeRecordConvertor.java rename to cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/EdgeRecordConvertor.java index b2796655..660efeef 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/EdgeRecordConvertor.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/EdgeRecordConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor; +package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util; import com.antgroup.openspg.builder.model.record.BaseAdvancedRecord; import com.antgroup.openspg.builder.model.record.RelationRecord; @@ -19,7 +19,6 @@ import com.antgroup.openspg.builder.model.record.property.SPGPropertyRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.EdgeRecord; import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName; import com.antgroup.openspg.core.schema.model.predicate.Relation; -import com.antgroup.openspg.core.schema.model.type.SPGTypeRef; import com.antgroup.openspg.server.common.model.exception.GraphStoreException; import java.util.Collections; import java.util.List; @@ -29,6 +28,7 @@ import java.util.stream.Collectors; /** Convertor for {@link EdgeRecord} and {@link RelationRecord}. */ public class EdgeRecordConvertor { + /** Convert the SPG record to an LPG record. */ public static EdgeRecord toEdgeRecord(RelationRecord relationRecord) { Relation relationType = relationRecord.getRelationType(); return new EdgeRecord( @@ -42,19 +42,9 @@ public class EdgeRecordConvertor { } /** - * Convert {@link SPGPropertyRecord} in {@link BaseAdvancedRecord} into edge records. - * Specifically, {@link BaseAdvancedRecord#getName()}, {@link BaseAdvancedRecord#getId()}, {@link - * SPGPropertyRecord#getName()}, {@link SPGTypeRef#getName()} from {@link SPGPropertyRecord}, and - * be assigned to start vertex's type name, start vertex's ID, edge label, and end vertex's type - * name of {@link EdgeRecord}s. And end vertex's IDs of {@link EdgeRecord}s will be split from - * {@link SPGPropertyRecord#getValue()} - * - * @param advancedRecord advanced record, such as {@link - * com.antgroup.openspg.builder.model.record.EventRecord EventRecord}, {@link - * com.antgroup.openspg.builder.model.record.EntityRecord EntityRecord} and {@link - * com.antgroup.openspg.builder.model.record.ConceptRecord ConceptRecord}. - * @param spgPropertyRecord record of SPG property. - * @return a list of {@link EdgeRecord}s + * For an advanced SPG record, when one of its property types is also an advanced type, an edge + * will be created between the current instance and the instance corresponding to that property + * value. */ public static List toEdgeRecords( BaseAdvancedRecord advancedRecord, SPGPropertyRecord spgPropertyRecord) { @@ -77,6 +67,10 @@ public class EdgeRecordConvertor { .collect(Collectors.toList()); } + /** + * Convert the LPG record to an SPG record, mainly used in the mapping or reasoning process of + * knowledge builder. + */ public static RelationRecord toRelationRecord( Relation relationType, String srcId, String dstId, Map properties) { return new RelationRecord( diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/PropertyRecordConvertor.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/PropertyRecordConvertor.java similarity index 95% rename from cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/PropertyRecordConvertor.java rename to cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/PropertyRecordConvertor.java index a53def6d..ea661983 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/PropertyRecordConvertor.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/PropertyRecordConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor; +package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util; import com.antgroup.openspg.builder.model.record.property.BasePropertyRecord; import com.antgroup.openspg.builder.model.record.property.SPGPropertyRecord; @@ -29,6 +29,7 @@ import org.apache.commons.collections4.MapUtils; /** Convertor for {@link LPGPropertyRecord} and {@link SPGPropertyRecord}. */ public class PropertyRecordConvertor { + /** Convert the SPG record to an LPG record. */ public static List toLPGProperties( List propertyRecords) { List resultProperties = new ArrayList<>(propertyRecords.size()); @@ -45,6 +46,7 @@ public class PropertyRecordConvertor { return resultProperties; } + /** Convert the LPG record to an SPG record. */ public static List toSPGProperties( Map properties, BaseSPGType spgType) { if (MapUtils.isEmpty(properties)) { @@ -64,6 +66,7 @@ public class PropertyRecordConvertor { return spgPropertyRecords; } + /** Convert the LPG record to an SPG record. */ public static List toSPGProperties( Map properties, Property spgProperty) { if (!spgProperty.hasSubProperty()) { diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/VertexRecordConvertor.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java similarity index 91% rename from cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/VertexRecordConvertor.java rename to cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java index 43bfd778..31bcb177 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/record/impl/convertor/VertexRecordConvertor.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.record.impl.convertor; +package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util; import com.antgroup.openspg.builder.model.record.BaseAdvancedRecord; import com.antgroup.openspg.builder.model.record.ConceptRecord; @@ -39,6 +39,7 @@ import java.util.stream.Collectors; */ public class VertexRecordConvertor { + /** Convert the SPG record to an LPG record. */ public static VertexRecord toVertexRecord(BaseAdvancedRecord advancedRecord) { return new VertexRecord( advancedRecord.getId(), @@ -46,6 +47,10 @@ public class VertexRecordConvertor { PropertyRecordConvertor.toLPGProperties(advancedRecord.getProperties())); } + /** + * Convert the SPG property record to an LPG record, which is currently used only for standard + * types, as a standard type will automatically generate an LPG record based on the property. + */ public static List toVertexRecords(SPGPropertyRecord propertyRecord) { if (propertyRecord.isBasicType()) { throw GraphStoreException.unexpectedSPGPropertyRecordType(propertyRecord); @@ -57,6 +62,10 @@ public class VertexRecordConvertor { .collect(Collectors.toList()); } + /** + * Convert the LPG record to an SPG record, mainly used in the mapping or reasoning process of + * knowledge builder. + */ public static BaseAdvancedRecord toAdvancedRecord( BaseSPGType baseSpgType, String bizId, Map properties) { BaseAdvancedRecord advancedRecord = null; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/BaseLPGRecord.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/BaseLPGRecord.java index 0c0b6f73..d8c56eeb 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/BaseLPGRecord.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/BaseLPGRecord.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; @@ -32,6 +33,7 @@ import org.apache.commons.collections4.CollectionUtils; * */ @Getter +@EqualsAndHashCode(callSuper = false) public abstract class BaseLPGRecord extends BaseValObj { private final LPGRecordTypeEnum recordType; diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/LPGPropertyRecord.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/LPGPropertyRecord.java index cf0c1824..4b54fb23 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/LPGPropertyRecord.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/LPGPropertyRecord.java @@ -15,6 +15,7 @@ package com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record; import com.antgroup.openspg.core.schema.model.type.BasicTypeEnum; import com.antgroup.openspg.server.common.model.base.BaseValObj; +import lombok.EqualsAndHashCode; import lombok.Getter; /** @@ -34,6 +35,7 @@ import lombok.Getter; * */ @Getter +@EqualsAndHashCode(callSuper = false) public class LPGPropertyRecord extends BaseValObj { /** The property name */ diff --git a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java index 8f623310..937384cb 100644 --- a/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java +++ b/cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java @@ -18,6 +18,8 @@ import com.antgroup.openspg.server.api.facade.ApiConstants; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -26,6 +28,7 @@ import lombok.Setter; * VertexType} in LPG, identified by vertex id (id). */ @Getter +@EqualsAndHashCode(callSuper = true) public class VertexRecord extends BaseLPGRecord { @Setter private String vertexType; @@ -45,13 +48,13 @@ public class VertexRecord extends BaseLPGRecord { return otherProperties; } - public Map> toPropertyMapWithIdAndVersion() { + public Map> toPropertyMapWithIdAndVersion() { Map otherProperties = toPropertyMapWithId(); - Map> results = new HashMap<>(otherProperties.size()); + Map> results = new HashMap<>(otherProperties.size()); otherProperties.forEach( (key, value) -> { - Map propertyVersion = new HashMap<>(1); + TreeMap propertyVersion = new TreeMap<>(); propertyVersion.put(ApiConstants.DEFAULT_VERSION, value); results.put(key, propertyVersion); }); diff --git a/cloudext/interface/graph-store/src/test/groovy/com/antgroup/openspg/cloudext/interfaces/graphstore/RiskMiningLPGRecord.groovy b/cloudext/interface/graph-store/src/test/groovy/com/antgroup/openspg/cloudext/interfaces/graphstore/RiskMiningLPGRecord.groovy new file mode 100644 index 00000000..d3407632 --- /dev/null +++ b/cloudext/interface/graph-store/src/test/groovy/com/antgroup/openspg/cloudext/interfaces/graphstore/RiskMiningLPGRecord.groovy @@ -0,0 +1,19 @@ +package com.antgroup.openspg.cloudext.interfaces.graphstore + +import com.antgroup.openspg.builder.test.RiskMiningRecord +import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGPropertyRecord +import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord +import spock.lang.Specification + +class RiskMiningLPGRecord extends Specification { + + public final static VertexRecord PERSON_RECORD1 = new VertexRecord( + RiskMiningRecord.PERSON_RECORD1.id, + RiskMiningRecord.PERSON_RECORD1.getName(), + [ + new LPGPropertyRecord("name", "裘**"), + new LPGPropertyRecord("age", 58L), + new LPGPropertyRecord("hasPhone", "154****7458"), + ] + ); +} diff --git a/cloudext/interface/graph-store/src/test/groovy/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertorTest.groovy b/cloudext/interface/graph-store/src/test/groovy/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertorTest.groovy new file mode 100644 index 00000000..7b061768 --- /dev/null +++ b/cloudext/interface/graph-store/src/test/groovy/com/antgroup/openspg/cloudext/interfaces/graphstore/adapter/util/VertexRecordConvertorTest.groovy @@ -0,0 +1,17 @@ +package com.antgroup.openspg.cloudext.interfaces.graphstore.adapter.util + +import com.antgroup.openspg.builder.test.RiskMiningRecord +import com.antgroup.openspg.cloudext.interfaces.graphstore.RiskMiningLPGRecord +import spock.lang.Specification + +class VertexRecordConvertorTest extends Specification { + + def testToVertexRecord() { + expect: + outputRecord == VertexRecordConvertor.toVertexRecord(inputRecord) + + where: + inputRecord || outputRecord + RiskMiningRecord.PERSON_RECORD1_NORMALIZED || RiskMiningLPGRecord.PERSON_RECORD1 + } +} diff --git a/cloudext/interface/object-store/pom.xml b/cloudext/interface/object-store/pom.xml deleted file mode 100644 index d9525030..00000000 --- a/cloudext/interface/object-store/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.cloudext - cloudext-parent - 0.0.1-SNAPSHOT - ../../pom.xml - - - cloudext-interface-object-store - - - - com.antgroup.openspg - common-util - - - diff --git a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriver.java b/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriver.java deleted file mode 100644 index d83db16b..00000000 --- a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriver.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.objectstore; - -import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver; -import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo; - -public interface ObjectStoreClientDriver - extends CloudExtClientDriver {} diff --git a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriverManager.java b/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriverManager.java deleted file mode 100644 index 1c996652..00000000 --- a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriverManager.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.objectstore; - -import com.antgroup.openspg.common.util.DriverManagerUtils; -import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.exception.CloudExtException; -import java.util.concurrent.CopyOnWriteArrayList; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ObjectStoreClientDriverManager { - - private static final CopyOnWriteArrayList registeredDrivers = - new CopyOnWriteArrayList<>(); - - private ObjectStoreClientDriverManager() {} - - static { - DriverManagerUtils.loadDrivers("cloudext.objectstore.drivers", ObjectStoreClientDriver.class); - log.info("object-store DriverManager initialized"); - } - - public static synchronized void registerDriver(ObjectStoreClientDriver driver) { - if (driver != null) { - registeredDrivers.addIfAbsent(driver); - } else { - throw new NullPointerException(); - } - log.info("registerDriver: {}", driver); - } - - public static ObjectStoreClient getClient(ObjectStoreConnectionInfo config) { - for (ObjectStoreClientDriver driver : registeredDrivers) { - if (driver.acceptsConfig(config)) { - return driver.connect(config); - } - } - throw CloudExtException.driverNotExist(config); - } -} diff --git a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/cmd/ObjectStoreSaveCmd.java b/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/cmd/ObjectStoreSaveCmd.java deleted file mode 100644 index b41dd3d4..00000000 --- a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/cmd/ObjectStoreSaveCmd.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.objectstore.cmd; - -import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath; -import com.antgroup.openspg.server.common.model.base.BaseCmd; -import java.io.InputStream; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public class ObjectStoreSaveCmd extends BaseCmd { - - private final ObjectStorePath path; - - private final InputStream inputStream; -} diff --git a/cloudext/interface/object-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriver b/cloudext/interface/object-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriver deleted file mode 100644 index 01c378fa..00000000 --- a/cloudext/interface/object-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriver +++ /dev/null @@ -1 +0,0 @@ -com.antgroup.openspg.cloudext.impl.objectstore.local.LocalObjectStoreClientDriver diff --git a/cloudext/interface/table-store/pom.xml b/cloudext/interface/table-store/pom.xml deleted file mode 100644 index 7246377e..00000000 --- a/cloudext/interface/table-store/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.cloudext - cloudext-parent - 0.0.1-SNAPSHOT - ../../pom.xml - - - cloudext-interface-table-store - - - - com.antgroup.openspg - common-util - - - diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriver.java b/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriver.java deleted file mode 100644 index 1dfaad73..00000000 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriver.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.tablestore; - -import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver; -import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo; - -public interface TableStoreClientDriver - extends CloudExtClientDriver {} diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriverManager.java b/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriverManager.java deleted file mode 100644 index 9d61ff82..00000000 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriverManager.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.tablestore; - -import com.antgroup.openspg.common.util.DriverManagerUtils; -import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.exception.CloudExtException; -import java.util.concurrent.CopyOnWriteArrayList; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class TableStoreClientDriverManager { - - private static final CopyOnWriteArrayList registeredDrivers = - new CopyOnWriteArrayList<>(); - - private TableStoreClientDriverManager() {} - - static { - DriverManagerUtils.loadDrivers("cloudext.tablestore.drivers", TableStoreClientDriver.class); - log.info("table-store DriverManager initialized"); - } - - public static synchronized void registerDriver(TableStoreClientDriver driver) { - if (driver != null) { - registeredDrivers.addIfAbsent(driver); - } else { - throw new NullPointerException(); - } - log.info("registerDriver: {}", driver); - } - - public static TableStoreClient getClient(TableStoreConnectionInfo config) { - for (TableStoreClientDriver driver : registeredDrivers) { - if (driver.acceptsConfig(config)) { - return driver.connect(config); - } - } - throw CloudExtException.driverNotExist(config); - } -} diff --git a/cloudext/interface/table-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriver b/cloudext/interface/table-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriver deleted file mode 100644 index 8c9e3f74..00000000 --- a/cloudext/interface/table-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriver +++ /dev/null @@ -1 +0,0 @@ -com.antgroup.openspg.cloudext.impl.tablestore.local.LocalTableStoreClientDriver diff --git a/cloudext/pom.xml b/cloudext/pom.xml index 9923c6a0..3a915e2e 100644 --- a/cloudext/pom.xml +++ b/cloudext/pom.xml @@ -36,12 +36,8 @@ interface/graph-store - interface/object-store - interface/table-store interface/search-engine impl/graph-store/tugraph - impl/object-store/local - impl/table-store/local impl/search-engine/elasticsearch diff --git a/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java b/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java index 19dbb4d1..7bdd8624 100644 --- a/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java +++ b/common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.common.util; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -22,7 +22,7 @@ public class JsonUtils { public static String toJsonString(Object object) { String result = null; try { - result = JSON.serialize(object); + result = SchemaJsonUtils.serialize(object); } catch (Exception e) { log.error("toJsonString error, object={}", object, e); result = StringUtils.toString(object); diff --git a/dev/release/docker-compose-no-es.yml b/dev/release/docker-compose-no-es.yml deleted file mode 100644 index 01fa39a2..00000000 --- a/dev/release/docker-compose-no-es.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: "3.7" -services: - openspg: - restart: always - image: baifuyu/openspg:latest - container_name: release-openspg - ports: - - "8887:8887" - depends_on: - - mysql - - tugraph - command: [ - '--cloudext.repository.impl.jdbc.host=mysql', - '--builder.operator.python.exec=/usr/bin/python3.8', - '--builder.operator.python.paths=/usr/lib/python3.8/site-packages;/usr/local/lib/python3.8/dist-packages;' - ] - environment: - - PYTHONPATH=/usr/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages - - mysql: - restart: always - image: baifuyu/openspg-mysql:latest - container_name: release-openspg-mysql - environment: - TZ: Asia/Shanghai - LANG: C.UTF-8 - ports: - - "3306:3306" - command: [ - '--character-set-server=utf8mb4', - '--collation-server=utf8mb4_general_ci' - ] - - tugraph: - image: tugraph/tugraph-runtime-centos7:4.0.1 - container_name: release-openspg-tugraph - # default username is admin and default password is 73@TuGraph - ports: - - "7070:7070" - - "9090:9090" - command: lgraph_server - - - - - diff --git a/dev/release/docker-compose.yml b/dev/release/docker-compose.yml index fc433276..c115b8b2 100644 --- a/dev/release/docker-compose.yml +++ b/dev/release/docker-compose.yml @@ -1,19 +1,44 @@ version: "3.7" services: openspg: - extends: - file: docker-compose-no-es.yml - service: openspg + restart: always + image: baifuyu/openspg:latest + container_name: release-openspg + ports: + - "8887:8887" + depends_on: + - mysql + - tugraph + command: [ + '--cloudext.repository.impl.jdbc.host=mysql', + '--builder.operator.python.exec=/usr/bin/python3.8', + '--builder.operator.python.paths=/usr/lib/python3.8/site-packages;/usr/local/lib/python3.8/dist-packages;' + ] + environment: + - PYTHONPATH=/usr/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages mysql: - extends: - file: docker-compose-no-es.yml - service: mysql + restart: always + image: baifuyu/openspg-mysql:latest + container_name: release-openspg-mysql + environment: + TZ: Asia/Shanghai + LANG: C.UTF-8 + ports: + - "3306:3306" + command: [ + '--character-set-server=utf8mb4', + '--collation-server=utf8mb4_general_ci' + ] tugraph: - extends: - file: docker-compose-no-es.yml - service: tugraph + image: tugraph/tugraph-runtime-centos7:4.0.1 + container_name: release-openspg-tugraph + # default username is admin and default password is 73@TuGraph + ports: + - "7070:7070" + - "9090:9090" + command: lgraph_server elasticsearch: image: elasticsearch:8.5.3 diff --git a/dev/test/docker-compose-no-es.yml b/dev/test/docker-compose-no-es.yml deleted file mode 100644 index 3715d615..00000000 --- a/dev/test/docker-compose-no-es.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: "3.7" -services: - mysql: - restart: always - image: mariadb:10.5.8 - container_name: test-openspg-mysql - environment: - MYSQL_ROOT_PASSWORD: openspg - MYSQL_DATABASE: openspg - TZ: Asia/Shanghai - LANG: C.UTF-8 - volumes: - - "./mysql/sqlscript:/docker-entrypoint-initdb.d" - ports: - - "3306:3306" - command: [ - '--character-set-server=utf8mb4', - '--collation-server=utf8mb4_general_ci' - ] - - tugraph: - image: tugraph/tugraph-runtime-centos7:4.0.1 - container_name: test-openspg-tugraph - # default username is admin and default password is 73@TuGraph - ports: - - "7070:7070" - - "9090:9090" - command: lgraph_server - - - - diff --git a/dev/test/docker-compose.yml b/dev/test/docker-compose.yml index 1af46703..a2e66231 100644 --- a/dev/test/docker-compose.yml +++ b/dev/test/docker-compose.yml @@ -1,14 +1,32 @@ version: "3.7" services: mysql: - extends: - file: docker-compose-no-es.yml - service: mysql + restart: always + image: mariadb:10.5.8 + container_name: test-openspg-mysql + environment: + MYSQL_ROOT_PASSWORD: openspg + MYSQL_DATABASE: openspg + TZ: Asia/Shanghai + LANG: C.UTF-8 + volumes: + - "./mysql/sqlscript:/docker-entrypoint-initdb.d" + ports: + - "3306:3306" + command: [ + '--character-set-server=utf8mb4', + '--collation-server=utf8mb4_general_ci' + ] tugraph: - extends: - file: docker-compose-no-es.yml - service: tugraph + image: tugraph/tugraph-runtime-centos7:4.0.1 + container_name: test-openspg-tugraph + # default username is admin and default password is 73@TuGraph + ports: + - "7070:7070" + - "9090:9090" + command: lgraph_server + elasticsearch: image: elasticsearch:8.5.3 diff --git a/pom.xml b/pom.xml index 9b3a3730..593d4ecf 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,26 @@ + + com.antgroup.openspg.builder + builder-testdata + ${project.version} + + + com.antgroup.openspg.server + testdata + ${project.version} + + + com.antgroup.openspg.reasoner + reasoner-local-runner + ${project.version} + + + com.antgroup.openspg.reasoner + reasoner-cloudext-warehouse + ${project.version} + com.antgroup.openspg.server common-model @@ -99,31 +119,11 @@ cloudext-interface-search-engine ${project.version} - - com.antgroup.openspg.cloudext - cloudext-interface-object-store - ${project.version} - - - com.antgroup.openspg.cloudext - cloudext-interface-table-store - ${project.version} - com.antgroup.openspg.cloudext cloudext-impl-graph-store-tugraph ${project.version} - - com.antgroup.openspg.cloudext - cloudext-impl-object-store-local - ${project.version} - - - com.antgroup.openspg.cloudext - cloudext-impl-table-store-local - ${project.version} - com.antgroup.openspg.cloudext cloudext-impl-search-engine-elasticsearch diff --git a/python/medical/.knext.cfg b/python/medical/.knext.cfg index d8bc8587..880f4725 100644 --- a/python/medical/.knext.cfg +++ b/python/medical/.knext.cfg @@ -1,7 +1,7 @@ [local] project_name = Medical1 description = Medical1 -project_id = 3 +project_id = 2 namespace = Medical1 project_dir = medical1 schema_dir = schema diff --git a/python/medical/schema/medical1_schema_helper.py b/python/medical/schema/medical1_schema_helper.py index 0688fac7..35e3c495 100644 --- a/python/medical/schema/medical1_schema_helper.py +++ b/python/medical/schema/medical1_schema_helper.py @@ -17,8 +17,8 @@ class Medical1: description = "description" id = "id" name = "name" - alias = "alias" stdId = "stdId" + alias = "alias" def __init__(self): pass @@ -28,11 +28,11 @@ class Medical1: description = "description" id = "id" name = "name" - complication = "complication" - department = "department" - applicableDrug = "applicableDrug" diseaseSite = "diseaseSite" + complication = "complication" + applicableDrug = "applicableDrug" commonSymptom = "commonSymptom" + department = "department" def __init__(self): pass @@ -51,8 +51,8 @@ class Medical1: description = "description" id = "id" name = "name" - stdId = "stdId" alias = "alias" + stdId = "stdId" def __init__(self): pass diff --git a/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/IVertexId.java b/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/IVertexId.java index 6a599d77..51372224 100644 --- a/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/IVertexId.java +++ b/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/IVertexId.java @@ -35,6 +35,9 @@ public interface IVertexId extends Serializable, Comparable { /** get kgstate internal id */ long getInternalId(); + /** get biz id */ + String getBizId(); + /** get vertex type id */ long getTypeId(); diff --git a/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/impl/VertexId.java b/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/impl/VertexId.java index 0363631a..b8e0fe85 100644 --- a/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/impl/VertexId.java +++ b/reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/impl/VertexId.java @@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; import java.util.Comparator; +import lombok.Getter; public class VertexId implements IVertexId { /** @@ -33,6 +34,8 @@ public class VertexId implements IVertexId { */ private final byte[] vertexIdBytes; + @Getter private String bizId; + /** create vertex id */ public VertexId(long internalId, String type) { this.vertexIdBytes = generateIdBytes(internalId, type); @@ -40,6 +43,7 @@ public class VertexId implements IVertexId { public VertexId(String bizId, String type) { this.vertexIdBytes = generateIdBytes(generateInternalId(bizId, type), type); + this.bizId = bizId; } public VertexId(byte[] bytes) { diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/ExprUtil.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/ExprUtil.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/ExprUtil.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/ExprUtil.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/PatternOps.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/PatternOps.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/PatternOps.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/PatternOps.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/SolvedModel.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/SolvedModel.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/SolvedModel.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/SolvedModel.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/Var.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/Var.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/Var.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/Var.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Aggregate.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Aggregate.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Aggregate.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Aggregate.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/BoundedVarLenExpand.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/BoundedVarLenExpand.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/BoundedVarLenExpand.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/BoundedVarLenExpand.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/DDL.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/DDL.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/DDL.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/DDL.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/ExpandInto.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/ExpandInto.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/ExpandInto.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/ExpandInto.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Filter.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Filter.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Filter.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Filter.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/LinkedExpand.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/LinkedExpand.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/LinkedExpand.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/LinkedExpand.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/LogicalOperator.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/LogicalOperator.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/LogicalOperator.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/LogicalOperator.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Optional.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Optional.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Optional.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Optional.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/OrderAndLimit.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/OrderAndLimit.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/OrderAndLimit.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/OrderAndLimit.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/PatternJoin.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/PatternJoin.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/PatternJoin.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/PatternJoin.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/PatternScan.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/PatternScan.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/PatternScan.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/PatternScan.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Project.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Project.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Project.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Project.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Select.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Select.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Select.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Select.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Start.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Start.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/Start.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/Start.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/SubQuery.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/SubQuery.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/operators/SubQuery.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/operators/SubQuery.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/LogicalOptimizer.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/LogicalOptimizer.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/LogicalOptimizer.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/LogicalOptimizer.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/Rule.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/Rule.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/Rule.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/Rule.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/AggregatePushDown.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/AggregatePushDown.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/AggregatePushDown.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/AggregatePushDown.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/DistinctGet.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/DistinctGet.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/DistinctGet.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/DistinctGet.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/EdgeToProperty.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/EdgeToProperty.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/EdgeToProperty.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/EdgeToProperty.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/ExpandIntoPure.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/ExpandIntoPure.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/ExpandIntoPure.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/ExpandIntoPure.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/FilterMerge.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/FilterMerge.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/FilterMerge.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/FilterMerge.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/FilterPushDown.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/FilterPushDown.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/FilterPushDown.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/FilterPushDown.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/IdFilterPushDown.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/IdFilterPushDown.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/IdFilterPushDown.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/IdFilterPushDown.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/ProjectMerge.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/ProjectMerge.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/ProjectMerge.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/ProjectMerge.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/Pure.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/Pure.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/Pure.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/Pure.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/SolvedModelPure.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/SolvedModelPure.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/optimizer/rules/SolvedModelPure.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/optimizer/rules/SolvedModelPure.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/LogicalPlanner.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/LogicalPlanner.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/LogicalPlanner.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/LogicalPlanner.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/LogicalPlannerContext.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/LogicalPlannerContext.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/LogicalPlannerContext.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/LogicalPlannerContext.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/PatternMatchPlanner.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/PatternMatchPlanner.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/PatternMatchPlanner.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/PatternMatchPlanner.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/ProjectPlanner.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/ProjectPlanner.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/ProjectPlanner.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/ProjectPlanner.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/SubQueryMerger.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/SubQueryMerger.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/SubQueryMerger.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/SubQueryMerger.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/SubQueryPlanner.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/SubQueryPlanner.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/planning/SubQueryPlanner.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/planning/SubQueryPlanner.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/Dag.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/Dag.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/Dag.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/Dag.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/Validator.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/Validator.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/Validator.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/Validator.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/Explain.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/Explain.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/Explain.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/Explain.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/SemanticExplainer.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/SemanticExplainer.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/SemanticExplainer.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/SemanticExplainer.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/rules/ConceptExplain.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/rules/ConceptExplain.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/rules/ConceptExplain.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/rules/ConceptExplain.scala diff --git a/reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/rules/SpatioTemporalExplain.scala b/reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/rules/SpatioTemporalExplain.scala similarity index 100% rename from reasoner/lube-logical/src/main/scala/com/antgroup/kg/reasoner/lube/logical/validate/semantic/rules/SpatioTemporalExplain.scala rename to reasoner/lube-logical/src/main/scala/com/antgroup/openspg/reasoner/lube/logical/validate/semantic/rules/SpatioTemporalExplain.scala diff --git a/reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/EmptyCatalog.scala b/reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/EmptyCatalog.scala similarity index 100% rename from reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/EmptyCatalog.scala rename to reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/EmptyCatalog.scala diff --git a/reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/ExprUtilTests.scala b/reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/ExprUtilTests.scala similarity index 100% rename from reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/ExprUtilTests.scala rename to reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/ExprUtilTests.scala diff --git a/reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/LogicalOperatorOps.scala b/reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/LogicalOperatorOps.scala similarity index 100% rename from reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/LogicalOperatorOps.scala rename to reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/LogicalOperatorOps.scala diff --git a/reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/LogicalOperatorTests.scala b/reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/LogicalOperatorTests.scala similarity index 100% rename from reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/LogicalOperatorTests.scala rename to reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/LogicalOperatorTests.scala diff --git a/reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/LogicalPlannerTests.scala b/reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/LogicalPlannerTests.scala similarity index 100% rename from reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/LogicalPlannerTests.scala rename to reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/LogicalPlannerTests.scala diff --git a/reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/OptimizerTests.scala b/reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/OptimizerTests.scala similarity index 100% rename from reasoner/lube-logical/src/test/scala/com/antgroup/kg/reasoner/lube/logical/OptimizerTests.scala rename to reasoner/lube-logical/src/test/scala/com/antgroup/openspg/reasoner/lube/logical/OptimizerTests.scala diff --git a/reasoner/pom.xml b/reasoner/pom.xml index c4d6ae41..49bc8177 100644 --- a/reasoner/pom.xml +++ b/reasoner/pom.xml @@ -31,7 +31,10 @@ lube-logical lube-physical warehouse/warehouse-common + warehouse/cloudext-warehouse kgdsl-parser + runner/runner-common + runner/local-runner catalog/openspg-catalog @@ -87,11 +90,31 @@ reasoner-kgdsl-parser ${project.version} + + com.antgroup.openspg.reasoner + reasoner-runner-common + ${project.version} + com.antgroup.openspg.reasoner reasoner-openspg-catalog ${project.version} + + com.antgroup.openspg.reasoner + reasoner-warehouse-common + ${project.version} + + + com.antgroup.openspg.reasoner + reasoner-cloudext-warehouse + ${project.version} + + + com.antgroup.openspg.reasoner + reasoner-local-runner + ${project.version} + org.antlr @@ -247,6 +270,12 @@ odps-sdk-commons ${odps.version} + + com.google.protobuf + protobuf-java + 3.8.0 + provided + @@ -298,93 +327,6 @@ - - - com.alipay.cloudsdk - cloud-sdk-common - 1.2.1.20221206 - - - org.apache.httpcomponents - httpcore - - - - - com.alipay.cloudsdk - logservice-cloud-sdk - 1.1.0-SNAPSHOT - - - com.alipay.cloudsdk - logservice-cloud-driver-sls - 1.1.0-SNAPSHOT - - - com.alipay.cloudsdk - logservice-cloud-driver-kafka - 1.1.0-SNAPSHOT - - - - com.alipay.cloudsdk - object-storage-cloud-sdk - 2.0.0.20221213-SNAPSHOT - - - - com.alipay.cloudsdk - object-storage-cloud-driver-oss - 2.0.0.20221115-SNAPSHOT - - - - com.alipay.cloudsdk - object-storage-cloud-driver-minio - 2.0.0.20221115-SNAPSHOT - - - - com.alipay.cloudsdk - object-storage-cloud-driver-s3 - 2.0.0.20221213-SNAPSHOT - - - io.netty - netty-transport - - - io.netty - netty-transport-classes-epoll - - - io.netty - netty-handler - - - io.netty - netty-common - - - io.netty - netty-codec - - - io.netty - netty-codec-http - - - io.netty - netty-codec-http2 - - - io.netty - netty-buffer - - - - - com.opencsv opencsv diff --git a/reasoner/runner/local-runner/pom.xml b/reasoner/runner/local-runner/pom.xml new file mode 100644 index 00000000..15e4f8b4 --- /dev/null +++ b/reasoner/runner/local-runner/pom.xml @@ -0,0 +1,119 @@ + + + + 4.0.0 + + com.antgroup.openspg.reasoner + reasoner-parent + 0.0.1-SNAPSHOT + ../../pom.xml + + + reasoner-local-runner + + + 8 + 8 + + + + + com.antgroup.openspg.reasoner + reasoner-common + + + + com.antgroup.openspg.reasoner + reasoner-lube-api + + + com.antgroup.openspg.reasoner + reasoner-kgdsl-parser + + + com.antgroup.openspg.reasoner + reasoner-lube-physical + + + com.antgroup.openspg.reasoner + reasoner-openspg-catalog + + + com.antgroup.openspg.reasoner + reasoner-runner-common + + + org.apache.hive + hive-jdbc + + + com.alipay.cloudsdk + object-storage-cloud-driver-minio + + + com.alipay.cloudsdk + object-storage-cloud-driver-s3 + + + org.apache.parquet + parquet-hadoop-bundle + + + + + com.antgroup.openspg.reasoner + reasoner-warehouse-common + + + + org.scala-lang + scala-compiler + + + org.scala-lang + scala-reflect + + + org.scala-lang + scala-library + + + + org.apache.commons + commons-lang3 + + + org.projectlombok + lombok + + + junit + junit + + + org.scalatest + scalatest_${scala.binary.version} + + + org.apache.hadoop + hadoop-common + + + com.antgroup.openspg.reasoner + reasoner-common + + + + diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/KGReasonerLocalRunner.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/KGReasonerLocalRunner.java new file mode 100644 index 00000000..61c3d15c --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/KGReasonerLocalRunner.java @@ -0,0 +1,193 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local; + +import com.antgroup.openspg.reasoner.catalog.CatalogFactory; +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface; +import com.antgroup.openspg.reasoner.lube.physical.operators.PhysicalOperator; +import com.antgroup.openspg.reasoner.lube.physical.operators.Select; +import com.antgroup.openspg.reasoner.lube.physical.operators.Start; +import com.antgroup.openspg.reasoner.lube.physical.util.PhysicalOperatorUtil; +import com.antgroup.openspg.reasoner.parser.KgDslParser; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.impl.LocalPropertyGraph; +import com.antgroup.openspg.reasoner.runner.local.impl.LocalReasonerSession; +import com.antgroup.openspg.reasoner.runner.local.impl.LocalRunnerThreadPool; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.runner.local.rdg.LocalRDG; +import com.antgroup.openspg.reasoner.runner.local.rdg.LocalRow; +import com.antgroup.openspg.reasoner.runner.local.rdg.TypeTags; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; + +@Slf4j +public class KGReasonerLocalRunner { + + /** run dsl task on local runner */ + public LocalReasonerResult run(LocalReasonerTask task) { + try { + return doRun(task); + } catch (Throwable e) { + log.error("KGReasonerLocalRunner,error", e); + return new LocalReasonerResult("KGReasonerLocalRunner,error " + e.getMessage()); + } + } + + private LocalReasonerResult doRun(LocalReasonerTask task) { + // load graph + GraphState graphState = loadGraph(task); + + // plan + LocalReasonerSession session = task.getSession(); + List> dslDagList = task.getDslDagList(); + if (CollectionUtils.isEmpty(dslDagList)) { + if (StringUtils.isEmpty(task.getDsl())) { + throw new RuntimeException("please provide dsl"); + } + + // create catalog + Catalog catalog = task.getCatalog(); + if (null == catalog) { + catalog = CatalogFactory.createCatalog(task.getParams(), task.getConnInfo()); + catalog.init(); + } + + if (!task.getParams().containsKey(Constants.START_ALIAS) + && !task.getParams().containsKey(Constants.START_LABEL) + && task.getStartIdList() != null + && task.getStartIdList().size() > 0) { + task.getParams().put(Constants.START_LABEL, task.getStartIdList().get(0)._2); + } + ParserInterface parser = new KgDslParser(); + session = new LocalReasonerSession(parser, catalog, TypeTags.rdgTypeTag(), graphState); + dslDagList = + Lists.newArrayList( + JavaConversions.asJavaCollection( + session.plan( + task.getDsl(), Convert2ScalaUtil.toScalaImmutableMap(task.getParams())))); + } + + LocalReasonerResult result = null; + for (int i = 0; i < dslDagList.size(); ++i) { + Map idFilterMaps = + JavaConversions.mapAsJavaMap(session.getIdFilterParameters()); + Map taskRunningContext = + RunnerUtil.getTaskRunningContext(session, task.getParams()); + + RuleRunner.getInstance().putRuleRunningContext(task.getId(), taskRunningContext); + boolean isLastDsl = (i + 1 == dslDagList.size()); + + if (isLastDsl) { + Start start = + PhysicalOperatorUtil.getStartOp( + dslDagList.get(i), + com.antgroup.openspg.reasoner.runner.local.rdg.TypeTags.rdgTypeTag()); + + if (idFilterMaps != null && idFilterMaps.size() != 0) { + String startAliasName = start.alias(); + if (idFilterMaps.containsKey(startAliasName)) { + String parameter = idFilterMaps.get(startAliasName); + Object obj = taskRunningContext.get(parameter); + List originIds = new ArrayList<>(); + if (obj instanceof Object[]) { + Object[] ids = (Object[]) obj; + for (Object id : ids) { + originIds.add(id.toString()); + } + } else { + originIds.add(obj.toString()); + } + List> startIdList = new ArrayList<>(); + + for (String type : JavaConversions.setAsJavaSet(start.types())) { + for (Object id : originIds) { + startIdList.add(new Tuple2<>(id.toString(), type)); + } + } + task.setStartIdList(startIdList); + } + } + } + PhysicalOperator physicalOpRoot = dslDagList.get(i); + LocalPropertyGraph localPropertyGraph = + (LocalPropertyGraph) + physicalOpRoot.context().graphSession().getGraph(Catalog.defaultGraphName()); + localPropertyGraph.setTask(task); + localPropertyGraph.setThreadPoolExecutor(LocalRunnerThreadPool.getThreadPoolExecutor(task)); + localPropertyGraph.setExecutorTimeoutMs(task.getExecutorTimeoutMs()); + if (CollectionUtils.isNotEmpty(task.getStartIdList()) && isLastDsl) { + localPropertyGraph.setStartIdTuple2List(task.getStartIdList()); + } else { + localPropertyGraph.setStartIdTuple2List(null); + } + + if (physicalOpRoot instanceof Select) { + String isGraphOutput = + String.valueOf( + task.getParams().computeIfAbsent(ConfigKey.KG_REASONER_OUTPUT_GRAPH, k -> "false")); + if ("true".equals(isGraphOutput)) { + LocalRDG rdg = ((Select) physicalOpRoot).in().rdg(); + result = rdg.getRDGGraph(); + } else { + LocalRow row = (LocalRow) ((Select) physicalOpRoot).row(); + result = row.getResult(); + } + } else { + LocalRDG rdg = physicalOpRoot.rdg(); + result = rdg.getResult(); + } + log.info("dsl,index=" + i + ",result=" + result.toString()); + } + + return result; + } + + protected GraphState loadGraph(LocalReasonerTask task) { + GraphState graphState = task.getGraphState(); + if (null != graphState) { + return graphState; + } + + String graphLoadClass = task.getGraphLoadClass(); + MemGraphState memGraphState = new MemGraphState(); + AbstractLocalGraphLoader graphLoader; + try { + graphLoader = + (AbstractLocalGraphLoader) Class.forName(graphLoadClass).getConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException("can not create graph loader from name " + graphLoadClass, e); + } + graphLoader.setGraphState(memGraphState); + graphLoader.load(); + task.setGraphState(memGraphState); + return memGraphState; + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/LocalRunnerMain.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/LocalRunnerMain.java new file mode 100644 index 00000000..1728fd42 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/LocalRunnerMain.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LocalRunnerMain { + + /** result */ + public static LocalReasonerResult result = null; + + /** KGReasoner main */ + public static void main(String[] args) { + String taskInfoJson = new String(Base64.getDecoder().decode(args[0]), StandardCharsets.UTF_8); + LocalReasonerTask task = JSON.parseObject(taskInfoJson, LocalReasonerTask.class); + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + result = runner.run(task); + if (null != result) { + log.info(result.toString()); + } + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalPropertyGraph.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalPropertyGraph.java new file mode 100644 index 00000000..aa9c6a02 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalPropertyGraph.java @@ -0,0 +1,198 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.impl; + +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.lube.logical.RepeatPathVar; +import com.antgroup.openspg.reasoner.lube.physical.PropertyGraph; +import com.antgroup.openspg.reasoner.recorder.EmptyRecorder; +import com.antgroup.openspg.reasoner.recorder.IExecutionRecorder; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.runner.local.rdg.LocalRDG; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import org.apache.commons.collections4.CollectionUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; +import scala.collection.immutable.Set; + +public class LocalPropertyGraph implements PropertyGraph { + /** task info */ + private LocalReasonerTask task; + + /** start id from input */ + private List> startIdTuple2List; + + /** graph state */ + private final GraphState graphState; + + /** executor */ + private ThreadPoolExecutor threadPoolExecutor; + + /** executor timeout ms */ + private long executorTimeoutMs = 40 * 1000; + + /** default path limit */ + private long defaultPathLimit = 3000; + + /** local property graph */ + public LocalPropertyGraph(GraphState graphState) { + this.graphState = graphState; + } + + @Override + public LocalRDG createRDG(String alias, Set types) { + LocalRDG result = + new LocalRDG( + graphState, + getStartIdList(types), + threadPoolExecutor, + executorTimeoutMs, + alias, + getTaskId(), + getExecutionRecorder()); + result.setMaxPathLimit(getMaxPathLimit()); + result.setStrictMaxPathLimit(getStrictMaxPathLimit()); + return result; + } + + @Override + public LocalRDG createRDG(String alias, LocalRDG rdg) { + java.util.Set startIdSet = new HashSet<>(); + for (KgGraph kgGraph : rdg.getKgGraphList()) { + List> vertexList = kgGraph.getVertex(alias); + for (IVertex vertex : vertexList) { + startIdSet.add(vertex.getId()); + } + } + LocalRDG result = + new LocalRDG( + graphState, + Lists.newArrayList(startIdSet), + threadPoolExecutor, + executorTimeoutMs, + alias, + getTaskId(), + getExecutionRecorder()); + result.setMaxPathLimit(getMaxPathLimit()); + result.setStrictMaxPathLimit(getStrictMaxPathLimit()); + return result; + } + + @Override + public LocalRDG createRDGFromPath(RepeatPathVar repeatVar, String alias, LocalRDG rdg) { + return null; + } + + private String getTaskId() { + if (task == null) { + return ""; + } + return task.getId(); + } + + private List getStartIdList(Set types) { + if (CollectionUtils.isNotEmpty(this.startIdTuple2List)) { + List startIdList = new ArrayList<>(); + for (Tuple2 tuple2 : this.startIdTuple2List) { + startIdList.add(IVertexId.from(tuple2._1(), tuple2._2())); + } + return startIdList; + } + List startIdList = new ArrayList<>(); + Iterator> it = + this.graphState.getVertexIterator(JavaConversions.setAsJavaSet(types)); + while (it.hasNext()) { + IVertex vertex = it.next(); + startIdList.add(vertex.getId()); + } + return startIdList; + } + + /** + * Setter method for property startIdList. + * + * @param startIdTuple2List value to be assigned to property startIdList + */ + public void setStartIdTuple2List(List> startIdTuple2List) { + this.startIdTuple2List = startIdTuple2List; + } + + /** + * Setter method for property threadPoolExecutor. + * + * @param threadPoolExecutor value to be assigned to property threadPoolExecutor + */ + public void setThreadPoolExecutor(ThreadPoolExecutor threadPoolExecutor) { + this.threadPoolExecutor = threadPoolExecutor; + } + + /** + * Setter method for property timeoutMillSeconds. + * + * @param executorTimeoutMs value to be assigned to property timeoutMillSeconds + */ + public void setExecutorTimeoutMs(long executorTimeoutMs) { + this.executorTimeoutMs = executorTimeoutMs; + } + + /** max path limit */ + private Long getMaxPathLimit() { + Object maxPathLimitObj = null; + if (null != task && null != this.task.getParams()) { + maxPathLimitObj = this.task.getParams().get(ConfigKey.KG_REASONER_MAX_PATH_LIMIT); + } + if (null == maxPathLimitObj) { + return defaultPathLimit; + } + return 2 * Long.parseLong(String.valueOf(maxPathLimitObj)); + } + + /** return strict max path limit */ + private Long getStrictMaxPathLimit() { + Object maxPathLimitObj = null; + if (null != task && null != this.task.getParams()) { + maxPathLimitObj = this.task.getParams().get(ConfigKey.KG_REASONER_STRICT_MAX_PATH_THRESHOLD); + } + if (null == maxPathLimitObj) { + return null; + } + return Long.parseLong(String.valueOf(maxPathLimitObj)); + } + + private IExecutionRecorder getExecutionRecorder() { + if (null == task) { + return new EmptyRecorder(); + } + return task.getExecutionRecorder(); + } + + /** + * Setter method for property task. + * + * @param task value to be assigned to property task + */ + public void setTask(LocalReasonerTask task) { + this.task = task; + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalReasonerSession.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalReasonerSession.java new file mode 100644 index 00000000..636c9822 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalReasonerSession.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.impl; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface; +import com.antgroup.openspg.reasoner.lube.physical.PropertyGraph; +import com.antgroup.openspg.reasoner.runner.local.rdg.LocalRDG; +import com.antgroup.openspg.reasoner.session.KGReasonerSession; +import com.antgroup.openspg.reasoner.warehouse.common.config.GraphLoaderConfig; +import scala.reflect.api.TypeTags.TypeTag; + +public class LocalReasonerSession extends KGReasonerSession { + + private final GraphState graphState; + + /** session implement */ + public LocalReasonerSession(ParserInterface parser, Catalog catalog, TypeTag typeTag) { + super(parser, catalog, typeTag); + this.graphState = new MemGraphState(); + } + + /** session implement */ + public LocalReasonerSession( + ParserInterface parser, + Catalog catalog, + TypeTag typeTag, + GraphState graphState) { + super(parser, catalog, typeTag); + this.graphState = graphState; + } + + @Override + public PropertyGraph loadGraph(GraphLoaderConfig graphLoaderConfig) { + return new LocalPropertyGraph(this.graphState); + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalRunnerThreadPool.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalRunnerThreadPool.java new file mode 100644 index 00000000..ca72ea72 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/impl/LocalRunnerThreadPool.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.impl; + +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; +import java.util.concurrent.TimeUnit; + +public class LocalRunnerThreadPool { + + private LocalRunnerThreadPool() {} + + /** + * when the number of threads is greater than the core, this is the maximum time that excess idle + * threads will wait for new tasks before terminating. + */ + private static final long KEEP_ALIVE_TIME_SECONDS = 5 * 60; + + private static final int DEFAULT_CORE_POOL_SIZE = 10; + private static final int DEFAULT_MAXIMUM_POOL_SIZE = 30; + + private static final int WORK_QUEUE_SIZE = 1000; + + private static volatile ThreadPoolExecutor DEFAULT_THREAD_POOL_EXECUTOR = null; + + /** get thread pool */ + public static ThreadPoolExecutor getThreadPoolExecutor(LocalReasonerTask task) { + if (null == task || null == task.getThreadPoolExecutor()) { + return getDefaultThreadPoolExecutor(); + } + return task.getThreadPoolExecutor(); + } + + private static ThreadPoolExecutor getDefaultThreadPoolExecutor() { + if (null != DEFAULT_THREAD_POOL_EXECUTOR) { + return DEFAULT_THREAD_POOL_EXECUTOR; + } + synchronized (LocalRunnerThreadPool.class) { + if (null != DEFAULT_THREAD_POOL_EXECUTOR) { + return DEFAULT_THREAD_POOL_EXECUTOR; + } + DEFAULT_THREAD_POOL_EXECUTOR = + new ThreadPoolExecutor( + DEFAULT_CORE_POOL_SIZE, + DEFAULT_MAXIMUM_POOL_SIZE, + KEEP_ALIVE_TIME_SECONDS, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(WORK_QUEUE_SIZE), + runnable -> new Thread(runnable, "LocalRDG-" + nextThreadNum()), + new CallerRunsPolicy()); + } + return DEFAULT_THREAD_POOL_EXECUTOR; + } + + private static int threadInitNumber = 0; + + private static synchronized int nextThreadNum() { + return threadInitNumber++; + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/load/graph/AbstractLocalGraphLoader.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/load/graph/AbstractLocalGraphLoader.java new file mode 100644 index 00000000..72b6020a --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/load/graph/AbstractLocalGraphLoader.java @@ -0,0 +1,50 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.load.graph; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.graphstate.generator.AbstractGraphGenerator; +import java.util.List; +import java.util.Map; +import scala.Tuple2; + +public abstract class AbstractLocalGraphLoader extends AbstractGraphGenerator { + + private GraphState graphState; + + /** set graph state */ + public void setGraphState(GraphState graphState) { + this.graphState = graphState; + } + + /** load graph */ + public void load() { + List> vertexList = getVertexList(); + for (IVertex vertex : vertexList) { + this.graphState.addVertex(vertex); + } + + Map>, List>>> + edgeMap = getEdgeAggregated(); + for (IVertexId id : edgeMap.keySet()) { + List> inEdgeList = edgeMap.get(id)._1(); + List> outEdgeList = edgeMap.get(id)._2(); + this.graphState.addEdges(id, inEdgeList, outEdgeList); + } + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/loader/EmptyLocalGraphLoader.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/loader/EmptyLocalGraphLoader.java new file mode 100644 index 00000000..911c42b4 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/loader/EmptyLocalGraphLoader.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import java.util.ArrayList; +import java.util.List; + +public class EmptyLocalGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return new ArrayList<>(); + } + + @Override + public List> genEdgeList() { + return new ArrayList<>(); + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/loader/MockLocalGraphLoader.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/loader/MockLocalGraphLoader.java new file mode 100644 index 00000000..c2a5441d --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/loader/MockLocalGraphLoader.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import java.util.ArrayList; +import java.util.List; + +public class MockLocalGraphLoader extends AbstractLocalGraphLoader { + private String demoGraphStr; + + /** + * mock local graph loader + * + * @param demoGraphStr + */ + public MockLocalGraphLoader(String demoGraphStr) { + this.demoGraphStr = demoGraphStr; + } + + @Override + public String getDemoGraph() { + return this.demoGraphStr; + } + + @Override + public List> genVertexList() { + return new ArrayList<>(); + } + + @Override + public List> genEdgeList() { + return new ArrayList<>(); + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/model/LocalReasonerResult.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/model/LocalReasonerResult.java new file mode 100644 index 00000000..55a08722 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/model/LocalReasonerResult.java @@ -0,0 +1,168 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.model; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; + +public class LocalReasonerResult { + + private final boolean graphResult; + + private final String errMsg; + + // select result + private final List columns; + private final List rows; + + // ddl result + private final List> vertexList; + private final List> edgeList; + + /** + * return with err msg + * + * @param errMsg + */ + public LocalReasonerResult(String errMsg) { + this.columns = null; + this.rows = null; + this.graphResult = false; + this.vertexList = new ArrayList<>(); + this.edgeList = new ArrayList<>(); + this.errMsg = errMsg; + } + /** select result */ + public LocalReasonerResult(List columns, List rows) { + this.columns = columns; + this.rows = rows; + this.graphResult = false; + this.vertexList = null; + this.edgeList = null; + this.errMsg = ""; + } + + /** ddl result */ + public LocalReasonerResult( + List> vertexList, + List> edgeList, + boolean graphResult) { + this.columns = null; + this.rows = null; + this.graphResult = graphResult; + this.vertexList = vertexList; + this.edgeList = edgeList; + this.errMsg = ""; + } + + /** + * Getter method for property ddlResult. + * + * @return property value of ddlResult + */ + public boolean isGraphResult() { + return graphResult; + } + + /** + * Getter method for property columns. + * + * @return property value of columns + */ + public List getColumns() { + return columns; + } + + /** + * Getter method for property rows. + * + * @return property value of rows + */ + public List getRows() { + return rows; + } + + /** + * Getter method for property vertexList. + * + * @return property value of vertexList + */ + public List> getVertexList() { + return vertexList; + } + + /** + * Getter method for property edgeList. + * + * @return property value of edgeList + */ + public List> getEdgeList() { + return edgeList; + } + + @Override + public String toString() { + if (graphResult) { + return ddlToString(); + } else { + return rowsToString(); + } + } + + protected String rowsToString() { + StringBuilder sb = new StringBuilder(); + assert this.columns != null; + sb.append("columns:").append(String.join(",", this.getColumns())); + for (int i = 0; i < rows.size(); ++i) { + Object[] row = rows.get(i); + sb.append("\n"); + sb.append("(").append(i).append(") ").append(Arrays.toString(row)); + } + return sb.toString(); + } + + public String getErrMsg() { + return errMsg; + } + + protected String ddlToString() { + StringBuilder sb = new StringBuilder(); + if (CollectionUtils.isNotEmpty(this.vertexList)) { + sb.append("vertex_list_start\n"); + for (int i = 0; i < this.vertexList.size(); ++i) { + IVertex vertex = this.getVertexList().get(i); + sb.append("(").append(i).append(") ").append(vertex).append("\n"); + } + sb.append("vertex_list_end"); + } + if (CollectionUtils.isNotEmpty(this.edgeList)) { + if (sb.length() > 0) { + sb.append("\n"); + } + sb.append("edge_list_start\n"); + for (int i = 0; i < this.edgeList.size(); ++i) { + IEdge edge = this.getEdgeList().get(i); + sb.append("(").append(i).append(") ").append(edge).append("\n"); + } + sb.append("edge_list_end"); + } + return sb.toString(); + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/model/LocalReasonerTask.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/model/LocalReasonerTask.java new file mode 100644 index 00000000..5fc67901 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/model/LocalReasonerTask.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.model; + +import com.antgroup.openspg.reasoner.catalog.impl.KgSchemaConnectionInfo; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.physical.operators.PhysicalOperator; +import com.antgroup.openspg.reasoner.recorder.IExecutionRecorder; +import com.antgroup.openspg.reasoner.runner.local.impl.LocalReasonerSession; +import com.antgroup.openspg.reasoner.runner.local.rdg.LocalRDG; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import lombok.Data; +import scala.Tuple2; + +@Data +public class LocalReasonerTask implements Serializable { + /** task id */ + private static final long serialVersionUID = 8591924774057455987L; + + private String id = ""; + /** Choose between dsl or dslDagList */ + private String dsl = null; + + private List> dslDagList = null; + private LocalReasonerSession session = null; + + /** pass catalog to runner or provide schema connection info */ + private Catalog catalog = null; + + private KgSchemaConnectionInfo connInfo = null; + + /** Choose between graphLoadClass or graphState */ + private String graphLoadClass = null; + + private GraphState graphState = null; + + /** start id from input */ + private List> startIdList; + + /** + * parameters ConfigKey.KG_REASONER_MAX_PATH_LIMIT, set max path, exceeding the threshold without + * reporting an error, will truncate path data + * ConfigKey.KG_REASONER_EXCEEDING_PATH_THRESHOLD_ERROR, set strict max path limit, exceeding the + * threshold will throw an exception + */ + private Map params = new HashMap<>(); + + /** + * thread pool can be null, will use default threadpool + * [com.antgroup.openspg.reasoner.runner.local.impl.LocalRunnerThreadPool] + */ + private ThreadPoolExecutor threadPoolExecutor = null; + + /** thread pool executor timeout ms */ + private long executorTimeoutMs = 3 * 1000; + + /** execution information recorder, for debug */ + private IExecutionRecorder executionRecorder = null; +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/rdg/LocalRDG.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/rdg/LocalRDG.java new file mode 100644 index 00000000..0ab9fce1 --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/rdg/LocalRDG.java @@ -0,0 +1,1127 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.rdg; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexProperty; +import com.antgroup.openspg.reasoner.common.graph.type.GraphItemType; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.block.AddPredicate; +import com.antgroup.openspg.reasoner.lube.block.AddProperty; +import com.antgroup.openspg.reasoner.lube.block.AddVertex; +import com.antgroup.openspg.reasoner.lube.block.DDLOp; +import com.antgroup.openspg.reasoner.lube.block.SortItem; +import com.antgroup.openspg.reasoner.lube.catalog.struct.Field; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.lube.common.rule.Rule; +import com.antgroup.openspg.reasoner.lube.logical.PropertyVar; +import com.antgroup.openspg.reasoner.lube.logical.RichVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.lube.physical.planning.JoinType; +import com.antgroup.openspg.reasoner.lube.physical.rdg.RDG; +import com.antgroup.openspg.reasoner.lube.physical.rdg.Row; +import com.antgroup.openspg.reasoner.pattern.PatternMatcher; +import com.antgroup.openspg.reasoner.rdg.common.ExtractRelationImpl; +import com.antgroup.openspg.reasoner.rdg.common.ExtractVertexImpl; +import com.antgroup.openspg.reasoner.rdg.common.FoldEdgeImpl; +import com.antgroup.openspg.reasoner.rdg.common.FoldRepeatEdgeInfo; +import com.antgroup.openspg.reasoner.rdg.common.GroupByKgGraphImpl; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphAddFieldsImpl; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphAggregateImpl; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphDropFieldsImpl; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphFirstEdgeAggImpl; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphJoinImpl; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphListProcess; +import com.antgroup.openspg.reasoner.rdg.common.KgGraphSortImpl; +import com.antgroup.openspg.reasoner.rdg.common.SinkRelationImpl; +import com.antgroup.openspg.reasoner.rdg.common.UnfoldEdgeImpl; +import com.antgroup.openspg.reasoner.rdg.common.UnfoldRepeatEdgeInfo; +import com.antgroup.openspg.reasoner.recorder.EmptyRecorder; +import com.antgroup.openspg.reasoner.recorder.IExecutionRecorder; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.udf.model.UdtfMeta; +import com.antgroup.openspg.reasoner.util.KgGraphSchema; +import com.antgroup.openspg.reasoner.utils.PredicateKgGraph; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.antgroup.openspg.reasoner.warehouse.utils.WareHouseUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Predicate; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; +import scala.collection.immutable.List; +import scala.collection.immutable.Map; +import scala.collection.immutable.Set; + +@Slf4j +public class LocalRDG extends RDG { + /** graph state */ + private final GraphState graphState; + + private final String startVertexAlias; + + /** kg graph and it's schema */ + private java.util.List> kgGraphList = new ArrayList<>(); + + private PartialGraphPattern kgGraphSchema = null; + + /** executor */ + private final ThreadPoolExecutor threadPoolExecutor; + + /** executor timeout ms */ + private final long executorTimeoutMs; + + /** max path limit */ + private Long maxPathLimit = null; + + /** strict path limit */ + private Long strictMaxPathLimit = null; + + /** rdg result list */ + private final java.util.Set> resultVertexSet = new HashSet<>(); + + private final java.util.Set> resultEdgeSet = new HashSet<>(); + + private final PatternMatcher patternMatcher; + + private final String taskId; + + private final IExecutionRecorder executionRecorder; + + /** local rdg with graph state */ + public LocalRDG( + GraphState graphState, + java.util.List startIdList, + ThreadPoolExecutor threadPoolExecutor, + long executorTimeoutMs, + String startVertexAlias, + String taskId, + IExecutionRecorder executionRecorder) { + this.graphState = graphState; + Pattern startIdPattern = new NodePattern(new PatternElement(startVertexAlias, null, null)); + for (IVertexId vertexId : startIdList) { + KgGraph startId = new KgGraphImpl(); + startId.init(new Vertex<>(vertexId), null, startIdPattern); + this.kgGraphList.add(startId); + } + this.threadPoolExecutor = threadPoolExecutor; + this.executorTimeoutMs = executorTimeoutMs; + this.startVertexAlias = startVertexAlias; + this.taskId = taskId; + this.patternMatcher = new PatternMatcher(this.taskId, graphState); + + if (null == executionRecorder) { + this.executionRecorder = new EmptyRecorder(); + } else { + this.executionRecorder = executionRecorder; + } + this.executionRecorder.entryRDG(startVertexAlias); + this.executionRecorder.stageResult( + "startId(" + startVertexAlias + ")", this.kgGraphList.size()); + } + + @Override + public LocalRDG patternScan(Pattern pattern) { + long startTime = System.currentTimeMillis(); + java.util.List rootVertexRuleList = WareHouseUtils.getVertexRuleList(pattern); + java.util.Map> dstVertexRuleMap = + WareHouseUtils.getDstVertexRuleList(pattern); + java.util.Map> edgeRuleMap = + WareHouseUtils.getEdgeRuleMap(pattern); + java.util.Map> edgeTypeRuleMap = + WareHouseUtils.getEdgeTypeRuleMap(pattern); + log.info( + "LocalRDG patternScan," + + ",pattern=" + + pattern + + ",rule=" + + JSON.toJSONString(rootVertexRuleList) + + ",edgeRule=" + + JSON.toJSONString(edgeRuleMap)); + + long count = 0; + java.util.List> newKgGraphList = new ArrayList<>(); + java.util.List>> futureList = new ArrayList<>(); + patternMatcher.resetInitTime(); + for (KgGraph kgGraphId : this.kgGraphList) { + IVertexId id = kgGraphId.getVertex(this.startVertexAlias).get(0).getId(); + CompletableFuture> future = + CompletableFuture.supplyAsync( + new Supplier>() { + @Override + public KgGraph get() { + return patternMatcher.patternMatch( + id, + null, + null, + pattern, + rootVertexRuleList, + dstVertexRuleMap, + edgeRuleMap, + new HashMap<>(), + pattern.root().rule(), + edgeTypeRuleMap, + maxPathLimit, + true, + 60 * 1000); + } + }, + threadPoolExecutor); + futureList.add(future); + } + for (CompletableFuture> future : futureList) { + KgGraph kgGraph; + try { + kgGraph = future.get(this.executorTimeoutMs, TimeUnit.MILLISECONDS); + } catch (Exception e) { + throw new RuntimeException("patternScan error " + e.getMessage(), e); + } + if (null == kgGraph) { + continue; + } + if (Utils.randomLog()) { + log.info( + "LocalRDG PatternScan,kgGraph=" + + kgGraph.getVertex(pattern.root().alias()).get(0).getId()); + } + count++; + newKgGraphList.add(kgGraph); + if (null != this.strictMaxPathLimit && newKgGraphList.size() > this.strictMaxPathLimit) { + throw new RuntimeException("exceeding strict max path limit " + this.strictMaxPathLimit); + } + } + this.kgGraphList = newKgGraphList; + this.kgGraphSchema = KgGraphSchema.convert2KgGraphSchema(pattern); + + log.info( + "LocalRDG patternScan,root=" + + pattern.root() + + ",matchCount=" + + count + + " cost time=" + + (System.currentTimeMillis() - startTime)); + this.executionRecorder.stageResultWithDesc( + "patternScan(" + RunnerUtil.getReadablePattern(pattern) + ")", + this.kgGraphList.size(), + "SubPattern"); + return this; + } + + @Override + public LocalRDG linkedExpand(EdgePattern pattern) { + java.util.List> newKgGraphList = new ArrayList<>(); + UdtfMeta udtfMeta = RunnerUtil.chooseUdtfMeta(pattern); + + KgGraphSplitStaticParameters staticParameters = + new KgGraphSplitStaticParameters(null, this.kgGraphSchema); + + long count = 0; + long targetVertexSize = 0; + for (KgGraph kgGraph : this.kgGraphList) { + java.util.List> splitedKgGraphList = + RunnerUtil.linkEdge( + this.taskId, kgGraph, this.kgGraphSchema, staticParameters, pattern, udtfMeta, null); + if (CollectionUtils.isNotEmpty(splitedKgGraphList)) { + KgGraph result = new KgGraphImpl(); + result.merge(splitedKgGraphList, null); + newKgGraphList.add(result); + count++; + targetVertexSize += splitedKgGraphList.size(); + } + } + this.kgGraphList = newKgGraphList; + this.kgGraphSchema = KgGraphSchema.expandSchema(this.kgGraphSchema, pattern); + + log.info( + "LinkedExpand, funcName=" + + pattern.edge().funcName() + + ",matchCount=" + + count + + ", linkedTargetVertexSize=" + + targetVertexSize); + this.executionRecorder.stageResult( + "linkedExpand(" + RunnerUtil.getReadablePattern(pattern) + ")", this.kgGraphList.size()); + return this; + } + + private CompletableFuture>> processKgGraphWithSameRoot( + IVertexId rootId, + java.util.List> sameRootKgGraphList, + java.util.Set intersectionAliasSet, + PartialGraphPattern matchPattern, + java.util.List vertexRuleList, + java.util.Map> dstVertexRuleMap, + java.util.Map> edgeRuleMap, + java.util.Map> edgeTypeRuleMap, + PartialGraphPattern afterKgGraphSchema, + ThreadPoolExecutor threadPoolExecutor) { + + PartialGraphPattern beforeKgGraphSchema = this.kgGraphSchema; + return CompletableFuture.supplyAsync( + new Supplier>>() { + @Override + public java.util.List> get() { + if (null == rootId) { + return null; + } + java.util.List> result = new ArrayList<>(); + + java.util.Map> edgeValidTargetIdSet = new HashMap<>(); + if (CollectionUtils.isNotEmpty(intersectionAliasSet)) { + edgeValidTargetIdSet = + RunnerUtil.getEdgeAlias2ValidTargetIdMap( + intersectionAliasSet, sameRootKgGraphList, matchPattern); + } + + KgGraph matchedKgGraph = + patternMatcher.patternMatch( + rootId, + null, + null, + matchPattern, + vertexRuleList, + dstVertexRuleMap, + edgeRuleMap, + edgeValidTargetIdSet, + matchPattern.root().rule(), + edgeTypeRuleMap, + maxPathLimit, + CollectionUtils.isEmpty(intersectionAliasSet), + 60 * 1000); + if (null == matchedKgGraph) { + return null; + } + + int matchCount = + RunnerUtil.getMinVertexCount(matchedKgGraph, matchPattern.root().alias()); + + long count = 0; + for (KgGraph kgGraph : sameRootKgGraphList) { + // expend and prune by intersection vertex + if (CollectionUtils.isEmpty(intersectionAliasSet)) { + kgGraph.expand(matchedKgGraph, afterKgGraphSchema); + } else { + matchCount = + kgGraph.expandAndPrune( + beforeKgGraphSchema, + matchedKgGraph, + matchPattern, + afterKgGraphSchema, + intersectionAliasSet); + if (matchCount <= 0) { + continue; + } + } + if (kgGraph.checkDuplicateVertex()) { + continue; + } + result.add(kgGraph); + count += matchCount; + if (null != maxPathLimit && count >= maxPathLimit) { + log.warn( + "ExpandInto,pathLimit,count=" + + count + + ",matchCount=" + + matchCount + + ",id=" + + rootId); + break; + } + } + return result; + } + }, + threadPoolExecutor); + } + + @Override + public LocalRDG expandInto(PatternElement target, Pattern pattern) { + log.info("ExpandInto __max_path_size__ = " + maxPathLimit); + long startTime = System.currentTimeMillis(); + + String rootAlias = pattern.root().alias(); + shuffleAndGroup(rootAlias, false); + + java.util.List vertexRuleList = WareHouseUtils.getVertexRuleList(pattern); + java.util.Map> dstVertexRuleMap = + WareHouseUtils.getDstVertexRuleList(pattern); + java.util.Map> edgeRuleMap = + WareHouseUtils.getEdgeRuleMap(pattern); + java.util.Map> edgeTypeRuleMap = + WareHouseUtils.getEdgeTypeRuleMap(pattern); + + log.info( + "LocalRDG expandInto," + + ",pattern=" + + pattern + + ",rule=" + + JSON.toJSONString(vertexRuleList) + + ",edgeRule=" + + JSON.toJSONString(edgeRuleMap)); + + PartialGraphPattern afterKgGraphSchema = + KgGraphSchema.expandSchema(this.kgGraphSchema, pattern); + + PartialGraphPattern matchPattern = KgGraphSchema.convert2KgGraphSchema(pattern); + java.util.Set intersectionAliasSet = + RunnerUtil.getIntersectionAliasSet(this.kgGraphSchema, matchPattern); + + java.util.List>>> futureList = + new ArrayList<>(); + + java.util.List> sameRootKgGraphList = new ArrayList<>(); + IVertexId lastVertexId = null; + patternMatcher.resetInitTime(); + for (KgGraph value : this.kgGraphList) { + IVertex vertex = value.getVertex(matchPattern.root().alias()).get(0); + if (vertex.getId().equals(lastVertexId)) { + sameRootKgGraphList.add(value); + } else { + futureList.add( + processKgGraphWithSameRoot( + lastVertexId, + sameRootKgGraphList, + intersectionAliasSet, + matchPattern, + vertexRuleList, + dstVertexRuleMap, + edgeRuleMap, + edgeTypeRuleMap, + afterKgGraphSchema, + threadPoolExecutor)); + sameRootKgGraphList = new ArrayList<>(); + sameRootKgGraphList.add(value); + lastVertexId = vertex.getId(); + } + } + if (null != lastVertexId) { + futureList.add( + processKgGraphWithSameRoot( + lastVertexId, + sameRootKgGraphList, + intersectionAliasSet, + matchPattern, + vertexRuleList, + dstVertexRuleMap, + edgeRuleMap, + edgeTypeRuleMap, + afterKgGraphSchema, + threadPoolExecutor)); + } + + long count = 0; + java.util.List> newKgGraphList = new ArrayList<>(); + for (CompletableFuture>> future : futureList) { + java.util.List> resultKgGraph; + try { + resultKgGraph = future.get(this.executorTimeoutMs, TimeUnit.MILLISECONDS); + } catch (Exception e) { + throw new RuntimeException("expandInto error", e); + } + + if (null == resultKgGraph) { + continue; + } + count += resultKgGraph.size(); + newKgGraphList.addAll(resultKgGraph); + if (null != this.strictMaxPathLimit && newKgGraphList.size() > this.strictMaxPathLimit) { + throw new RuntimeException("exceeding strict max path limit " + this.strictMaxPathLimit); + } + } + this.kgGraphSchema = afterKgGraphSchema; + this.kgGraphList = newKgGraphList; + log.info( + "LocalRDG ExpandInto,patternRoot=" + + pattern.root() + + ",matchCount=" + + count + + " cost time=" + + (System.currentTimeMillis() - startTime)); + this.executionRecorder.stageResult( + "expandInto(" + RunnerUtil.getReadablePattern(pattern) + ")", this.kgGraphList.size()); + return this; + } + + @Override + public Row select(List cols, List as) { + java.util.List crossBorderRuleList = new ArrayList<>(); + java.util.List columns = Lists.newArrayList(JavaConversions.asJavaCollection(cols)); + + KgGraphSplitStaticParameters staticParameters = + new KgGraphSplitStaticParameters(null, this.kgGraphSchema); + java.util.Map initRuleContext = + RunnerUtil.getKgGraphInitContext(this.kgGraphSchema); + + java.util.List rows = new ArrayList<>(); + for (KgGraph kgGraph : this.kgGraphList) { + Predicate> filter = null; + if (CollectionUtils.isNotEmpty(crossBorderRuleList)) { + filter = new PredicateKgGraph(this.kgGraphSchema, crossBorderRuleList); + } + Iterator> pathIt = kgGraph.getPath(staticParameters, filter); + long count = 0; + while (pathIt.hasNext()) { + KgGraph path = pathIt.next(); + if (null == path) { + continue; + } + java.util.Map context = RunnerUtil.kgGraph2Context(initRuleContext, path); + java.util.Map flattenContext = RunnerUtil.flattenContext(context); + Object[] row = new Object[columns.size()]; + for (int i = 0; i < columns.size(); ++i) { + Var var = columns.get(i); + PropertyVar propertyVar = (PropertyVar) var; + String key = + propertyVar.name() + RunnerUtil.FLATTEN_SEPARATOR + propertyVar.field().name(); + row[i] = Utils.objValue2Str(flattenContext.get(key)); + } + rows.add(row); + count++; + if (null != maxPathLimit && count >= maxPathLimit) { + break; + } + } + } + log.info("LocalRDG select,,matchCount=" + rows.size()); + this.executionRecorder.stageResultWithDesc( + "select(" + RunnerUtil.getReadableAsList(as) + ")", this.kgGraphList.size(), "select"); + return new LocalRow(cols, this, as, rows); + } + + @Override + public LocalRDG filter(Rule rule) { + Tuple2, java.util.Set> tuple2 = + RunnerUtil.getRuleUseVertexAndEdgeSet(rule, this.kgGraphSchema); + java.util.Set vertexSet = new HashSet<>(tuple2._1()); + for (Connection pc : tuple2._2()) { + vertexSet.add(pc.source()); + vertexSet.add(pc.target()); + } + + java.util.List exprStringSet = WareHouseUtils.getRuleList(rule); + + KgGraphSplitStaticParameters staticParameters = + new KgGraphSplitStaticParameters(vertexSet, this.kgGraphSchema); + java.util.List> newKgGraphList = new ArrayList<>(); + long count = 0; + for (KgGraph kgGraph : this.kgGraphList) { + java.util.List> resultList = + RunnerUtil.filterKgGraph( + kgGraph, + vertexSet, + this.kgGraphSchema, + staticParameters, + exprStringSet, + this.maxPathLimit); + count += resultList.size(); + newKgGraphList.addAll(resultList); + } + this.kgGraphList = newKgGraphList; + log.info("Filter,rule=" + exprStringSet + ",matchCount=" + count); + this.executionRecorder.stageResult("filter(" + exprStringSet + ")", this.kgGraphList.size()); + return this; + } + + private java.util.List convertGroupByVar2AliasSet(List by) { + java.util.List byAliasList = new ArrayList<>(by.size()); + JavaConversions.seqAsJavaList(by.toSeq()).forEach(var -> byAliasList.add(var.name())); + return byAliasList; + } + + private void groupByVariableThenAggregate( + java.util.List byAliasList, KgGraphListProcess kgGraphListProcess) { + boolean isGlobalTopK = byAliasList.isEmpty(); + if (!byAliasList.isEmpty()) { + if (!byAliasList.contains(this.kgGraphSchema.rootAlias())) { + // KgGraph not key by expected alias + String grouByAlias = byAliasList.remove(0); + shuffleAndGroup(grouByAlias, false); + } else { + String nowRootAlias = this.kgGraphSchema.rootAlias(); + byAliasList.remove(nowRootAlias); + } + } + + String nowRootAlias = this.kgGraphSchema.rootAlias(); + + if (byAliasList.isEmpty() && null == kgGraphListProcess) { + this.kgGraphList = doMerge(this.kgGraphList, this.kgGraphSchema); + } else { + java.util.List> newKgGraphList = new ArrayList<>(); + GroupByKgGraphImpl impl = + new GroupByKgGraphImpl( + byAliasList, kgGraphListProcess, this.kgGraphSchema, this.maxPathLimit); + java.util.List> sameRootKgGraphList = new ArrayList<>(); + IVertexId lastKgGraphId = null; + if (isGlobalTopK) { + java.util.List> kgGraphList = impl.groupReduce(this.kgGraphList); + newKgGraphList.addAll(kgGraphList); + } else { + for (KgGraph kgGraph : this.kgGraphList) { + IVertexId nowVertexId = kgGraph.getVertex(nowRootAlias).get(0).getId(); + if (nowVertexId.equals(lastKgGraphId)) { + sameRootKgGraphList.add(kgGraph); + } else { + if (null != lastKgGraphId) { + java.util.List> kgGraphList = + impl.groupReduce(sameRootKgGraphList); + newKgGraphList.addAll(kgGraphList); + } + + sameRootKgGraphList = new ArrayList<>(); + sameRootKgGraphList.add(kgGraph); + lastKgGraphId = nowVertexId; + } + } + if (null != lastKgGraphId) { + java.util.List> kgGraphList = impl.groupReduce(sameRootKgGraphList); + newKgGraphList.addAll(kgGraphList); + } + } + + this.kgGraphList = newKgGraphList; + } + } + + @Override + public LocalRDG orderBy(List groupKey, List sortItems, int limit) { + KgGraphSortImpl impl = new KgGraphSortImpl(groupKey, sortItems, this.kgGraphSchema, limit); + this.kgGraphList = impl.reduce(this.kgGraphList); + this.executionRecorder.stageResult( + "orderBy(" + RunnerUtil.getReadableByKey(groupKey) + ").limit(" + limit + ")", + this.kgGraphList.size()); + return this; + } + + @Override + public LocalRDG groupBy(List by, Map aggregations) { + java.util.List byAliasList = convertGroupByVar2AliasSet(by); + // agg first edge + java.util.List firstEdgeAliasList = RunnerUtil.getFirstEdgeAliasList(aggregations); + if (null != firstEdgeAliasList && byAliasList.contains(this.kgGraphSchema.rootAlias())) { + return aggregateFirstEdge(firstEdgeAliasList); + } + + KgGraphListProcess kgGraphListProcess; + java.util.Map aggregatorMap; + if (null == aggregations) { + // no aggregator, only merge + kgGraphListProcess = + new KgGraphListProcess() { + @Override + public java.util.List> reduce( + Collection> kgGraphs) { + KgGraphImpl kgGraph = new KgGraphImpl(); + assert kgGraphs instanceof java.util.List; + kgGraph.merge((java.util.List>) kgGraphs, kgGraphSchema); + return Lists.newArrayList(kgGraph); + } + }; + } else { + // do aggregate action + aggregatorMap = JavaConversions.mapAsJavaMap(aggregations); + KgGraphAggregateImpl impl = + new KgGraphAggregateImpl( + this.taskId, + byAliasList.get(0), + byAliasList, + this.kgGraphSchema, + aggregatorMap, + this.maxPathLimit); + impl.init(); + kgGraphListProcess = + new KgGraphListProcess() { + @Override + public java.util.List> reduce( + Collection> kgGraphs) { + return Lists.newArrayList(impl.map(kgGraphs)); + } + }; + } + + groupByVariableThenAggregate(byAliasList, kgGraphListProcess); + this.executionRecorder.stageResult( + "groupBy(" + RunnerUtil.getReadableByKey(by) + ")", this.kgGraphList.size()); + return this; + } + + private LocalRDG aggregateFirstEdge(java.util.List firstEdgeAliasList) { + KgGraphFirstEdgeAggImpl impl = new KgGraphFirstEdgeAggImpl(firstEdgeAliasList); + java.util.List> newKgGraphList = new ArrayList<>(); + for (KgGraph kgGraph : this.kgGraphList) { + newKgGraphList.add(impl.map(kgGraph)); + } + this.kgGraphList = newKgGraphList; + return this; + } + + private java.util.List> doMerge( + java.util.List> kgGraphList, PartialGraphPattern kgGraphSchema) { + java.util.Map> mergeMap = new java.util.TreeMap<>(); + for (KgGraph kgGraph : kgGraphList) { + IVertexId id = kgGraph.getVertex(kgGraphSchema.rootAlias()).get(0).getId(); + KgGraph idKgGraph = mergeMap.computeIfAbsent(id, k -> new KgGraphImpl()); + idKgGraph.merge(Lists.newArrayList(kgGraph), kgGraphSchema); + } + return Lists.newArrayList(mergeMap.values()); + } + + private java.util.Map>> groupByAlias( + String vertexAlias) { + KgGraphSplitStaticParameters staticParameters = + new KgGraphSplitStaticParameters(Sets.newHashSet(vertexAlias), this.kgGraphSchema); + java.util.Map>> mergeMap = + new java.util.TreeMap<>(); + for (KgGraph kgGraph : this.kgGraphList) { + java.util.List> splitList = + kgGraph.split( + Sets.newHashSet(vertexAlias), + this.kgGraphSchema, + staticParameters, + null, + this.maxPathLimit); + for (KgGraph kgGraphSpited : splitList) { + IVertexId id = kgGraphSpited.getVertex(vertexAlias).get(0).getId(); + java.util.List> sameIdKgGraph = + mergeMap.computeIfAbsent(id, k -> new ArrayList<>()); + sameIdKgGraph.add(kgGraphSpited); + } + } + return mergeMap; + } + + private void shuffleAndGroup(String vertexAlias, boolean merge) { + if (vertexAlias.equals(this.kgGraphSchema.rootAlias())) { + // The current root Alias of KgGraph is the same as the root Alias that requires shuffling; no + // action is needed. + return; + } + + this.kgGraphSchema = KgGraphSchema.schemaChangeRoot(this.kgGraphSchema, vertexAlias); + java.util.Map>> mergeMap = + groupByAlias(vertexAlias); + + java.util.List> newKgGraphList = new ArrayList<>(); + for (java.util.List> kgGraphs : mergeMap.values()) { + KgGraph mergedKgGraph = new KgGraphImpl(); + if (merge) { + mergedKgGraph.merge(kgGraphs, this.kgGraphSchema); + newKgGraphList.add(mergedKgGraph); + } else { + newKgGraphList.addAll(kgGraphs); + } + } + this.kgGraphList = newKgGraphList; + } + + @Override + public LocalRDG addFields(Map fields) { + java.util.Map> addFieldsInfo = new HashMap<>(); + JavaConversions.mapAsJavaMap(fields) + .forEach( + new BiConsumer() { + @Override + public void accept(Var var, Expr expr) { + addFieldsInfo.put(var, WareHouseUtils.getRuleList(expr)); + } + }); + + KgGraphAddFieldsImpl kgGraphAddFields = + new KgGraphAddFieldsImpl(addFieldsInfo, this.kgGraphSchema, 0, this.taskId); + java.util.List> newKgGraphList = new ArrayList<>(); + for (KgGraph kgGraph : this.kgGraphList) { + newKgGraphList.addAll(kgGraphAddFields.map(kgGraph)); + } + this.kgGraphList = newKgGraphList; + this.executionRecorder.stageResult( + "addFields(" + RunnerUtil.getReadableAddFields(addFieldsInfo) + ")", + this.kgGraphList.size()); + return this; + } + + @Override + public LocalRDG dropFields(Set fields) { + java.util.Set dropFieldSet = new HashSet<>(JavaConversions.asJavaCollection(fields)); + if (CollectionUtils.isEmpty(dropFieldSet)) { + return this; + } + + KgGraphDropFieldsImpl impl = new KgGraphDropFieldsImpl(dropFieldSet); + for (KgGraph kgGraph : this.kgGraphList) { + impl.doDropFields(kgGraph); + } + return this; + } + + @Override + public LocalRDG limit(long n) { + if (this.kgGraphList.size() > n) { + this.kgGraphList.subList(0, (int) n); + } + return this; + } + + @Override + public void show(int rows) { + log.info("###############LocalRDGShowStart#############"); + try { + if (null == this.kgGraphList) { + log.info("null"); + return; + } + this.limit(rows); + java.util.List> result = this.kgGraphList; + if (result.size() > rows) { + result = Lists.newArrayList(result.subList(0, rows)); + } + for (KgGraph kgGraph : result) { + kgGraph.show(); + } + } finally { + log.info("###############LocalRDGShowEnd###############"); + } + } + + @Override + public LocalRDG ddl(List ddlOps) { + KgGraphSplitStaticParameters staticParameters = + new KgGraphSplitStaticParameters(null, this.kgGraphSchema); + // convert to path + java.util.List> newKgGraphList = new ArrayList<>(); + for (KgGraph kgGraph : this.kgGraphList) { + Iterator> it = kgGraph.getPath(staticParameters, null); + while (it.hasNext()) { + KgGraph path = it.next(); + if (null == path) { + continue; + } + newKgGraphList.add(path); + } + } + this.kgGraphList = newKgGraphList; + + java.util.List ddlOpList = Lists.newArrayList(JavaConversions.asJavaCollection(ddlOps)); + for (DDLOp ddlOp : ddlOpList) { + if (ddlOp instanceof AddProperty) { + AddProperty addProperty = (AddProperty) ddlOp; + addProperty( + addProperty.s().alias(), + new Field(addProperty.propertyName(), addProperty.propertyType(), true)); + } else if (ddlOp instanceof AddPredicate) { + AddPredicate addPredicate = (AddPredicate) ddlOp; + addRelation(addPredicate); + } else if (ddlOp instanceof AddVertex) { + AddVertex addVertex = (AddVertex) ddlOp; + addVertex(addVertex); + } + } + this.kgGraphList = null; + return this; + } + + private void addProperty(String alias, Field property) { + java.util.Map alias2TypeMap = + new HashMap<>(JavaConversions.mapAsJavaMap(KgGraphSchema.alias2Type(this.kgGraphSchema))); + GraphItemType type = alias2TypeMap.get(alias); + long count = 0; + if (!GraphItemType.VERTEX.equals(type)) { + throw new NotImplementedException("only support add property on vertex", null); + } + + // check now schema is root by vertex alias + if (!alias.equals(this.kgGraphSchema.rootAlias())) { + // need shuffle KgGraph + shuffleAndGroup(alias, true); + } else { + this.kgGraphList = doMerge(this.kgGraphList, this.kgGraphSchema); + } + + for (KgGraph kgGraph : this.kgGraphList) { + IVertex vertex = kgGraph.getVertex(alias).get(0); + Object value = vertex.getValue().get(property.name()); + java.util.Map propertyMap = new HashMap<>(); + propertyMap.put(property.name(), value); + count++; + graphState.mergeVertexProperty(vertex.getId(), propertyMap, MergeTypeEnum.REPLACE, 0L); + + // add to result list + this.resultVertexSet.add( + new Vertex<>(vertex.getId(), new VertexProperty(new HashMap<>(propertyMap)))); + } + + this.executionRecorder.stageResultWithDesc( + "addProperty(" + alias + "." + property.toString() + ")", + count, + "addProperty(" + alias + "." + property.toString() + ")"); + log.info( + "LocalRDG.addProperty,sinkCount=" + + count + + " addProperty=" + + alias + + "." + + property.toString()); + } + + private void addRelation(AddPredicate addPredicate) { + ExtractRelationImpl impl = + new ExtractRelationImpl(addPredicate, this.kgGraphSchema, 0, this.taskId); + java.util.Set> allEdgeSet = new HashSet<>(); + for (KgGraph kgGraph : this.kgGraphList) { + IEdge edge = impl.extractEdge(kgGraph); + allEdgeSet.add(edge); + + // add to result list + this.resultEdgeSet.add(edge); + + if (impl.withReverseEdge()) { + // 反向边 + allEdgeSet.add(impl.createReverseEdge(edge)); + } + } + + SinkRelationImpl sinkImpl = new SinkRelationImpl(this.graphState, 0); + long sinkCount = sinkImpl.sink(allEdgeSet); + this.executionRecorder.stageResultWithDesc( + "addRelation(" + RunnerUtil.getReadableAddPredicate(addPredicate) + ")", + sinkCount, + "addRelation(" + addPredicate.predicate().label() + ")"); + log.info( + "LocalRDG.addRelation,sinkCount=" + sinkCount + " addPredicate=" + addPredicate.toString()); + } + + private void addVertex(AddVertex addVertex) { + ExtractVertexImpl impl = new ExtractVertexImpl(addVertex, this.kgGraphSchema, 0, this.taskId); + + long count = 0; + for (KgGraph kgGraph : this.kgGraphList) { + IVertex willAddedVertex = impl.extractVertex(kgGraph); + + // add to result list + this.resultVertexSet.add(willAddedVertex); + count++; + } + + this.executionRecorder.stageResultWithDesc( + "addVertex(" + RunnerUtil.getReadableAddVertex(addVertex) + ")", count, "addVertex"); + log.info("LocalRDG.addVertex,sinkCount=" + count); + } + + @Override + public LocalRDG cache() { + // local runner do nothing + return this; + } + + @Override + public LocalRDG join( + LocalRDG other, + JoinType joinType, + List> onAlias, + Map lhsSchemaMapping, + Map rhsSchemaMapping) { + try { + if (null == other.getKgGraphList()) { + // other KgGraph is ddl result, nothing to join + return this; + } + + java.util.List leftJoinAliasList = new ArrayList<>(); + java.util.List rightJoinAliasList = new ArrayList<>(); + for (Tuple2 tuple2 : JavaConversions.seqAsJavaList(onAlias)) { + leftJoinAliasList.add(tuple2._1()); + rightJoinAliasList.add(tuple2._2()); + } + + String rightJoinAlias = rightJoinAliasList.get(0); + java.util.Map>> rightDataMap = + new HashMap<>(); + other.groupByVariableThenAggregate( + rightJoinAliasList, + new KgGraphListProcess() { + @Override + public java.util.List> reduce( + Collection> kgGraphs) { + IVertexId id = kgGraphs.iterator().next().getVertex(rightJoinAlias).get(0).getId(); + rightDataMap.put(id, kgGraphs); + return new ArrayList<>(); + } + }); + other.kgGraphSchema = KgGraphSchema.schemaChangeRoot(other.kgGraphSchema, rightJoinAlias); + + String leftJoinAlias = leftJoinAliasList.get(0); + java.util.Map>> leftDataMap = + new HashMap<>(); + this.groupByVariableThenAggregate( + leftJoinAliasList, + new KgGraphListProcess() { + @Override + public java.util.List> reduce( + Collection> kgGraphs) { + IVertexId id = kgGraphs.iterator().next().getVertex(leftJoinAlias).get(0).getId(); + leftDataMap.put(id, kgGraphs); + return new ArrayList<>(); + } + }); + this.kgGraphSchema = KgGraphSchema.schemaChangeRoot(this.kgGraphSchema, leftJoinAlias); + + java.util.List> newKgGraphList = new ArrayList<>(); + KgGraphJoinImpl joinImpl = + new KgGraphJoinImpl( + joinType, + onAlias, + lhsSchemaMapping, + rhsSchemaMapping, + other.kgGraphSchema, + this.maxPathLimit); + + Iterator>>> it = + leftDataMap.entrySet().iterator(); + while (it.hasNext()) { + java.util.Map.Entry>> entry = it.next(); + IVertexId id = entry.getKey(); + java.util.Collection> left = entry.getValue(); + it.remove(); + java.util.Collection> right = rightDataMap.remove(id); + java.util.List> rst = joinImpl.join(left, right); + newKgGraphList.addAll(rst); + } + this.kgGraphList = newKgGraphList; + this.kgGraphSchema = + RunnerUtil.getAfterJoinSchema( + this.kgGraphSchema, other.kgGraphSchema, onAlias, rhsSchemaMapping); + } finally { + other.clear(); + this.executionRecorder.leaveRDG(); + this.executionRecorder.stageResult( + "join(" + RunnerUtil.getReadableJoinString(onAlias, rhsSchemaMapping) + ")", + this.kgGraphList.size()); + } + return this; + } + + @Override + public LocalRDG unfold(List>> mapping) { + UnfoldRepeatEdgeInfo unfoldRepeatEdgeInfo = + RunnerUtil.getUnfoldEdgeInfo(mapping, this.kgGraphSchema); + UnfoldEdgeImpl impl = new UnfoldEdgeImpl(unfoldRepeatEdgeInfo); + java.util.List> newKgGraphList = new ArrayList<>(); + for (KgGraph kgGraph : this.kgGraphList) { + java.util.List> rst = impl.unfold((KgGraphImpl) kgGraph); + newKgGraphList.addAll(rst); + } + this.kgGraphList = newKgGraphList; + this.kgGraphSchema = KgGraphSchema.schemaChangeRoot(this.kgGraphSchema, null); + return this; + } + + @Override + public LocalRDG fold(List, RichVar>> foldMapping) { + FoldRepeatEdgeInfo foldRepeatEdgeInfo = + RunnerUtil.getFoldRepeatEdgeInfo(foldMapping, this.kgGraphSchema); + this.kgGraphSchema = KgGraphSchema.foldPathEdgeSchema(this.kgGraphSchema, foldRepeatEdgeInfo); + + FoldEdgeImpl impl = new FoldEdgeImpl(kgGraphSchema, foldRepeatEdgeInfo); + java.util.List> newKgGraphList = new ArrayList<>(); + for (KgGraph kgGraph : this.kgGraphList) { + java.util.List> rst = impl.fold((KgGraphImpl) kgGraph); + newKgGraphList.addAll(rst); + } + this.kgGraphList = newKgGraphList; + return this; + } + + private void clear() { + this.kgGraphList = null; + this.kgGraphSchema = null; + this.resultVertexSet.clear(); + this.resultEdgeSet.clear(); + } + + // use for sub rdg + public java.util.List> getKgGraphList() { + return this.kgGraphList; + } + + public PartialGraphPattern getKgGraphSchema() { + return this.kgGraphSchema; + } + + /** get ddl result */ + public LocalReasonerResult getResult() { + return new LocalReasonerResult( + Lists.newArrayList(resultVertexSet), Lists.newArrayList(resultEdgeSet), true); + } + + /** + * get all RDG Edges and Nodes + * + * @return + */ + public LocalReasonerResult getRDGGraph() { + LocalReasonerResult localReasonerResult = this.getResult(); + this.kgGraphList.forEach( + graph -> { + for (String alias : graph.getVertexAlias()) { + localReasonerResult.getVertexList().addAll(graph.getVertex(alias)); + } + for (String alias : graph.getEdgeAlias()) { + localReasonerResult.getEdgeList().addAll(graph.getEdge(alias)); + } + }); + return localReasonerResult; + } + + /** + * Setter method for property maxPathLimit. + * + * @param maxPathLimit value to be assigned to property maxPathLimit + */ + public void setMaxPathLimit(Long maxPathLimit) { + this.maxPathLimit = maxPathLimit; + } + + /** + * Setter method for property strictMaxPathLimit. + * + * @param strictMaxPathLimit value to be assigned to property strictMaxPathLimit + */ + public void setStrictMaxPathLimit(Long strictMaxPathLimit) { + this.strictMaxPathLimit = strictMaxPathLimit; + } +} diff --git a/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/rdg/LocalRow.java b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/rdg/LocalRow.java new file mode 100644 index 00000000..5f78a41b --- /dev/null +++ b/reasoner/runner/local-runner/src/main/java/com/antgroup/openspg/reasoner/runner/local/rdg/LocalRow.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.rdg; + +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.lube.physical.rdg.Row; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import scala.collection.JavaConversions; + +@Slf4j +public class LocalRow extends Row { + private final List columns; + private final List rowList; + + /** row implement */ + public LocalRow( + scala.collection.immutable.List orderedFields, + LocalRDG rdg, + scala.collection.immutable.List as, + List rows) { + super(orderedFields, rdg); + this.columns = new ArrayList<>(); + this.columns.addAll(Lists.newArrayList(JavaConversions.asJavaCollection(as))); + this.rowList = rows; + } + + @Override + public void show(int rows) { + log.info("###############GeaflowRowShow###############"); + for (int i = 0; i < rowList.size(); ++i) { + Object[] row = rowList.get(i); + log.info("(" + i + ") " + Arrays.toString(row)); + } + log.info("###############GeaflowRowShow###############"); + } + + /** get select result */ + public LocalReasonerResult getResult() { + return new LocalReasonerResult(columns, rowList); + } +} diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobCanSubmitQuery.java b/reasoner/runner/local-runner/src/main/scala/com/antgroup/openspg/reasoner/runner/local/rdg/TypeTags.scala similarity index 72% rename from server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobCanSubmitQuery.java rename to reasoner/runner/local-runner/src/main/scala/com/antgroup/openspg/reasoner/runner/local/rdg/TypeTags.scala index eaab6772..f611ee9a 100644 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobCanSubmitQuery.java +++ b/reasoner/runner/local-runner/src/main/scala/com/antgroup/openspg/reasoner/runner/local/rdg/TypeTags.scala @@ -11,8 +11,10 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.computing.cmd; +package com.antgroup.openspg.reasoner.runner.local.rdg -import com.antgroup.openspg.server.common.model.base.BaseQuery; +import scala.reflect.runtime.universe -public class ReasonerJobCanSubmitQuery extends BaseQuery {} +object TypeTags { + val rdgTypeTag: universe.TypeTag[LocalRDG] = universe.typeTag[LocalRDG] +} diff --git a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/model/ObjectStorePath.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalPropertyGraphTest.java similarity index 55% rename from cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/model/ObjectStorePath.java rename to reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalPropertyGraphTest.java index e2e599f4..e5162ed2 100644 --- a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/model/ObjectStorePath.java +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalPropertyGraphTest.java @@ -11,20 +11,15 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.objectstore.model; +package com.antgroup.openspg.reasoner.runner.local; -import com.antgroup.openspg.server.common.model.base.BaseValObj; -import lombok.Getter; -import lombok.Setter; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.runner.local.impl.LocalPropertyGraph; +import org.junit.Test; -@Getter -public class ObjectStorePath extends BaseValObj { - - private final String relativePath; - - @Setter private String absolutePath; - - public ObjectStorePath(String relativePath) { - this.relativePath = relativePath; +public class LocalPropertyGraphTest { + @Test + public void localPropertyGraphTest() { + LocalPropertyGraph localPropertyGraph = new LocalPropertyGraph(new MemGraphState()); } } diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalRDGTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalRDGTest.java new file mode 100644 index 00000000..19bfc86a --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalRDGTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.lube.common.rule.LogicRule; +import com.antgroup.openspg.reasoner.parser.expr.RuleExprParser; +import com.antgroup.openspg.reasoner.runner.local.impl.LocalRunnerThreadPool; +import com.antgroup.openspg.reasoner.runner.local.rdg.LocalRDG; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Test; +import scala.collection.JavaConversions; + +public class LocalRDGTest { + + @Test + public void localRDGTest() { + LocalRDG localRDG = + new LocalRDG( + new MemGraphState(), + new ArrayList<>(), + LocalRunnerThreadPool.getThreadPoolExecutor(null), + 1000, + "", + "", + null); + localRDG.limit(1); + localRDG.show(10); + } + + @Test + public void testFilter() throws Exception { + LocalRDG localRDG = + new LocalRDG( + new MemGraphState(), + new ArrayList<>(), + LocalRunnerThreadPool.getThreadPoolExecutor(null), + 1000, + "", + "", + null); + + List> kgGraphList = new ArrayList<>(); + Map>> alias2VertexMap = new HashMap<>(); + alias2VertexMap.put( + "A", + Sets.newHashSet( + new Vertex<>(IVertexId.from(0L, "type1")), new Vertex<>(IVertexId.from(1L, "type1")))); + Map>> alias2EdgeMap = new HashMap<>(); + KgGraphImpl kgGraph = new KgGraphImpl(alias2VertexMap, alias2EdgeMap); + kgGraphList.add(kgGraph); + + Field kgGraphField = LocalRDG.class.getDeclaredField("kgGraphList"); + kgGraphField.setAccessible(true); + kgGraphField.set(localRDG, kgGraphList); + + Field schemaField = LocalRDG.class.getDeclaredField("kgGraphSchema"); + schemaField.setAccessible(true); + schemaField.set(localRDG, loadPattern()); + + localRDG.setMaxPathLimit(1L); + RuleExprParser ruleParser = new RuleExprParser(); + localRDG.filter(new LogicRule("R1", "", ruleParser.parse("19 > 18"))); + + List> kgGraphListAfterFilter = + (List>) kgGraphField.get(localRDG); + Assert.assertEquals(1, kgGraphListAfterFilter.size()); + } + + public Pattern loadPattern() { + PatternElement A = new PatternElement("A", null, null); + Map> topology = new HashMap<>(); + Pattern schema = + new PartialGraphPattern( + A.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + return schema; + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalReasonerResultTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalReasonerResultTest.java new file mode 100644 index 00000000..7f09356e --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalReasonerResultTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.google.common.collect.Lists; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; + +public class LocalReasonerResultTest { + + @Test + public void localReasonerResultTest() { + java.util.List> vertexList = + Lists.newArrayList( + new Vertex<>(IVertexId.from("id1", "type1"), new VertexProperty()), + new Vertex<>(IVertexId.from("id2", "type1"), new VertexProperty())); + List> edgeList = + Lists.newArrayList( + new Edge<>( + IVertexId.from("id1", "type1"), + IVertexId.from("id2", "type1"), + new EdgeProperty(), + Direction.OUT)); + boolean ddlResult = true; + LocalReasonerResult localReasonerResult = + new LocalReasonerResult(vertexList, edgeList, ddlResult); + Assert.assertTrue(localReasonerResult.isGraphResult()); + + System.out.println(localReasonerResult); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalRunnerTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalRunnerTest.java new file mode 100644 index 00000000..62b2872d --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/LocalRunnerTest.java @@ -0,0 +1,932 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.utils.PropertyUtil; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.progress.ProgressReport; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.antgroup.openspg.reasoner.utils.SimpleObjSerde; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import scala.Tuple2; + +public class LocalRunnerTest { + + @Test + public void doTestAvg2() { + String dsl = + "Define (s: AttributePOC.User)-[p: holdPMProduct]->(o: AttributePOC.PortfolioManager) {\n" + + " GraphStructure {\n" + + "\t(s)-[p1:holdProduct]->(u1:AttributePOC.Fund)\n" + + " (o)-[p2:underControl]->(u1)\n" + + " }\n" + + " Rule {\n" + + " }\n" + + "}"; + dsl = + dsl + + "GraphStructure {\n" + + "(s:AttributePOC.User)-[p:followPM]->(o:AttributePOC.PortfolioManager),\n" + + "(s)-[p2:holdPMProduct]->(o:AttributePOC.PortfolioManager)\n" + + " }\n" + + " Rule {\n" + + "\n" + + " }\n" + + "Action {\n" + + " get(s.id, p.times, p.holdRet) // 获取A和B两个导演的名字,调试和在线执行时直接看渲染出来的子图\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass("com.antgroup.openspg.reasoner.runner.local.loader.TestFundGraphLoader"); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("u1", "AttributePOC.User"))); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("AttributePOC.User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.PortfolioManager", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("AttributePOC.Fund", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.User_holdProduct_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + schema.put( + "AttributePOC.PortfolioManager_underControl_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + schema.put( + "AttributePOC.User_followPM_AttributePOC.PortfolioManager", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet", "times"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "4"); + clear(); + } + + @Test + public void doTestAvg() { + String dsl = + "Define (s: AttributePOC.User)-[p: holdPMProduct2]->(o: AttributePOC.PortfolioManager) {\n" + + " GraphStructure {\n" + + "\t(s)-[p1:holdProduct]->(u1:AttributePOC.Fund)<-[p2:underControl]-(o)\n" + + " }\n" + + " Rule {\n" + + " avgProfit = group(s,o).avg(p1.holdRet)\n" + + " p.avgProfit = avgProfit\n" + + " }\n" + + "}"; + dsl += + "GraphStructure {\n" + + " (s: AttributePOC.User)-[p: holdPMProduct2]->(o: AttributePOC.PortfolioManager)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, p.avgProfit)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass("com.antgroup.openspg.reasoner.runner.local.loader.TestFundGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("AttributePOC.User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.PortfolioManager", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("AttributePOC.Fund", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.User_holdProduct_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + schema.put( + "AttributePOC.PortfolioManager_underControl_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "0.02"); + clear(); + } + + @Test + public void doTestMin() { + String dsl = + "Define (s: AttributePOC.User)-[p: holdPMProduct2]->(o: AttributePOC.PortfolioManager) {\n" + + " GraphStructure {\n" + + "\t(s)-[p1:holdProduct]->(u1:AttributePOC.Fund)<-[p2:underControl]-(o)\n" + + " }\n" + + " Rule {\n" + + " avgProfit = group(s,o).min(p1.holdRet)\n" + + " p.avgProfit = avgProfit\n" + + " }\n" + + "}"; + dsl += + "GraphStructure {\n" + + " (s: AttributePOC.User)-[p: holdPMProduct2]->(o: AttributePOC.PortfolioManager)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id,p.avgProfit)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass("com.antgroup.openspg.reasoner.runner.local.loader.TestFundGraphLoader"); + + // use test catalog + /* + dslParams.put("projId", "363000092"); + Catalog catalog = CatalogFactory.createCatalog(dslParams); + */ + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("AttributePOC.User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.PortfolioManager", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("AttributePOC.Fund", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.User_holdProduct_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + schema.put( + "AttributePOC.PortfolioManager_underControl_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "0.02"); + clear(); + } + + @Test + public void doTestMax() { + String dsl = + "Define (s: AttributePOC.User)-[p: holdPMProduct2]->(o: AttributePOC.PortfolioManager) {\n" + + " GraphStructure {\n" + + "\t(s)-[p1:holdProduct]->(u1:AttributePOC.Fund)<-[p2:underControl]-(o)\n" + + " }\n" + + " Rule {\n" + + " avgProfit = group(s,o).max(p1.holdRet)\n" + + " p.avgProfit = avgProfit\n" + + " }\n" + + "}"; + dsl += + "GraphStructure {\n" + + " (s: AttributePOC.User)-[p: holdPMProduct2]->(o: AttributePOC.PortfolioManager)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id,p.avgProfit)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass("com.antgroup.openspg.reasoner.runner.local.loader.TestFundGraphLoader"); + + // use test catalog + /* + dslParams.put("projId", "363000092"); + Catalog catalog = CatalogFactory.createCatalog(dslParams); + */ + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("AttributePOC.User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.PortfolioManager", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("AttributePOC.Fund", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.User_holdProduct_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + schema.put( + "AttributePOC.PortfolioManager_underControl_AttributePOC.Fund", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("holdRet"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "0.02"); + clear(); + } + + @Test + public void doTestLocalRunnerDependency() { + + String dsl = + "Define (user:TuringCore.AlipayUser)-[teCount:teCount]->(o:Long) {\n" + + "\tGraphStructure {\n" + + "\t\t(user) -[pwl:workLoc]-> (aa1:CKG.AdministrativeArea)\n" + + "\t\t(te:TuringCore.TravelEvent) -[ptler:traveler]-> (user)\n" + + "\t\t(te) -[ptm:travelMode]-> (tm:TuringCore.TravelMode)\n" + + "\t\t(te) -[pte:travelEndpoint]-> (aa1:CKG.AdministrativeArea)\n" + + "\t}\n" + + " Rule {\n" + + " R1('常驻地在杭州'): aa1.id == '中国-浙江省-杭州市'\n" + + " \tR2('工作日上班时间通勤用户'): dayOfWeek(te.eventTime) in [1, 2, 3, 4, 5] \n" + + " and hourOfDay(te.eventTime) in [6, 7, 8, 9, 10, 17, 18, 19, 20, 21] \n" + + " R3('公交地铁'): tm.id in ['bus', 'train']\n" + + " teCount('出行次数') = group(user).count(te.id)\n" + + " o=teCount\n" + + " }\n" + + "}"; + + dsl += + "\n" + + "GraphStructure {\n" + + " (user:TuringCore.AlipayUser)\n" + + "}\n" + + "Rule{\n" + + " R1: user.teCount >= 3\n" + + "}\n" + + "Action{\n" + + " get(user.id, user.teCount)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestCrowdGraphLoader"); + + // use test catalog + /* + dslParams.put("projId", "363000092"); + Catalog catalog = CatalogFactory.createCatalog(dslParams); + */ + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "CKG.AdministrativeArea", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "TuringCore.TravelMode", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "TuringCore.AlipayUser", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "TuringCore.TravelEvent", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "eventTime"))); + schema.put( + "TuringCore.AlipayUser_workLoc_CKG.AdministrativeArea", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "TuringCore.TravelEvent_traveler_TuringCore.AlipayUser", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "TuringCore.TravelEvent_travelEndpoint_CKG.AdministrativeArea", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "TuringCore.TravelEvent_travelMode_TuringCore.TravelMode", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "3"); + + task.getParams().put(ConfigKey.KG_REASONER_OUTPUT_GRAPH, true); + LocalReasonerResult result2 = runner.run(task); + System.out.println(result2); + Assert.assertEquals(result2.isGraphResult(), true); + Assert.assertEquals(result2.getVertexList().size(), 1); + clear(); + } + + @Test + public void testComplexMultiDefineDsl() { + String dsl = + "// 当月交易量\n" + + "Define (s:CustFundKG.Account)-[p:cur_month_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"当月交易量\"): date_diff(from_unix_time(now(), 'yyyyMMdd'),t.transDate) <= 20\n" + + " o = group(s).count(u.id)\n" + + " }\n" + + "}\n" + + "// 次月交易量\n" + + "Define (s:CustFundKG.Account)-[p:last_month_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tdate_delta = date_diff(from_unix_time(now(), 'yyyyMMdd'),t.transDate)\n" + + " \tR1(\"次月交易量\"): date_delta > 20 && date_delta <=40\n" + + " o = group(s).count(u.id)\n" + + " }\n" + + "}\n" + + "\n" + + "// 次次月交易量\n" + + "Define (s:CustFundKG.Account)-[p:last_last_month_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tdate_delta = date_diff(from_unix_time(now(), 'yyyyMMdd'),t.transDate)\n" + + " \tR1(\"次月交易量\"): date_delta > 40 && date_delta <=60\n" + + " o = group(s).count(u.id)\n" + + " }\n" + + "}\n" + + "// 倍数\n" + + "Define (s:CustFundKG.Account)-[p:last_trans_multiple]->(o:Float) {\n" + + "\t\tGraphStructure {\n" + + " (s)\n" + + " }\n" + + "\tRule {\n" + + " \tmultiple = s.last_month_num*1.0 / s.last_last_month_num\n" + + " o = multiple\n" + + " }\n" + + "}\n" + + "\n" + + "// 倍数\n" + + "Define (s:CustFundKG.Account)-[p:cur_trans_multiple]->(o:Float) {\n" + + "\t\tGraphStructure {\n" + + " (s)\n" + + " }\n" + + "\tRule {\n" + + " \tmultiple = s.cur_month_num*1.0 / s.last_month_num\n" + + " o = multiple\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s:CustFundKG.Account)-[p:is_trans_raise_more_after_down]->(o:Boolean) {\n" + + "\t\tGraphStructure {\n" + + " (s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"T月交易量级超过T-1月交易量级3倍\"): s.last_trans_multiple >=3\n" + + " \tR2(\"T+1月交易量级小于T月交易量级的1/2\"): s.cur_trans_multiple <0.5\n" + + " \to = rule_value(R1 && R2, true, false)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s:CustFundKG.Account)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.is_trans_raise_more_after_down, s.cur_trans_multiple, s.last_trans_multiple, s.last_last_month_num, s.last_month_num, s.cur_month_num)\n" + + "}\n"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestMultiVersionGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("sumAmt", "transDate"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("1", "CustFundKG.Account"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + + clear(); + } + + @Test + public void testEdgePropertyDefineDsl2() { + String dsl = + "Define (s:InsProduct.Product)-[p:liabilityLight]->(o:Boolean) {\n" + + "GraphStructure {\n" + + " (s)-[p:includeLiability]->(l:InsProduct.Liability)\n" + + "}\n" + + "Rule {\n" + + " R1: l.name like \"%轻症疾病保险金%\"\n" + + " o = true\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:InsProduct.Product)-[p:liabilityMid]->(o:Boolean) {\n" + + "GraphStructure {\n" + + " (s)-[p:includeLiability]->(l:InsProduct.Liability)\n" + + "}\n" + + "Rule {\n" + + " R1: l.name like \"%中症疾病保险金%\"\n" + + " o = true\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " A [InsProduct.Product]\n" + + "}\n" + + "Rule {\n" + + " R1: A.liabilityLight == null || A.liabilityMid == null\n" + + " output = rule_value(R1, true, false)\n" + + "}\n" + + "Action {\n" + + " get(A.name,output,A.liabilityLight, A.liabilityMid) \n" + + "}"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestInsProductGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "InsProduct.Product", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "name"))); + schema.put( + "InsProduct.Liability", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "name"))); + schema.put( + "InsProduct.Product_includeLiability_InsProduct.Liability", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + task.getParams().put(ConfigKey.KG_REASONER_BINARY_PROPERTY, false); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("保险产品", "InsProduct.Product"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + + clear(); + } + + @Test + public void testEdgePropertyDefineDsl() { + String dsl = + "//1 先定义每两个用户间进行聚合交易\n" + + "Define (s:CustFundKG.Account)-[p:tranTargetUser]->(o:CustFundKG.Account) {\n" + + " GraphStructure {\n" + + " (o)<-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " tran_count = group(s,o).count(t.sumAmt)\n" + + " p.tran_count = tran_count\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "Define (s:CustFundKG.Account)-[p:is_repeat_tran_user]->(o:Int) {\n" + + " GraphStructure {\n" + + " (s)-[t:tranTargetUser]->(u:CustFundKG.Account)\n" + + " }\n" + + "\tRule {\n" + + " user_num(\"交易笔数大于3的重复用户个数\") = group(s).countIf(t.tran_count>=3, u.id)\n" + + " R1(\"超过10个\"): user_num > 10\n" + + " o = rule_value(R1, true, false)\n" + + " }\n" + + "}\n" + + "\n" + + "//获取结果\n" + + "GraphStructure {\n" + + "\ts [CustFundKG.Account]\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action{\n" + + "\tget(s.id, s.is_repeat_tran_user)\n" + + "}"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestMultiVersionGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("sumAmt"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("1", "CustFundKG.Account"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + + clear(); + } + + @Test + public void testComplexMultiTimeUDFDefineDsl() { + String dsl = + "//1 先定义流入资金总数\n" + + "Define (s:CustFundKG.Account)-[p:total_in_trans_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]->(s)\n" + + " }\n" + + "\tRule {\n" + + " o = group(s).count(t.sumAmt)\n" + + " }\n" + + "}\n" + + "\n" + + "//2 定义整百流入笔数\n" + + "Define (s:CustFundKG.Account)-[p:multiples_hundred_in_trans_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]->(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"必须是整百交易\"): t.sumAmt % 100 == 0\n" + + " o = group(s).count(t.sumAmt)\n" + + " }\n" + + "}\n" + + "\n" + + "// 判断是否汇聚赌博\n" + + "Define (s:CustFundKG.Account)-[p:is_pooling_gambling_funds]->(o:Float) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + "\tRule {\n" + + " R0(\"存在流入和整百资金\"): s.multiples_hundred_in_trans_num != null && s.total_in_trans_num != null\n" + + " \tR1(\"流入整百金额笔数大于2比\"): s.multiples_hundred_in_trans_num > 2\n" + + " R2(\"整百交易占比大于2%\"): s.multiples_hundred_in_trans_num /cast_type(s.total_in_trans_num,'double') > 0.02\n" + + " o = rule_value(R2, true, false)\n" + + " }\n" + + "}\n" + + "\n" + + "//获取结果\n" + + "GraphStructure {\n" + + "\ts [CustFundKG.Account]\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action{\n" + + "\tget(s.id, s.is_pooling_gambling_funds, s.multiples_hundred_in_trans_num, s.total_in_trans_num)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestMultiVersionGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("sumAmt"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("1", "CustFundKG.Account"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + + clear(); + } + + @Test + public void doTestComplexDefineDsl() { + String dsl = + "// 1\n" + + "Define (s:DomainFamily)-[p:blackRelateRate]->(o:Pkg) {\n" + + " GraphStructure {\n" + + " (o)-[:use]->(d:Domain)-[:belong]->(s)\n" + + " }\n" + + " Rule {\n" + + " R1: o.is_black == true\n" + // + " domain_num = group(s,o).count(d)\n" + // + " p.same_domain_num = domain_num\n" + + " }\n" + + "}\n" + + "\n" + + "// 2\n" + + "Define (s:DomainFamily)-[p:total_domain_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (s)<-[:belong]-(d:Domain)\n" + + " }\n" + + " Rule {\n" + + " o = group(s).count(d.id)\n" + + " }\n" + + "}\n" + + "\n" + + "// 3\n" + + "Define (s:Pkg)-[p:target]->(o:User) {\n" + + " GraphStructure {\n" + + " (s)<-[p1:blackRelateRate]-(df:DomainFamily),\n" + + " (df)<-[:belong]-(d:Domain),\n" + + " (o)-[:visit]->(d)\n" + + " } Rule {\n" + + " visit_time = group(o, df).count(d.id)\n" + + " R1(\"必须大于1次\"): visit_time >= 1\n" + + " R2(\"必须占比大于50%\"): 1.0 * visit_time / df.total_domain_num > 0.2\n" + + " }\n" + + "}\n" + + "\n" + + "// 4\n" + + "GraphStructure {\n" + + " (s:Pkg)-[p:target]->(o:User)\n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id,o.id)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestDefineGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "DomainFamily", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "total_domain_num"))); + schema.put("Pkg", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "is_black"))); + schema.put("Domain", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("Pkg_use_Domain", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "Domain_belong_DomainFamily", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put("User_visit_Domain", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + + // define + schema.put( + "DomainFamily_blackRelateRate_Pkg", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put("Pkg_target_User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("black_app_1", "Pkg"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + + clear(); + } + + @Test + public void doTestWithStart() { + String dsl = + "// 查找使用了相同主演的两个导演\n" + + "GraphStructure {\n" + + " (a:ABM.User)-[hasIp:acc2ip]->(ip:ABM.IP)\n" + + "}\n" + + "Rule {\n" + + " \tR1(\"必选180天以内\"): hasIp.ipUse180dCnt >=2\n" + + " R2(\"必须是高危地区\"): ip.country in ['菲律宾','柬埔寨','老挝','日本','香港','台湾','泰国','澳门','越南','马来西亚','印度尼西亚']\n" + + " result = rule_value(R2, true, false)\n" + + "}\n" + + "Action {\n" + + " get(a.id, result, ip.country , hasIp.ipUse180dCnt) // 获取A和B两个导演的名字,调试和在线执行时直接看渲染出来的子图\n" + + "}"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestDefineGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("ABM.User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ABM.IP", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "country"))); + schema.put( + "ABM.User_acc2ip_ABM.IP", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("ipUse180dCnt"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("user1", "ABM.User"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + + clear(); + } + + @Test + public void doTestComplexLabelDsl() { + String dsl = + "// 查找使用了相同主演的两个导演\n" + + "GraphStructure {\n" + + " S [CustFundKG.Account, __start__='true']\n" + + " A,E [STD.AlipayAccount]\n" + + " S->A [accountId]\n" + + " B,C [CustFundKG.Alipay,CustFundKG.BankCard]\n" + + " B->A [accountId]\n" + + " B->C [transfer,consume] as t\n" + + " D [CustFundKG.Account]\n" + + " D->E [accountId]\n" + + " C->E [accountId]\n" + + "}\n" + + "Rule {\n" + + "transNum = group(S,D).sum(t.amount)\n" + + "}\n" + + "Action {\n" + + " get(S.id,D.id, transNum) \n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestMultiLabelGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "accountId"))); + schema.put( + "CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "accountId"))); + schema.put( + "CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "accountId"))); + schema.put("STD.AlipayAccount", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + + // define + schema.put( + "CustFundKG.Account_accountId_STD.AlipayAccount", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "CustFundKG.Alipay_accountId_STD.AlipayAccount", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "CustFundKG.BankCard_accountId_STD.AlipayAccount", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "CustFundKG.BankCard_transfer_CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.BankCard_consume_CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.BankCard_transfer_CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.BankCard_consume_CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.Alipay_transfer_CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.Alipay_consume_CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.Alipay_transfer_CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + schema.put( + "CustFundKG.Alipay_consume_CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("amount"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("S", "CustFundKG.Account"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + Assert.assertEquals(1, result.getRows().size()); + clear(); + } + + @Test + public void doTestSpatioTemporalDsl() { + String nearbyDsl = + "GraphStructure{\n" + + " (s:PE.JiuZhi)-[e1:nearby(s.shape4, o.shape2, 100)]->(o:PE.JiuZhi)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id)\n" + + "}"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(nearbyDsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.loader.TestSpatioTemporalGraphLoader"); + + Map> schema = new HashMap<>(); + schema.put( + "PE.JiuZhi", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "shape4", "shape2"))); + schema.put("STD.S2CellId", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "PE.JiuZhi_shape2S2CellId_STD.S2CellId", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.getParams().put(ConfigKey.KG_REASONER_CATALOG, SimpleObjSerde.ser(catalog)); + task.getParams().put(Constants.START_ALIAS, "s"); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("MOCK1", "PE.JiuZhi"))); + + LocalRunnerMain.main( + new String[] {new String(Base64.getEncoder().encode(JSON.toJSONBytes(task)))}); + LocalReasonerResult result = LocalRunnerMain.result; + System.out.println("##########################"); + System.out.println(result); + System.out.println("##########################"); + Assert.assertEquals(1, result.getRows().size()); + clear(); + } + + private void clear() { + PropertyUtil.reset(); + ProgressReport.clear(); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/MedicalGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/MedicalGraphLoader.java new file mode 100644 index 00000000..6b5704a5 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/MedicalGraphLoader.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class MedicalGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("u1", "ProfMedV1.Patient"), + constructionVertex("index1", "ProfMedV1.PatientIndex", "entity", "影像学检查"), + constructionVertex("前列腺癌", "ProfMedV1.Disease")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList(constructionEdge("u1", "inspectionIndex", "index1")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestCrowdGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestCrowdGraphLoader.java new file mode 100644 index 00000000..d6c8a978 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestCrowdGraphLoader.java @@ -0,0 +1,113 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestCrowdGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("中国-浙江省-杭州市", "CKG.AdministrativeArea"), + constructionVertex("中国-浙江省-温州市", "CKG.AdministrativeArea"), + constructionVertex("bus", "TuringCore.TravelMode"), + constructionVertex("train", "TuringCore.TravelMode"), + constructionVertex("u1", "TuringCore.AlipayUser", "workLoc", "中国-浙江省-杭州市"), + constructionVertex( + "u1_te1", + "TuringCore.TravelEvent", + "eventTime", + "1681208495011", + "travelEndpoint", + "中国-浙江省-杭州市", + "travelMode", + "bus"), + constructionVertex( + "u1_te2", + "TuringCore.TravelEvent", + "eventTime", + "1681208495012", + "travelEndpoint", + "中国-浙江省-杭州市", + "travelMode", + "train"), + constructionVertex( + "u1_te3", + "TuringCore.TravelEvent", + "eventTime", + "1681208495013", + "travelEndpoint", + "中国-浙江省-杭州市", + "travelMode", + "train"), + constructionVertex("u2", "TuringCore.AlipayUser", "workLoc", "中国-浙江省-杭州市"), + constructionVertex( + "u2_te1", + "TuringCore.TravelEvent", + "eventTime", + "1681208495021", + "travelEndpoint", + "中国-浙江省-温州市", + "travelMode", + "bus"), + constructionVertex( + "u2_te2", + "TuringCore.TravelEvent", + "eventTime", + "1681208495021", + "travelEndpoint", + "中国-浙江省-杭州市", + "travelMode", + "bus"), + constructionVertex( + "u2_te3", + "TuringCore.TravelEvent", + "eventTime", + "1681208495021", + "travelEndpoint", + "中国-浙江省-杭州市", + "travelMode", + "bus")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("u1", "workLoc", "中国-浙江省-杭州市"), + constructionEdge("u1_te1", "traveler", "u1"), + constructionEdge("u1_te1", "travelMode", "bus"), + constructionEdge("u1_te1", "travelEndpoint", "中国-浙江省-杭州市"), + constructionEdge("u1_te2", "traveler", "u1"), + constructionEdge("u1_te2", "travelMode", "train"), + constructionEdge("u1_te2", "travelEndpoint", "中国-浙江省-杭州市"), + constructionEdge("u1_te3", "traveler", "u1"), + constructionEdge("u1_te3", "travelMode", "train"), + constructionEdge("u1_te3", "travelEndpoint", "中国-浙江省-杭州市"), + constructionEdge("u2", "workLoc", "中国-浙江省-杭州市"), + constructionEdge("u2_te1", "traveler", "u2"), + constructionEdge("u2_te1", "travelMode", "bus"), + constructionEdge("u2_te1", "travelEndpoint", "中国-浙江省-温州市"), + constructionEdge("u2_te2", "traveler", "u2"), + constructionEdge("u2_te2", "travelMode", "bus"), + constructionEdge("u2_te2", "travelEndpoint", "中国-浙江省-杭州市"), + constructionEdge("u2_te3", "traveler", "u2"), + constructionEdge("u2_te3", "travelMode", "bus"), + constructionEdge("u2_te3", "travelEndpoint", "中国-浙江省-杭州市")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestDefineGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestDefineGraphLoader.java new file mode 100644 index 00000000..815b679e --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestDefineGraphLoader.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestDefineGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("user1", "ABM.User"), + constructionVertex("192.168.1.1", "ABM.IP", "country", "柬埔寨"), + constructionVertex("black.net", "DomainFamily"), + constructionVertex("one.black.net", "Domain"), + constructionVertex("two.black.net", "Domain"), + constructionVertex("black_app_1", "Pkg", "is_black", true), + constructionVertex("black_app_2", "Pkg", "is_black", true), + constructionVertex("u1", "User"), + constructionVertex("u2", "User"), + constructionVertex("u3", "User"), + constructionVertex("u4", "User")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("user1", "acc2ip", "192.168.1.1", "ipUse180dCnt", 5), + constructionEdge("one.black.net", "belong", "black.net"), + constructionEdge("two.black.net", "belong", "black.net"), + constructionEdge("black_app_1", "use", "one.black.net"), + constructionEdge("black_app_2", "use", "two.black.net"), + constructionEdge("u1", "visit", "one.black.net"), + constructionEdge("u2", "visit", "one.black.net"), + constructionEdge("u3", "visit", "two.black.net"), + constructionEdge("u4", "visit", "two.black.net")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestFundGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestFundGraphLoader.java new file mode 100644 index 00000000..974302ac --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestFundGraphLoader.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestFundGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("u1", "AttributePOC.User"), + constructionVertex("m1", "AttributePOC.PortfolioManager"), + constructionVertex("f1", "AttributePOC.Fund"), + constructionVertex("f2", "AttributePOC.Fund")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("u1", "holdProduct", "f1", "holdRet", 0.02), + constructionEdge("m1", "underControl", "f1"), + constructionEdge("u1", "followPM", "m1", "holdRet", 0.01, "times", 4), + constructionEdge("u1", "holdProduct", "f2", "holdRet", 0.02), + constructionEdge("m1", "underControl", "f2")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestInsProductGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestInsProductGraphLoader.java new file mode 100644 index 00000000..c0e55524 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestInsProductGraphLoader.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestInsProductGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("保险产品", "InsProduct.Product", "name", "保险产品"), + constructionVertex("轻症疾病保险金", "InsProduct.Liability", "name", "轻症疾病保险金"), + constructionVertex("中症疾病保险金", "InsProduct.Liability", "name", "中症疾病保险金"), + constructionVertex("重症疾病保险金", "InsProduct.Liability", "name", "重症疾病保险金")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("保险产品", "includeLiability", "轻症疾病保险金"), + constructionEdge("保险产品", "includeLiability", "中症疾病保险金"), + constructionEdge("保险产品", "includeLiability", "重症疾病保险金")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMockDataLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMockDataLoader.java new file mode 100644 index 00000000..fa5f6527 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMockDataLoader.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import org.junit.Assert; +import org.junit.Test; + +public class TestMockDataLoader { + @Test + public void test() { + String demoGraph = + "Graph {\n" + + " C_37091 [CRO.Company, regNo='C_37091']\n" + + " C_7661 [CRO.Company, regNo='C_7661']\n" + + " C_8125 [CRO.Company, regNo='C_8125'] \n" + + " P1 [CRO.Person]\n" + + " P2 [CRO.Person]\n" + + " P3 [CRO.Person]\n" + + "\n" + + "\n" + + " P1->C_37091[corporate]\n" + + " P1->C_7661 [control]\n" + + " P1->C_7661 [superviseDirctor]\n" + + "\n" + + " P2->C_7661[corporate]\n" + + " P2->C_7661 [control]\n" + + " P2->C_7661 [superviseDirctor]\n" + + "\n" + + " P3->C_8125[corporate]\n" + + " P3->C_7661[superviseDirctor]\n" + + "\n" + + "}"; + MockLocalGraphLoader mockLocalGraphLoader = new MockLocalGraphLoader(demoGraph); + mockLocalGraphLoader.setGraphState(new MemGraphState()); + mockLocalGraphLoader.load(); + Assert.assertEquals(0, mockLocalGraphLoader.genVertexList().size()); + Assert.assertEquals(0, mockLocalGraphLoader.genEdgeList().size()); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMultiLabelGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMultiLabelGraphLoader.java new file mode 100644 index 00000000..d9d5a347 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMultiLabelGraphLoader.java @@ -0,0 +1,55 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestMultiLabelGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("S", "CustFundKG.Account", "accountId", "20881"), + constructionVertex("D", "CustFundKG.Account", "accountId", "20882"), + constructionVertex("B_pay", "CustFundKG.Alipay", "accountId", "20881"), + constructionVertex("B_card", "CustFundKG.BankCard", "accountId", "20881"), + constructionVertex("C_pay", "CustFundKG.Alipay", "accountId", "20882"), + constructionVertex("C_card", "CustFundKG.BankCard", "accountId", "20882"), + constructionVertex("20882", "STD.AlipayAccount"), + constructionVertex("20881", "STD.AlipayAccount")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("S", "accountId", "20881"), + constructionEdge("D", "accountId", "20882"), + constructionEdge("B_pay", "accountId", "20881"), + constructionEdge("B_card", "accountId", "20881"), + constructionEdge("C_pay", "accountId", "20882"), + constructionEdge("C_card", "accountId", "20882"), + constructionEdge("B_pay", "transfer", "C_pay", "amount", 100), + constructionEdge("B_pay", "transfer", "C_card", "amount", 100), + constructionEdge("B_card", "transfer", "C_pay", "amount", 100), + constructionEdge("B_card", "transfer", "C_card", "amount", 100), + constructionEdge("B_pay", "consume", "C_pay", "amount", 100), + constructionEdge("B_pay", "consume", "C_card", "amount", 100), + constructionEdge("B_card", "consume", "C_pay", "amount", 100), + constructionEdge("B_card", "consume", "C_card", "amount", 100)); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMultiVersionGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMultiVersionGraphLoader.java new file mode 100644 index 00000000..5045fbc1 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestMultiVersionGraphLoader.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestMultiVersionGraphLoader extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("1", "CustFundKG.Account"), + constructionVertex("2", "CustFundKG.Account"), + constructionVertex("3", "CustFundKG.Account"), + constructionVertex("4", "CustFundKG.Account"), + constructionVertex("5", "CustFundKG.Account")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge( + "2", "accountFundContact", "1", 123, "sumAmt", 1000, "transDate", "20230630"), + constructionVersionEdge( + "3", "accountFundContact", "1", 123, "sumAmt", 1000, "transDate", "20230601"), + constructionVersionEdge( + "4", "accountFundContact", "1", 123, "sumAmt", 1000, "transDate", "20230602"), + constructionVersionEdge( + "5", "accountFundContact", "1", 123, "sumAmt", 1000, "transDate", "20230514")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestSpatioTemporalGraphLoader.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestSpatioTemporalGraphLoader.java new file mode 100644 index 00000000..0e2f93c6 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/loader/TestSpatioTemporalGraphLoader.java @@ -0,0 +1,70 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.loader; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.google.common.collect.Lists; +import java.util.List; + +public class TestSpatioTemporalGraphLoader extends AbstractLocalGraphLoader { + + /** please provide your mock vertex list */ + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex( + "MOCK1", + "PE.JiuZhi", + "shape4", + "MULTIPOINT(116.506619 39.945368,116.509562 39.945402,116.509474 39.943348,116.506648 39" + + ".943247)", + "shape2", + "LINESTRING(116.506619 39.945368,116.509562 39.945402,116.509474 39.943348,116.506648 39" + + ".943247)", + "id", + "1"), + constructionVertex( + "MOCK2", + "PE.JiuZhi", + "shape4", + "MULTIPOINT(116.506619 39.945368,116.509562 39.945402,116.509474 39.943348,116.506648 39" + + ".943247)", + "shape2", + "LINESTRING(116.506619 39.945368,116.509562 39.945402,116.509474 39.943348,116.506648 39" + + ".943247)", + "id", + "1"), + constructionVertex("35f1abf9", "STD.S2CellId"), + constructionVertex("35f1abfb", "STD.S2CellId"), + constructionVertex("35f1abfd", "STD.S2CellId"), + constructionVertex("35f1abff", "STD.S2CellId")); + } + + /** please provide your mock edge list */ + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("MOCK1", "shape2S2CellId", "35f1abf9"), + constructionEdge("MOCK1", "shape2S2CellId", "35f1abfb"), + constructionEdge("MOCK1", "shape2S2CellId", "35f1abfd"), + constructionEdge("MOCK1", "shape2S2CellId", "35f1abff"), + constructionEdge("MOCK2", "shape2S2CellId", "35f1abf9"), + constructionEdge("MOCK2", "shape2S2CellId", "35f1abfb"), + constructionEdge("MOCK2", "shape2S2CellId", "35f1abfd"), + constructionEdge("MOCK2", "shape2S2CellId", "35f1abff")); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerABMLocalTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerABMLocalTest.java new file mode 100644 index 00000000..1fd7922e --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerABMLocalTest.java @@ -0,0 +1,990 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.recorder.DefaultRecorder; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Test; +import scala.Tuple2; + +public class KgReasonerABMLocalTest { + + @Test + public void test11() { + String dsl = + "Define (s:Attribute1.Name138)-[p:xuanguFeature]->(o:Float) {\n" + + "\tGraphStructure {\n" + + " \t(s)-[:p3]->(t:Attribute1.Name142)\n" + + " }\n" + + " Rule {\n" + + " \tv = t.stock/t.total\n" + + " R1(\"必须大于20%\"): v > 0.2\n" + + " o = v\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Attribute1.Name138)\n" + + "}\n" + + "Rule {\n" + + " v2 = s.xuanguFeature\n" + + "}\n" + + "Action {\n" + + " get(s.id, s.xuanguFeature, v2, \"\" as out)\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("Attribute1.Name138", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "Attribute1.Name142", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "stock", "total"))); + schema.put( + "Attribute1.Name138_p3_Attribute1.Name142", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader8"); + + // enable subquery + Map params = new HashMap<>(); + // 开启子查询 + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getRows().size()); + Assert.assertTrue(Double.parseDouble(result.getRows().get(0)[1].toString()) - 0.3999 < 0.01); + Assert.assertTrue(StringUtils.isEmpty(result.getRows().get(0)[3].toString())); + } + + @Test + public void test9() { + String dsl = + "Define (s:Attribute1.Name138)-[p:xuanguFeature]->(o:Float) {\n" + + "\tGraphStructure {\n" + + " \t(s)-[:p3]->(t:Attribute1.Name142)\n" + + " }\n" + + " Rule {\n" + + " \tv = t.stock/t.total\n" + + " R1(\"必须大于20%\"): v > 0.2\n" + + " o = v\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Attribute1.Name138)\n" + + "}\n" + + "Rule {\n" + + " v2 = s.xuanguFeature\n" + + "}\n" + + "Action {\n" + + " get(s.id, s.xuanguFeature, v2)\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("Attribute1.Name138", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "Attribute1.Name142", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "stock", "total"))); + schema.put( + "Attribute1.Name138_p3_Attribute1.Name142", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader8"); + + // enable subquery + Map params = new HashMap<>(); + // 开启子查询 + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getRows().size()); + Assert.assertTrue(Double.parseDouble(result.getRows().get(0)[1].toString()) - 0.3999 < 0.01); + } + + public static class GraphLoader8 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A", "Attribute1.Name138"), + constructionVertex("B", "Attribute1.Name142", "stock", 0.02, "total", 0.05)); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList(constructionEdge("A", "p3", "B")); + } + } + + @Test + public void test10() { + String dsl = + "//定义额外分支\n" + + "Define (s:RAC.Param)-[p:hasOtherFeature]->(o:Boolean) {\n" + + "\tGraphStructure {\n" + + " \t(A:RAC.Param)-[:param2link]->(L:RAC.LinkID)-[:link2param]->(s),\n" + + " (A)-[:param2feature]->(F:RAC.Feature)\n" + + " }\n" + + " Rule {\n" + + " \tR1: F.featureCode == \"SESSION\"\n" + + " \t \to = rule_value(s.resultFrom == \"RAC\", true, false)\n\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " ServiceApi [RAC.ServiceApi, __start__=\"true\"]\n" + + " A [RAC.Param]\n" + + " B [RAC.Param]\n" + + " ServiceApi -> A [api2param]\n" + + " L [RAC.LinkID]\n" + + " A->L [param2link]\n" + + " L->B [link2param]\n" + + " F [RAC.Feature]\n" + + " B -> F [param2feature]\n" + + "}\n" + + "Rule {\n" + + "\tR1: B.hasOtherFeature != true\n" + + "}\n" + + "Action {\n" + + "\tget(ServiceApi.id, B.id, B.hasOtherFeature)\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "RAC.Param", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "resultFrom"))); + schema.put( + "RAC.LinkID", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "stock", "total"))); + schema.put( + "RAC.ServiceApi", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "stock", "total"))); + schema.put( + "RAC.Feature", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "stock", "featureCode"))); + schema.put( + "RAC.Param_param2link_RAC.LinkID", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + schema.put( + "RAC.LinkID_link2param_RAC.Param", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + schema.put( + "RAC.ServiceApi_api2param_RAC.Param", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + schema.put( + "RAC.Param_param2feature_RAC.Feature", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader10"); + + // enable subquery + Map params = new HashMap<>(); + // 开启子查询 + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(0, result.getRows().size()); + } + + public static class GraphLoader10 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A", "RAC.Param", "resultFrom", "RAC"), + constructionVertex("A1", "RAC.Param", "resultFrom", "RAC"), + constructionVertex("B", "RAC.Param", "resultFrom", "RAC"), + constructionVertex("L", "RAC.LinkID"), + constructionVertex("L1", "RAC.LinkID"), + constructionVertex("F", "RAC.Feature"), + constructionVertex("F1", "RAC.Feature", "featureCode", "SESSION"), + constructionVertex("BF", "RAC.Feature"), + constructionVertex("S", "RAC.ServiceApi")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("S", "api2param", "A"), + constructionEdge("A", "param2link", "L"), + constructionEdge("L", "link2param", "B"), + constructionEdge("B", "param2feature", "BF"), + constructionEdge("L1", "link2param", "B"), + constructionEdge("A1", "param2link", "L1"), + constructionEdge("A", "param2feature", "F"), + constructionEdge("A1", "param2feature", "F1")); + } + } + + @Test + public void test7() { + String dsl = + "GraphStructure {\n" + + "(A:CustFundKG.Account)-[:expand_linked_alipay_id(A.id)]->(B:CustFundKG.Alipay)-[t:consume]->(C:CustFundKG.Alipay|CustFundKG.BankCard)\n" + + "}\n" + + "Rule {\n" + + "\t\n" + + "}\n" + + "Action {\n" + + " get(B.id, t.payDate) \n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("CustFundKG.Alipay", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("CustFundKG.BankCard", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Alipay_consume_CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payDate"))); + schema.put( + "CustFundKG.Alipay_consume_CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payDate"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader6"); + + // enable subquery + Map params = new HashMap<>(); + task.setParams(params); + List> startIdList = new ArrayList<>(); + startIdList.add(new Tuple2<>("2088xxx", "CustFundKG.Account")); + task.setStartIdList(startIdList); + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(2, result.getRows().size()); + } + + public static class GraphLoader6 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("2088xxx", "CustFundKG.Account"), + constructionVertex("2088xxx", "CustFundKG.BankCard"), + constructionVertex("2088xxx", "CustFundKG.Alipay"), + constructionVertex("2088xx1", "CustFundKG.BankCard"), + constructionVertex("2088xx1", "CustFundKG.Alipay")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge( + "2088xxx_CustFundKG.Alipay", + "consume", + "2088xx1_CustFundKG.Alipay", + 1L, + "payDate", + 123), + constructionVersionEdge( + "2088xxx_CustFundKG.Alipay", + "consume", + "2088xx1_CustFundKG.BankCard", + 1L, + "payDate", + 321)); + } + } + + @Test + public void test711() { + String dsl = + "GraphStructure {\n" + + "(A:CustFundKG.Account)-[:expand_linked_alipay_id(A.prop)]->(B:CustFundKG.Alipay)-[t:consume]->(C:CustFundKG.Alipay|CustFundKG.BankCard)\n" + + "}\n" + + "Rule {\n" + + "\t\n" + + "}\n" + + "Action {\n" + + " get(B.id, t.payDate) \n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "prop"))); + schema.put("CustFundKG.Alipay", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("CustFundKG.BankCard", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Alipay_consume_CustFundKG.Alipay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payDate"))); + schema.put( + "CustFundKG.Alipay_consume_CustFundKG.BankCard", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payDate"))); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader6"); + + // enable subquery + Map params = new HashMap<>(); + task.setParams(params); + List> startIdList = new ArrayList<>(); + startIdList.add(new Tuple2<>("2088xxx", "CustFundKG.Account")); + task.setStartIdList(startIdList); + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(0, result.getRows().size()); + } + + @Test + public void test3() { + String dsl = + "Define (s:CustFundKG.Account)-[p:aggTransAmountNumByDay]->(o:Boolean) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)<-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"当月交易\"): date_diff(from_unix_time(now(), 'yyyyMMdd'),from_unix_time(cast_type(t.transDate,'bigint'), 'yyyyMMdd')) <= 30\n" + + " \tao = group(s).trans_count_by_day(t, \"transDate\",\"s\", 1, \"large\")\n" + + " \t o = rule_value(ao>1,true,false) \n" + + " }\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader2"); + + // enable subquery + Map params = new HashMap<>(); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getVertexList().size()); + Assert.assertEquals( + result.getVertexList().get(0).getValue().get("aggTransAmountNumByDay"), true); + } + + @Test + public void test4() { + String dsl = + "Define (s:CustFundKG.Account)-[p:aggTransAmountNumByDay]->(o:Boolean) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)<-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"当月交易\"): date_diff(from_unix_time(now(), 'yyyyMMdd'),from_unix_time(cast_type(t.transDate,'bigint')/1000, 'yyyyMMdd')) <= 30\n" + + " \tao = group(s).trans_count_by_day(t, \"transDate\",\"ms\", 3, \"small\")\n" + + " \t o = rule_value(ao>1,true,false) \n" + + " }\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader3"); + + // enable subquery + Map params = new HashMap<>(); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getVertexList().size()); + Assert.assertEquals( + result.getVertexList().get(0).getValue().get("aggTransAmountNumByDay"), true); + } + + @Test + public void test5() { + String dsl = + "Define (s:CustFundKG.Account)-[p:aggTransAmountNumByDay]->(o:Boolean) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)<-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"当月交易\"): date_diff(from_unix_time(now(), 'yyyyMMdd'),from_unix_time(cast_type(t.transDate,'bigint')/1000/1000, 'yyyyMMdd')) <= 30\n" + + " \tao = group(s).trans_count_by_day(t, \"transDate\",\"us\", 3, \"small\")\n" + + " \t o = rule_value(ao>1,true,false) \n" + + " }\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader4"); + + // enable subquery + Map params = new HashMap<>(); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getVertexList().size()); + Assert.assertEquals( + result.getVertexList().get(0).getValue().get("aggTransAmountNumByDay"), false); + } + + @Test + public void test6() { + String dsl = + "Define (s:CustFundKG.Account)-[p:aggTransAmountNumByDay]->(o:Boolean) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)<-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"当月交易\"): date_diff(from_unix_time(now(), 'yyyyMMdd'),from_unix_time(cast_type(t.transDate,'bigint')/1000/1000, 'yyyyMMdd')) <= 30\n" + + " \tao = group(s).trans_count_by_day(t, \"transDate\",\"xx\", 3, \"small\")\n" + + " \t o = rule_value(ao>1,true,false) \n" + + " }\n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader4"); + + // enable subquery + Map params = new HashMap<>(); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertTrue(result.getErrMsg().contains("time unit need in s/ms/us, but this is xx")); + } + + @Test + public void test8() { + String dsl = + "Define (s:CustFundKG.Account)-[p:transInAmount]->(o:CustFundKG.Account) {\n" + + " GraphStructure {\n" + + " \t(o)<-[t:accountFundContact]-(s)\n" + + " }\n" + + " Rule {\n" + + " transAmount = group(s,o).sum(t.amount)\n" + + " \tp.transAmount = transAmount\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s:CustFundKG.Account)-[p:centralizedTransfer]->(o:Boolean) {\n" + + " GraphStructure {\n" + + " \t(s)-[t:transInAmount]->(u:CustFundKG.Account)\n" + + " }\n" + + " Rule {\n" + + " \ttotalAmount = group(s).sum(t.transAmount)\n" + + " \ttop5Amount = group(s).order_edge_and_slice_sum(t.transAmount, \"desc\", 5)\n" + + " \tR2(\"top5流入资金占比\"): top5Amount*1.0/totalAmount > 0.5\n" + + " \to = rule_value(R2, true, false)\n" + + " }\n" + + "}\n" + + "GraphStructure {\n" + + "(A:CustFundKG.Account)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(A.id, A.centralizedTransfer) \n" + + "}"; + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate", "amount"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader4"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(ConfigKey.KG_REASONER_OUTPUT_GRAPH, true); + task.setParams(params); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("A", "CustFundKG.Account"))); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertTrue(result != null); + } + + public static class GraphLoader4 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A", "CustFundKG.Account"), + constructionVertex("B", "CustFundKG.Account"), + constructionVertex("C1", "CustFundKG.Account"), + constructionVertex("C2", "CustFundKG.Account"), + constructionVertex("C3", "CustFundKG.Account"), + constructionVertex("C4", "CustFundKG.Account"), + constructionVertex("C5", "CustFundKG.Account")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 1L, + "transDate", + (System.currentTimeMillis() - 3600 * 48 * 1000) * 1000, + "amount", + 10), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 2L, + "transDate", + (System.currentTimeMillis() - 3600 * 48 * 1000) * 1000, + "amount", + 10), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 3L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 10), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 10), + constructionVersionEdge( + "A", + "accountFundContact", + "C1", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 10), + constructionVersionEdge( + "A", + "accountFundContact", + "C1", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 10), + constructionVersionEdge( + "A", + "accountFundContact", + "C2", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 15), + constructionVersionEdge( + "A", + "accountFundContact", + "C3", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 14), + constructionVersionEdge( + "A", + "accountFundContact", + "C4", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 13), + constructionVersionEdge( + "A", + "accountFundContact", + "C5", + 4L, + "transDate", + System.currentTimeMillis() * 1000 - 400, + "amount", + 1)); + } + } + + public static class GraphLoader3 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A", "CustFundKG.Account"), + constructionVertex("B", "CustFundKG.Account")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge( + "A", "accountFundContact", "B", 1L, "transDate", System.currentTimeMillis() - 400), + constructionVersionEdge( + "A", "accountFundContact", "B", 2L, "transDate", System.currentTimeMillis() - 400), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 3L, + "transDate", + System.currentTimeMillis() - 3600 * 48 * 1000), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 4L, + "transDate", + System.currentTimeMillis() - 3600 * 48 * 1000)); + } + } + + public static class GraphLoader2 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A", "CustFundKG.Account"), + constructionVertex("B", "CustFundKG.Account")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 1L, + "transDate", + System.currentTimeMillis() / 1000 - 3600), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 2L, + "transDate", + System.currentTimeMillis() / 1000 - 3600), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 3L, + "transDate", + System.currentTimeMillis() / 1000 - 3600 * 48), + constructionVersionEdge( + "A", + "accountFundContact", + "B", + 4L, + "transDate", + System.currentTimeMillis() / 1000 - 3600 * 48)); + } + } + + private Catalog initABMSchema() { + Map> schema = new HashMap<>(); + schema.put( + "ABM.Pkg", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "markResult", "algoMarkResult"))); + schema.put("ABM.BundleApp", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ABM.BundleAppFamily", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ABM.Apdid", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "insPkgToolsLabel", "insPkgToolsLabelCnt"))); + schema.put( + "ABM.User", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "hunterLabelCount", "hunterLabel"))); + schema.put( + "ABM.Pkg_pkg2bundleApp_ABM.BundleApp", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "ABM.BundleApp_belong_ABM.BundleAppFamily", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "ABM.Apdid_install_ABM.Pkg", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("rn"))); + schema.put( + "ABM.User_acc2apdid_ABM.Apdid", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "ABM.Apdid_acc2apdid_ABM.User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + return catalog; + } + + @Test + public void test2() { + String dsl = + "Define (s:ABM.Pkg)-[p:pkg2Family]->(o:ABM.BundleAppFamily) {\n" + + " GraphStructure {\n" + + " (s)-[:pkg2bundleApp]->(b:ABM.BundleApp)-[:belong]->(o)\n" + + " }\n" + + "\tRule{\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s:ABM.Pkg)-[p:blackSameFamilyPkg]->(o:ABM.Pkg) {\n" + + " GraphStructure {\n" + + " (s)-[:pkg2Family]->(b:ABM.BundleAppFamily)<-[:pkg2Family]-(o)\n" + + " }\n" + + "\tRule{\n" + + " R1(\"必须是黑包\"): s.algoMarkResult == 'UNSAFE'\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s:ABM.Pkg)-[p:blackRelateApdid]->(o:ABM.Apdid) {\n" + + " GraphStructure {\n" + + " (s)-[:blackSameFamilyPkg]->(p1:ABM.Pkg),\n" + + " (o)-[r:install]->(p1)\n" + + " }\n" + + "\tRule {\n" + + " R0(\"前几位安装\"): r.rn <= 20\n" + + " R3: o.insPkgToolsLabel != ''\n" + + " R4: o.insPkgToolsLabelCnt >= 2\n" + + " \n" + + " num = group(o).count(p1.id)\n" + + " R1(\"数目必须大于2\"): num >=2\n" + + " //排序暂未实现\n" + + " p.pkgNum = num\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s:ABM.Pkg)-[p:relateUser]->(o:ABM.User) {\n" + + " GraphStructure {\n" + + " (s)-[:blackRelateApdid]->(d:ABM.Apdid),\n" + + " (o)-[:acc2apdid]->(d)\n" + + " }\n" + + "\tRule {\n" + + " R1: o.hunterLabel != ''\n" + + " R2: o.hunterLabelCount >=2\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(pkg:ABM.Pkg)-[p:blackRelateApdid]->(did:ABM.Apdid),\n" + + " (pkg)-[:relateUser]->(u:ABM.User)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(pkg.id, p.pkgNum, did.id, u.id)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + Map dslParams = new HashMap<>(); + // use test catalog + Catalog catalog = initABMSchema(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerABMLocalTest$GraphLoader"); + task.setExecutionRecorder(new DefaultRecorder()); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(1, result.getRows().size()); + Assert.assertEquals(4, result.getRows().get(0).length); + Assert.assertEquals("Pkg", result.getRows().get(0)[0]); + Assert.assertEquals("3", result.getRows().get(0)[1]); + Assert.assertEquals("Apdid", result.getRows().get(0)[2]); + Assert.assertEquals("user", result.getRows().get(0)[3]); + + System.out.println(task.getExecutionRecorder().toReadableString()); + } + + public static class GraphLoader extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("BundleAppFamily", "ABM.BundleAppFamily"), + constructionVertex("BundleApp", "ABM.BundleApp"), + constructionVertex("Pkg1", "ABM.Pkg"), + constructionVertex("Pkg2", "ABM.Pkg"), + constructionVertex("Pkg3", "ABM.Pkg"), + constructionVertex("Pkg4", "ABM.Pkg"), + constructionVertex( + "Apdid", "ABM.Apdid", "insPkgToolsLabel", "abc", "insPkgToolsLabelCnt", 3), + constructionVertex("user", "ABM.User", "hunterLabel", "abc", "hunterLabelCount", 5), + constructionVertex("Pkg", "ABM.Pkg", "algoMarkResult", "UNSAFE")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("Pkg", "pkg2bundleApp", "BundleApp"), + constructionEdge("Pkg1", "pkg2bundleApp", "BundleApp"), + constructionEdge("Pkg2", "pkg2bundleApp", "BundleApp"), + constructionEdge("Pkg3", "pkg2bundleApp", "BundleApp"), + constructionEdge("Pkg4", "pkg2bundleApp", "BundleApp"), + constructionEdge("BundleApp", "belong", "BundleAppFamily"), + constructionEdge("Apdid", "install", "Pkg1", "rn", 1), + constructionEdge("Apdid", "install", "Pkg2", "rn", 1000), + constructionEdge("Apdid", "install", "Pkg3", "rn", 3), + constructionEdge("Apdid", "install", "Pkg4", "rn", 2), + constructionEdge("user", "acc2apdid", "Apdid")); + } + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAddEdgeWithPropertyTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAddEdgeWithPropertyTest.java new file mode 100644 index 00000000..10ac36ae --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAddEdgeWithPropertyTest.java @@ -0,0 +1,132 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +public class KgReasonerAddEdgeWithPropertyTest { + @Test + public void testAddEdgeWithProperty() { + String dsl = + "Define (s:User)-[p:tradeInfo]->(o:User) {\n" + + " GraphStructure {\n" + + " (s:User)-[t:trade]->(o:User)\n" + + " } Rule {\n" + + " R1(\"交易时间在90天内\"): t.trade_time < now()\n" + + " trade_num(\"计算每个交易对象的交易次数\") = group(s,o).count(t)\n" + + " p.trade_num = trade_num\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s:User)-[p:belongTo]->(o:CrowdType/RepeatTradeUser) {\n" + + " GraphStructure {\n" + + " (s:User)-[t:tradeInfo]->(u:User)\n" + + " } Rule {\n" + + " trade_user_count = group(s).count(u.id)\n" + + " R2(\"至少有3个交易对手\"): trade_user_count >= 3\n" + + " every_user_trade_more_then3 = group(s).countIf(t.trade_num > 3, t.trade_num)\n" + + " R3(\"每个交于对手交易大于3比\"): every_user_trade_more_then3 >= 1\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s:CrowdType/`RepeatTradeUser`)\n" + + "}\n" + + "Rule{\n" + + "}\n" + + "Action {\n" + + " get(s.id)\n" + + "}\n"; + + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("CrowdType", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "User_trade_User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("trade_time"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerAddEdgeWithPropertyTest$GraphLoader"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getRows().size()); + Assert.assertEquals("u1", result.getRows().get(0)[0]); + } + + public static class GraphLoader extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("u1", "User"), + constructionVertex("u2", "User"), + constructionVertex("u3", "User"), + constructionVertex("u4", "User"), + constructionVertex("RepeatTradeUser", "CrowdType")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge("u1", "trade", "u2", 1, "trade_time", 1), + constructionVersionEdge("u1", "trade", "u2", 2, "trade_time", 2), + constructionVersionEdge("u1", "trade", "u2", 3, "trade_time", 3), + constructionVersionEdge("u1", "trade", "u3", 1, "trade_time", 1), + constructionVersionEdge("u1", "trade", "u3", 2, "trade_time", 2), + constructionVersionEdge("u1", "trade", "u3", 3, "trade_time", 3), + constructionVersionEdge("u1", "trade", "u3", 4, "trade_time", 4), + constructionVersionEdge("u1", "trade", "u4", 1, "trade_time", 1), + constructionVersionEdge("u1", "trade", "u4", 2, "trade_time", 2), + constructionVersionEdge("u1", "trade", "u4", 3, "trade_time", 3), + constructionVersionEdge("u2", "trade", "u3", 1, "trade_time", 1), + constructionVersionEdge("u2", "trade", "u3", 2, "trade_time", 2), + constructionVersionEdge("u2", "trade", "u3", 3, "trade_time", 3), + constructionVersionEdge("u2", "trade", "u4", 1, "trade_time", 1), + constructionVersionEdge("u2", "trade", "u4", 2, "trade_time", 2), + constructionVersionEdge("u2", "trade", "u4", 3, "trade_time", 3)); + } + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAggTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAggTest.java new file mode 100644 index 00000000..a01f7e50 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAggTest.java @@ -0,0 +1,471 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.recorder.DefaultRecorder; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +public class KgReasonerAggTest { + + @Test + public void testAgg3() { + String dsl = + "GraphStructure {\n" + + " s [AMLz50.Custid, __start__='true']\n" + + " s2 [AMLz50.Custid]\n" + + "\n" + + " u1,u2 [AMLz50.Userinfo]\n" + + " test [Test]\n" + + " s->u1 [has]\n" + + " s2->u2 [has]\n" + + " u1->u2 [aml90dTradeEdge] as e\n" + + " s->test [aml90dTradeCrimeReport]\n" + + " s2->test [aml90dTradeCrimeReport]\n" + + "} \n" + + "Rule {\n" + + " totalAmt = group(s, s2).sum(e.payAmt90d)\n" + + " R1(\"必须大于1万\"): totalAmt > 1000\n" + + " nums = group(s).count(s2)\n" + + " result = rule_value(nums >= 1, true, false)\n" + + "}\n" + + "Action {\n" + + " get(s.id, result, nums)\n" + + "}"; + + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("AMLz50.Custid", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("AMLz50.Userinfo", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("Test", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AMLz50.Userinfo_aml90dTradeEdge_AMLz50.Userinfo", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payAmt90d"))); + schema.put( + "AMLz50.Custid_aml90dTradeCrimeReport_Test", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payAmt90d"))); + schema.put( + "AMLz50.Custid_has_AMLz50.Userinfo", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerAggTest$GraphLoader223"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_MULTI_VERSION_ENABLE, "true"); + task.setParams(params); + task.setExecutorTimeoutMs(99999999999999999L); + + task.setExecutionRecorder(new DefaultRecorder()); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + System.out.println(task.getExecutionRecorder().toReadableString()); + + // only u1 + Assert.assertEquals(1, result.getRows().size()); + Assert.assertEquals("s", result.getRows().get(0)[0]); + Assert.assertEquals("true", result.getRows().get(0)[1]); + Assert.assertEquals("1", result.getRows().get(0)[2]); + } + + public static class GraphLoader223 extends AbstractLocalGraphLoader { + @Override + public String getDemoGraph() { + return "Graph {\n" + + " s [AMLz50.Custid]\n" + + " s2 [AMLz50.Custid]\n" + + "\n" + + " test [Test]\n" + + " u1,u12,u13,u14,u15,u2 [AMLz50.Userinfo]\n" + + " s->u1 [has]\n" + + " s->u12 [has]\n" + + " s->u13 [has]\n" + + " s->u14 [has]\n" + + " s->u15 [has]\n" + + " s2->u2 [has]\n" + + " u1->u2 [aml90dTradeEdge, payAmt90d=10000] as e\n" + + " s->test [aml90dTradeCrimeReport]" + + " s2->test [aml90dTradeCrimeReport]" + + "}"; + } + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("C_37091", "CRO.Company", "regNo", "C_37091"), + constructionVertex("C_7661", "CRO.Company", "regNo", "C_7661"), + constructionVertex("C_8125", "CRO.Company", "regNo", "C_8125"), + constructionVertex("P1", "CRO.Person"), + constructionVertex("P2", "CRO.Person"), + constructionVertex("P3", "CRO.Person")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("P1", "corporate", "C_37091"), + constructionEdge("P1", "control", "C_7661"), + constructionEdge("P1", "superviseDirctor", "C_7661"), + constructionEdge("P2", "control", "C_7661"), + constructionEdge("P2", "corporate", "C_7661"), + constructionEdge("P2", "superviseDirctor", "C_7661"), + constructionEdge("P2", "corporate", "C_8125"), + constructionEdge("P3", "superviseDirctor", "C_7661"), + constructionEdge("P3", "corporate", "C_8125")); + } + } + + @Test + public void testAgg2() { + String dsl = + "GraphStructure {\n" + + " s [AMLz50.Custid, __start__='true']\n" + + " s2 [AMLz50.Custid]\n" + + "\n" + + " u1,u2 [AMLz50.Userinfo]\n" + + " s->u1 [has]\n" + + " s2->u2 [has]\n" + + " u1->u2 [aml90dTradeEdge] as e\n" + + " s->s2 [aml90dTradeCrimeReport]\n" + + "} \n" + + "Rule {\n" + + " totalAmt = group(s, s2).sum(e.payAmt90d)\n" + + " R1(\"必须大于1万\"): totalAmt > 1000\n" + + " nums = group(s).count(s2)\n" + + " result = rule_value(nums >= 1, true, false)\n" + + "}\n" + + "Action {\n" + + " get(s.id, result, nums)\n" + + "}"; + + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("AMLz50.Custid", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("AMLz50.Userinfo", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AMLz50.Userinfo_aml90dTradeEdge_AMLz50.Userinfo", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payAmt90d"))); + schema.put( + "AMLz50.Custid_aml90dTradeCrimeReport_AMLz50.Custid", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payAmt90d"))); + schema.put( + "AMLz50.Custid_has_AMLz50.Userinfo", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerAggTest$GraphLoader22"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_MULTI_VERSION_ENABLE, "true"); + task.setParams(params); + task.setExecutorTimeoutMs(99999999999999999L); + + task.setExecutionRecorder(new DefaultRecorder()); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + System.out.println(task.getExecutionRecorder().toReadableString()); + + // only u1 + Assert.assertEquals(1, result.getRows().size()); + Assert.assertEquals("s", result.getRows().get(0)[0]); + Assert.assertEquals("true", result.getRows().get(0)[1]); + Assert.assertEquals("1", result.getRows().get(0)[2]); + } + + public static class GraphLoader22 extends AbstractLocalGraphLoader { + @Override + public String getDemoGraph() { + return "Graph {\n" + + " s [AMLz50.Custid]\n" + + " s2 [AMLz50.Custid]\n" + + "\n" + + " u1,u12,u13,u14,u15,u2 [AMLz50.Userinfo]\n" + + " s->u1 [has]\n" + + " s->u12 [has]\n" + + " s->u13 [has]\n" + + " s->u14 [has]\n" + + " s->u15 [has]\n" + + " s2->u2 [has]\n" + + " u1->u2 [aml90dTradeEdge, payAmt90d=10000] as e\n" + + " s->s2 [aml90dTradeCrimeReport]" + + "}"; + } + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("C_37091", "CRO.Company", "regNo", "C_37091"), + constructionVertex("C_7661", "CRO.Company", "regNo", "C_7661"), + constructionVertex("C_8125", "CRO.Company", "regNo", "C_8125"), + constructionVertex("P1", "CRO.Person"), + constructionVertex("P2", "CRO.Person"), + constructionVertex("P3", "CRO.Person")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("P1", "corporate", "C_37091"), + constructionEdge("P1", "control", "C_7661"), + constructionEdge("P1", "superviseDirctor", "C_7661"), + constructionEdge("P2", "control", "C_7661"), + constructionEdge("P2", "corporate", "C_7661"), + constructionEdge("P2", "superviseDirctor", "C_7661"), + constructionEdge("P2", "corporate", "C_8125"), + constructionEdge("P3", "superviseDirctor", "C_7661"), + constructionEdge("P3", "corporate", "C_8125")); + } + } + + @Test + public void testAgg() { + String dsl = + "GraphStructure {\n" + + " // 公司通过人间接关系,两度\n" + + " B [CRO.Company,__start__='true'] \n" + + " A, C [CRO.Company] \n" + + " E, F [CRO.Person] \n" + + " E -> A [superviseDirctor, corporate, control] as E1\n" + + " E -> B [superviseDirctor, corporate, control] as E2\n" + + " F -> B [superviseDirctor, corporate, control] as E3\n" + + " F -> C [superviseDirctor, corporate, control] as E4\n" + + "}\n" + + "Rule {\n" + + " R1: A.id> schema = new HashMap<>(); + schema.put( + "CRO.Company", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "regNo"))); + schema.put("CRO.Person", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CRO.Person_superviseDirctor_CRO.Company", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "CRO.Person_corporate_CRO.Company", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "CRO.Person_control_CRO.Company", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerAggTest$GraphLoader"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_MULTI_VERSION_ENABLE, "true"); + task.setParams(params); + + task.setExecutionRecorder(new DefaultRecorder()); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + Assert.assertEquals(1, result.getRows().size()); + Assert.assertEquals("C_37091", result.getRows().get(0)[0]); + Assert.assertEquals("C_7661", result.getRows().get(0)[1]); + Assert.assertEquals("C_8125", result.getRows().get(0)[2]); + + System.out.println(task.getExecutionRecorder().toReadableString()); + } + + public static class GraphLoader extends AbstractLocalGraphLoader { + @Override + public String getDemoGraph() { + return "Graph {\n" + + " C_37091 [CRO.Company, regNo='C_37091']\n" + + " C_7661 [CRO.Company, regNo='C_7661']\n" + + " C_8125 [CRO.Company, regNo='C_8125'] \n" + + " P1 [CRO.Person]\n" + + " P2 [CRO.Person]\n" + + " P3 [CRO.Person]\n" + + "\n" + + "\n" + + " P1->C_37091[corporate]\n" + + " P1->C_7661 [control]\n" + + " P1->C_7661 [superviseDirctor]\n" + + "\n" + + " P2->C_7661[corporate]\n" + + " P2->C_7661 [control]\n" + + " P2->C_7661 [superviseDirctor]\n" + + "\n" + + " P3->C_8125[corporate]\n" + + " P3->C_7661[superviseDirctor]\n" + + "\n" + + "}"; + } + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("C_37091", "CRO.Company", "regNo", "C_37091"), + constructionVertex("C_7661", "CRO.Company", "regNo", "C_7661"), + constructionVertex("C_8125", "CRO.Company", "regNo", "C_8125"), + constructionVertex("P1", "CRO.Person"), + constructionVertex("P2", "CRO.Person"), + constructionVertex("P3", "CRO.Person")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("P1", "corporate", "C_37091"), + constructionEdge("P1", "control", "C_7661"), + constructionEdge("P1", "superviseDirctor", "C_7661"), + constructionEdge("P2", "control", "C_7661"), + constructionEdge("P2", "corporate", "C_7661"), + constructionEdge("P2", "superviseDirctor", "C_7661"), + constructionEdge("P2", "corporate", "C_8125"), + constructionEdge("P3", "superviseDirctor", "C_7661"), + constructionEdge("P3", "corporate", "C_8125")); + } + } + + @Test + public void testDistinct() { + String dsl = + "GraphStructure {\n" + + " A, B [FilmDirector] // 定义了A,B两个导演\n" + + " C, D [Film] // 定义了C,D两部电影\n" + + " E [FilmStar] // E是电影明星\n" + + " C->A [directFilm] as F1 // C的导演是A\n" + + " D->B [directFilm] as F2 // D的导演是B\n" + + " C->E [starOfFilm] as F3 // C的主演是E\n" + + " D->E [starOfFilm] as F4 // D的主演也是E\n" + + "}\n" + + "Rule {\n" + + " R1: A.id> schema = new HashMap<>(); + schema.put("Film", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("FilmDirector", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("FilmStar", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "Film_directFilm_FilmDirector", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "Film_starOfFilm_FilmStar", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerAggTest$GraphLoader2"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_MULTI_VERSION_ENABLE, "true"); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + Assert.assertEquals(result.getRows().size(), 1); + } + + public static class GraphLoader2 extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("D1", "FilmDirector"), + constructionVertex("D2", "FilmDirector"), + constructionVertex("F1", "Film"), + constructionVertex("F2", "Film"), + constructionVertex("F3", "Film"), + constructionVertex("Star", "FilmStar")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("F1", "directFilm", "D1"), + constructionEdge("F1", "starOfFilm", "Star"), + constructionEdge("F2", "directFilm", "D1"), + constructionEdge("F2", "starOfFilm", "Star"), + constructionEdge("F3", "directFilm", "D2"), + constructionEdge("F3", "starOfFilm", "Star")); + } + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAliasSetKFilmTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAliasSetKFilmTest.java new file mode 100644 index 00000000..49e6dd4f --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerAliasSetKFilmTest.java @@ -0,0 +1,158 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.runner.local.main.basetest.TransBaseTestData; +import java.util.HashMap; +import java.util.List; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class KgReasonerAliasSetKFilmTest { + + public interface TestFunction { + void run(); + } + + public static class FileMutex { + public static void runTestWithMutex(TestFunction function) { + function.run(); + } + } + + @Before + public void init() {} + + @Test + public void test1() { + FileMutex.runTestWithMutex(this::doTest1); + } + + private void doTest1() { + String dsl = + "\n" + + "GraphStructure {\n" + + " (A:User)-[p1:trans]->(B:User)-[p2:trans]->(C:User)-[p3:trans]->(A)\n" + + "}\n" + + "Rule {\n" + + "R1: A.id == $idSet1\n" + + "R2: B.id in $idSet2\n" + + "R3: C.id in $idSet2\n" + + "totalTrans1 = group(A,B,C).sum(p1.amount)\n" + + "totalTrans2 = group(A,B,C).sum(p2.amount)\n" + + "totalTrans3 = group(A,B,C).sum(p3.amount)\n" + + "totalTrans = totalTrans1 + totalTrans2 + totalTrans3\n" + + "R2('取top2'): top(totalTrans, 2)" + + "}\n" + + "Action {\n" + + " get(A.id, B.id, C.id, totalTrans)\n" + + "}"; + List result = + TransBaseTestData.runTestResult( + dsl, + new HashMap() { + { + put("idSet1", "'1'"); + put("idSet2", "['2', '3']"); + } + }); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("1", result.get(0)[0]); + Assert.assertEquals("2", result.get(0)[1]); + Assert.assertEquals("3", result.get(0)[2]); + } + + @Test + public void test2() { + FileMutex.runTestWithMutex(this::doTest2); + } + + private void doTest2() { + String dsl = + "\n" + + "GraphStructure {\n" + + " (A:User)-[p1:trans]->(B:User)-[p2:trans]->(C:User)-[p3:trans]->(A)\n" + + "}\n" + + "Rule {\n" + + "R1: A.id in $idSet1\n" + + "R2: B.id in $idSet2\n" + + "R3: C.id in $idSet2\n" + + "totalTrans = p1.amount + p2.amount + p3.amount\n" + + "R2('取top2'): top(totalTrans, 3)" + + "}\n" + + "Action {\n" + + " get(A.id, B.id, C.id, totalTrans)\n" + + "}"; + List result = + TransBaseTestData.runTestResult( + dsl, + new HashMap() { + { + put("idSet1", "['1', '4', '5']"); + put("idSet2", "['2', '3']"); + } + }); + Assert.assertEquals(3, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("500", result.get(0)[3]); + Assert.assertEquals("500", result.get(1)[3]); + Assert.assertEquals("500", result.get(2)[3]); + } + + @Test + public void test4() { + FileMutex.runTestWithMutex(this::doTest4); + } + + private void doTest4() { + String dsl = + "\n" + + "GraphStructure {\n" + + " A [User, __start__='true']\n" + + " B,C [User]\n" + + " A->B [trans] as p1\n" + + " B->C [trans] as p2\n" + + " C->A [trans] as p3" + + "}\n" + + "Rule {\n" + + "R1: A.id in $idSet1\n" + + "R2: B.id in $idSet2\n" + + "R3: C.id in $idSet2\n" + + "t1 = group(A,B,C).sum(p1.amount)\n" + + "t2 = group(A,B,C).sum(p2.amount)\n" + + "t3 = group(A,B,C).sum(p3.amount)\n" + + "totalSum = t1 + t2 + t3" + + "}\n" + + "Action {\n" + + " get(A.id, B.id, C.id, totalSum)\n" + + "}"; + List result = + TransBaseTestData.runTestResult( + dsl, + new HashMap() { + { + put("idSet1", "['1']"); + put("idSet2", "['2', '3']"); + } + }); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("1", result.get(0)[0]); + Assert.assertEquals("2", result.get(0)[1]); + Assert.assertEquals("3", result.get(0)[2]); + Assert.assertEquals("700.0", result.get(0)[3]); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerDependencyFilmTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerDependencyFilmTest.java new file mode 100644 index 00000000..a9245cac --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerDependencyFilmTest.java @@ -0,0 +1,1080 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.runner.local.main.basetest.FilmBaseTestData; +import java.util.List; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class KgReasonerDependencyFilmTest { + + public interface TestFunction { + void run(); + } + + public static class FileMutex { + public static void runTestWithMutex(TestFunction function) { + function.run(); + } + } + + @Before + public void init() {} + + public Catalog getMockCatalogSchema() { + return FilmBaseTestData.getMockCatalogSchema(); + } + + private List runTestResult(String dsl) { + return FilmBaseTestData.runTestResult(dsl); + } + + @Test + public void test1() { + FileMutex.runTestWithMutex(this::doTest1); + } + + private void doTest1() { + String dsl = + "GraphStructure {\n" + + "\t(s:Film)-[p]->(u)\n" + + "}\n" + + "Rule {\n" + + "\tnums = group(s).count(u)\n" + + "}\n" + + "Action {\n" + + "\tget(s.id as id_num, nums as tmp)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("4", result.get(0)[1]); + } + + @Test + public void test2() { + FileMutex.runTestWithMutex(this::doTest2); + } + + private void doTest2() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.starNum as starNum)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + } + + @Test + public void test3() { + FileMutex.runTestWithMutex(this::doTest3); + } + + private void doTest3() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.starNum, s.directNums)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + Assert.assertEquals("1", result.get(0)[2]); + } + + @Test + public void test4() { + FileMutex.runTestWithMutex(this::doTest4); + } + + private void doTest4() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:workerNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p]->(u)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.starNum, s.directNums, s.workerNums)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + Assert.assertEquals("1", result.get(0)[2]); + Assert.assertEquals("4", result.get(0)[3]); + } + + @Test + public void test5() { + FileMutex.runTestWithMutex(this::doTest5); + } + + private void doTest5() { + String dsl = + "Define (s:FilmStar)-[p:isHosStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).count(u)\n" + + "\to = rule_value(filmNum > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:hotStarNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).countIf(u.isHosStar == true, u)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.hotStarNum)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + } + + @Test + public void test6() { + FileMutex.runTestWithMutex(this::doTest6); + } + + private void doTest6() { + String dsl = + "Define (s:FilmStar)-[p:isHosStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).count(u)\n" + + "\to = rule_value(filmNum > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:hotStarNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).countIf(u.isHosStar == true, u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.hotStarNum, s.directNums)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + Assert.assertEquals("1", result.get(0)[2]); + } + + @Test + public void test7() { + FileMutex.runTestWithMutex(this::doTest7); + } + + private void doTest7() { + String dsl = + "Define (s:FilmStar)-[p:isHosStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).count(u)\n" + + "\to = rule_value(filmNum > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:hotStarNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).countIf(u.isHosStar == true, u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:FilmDirector)-[p:isHotDirector]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:directOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).count(u)\n" + + "\to = rule_value(filmNum > 1, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:hasHotDirector]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\tnum = group(s).countIf(u.isHotDirector == true, u)\n" + + "\to = rule_value(num > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.hotStarNum, s.hasHotDirector)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + Assert.assertEquals("true", result.get(0)[2]); + } + + @Test + public void test8() { + FileMutex.runTestWithMutex(this::doTest8); + } + + private void doTest8() { + String dsl = + "Define (s:Film)-[p:hasManyStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\tnum = group(s).count(u)\n" + + "\to = rule_value(num > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "Define (s:FilmStar)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.hasManyStar == true, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:FilmDirector)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:directOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.hasManyStar == true, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm|directOfFilm]->(u:FilmStar|FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, u.partInFamous)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(5, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("true", result.get(0)[1]); + } + + @Test + public void test9() { + FileMutex.runTestWithMutex(this::doTest9); + } + + private void doTest9() { + String dsl = + "Define (s:FilmStar)-[p:isHosStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).count(u)\n" + + "\to = rule_value(filmNum > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:hotStarNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).countIf(u.isHosStar == true, u)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.hotStarNum, s.starNum, s.directNums)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("3", result.get(0)[1]); + Assert.assertEquals("3", result.get(0)[2]); + Assert.assertEquals("1", result.get(0)[3]); + } + + @Test + public void test10() { + FileMutex.runTestWithMutex(this::doTest10); + } + + private void doTest10() { + String dsl = + "Define (s:Film)-[p:hasManyStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\tnum = group(s).count(u)\n" + + "\to = rule_value(num > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "Define (s:FilmStar)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.hasManyStar == true, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:FilmDirector)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:directOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.hasManyStar == true, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm|directOfFilm]->(u:FilmStar|FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, u.partInFamous, s.directNums)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(5, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("true", result.get(0)[1]); + Assert.assertEquals("1", result.get(0)[2]); + } + + @Test + public void test11() { + FileMutex.runTestWithMutex(this::doTest11); + } + + private void doTest11() { + String dsl = + "Define (s:Film)-[p:hasManyStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\tnum = group(s).count(u)\n" + + "\to = rule_value(num > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "Define (s:FilmStar)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.hasManyStar == true, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:FilmDirector)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:directOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.hasManyStar == true, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p]->(u)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.hasManyStar, u.partInFamous)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(5, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("true", result.get(0)[1]); + Assert.assertEquals("true", result.get(0)[2]); + } + + @Test + public void test12() { + FileMutex.runTestWithMutex(this::doTest12); + } + + private void doTest12() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + } + + @Test + public void test13() { + FileMutex.runTestWithMutex(this::doTest13); + } + + private void doTest13() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:FilmStar)-[p:isHosStar]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).count(u)\n" + + "\to = rule_value(filmNum > 2, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, u.isHosStar)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(4, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("true", result.get(0)[2]); + } + + @Test + public void test14() { + FileMutex.runTestWithMutex(this::doTest14); + } + + private void doTest14() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:FilmStar)-[p:partInFamous]->(o:Boolean) {\n" + + "GraphStructure {\n" + + "\t(s)<-[p:starOfFilm]-(u:Film)\n" + + "}\n" + + "Rule {\n" + + "\tfilmNum = group(s).countIf(u.starNum>1, u)\n" + + "\to = rule_value(filmNum > 0, true, false)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, u.partInFamous)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(4, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("true", result.get(0)[2]); + } + + @Test + public void test15() { + FileMutex.runTestWithMutex(this::doTest15); + } + + private void doTest15() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo2]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, s.partiInfo2)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("star:3 director:1", result.get(0)[2]); + } + + @Test + public void test16() { + FileMutex.runTestWithMutex(this::doTest16); + } + + private void doTest16() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, s.starNum, s.directNums)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("3", result.get(0)[2]); + Assert.assertEquals("1", result.get(0)[3]); + } + + @Test + public void test17() { + FileMutex.runTestWithMutex(this::doTest17); + } + + private void doTest17() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:starInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"starInfo\", s.starNum)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, s.starNum, s.starInfo)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("3", result.get(0)[2]); + Assert.assertEquals("starInfo3", result.get(0)[3]); + } + + @Test + public void test18() { + FileMutex.runTestWithMutex(this::doTest18); + } + + private void doTest18() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo2]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, s.starNum, s.partiInfo2)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("3", result.get(0)[2]); + Assert.assertEquals("star:3 director:1", result.get(0)[3]); + } + + @Test + public void test19() { + FileMutex.runTestWithMutex(this::doTest19); + } + + private void doTest19() { + String dsl = + "Define (s:Film)-[p:starNum]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:directNums]->(o:Int) {\n" + + "GraphStructure {\n" + + "\t(s)-[p:directOfFilm]->(u:FilmDirector)\n" + + "}\n" + + "Rule {\n" + + "\to = group(s).count(u)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo2]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "Define (s:Film)-[p:partiInfo3]->(o:String) {\n" + + "GraphStructure {\n" + + "\t(s)\n" + + "}\n" + + "Rule {\n" + + "\to = concat(\"star:\", s.starNum, \" director:\", s.directNums)\n" + + "}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t(s:Film)-[p:starOfFilm]->(u:FilmStar)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,s.partiInfo, s.partiInfo3, s.partiInfo2)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(4, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("star:3 director:1", result.get(0)[1]); + Assert.assertEquals("star:3 director:1", result.get(0)[2]); + Assert.assertEquals("star:3 director:1", result.get(0)[2]); + } + + @Test + public void test20() { + FileMutex.runTestWithMutex(this::doTest20); + } + + private void doTest20() { + String dsl = + "GraphStructure {\n" + + "\t(s:Film)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id as id_num, s.filmBudget)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("100", result.get(0)[1]); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerLeadToTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerLeadToTest.java new file mode 100644 index 00000000..0a6232ea --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerLeadToTest.java @@ -0,0 +1,231 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +public class KgReasonerLeadToTest { + @Test + public void addVertexTest1() { + String dsl = + "Define (s:DomainFamily)-[p:total_domain_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (s)<-[e:belong]-(d:Domain)\n" + + " }\n" + + " Rule {\n" + + " num = group(s).count(d)\n" + + " o = num\n" + + " }\n" + + " Action {\n" + + " \tcreateNodeInstance(\n" + + " \t type=DomainFamilyCount,\n" + + " \t value={\n" + + " \t\t id = s.id\n" + + " \t\t 主体 = s.hangye\n" + + " \t\t 客体 = o\n" + + " \t\t 时间 = e.occurTime\n" + + " \t\t 空间 = e.occurSpace\n" + + " }\n" + + " )\n" + + " }" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "DomainFamily", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "hangye"))); + schema.put("Domain", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "Domain_belong_DomainFamily", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("occurTime", "occurSpace"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerLeadToTest$GraphLoaderForAddVertex"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(1, result.getVertexList().size()); + } + + public static class GraphLoaderForAddVertex extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("df1", "DomainFamily"), + constructionVertex("d1", "Domain"), + constructionVertex("d2", "Domain")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("d1", "belong", "df1"), constructionEdge("d2", "belong", "df1")); + } + } + + @Test + public void leadToTest1() { + String dsl = + "Define (s: `HengSheng.TaxonomyOfCompanyAccident`/`周期性行业头部上市公司停产事故`)-[p: leadTo]->(o: `HengSheng.TaxonomyOfCompanyInfluence`/`利润下降`) {\n" + + " GraphStructure {\n" + + " \n }\n" + + " Rule {\n" + + " }\n" + + " Action {\n" + + " \tcompanyInfluenceEvent = createNodeInstance(\n" + + " \ttype=HengSheng.CompanyInfluence,\n" + + " value={\n" + + " \tname='公司利润下降'\n" + + " influenceDegree='下降'\n" + + " indexTag='利润'\n" + + " subject=s.subject\n" + + " objectWho='下降'\n" + + " }\n" + + " )\n" + + " createEdgeInstance(\n" + + " src=s,\n" + + " dst=companyInfluenceEvent,\n" + + " type=leadTo,\n" + + " value={\n" + + " \n" + + " }\n" + + " )\n" + + " }"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "HengSheng.TaxonomyOfCompanyAccident", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "HengSheng.TaxonomyOfCompanyInfluence", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "HengSheng.CompanyAccident", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet( + "id", + "companyIndustryType", + "companyType", + "companyMarketShare", + "accidentImpactType", + "subject"))); + schema.put("HengSheng.Industry", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("HengSheng.Company", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "HengSheng.CompanyInfluence", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "name", "influenceDegree", "indexTag", "subject", "objectWho"))); + schema.put( + "HengSheng.CompanyAccident_subject_HengSheng.Industry", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "HengSheng.Company_belongIndustry_HengSheng.Industry", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "HengSheng.CompanyAccident_belongTo_HengSheng.TaxonomyOfCompanyAccident", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "HengSheng.CompanyInfluence_belongTo_HengSheng.TaxonomyOfCompanyInfluence", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "HengSheng.CompanyAccident_leadTo_HengSheng.CompanyInfluence", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerLeadToTest$GraphLoader"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(Constants.START_ALIAS, "s"); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(1, result.getVertexList().size()); + } + + public static class GraphLoader extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("周期性行业头部上市公司停产事故", "HengSheng.TaxonomyOfCompanyAccident"), + constructionVertex("利润下降", "HengSheng.TaxonomyOfCompanyInfluence"), + constructionVertex( + "s1", + "HengSheng.CompanyAccident", + "companyIndustryType", + "cyclical", + "companyType", + "listedCompany", + "companyMarketShare", + 0.1, + "accidentImpactType", + "productionHalt", + "subject", + "c1"), + constructionVertex("i1", "HengSheng.Industry"), + constructionVertex("c1", "HengSheng.Company")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("s1", "subject", "i1"), + constructionEdge("c1", "belongIndustry", "i1") + + // add belongTo + , + constructionEdge("s1", "belongTo", "周期性行业头部上市公司停产事故")); + } + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerTopKFilmTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerTopKFilmTest.java new file mode 100644 index 00000000..56987d54 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerTopKFilmTest.java @@ -0,0 +1,419 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.runner.local.main.basetest.FilmBaseTestData; +import java.util.List; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class KgReasonerTopKFilmTest { + + public interface TestFunction { + void run(); + } + + public static class FileMutex { + public static void runTestWithMutex(TestFunction function) { + function.run(); + } + } + + @Before + public void init() {} + + private List runTestResult(String dsl) { + return FilmBaseTestData.runTestResult( + dsl, + "com.antgroup.openspg.reasoner.runner.local.main.basetest.FilmBaseTestData$FilmGraphGeneratorTopK"); + } + + @Test + public void test1() { + FileMutex.runTestWithMutex(this::doTest1); + } + + private void doTest1() { + String dsl = + "Define (s:Film)-[p:olderStar]->(o:FilmStar) {\n" + + "\tGraphStructure {\n" + + " (s)-[:starOfFilm]->(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"只取年纪最大的演员\"): group(s).desc(o.age).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:olderStar]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("L1_1_star", result.get(0)[1]); + } + + @Test + public void test2() { + FileMutex.runTestWithMutex(this::doTest2); + } + + private void doTest2() { + String dsl = + "Define (s:Film)-[p:youngStar]->(o:FilmStar) {\n" + + "\tGraphStructure {\n" + + " (s)-[:starOfFilm]->(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"只取年纪最小的演员\"): group(s).asc(o.age).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:youngStar]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("L1_3_star", result.get(0)[1]); + } + + @Test + public void test3() { + FileMutex.runTestWithMutex(this::doTest3); + } + + private void doTest3() { + String dsl = + "Define (s:Film)-[p:firstJoinStar]->(o:FilmStar) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"只取最早加入的演员\"): group(s).asc(sf.joinTs).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:firstJoinStar]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("L1_1_star", result.get(0)[1]); + } + + @Test + public void test4() { + FileMutex.runTestWithMutex(this::doTest4); + } + + private void doTest4() { + String dsl = + "Define (s:Film)-[p:lastJoinStar]->(o:FilmStar) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"只取最晚加入的演员\"): group(s).desc(sf.joinTs).limit(1)\n" + + " p.joinTs = sf.joinTs \n" + + " }\n" + + "}\n" + + "GraphStructure {\n" + + " (s)-[p:lastJoinStar]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id, p.joinTs)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("L1_3_star", result.get(0)[1]); + Assert.assertEquals("400", result.get(0)[2]); + } + + @Test + public void test5() { + FileMutex.runTestWithMutex(this::doTest5); + } + + private void doTest5() { + String dsl = + "Define (s:Film)-[p:mostSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " sameStarNum(\"得到相同演员数目\") = group(s,o).count(star)\n" + + " R1(\"值取最大值\"): group(s).desc(sameStarNum).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:mostSameStarFilm]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f1", result.get(0)[1]); + } + + @Test + public void test6() { + FileMutex.runTestWithMutex(this::doTest6); + } + + private void doTest6() { + String dsl = + "Define (s:Film)-[p:lessSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " sameStarNum(\"得到相同演员数目\") = group(s,o).count(star)\n" + + " R1(\"值取最大值\"): group(s).asc(sameStarNum).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:lessSameStarFilm]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f3", result.get(0)[1]); + } + + @Test + public void test7() { + FileMutex.runTestWithMutex(this::doTest7); + } + + private void doTest7() { + String dsl = + "Define (s:Film)-[p:mostSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " sameStarNum(\"得到相同男演员数目\") = group(s,o).countIf(star.gender == '男', star)\n" + + " R1(\"值取最大值\"): group(s).desc(sameStarNum).limit(1)\n" + + " }\n" + + "}\n" + + "GraphStructure {\n" + + " (s)-[:mostSameStarFilm]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f1", result.get(0)[1]); + } + + @Test + public void test8() { + FileMutex.runTestWithMutex(this::doTest8); + } + + private void doTest8() { + String dsl = + "Define (s:Film)-[p:lessSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " sameStarNum(\"得到相同男演员数目\") = group(s,o).countIf(star.gender == '男', star)\n" + + " R1(\"值取最大值\"): group(s).asc(sameStarNum).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:lessSameStarFilm]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f3", result.get(0)[1]); + } + + @Test + public void test9() { + FileMutex.runTestWithMutex(this::doTest9); + } + + private void doTest9() { + String dsl = + "Define (s:Film)-[p:firstSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"值取最大值\"): group(s).asc(sf2.joinTs).limit(1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:firstSameStarFilm]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f3", result.get(0)[1]); + } + + @Test + public void test10() { + FileMutex.runTestWithMutex(this::doTest10); + } + + private void doTest10() { + String dsl = + "Define (s:Film)-[p:lastSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"值取最大值\"): group(s).top(sf2.joinTs, 1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " s [Film, __start__='true']\n" + + " o [Film]\n" + + " s->o [lastSameStarFilm]\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f1", result.get(0)[1]); + } + + @Test + public void test11() { + FileMutex.runTestWithMutex(this::doTest11); + } + + private void doTest11() { + String dsl = + "Define (s:Film)-[p:lastSameStarFilm]->(o:Film) {\n" + + "\tGraphStructure {\n" + + " (s)-[sf:starOfFilm]->(star:FilmStar)<-[sf2:starOfFilm]-(o)\n" + + " }\n" + + "\tRule {\n" + + " R1(\"值取最大值\"): top(sf2.joinTs, 1)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s)-[:lastSameStarFilm]->(o)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(2, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f1", result.get(0)[1]); + } + + @Test + public void test12() { + FileMutex.runTestWithMutex(this::doTest12); + } + + private void doTest12() { + String dsl = + "\n" + + "GraphStructure {\n" + + " s [Film, __start__='true']\n" + + " star [FilmStar]\n" + + " o [Film]\n" + + " s->star [starOfFilm] as sf \n" + + " o->star [starOfFilm] as sf2\n" + + "}\n" + + "Rule {\n" + + "total = sf.joinTs + sf2.joinTs\n" + + "R2: top(total, 1)\n" + + "}\n" + + "Action {\n" + + " get(s.id, o.id, total)\n" + + "}"; + List result = runTestResult(dsl); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(3, result.get(0).length); + Assert.assertEquals("root", result.get(0)[0]); + Assert.assertEquals("f1", result.get(0)[1]); + Assert.assertEquals("700", result.get(0)[2]); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerZijinLocalTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerZijinLocalTest.java new file mode 100644 index 00000000..6575820a --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/KgReasonerZijinLocalTest.java @@ -0,0 +1,404 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Test; +import scala.Tuple2; + +public class KgReasonerZijinLocalTest { + + @Test + public void test3() { + String dsl = + "// 当月交易量\n" + + "Define (s:CustFundKG.Account)-[p:cur_month_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tR1(\"当月交易量\"): date_diff(from_unix_time(now(), 'yyyyMMdd'),t.transDate) <= 30\n" + + " o = group(s).count(t.transDate)\n" + + " }\n" + + "}\n" + + "// 次月交易量\n" + + "Define (s:CustFundKG.Account)-[p:last_month_num]->(o:Int) {\n" + + " GraphStructure {\n" + + " (u:CustFundKG.Account)-[t:accountFundContact]-(s)\n" + + " }\n" + + "\tRule {\n" + + " \tdate_delta = date_diff(from_unix_time(now(), 'yyyyMMdd'),t.transDate)\n" + + " \tR1(\"次月交易量\"): date_delta > 30 && date_delta <=60\n" + + " o = group(s).count(t.transDate)\n" + + " }\n" + + "}\n" + + "\n" + + "// 倍数\n" + + "Define (s:CustFundKG.Account)-[p:trans_multiple]->(o:Float) {\n" + + "\t\tGraphStructure {\n" + + " (s)\n" + + " }\n" + + "\tRule {\n" + + " \tcur_month = rule_value(s.cur_month_num==null, 0.0, s.cur_month_num*1.0)\n" + + " last_month = rule_value(s.last_month_num == null, 1, s.last_month_num)\n" + + " \tmultiple = cur_month / last_month\n" + + " o = multiple\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s:CustFundKG.Account)-[p:is_trans_raise_more]->(o:Boolean) {\n" + + "\t\tGraphStructure {\n" + + " (s)\n" + + " }\n" + + "\tRule {\n" + + " \to = rule_value(s.trans_multiple >= 3, true, false)\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s:CustFundKG.Account)\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + "\tget(s.id, s.is_trans_raise_more, s.trans_multiple, s.last_month_num, s.cur_month_num)\n" + + "}\n"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("2088xx3", "CustFundKG.Account"))); + + task.setExecutorTimeoutMs(60 * 1000 * 100); + task.setDsl(dsl); + + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("transDate"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerZijinLocalTest$GraphLoader"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(1, result.getRows().size()); + } + + @Test + public void test4() { + String dsl = + "GraphStructure {\n" + + "s [CustFundKG.Account, __start__='true']\n" + + "u [CustFundKG.Account] \n" + + "u->s [accountFundContact] as t\n" + + "}\n" + + "Rule {\n" + + "o = group(s).count(t)\n" + + "}\n" + + "Action {\n" + + "\tget(s.id,t.sumAmt,t.transCount)\n" + + "}\n"; + LocalReasonerTask task = new LocalReasonerTask(); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("2088xx4", "CustFundKG.Account"))); + + task.setExecutorTimeoutMs(60 * 1000 * 100); + task.setDsl(dsl); + + Map> schema = new HashMap<>(); + schema.put("CustFundKG.Account", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "CustFundKG.Account_accountFundContact_CustFundKG.Account", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("transDate", "sumAmt", "transCount"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerZijinLocalTest$GraphLoader"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(1, result.getRows().size()); + } + + public static class GraphLoader extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("2088xx", "CustFundKG.Alipay"), + constructionVertex("2088xxx1", "CustFundKG.BankCard"), + constructionVertex("2088xxx2", "CustFundKG.Alipay"), + constructionVertex("2088xx3", "CustFundKG.Account"), + constructionVertex("2088xx4", "CustFundKG.Account")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("2088xx", "transfer", "2088xxx1", "payDate", 1691950562L), + constructionEdge("2088xx", "transfer", "2088xxx2", "payDate", 1691950562L), + constructionEdge("2088xx3", "accountFundContact", "2088xx4", "transDate", "20230801")); + } + } + + @Test + public void test5() { + String dsl = + "Define (u1:AttributePOC.BrinsonAttribute)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfBrinsonAttribute`/`市场贡献大`) {\n" + + " GraphStructure {\n" + + " (s:AttributePOC.BrinsonAttribute)-[p1:factorValue]->(u1)\n" + + " }\n" + + "\n" + + " Rule {\n" + + " R1: u1.factorType == \"market\"\n" + + " R4: s.factorType == \"total\"\n" + + " v = (u1.factorValue/ s.factorValue)\n" + + " R2(\"必须大于50%\"): v > 0.5\n" + + " }\n" + + "}\n" + + "\n" + + "Define (u2:AttributePOC.BrinsonAttribute)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfBrinsonAttribute`/`选股贡献大`) {\n" + + " GraphStructure {\n" + + " (s:AttributePOC.BrinsonAttribute)-[p1:factorValue]->(u1:AttributePOC.BrinsonAttribute)\n" + + " (s)-[p2:factorValue]->(u2)\n" + + " (s)-[p3:factorValue]->(u3:AttributePOC.BrinsonAttribute)\n" + + " }\n" + + "\n" + + " Rule {\n" + + " R1: u1.factorType == \"cluster\"\n" + + " R2: u2.factorType == \"stock\"\n" + + " R3: u3.factorType == \"trade\"\n" + + " R4: s.factorType == \"total\"\n" + + " v = (u1.factorValue/ s.factorValue + u3.factorValue / s.factorValue)\n" + + " R6(\"必须大于50%\"): v < 0.5\n" + + " R5(\"交易收益大于选股\"): u2.factorValue > u3.factorValue\n" + + " }\n" + + "}\n" + + "\n" + + "Define (u2:AttributePOC.BrinsonAttribute)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfBrinsonAttribute`/`交易贡献大`) {\n" + + " GraphStructure {\n" + + " (s:AttributePOC.BrinsonAttribute)-[p1:factorValue]->(u1:AttributePOC.BrinsonAttribute)\n" + + " (s)-[p2:factorValue]->(u2)\n" + + " (s)-[p3:factorValue]->(u3:AttributePOC.BrinsonAttribute)\n" + + " }\n" + + "\n" + + " Rule {\n" + + " R1: u1.factorType == \"cluster\"\n" + + " R2: u2.factorType == \"trade\"\n" + + " R3: u3.factorType == \"stock\"\n" + + " R4: s.factorType == \"total\"\n" + + " v = (u1.factorValue/ s.factorValue + u2.factorValue / s.factorValue)\n" + + " R5(\"必须大于50%\"): v > 0.5\n" + + " R6(\"交易收益大于选股\"): u2.factorValue > u3.factorValue\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: AttributePOC.TracebackDay)-[p: market]->(o: Float) {\n" + + " GraphStructure {\n" + + " (s:AttributePOC.TracebackDay)-[:day]->(f: AttributePOC.BrinsonAttribute)-[:factorValue]->(u1:`AttributePOC.TaxonomyOfBrinsonAttribute`/`市场贡献大`)\n" + + "\t}\n" + + " Rule {\n" + + " o = u1.factorValue\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: AttributePOC.TracebackDay)-[p: stock]->(o: Float) {\n" + + " GraphStructure {\n" + + " (s:AttributePOC.TracebackDay)-[:day]->(f: AttributePOC.BrinsonAttribute)-[:factorValue]->(u1:`AttributePOC.TaxonomyOfBrinsonAttribute`/`选股贡献大`)\n" + + "\t}\n" + + " Rule {\n" + + " o = u1.factorValue\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: AttributePOC.TracebackDay)-[p: trade]->(o: Float) {\n" + + " GraphStructure {\n" + + " (s:AttributePOC.TracebackDay)-[:day]->(f: AttributePOC.BrinsonAttribute)-[:factorValue]->(u1:`AttributePOC.TaxonomyOfBrinsonAttribute`/`交易贡献大`)\n" + + "\t}\n" + + " Rule {\n" + + " o = u1.factorValue\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: AttributePOC.TracebackDay)-[p: result]->(o: Text) {\n" + + " GraphStructure {\n" + + " (s)\n" + + "}\n" + + "Rule {\n" + + "// 按照选股、交易、市场的顺序输出\n" + + " str1 = rule_value(s.stock == null, \"\", concat(\"选股\", \": \", s.stock, ', '))\n" + + " str2 = concat(str1, rule_value(s.trade == null, \"\", concat(\"交易\", \": \", s.trade, ', ')))\n" + + " str3 = concat(str2, rule_value(s.market == null, \"\", concat(\"市场\", \": \", s.market)))\n" + + " o = str3\n" + + "}\n" + + "}\n" + + "\n" + + "Define (u1:AttributePOC.Scenario)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfScenario`/`基金收益分析`) {\n" + + " GraphStructure {\n" + + " \t(u1)<-[p1:scConfig]-(s:AttributePOC.TracebackDay)\n" + + " }\n" + + " Rule {\n" + + " R1: s.result != null\n" + + " }\n" + + "}"; + dsl = + dsl + + "GraphStructure {\n" + + " (s: AttributePOC.TracebackDay)\n" + + "}\n" + + "Rule {\n" + + "// 按照选股、交易、市场的顺序输出\n" + + " str1 = rule_value(s.stock == null, \"\", concat(\"选股\", \": \", s.stock, ', '))\n" + + " str2 = rule_value(s.trade == null, \"\", concat(\"交易\", \": \", s.trade, ', '))\n" + + "}\n" + + "Action {\n" + + " get(s.id, str1, str2) \n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setExecutorTimeoutMs(60 * 1000 * 100); + task.setDsl(dsl); + + Map> schema = new HashMap<>(); + schema.put( + "AttributePOC.TracebackDay", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.Scenario", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.BrinsonAttribute", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "factorValue", "factorType"))); + schema.put( + "AttributePOC.TaxonomyOfBrinsonAttribute", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.TaxonomyOfScenario", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "AttributePOC.TracebackDay_day_AttributePOC.BrinsonAttribute", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "AttributePOC.BrinsonAttribute_factorValue_AttributePOC.BrinsonAttribute", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "AttributePOC.Scenario_scConfig_AttributePOC.TracebackDay", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "AttributePOC.TracebackDay_scConfig_AttributePOC.Scenario", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.KgReasonerZijinLocalTest$GraphLoader2"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(1, result.getRows().size()); + Assert.assertTrue(StringUtils.isNotBlank(result.getRows().get(0)[1].toString())); + } + + public static class GraphLoader2 extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("day", "AttributePOC.TracebackDay"), + constructionVertex( + "s", "AttributePOC.BrinsonAttribute", "factorType", "total", "factorValue", 0.01), + constructionVertex( + "stock", + "AttributePOC.BrinsonAttribute", + "factorType", + "stock", + "factorValue", + 0.004), + constructionVertex( + "trade", + "AttributePOC.BrinsonAttribute", + "factorType", + "trade", + "factorValue", + -0.005), + constructionVertex( + "market", + "AttributePOC.BrinsonAttribute", + "factorType", + "market", + "factorValue", + 0.006), + constructionVertex( + "cluster", + "AttributePOC.BrinsonAttribute", + "factorType", + "cluster", + "factorValue", + -0.003), + constructionVertex("市场贡献大", "AttributePOC.TaxonomyOfBrinsonAttribute"), + constructionVertex("选股贡献大", "AttributePOC.TaxonomyOfBrinsonAttribute"), + constructionVertex("交易贡献大", "AttributePOC.TaxonomyOfBrinsonAttribute")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("day", "day", "s"), + constructionEdge("s", "factorValue", "stock"), + constructionEdge("s", "factorValue", "trade"), + constructionEdge("s", "factorValue", "market"), + constructionEdge("s", "factorValue", "cluster")); + } + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/LocalOptionalTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/LocalOptionalTest.java new file mode 100644 index 00000000..4ae7b390 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/LocalOptionalTest.java @@ -0,0 +1,1459 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.recorder.DefaultRecorder; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import scala.Tuple2; +import scala.collection.immutable.Set; + +public class LocalOptionalTest { + @Test + public void doTestOptional4() { + String dsl = + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`PI-RADs`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // PI-RADs:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"PI-RAD\", \"PIRAD\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // PI-RADs:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"PI-RAD\", \"PIRAD\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-低回声`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺-低回声:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"外周带\", \"移行带\", \"周围带\", \"\", \"外腺\", \"内腺\"]) and contains(s.inspection, [\"b超\",\"MRI\"]) and contains(s.status, [\"低回声结节\", \"混杂回声\", \"低回声占位\", \"混杂信号\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺-低回声:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"外周带\", \"移行带\", \"周围带\", \"\", \"外腺\", \"内腺\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-体积`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺-体积:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"体积\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺-体积:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"体积\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-外周带异常`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺-外周带异常:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"外腺\", \"外周带\", \"周围带\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"形态欠规则\", \"内外腺分界模糊\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺-外周带异常:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"外腺\", \"外周带\", \"周围带\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"轮廓规则\", \"分界清晰\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-强回声`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺-强回声:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"尿道周围\", \"内外腺\", \"外周带\", \"移行带\", \"周围带\", \"\", \"外腺\", \"内腺\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"强回声团\", \"内见散在斑点状强回声\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺-强回声:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"尿道周围\", \"内外腺\", \"外周带\", \"移行带\", \"周围带\", \"\", \"外腺\", \"内腺\"]) and contains(s.inspection, [\"b超\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-影像异常`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺-影像异常:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"累及中央带\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"外周带\", \"移行带\", \"周围带\", \"\", \"外腺\", \"内腺\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"略低信号结节\", \"\", \"分界不清\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0 or Rpos1\n" + + "\n" + + " // 前列腺-影像异常:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"外周带\", \"移行带\", \"周围带\", \"\", \"外腺\", \"内腺\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"未见明显异常\", \"未见异常信号\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0 or Rneg1\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-侵犯精囊`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-侵犯精囊:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"精囊腺\", \"精囊\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"见强化影\", \"异常信号结节影\", \"点状高信号影\", \"分界不清\", \"信号不均\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"累及精囊腺\", \"与精囊腺分界不清\"])\n" + + " Rpos2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱精囊角\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"消失\"])\n" + + " Rpos: Rpos0 or Rpos1 or Rpos2\n" + + "\n" + + " // 前列腺肿瘤-侵犯精囊:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"精囊腺\", \"精囊\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"未见异常\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱精囊角\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"存在\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0 or Rneg1 or Rneg2\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-区域淋巴结转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-区域淋巴结转移:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"髂血管旁淋巴结\", \"盆腔淋巴结\", \"盆底淋巴结\", \"腹股沟淋巴结\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\">10mm\", \"多发结节样\", \"稍高信号影\", \"散在淋巴结\", \"斑片状异常信号影\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺肿瘤-区域淋巴结转移:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"髂血管旁淋巴结\", \"盆腔淋巴结\", \"盆底淋巴结\", \"腹股沟淋巴结\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"未见明显肿大淋巴结影\", \"\", \"多发小淋巴结\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-固定或侵犯除精囊外的其他邻近组织`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-固定或侵犯除精囊外的其他邻近组织:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"病灶与膀胱颈部分界\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"无法分界\", \"分界不清\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"累及膀胱\", \"与膀胱分界不清\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"累及直肠\", \"与直肠分界不清\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos3: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"直肠膀胱间隙\", \"前列腺直肠分界\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"消失\", \"分界不清\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos4: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"结节样低信号影\", \"多发囊袋状突起\", \"团块状异常信号\", \"异常强化灶\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0 or Rpos1 or Rpos2 or Rpos3 or Rpos4\n" + + "\n" + + " // 前列腺肿瘤-固定或侵犯除精囊外的其他邻近组织:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"病灶与膀胱颈部分界\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"清晰\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg3: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"直肠膀胱间隙\", \"前列腺直肠分界\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"清晰\", \"存在\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg4: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"光滑\", \"未见异常\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0 or Rneg1 or Rneg2 or Rneg3 or Rneg4\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-突破前列腺包膜:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"包膜\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"突破\", \"侵犯\", \"累及\", \"不清\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"突破包膜\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0 or Rpos1\n" + + "\n" + + " // 前列腺肿瘤-突破前列腺包膜:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"包膜\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"完整\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0 or Rneg1\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-肝转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-肝转移:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"腹部\"]) and contains(s.entity, [\"肝脏\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"多发转移\", \"多发病灶\", \"可疑转移\", \"多发结节灶\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺肿瘤-肝转移:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"腹部\"]) and contains(s.entity, [\"肝脏\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-肺转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-肺转移:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"胸部\"]) and contains(s.entity, [\"肺\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"多发转移\", \"多发病灶\", \"可疑转移\", \"多发结节灶\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺肿瘤-肺转移:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"胸部\"]) and contains(s.entity, [\"肺\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-脑转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-脑转移:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"头颅\"]) and contains(s.entity, [\"脑\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"多发转移\", \"多发病灶\", \"可疑转移\", \"多发结节灶\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺肿瘤-脑转移:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"头颅\"]) and contains(s.entity, [\"脑\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-非区域淋巴结转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-非区域淋巴结转移:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"盆腔\", \"腹部\", \"胸部\", \"头颅\"]) and contains(s.entity, [\"锁骨上\", \"纵膈\", \"腹主动脉旁\", \"腹股沟\", \"淋巴结\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\">10mm\", \"多发结节样\", \"稍高信号影\", \"散在淋巴结\", \"斑片状异常信号影\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 前列腺肿瘤-非区域淋巴结转移:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"盆腔\", \"腹部\", \"胸部\", \"头颅\"]) and contains(s.entity, [\"锁骨上\", \"纵膈\", \"腹主动脉旁\", \"腹股沟\", \"淋巴结\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"未见明显肿大淋巴结影\", \"\", \"多发小淋巴结\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-骨转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-骨转移:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"骨\"]) and contains(s.entity, [\"全身骨骼\", \"骨盆各骨骨质\", \"骨盆\", \"坐骨\", \"耻骨\", \"髋骨\", \"骶骨\", \"骶椎\", \"骶髂骨\", \"骶椎\", \"肋骨\", \"颅骨\", \"腰椎\", \"胸椎\", \"脊柱\", \"股骨\", \"肱骨\", \"胫骨\", \"腓骨\", \"尺骨\", \"桡骨\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"骨转移\", \"\", \"多发骨异常信号/病灶\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"骨盆各骨骨质\", \"骨盆\", \"坐骨\", \"耻骨\", \"髋骨\", \"骶骨\", \"骶椎\", \"骶髂骨\", \"骶椎\", \"腰椎\", \"脊柱\", \"股骨\", \"肱骨\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"骨质结构破坏\", \"多发结节样稍高信号影\", \"散在小斑片诸序列低信号影\", \"多发片状信号异常区\", \"斑片状异常信号影\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos2: contains(s.bodypart, [\"全身骨\"]) and contains(s.entity, [\"全身骨骼\", \"骨盆\", \"各骨骨质\", \"骨盆\", \"坐骨\", \"耻骨\", \"髋骨\", \"骶骨\", \"骶椎\", \"骶髂骨\", \"骶椎\", \"肋骨\", \"颅骨\", \"腰椎\", \"胸椎\", \"脊柱\", \"股骨\", \"肱骨\", \"胫骨\", \"腓骨\", \"尺骨\", \"桡骨\"]) and contains(s.inspection, [\"骨扫描\"]) and contains(s.status, [\"肿瘤骨转移\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0 or Rpos1 or Rpos2\n" + + "\n" + + " // 前列腺肿瘤-骨转移:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"骨\"]) and contains(s.entity, [\"全身骨骼\", \"骨盆各骨骨质\", \"骨盆\", \"坐骨\", \"耻骨\", \"髋骨\", \"骶骨\", \"骶椎\", \"骶髂骨\", \"骶椎\", \"肋骨\", \"颅骨\", \"腰椎\", \"胸椎\", \"脊柱\", \"股骨\", \"肱骨\", \"胫骨\", \"腓骨\", \"尺骨\", \"桡骨\"]) and contains(s.inspection, [\"CT\", \"MRI\", \"PET\"]) and contains(s.status, [\"良性病变\", \"退变\", \"退行性变\", \"外伤\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"骨盆各骨骨质\", \"骨盆\", \"坐骨\", \"耻骨\", \"髋骨\", \"骶骨\", \"骶椎\", \"骶髂骨\", \"骶椎\", \"腰椎\", \"脊柱\", \"股骨\", \"肱骨\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"未见明显异常信号改变\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg2: contains(s.bodypart, [\"全身骨\"]) and contains(s.entity, [\"全身骨骼\", \"骨盆\", \"各骨骨质\", \"骨盆\", \"坐骨\", \"耻骨\", \"髋骨\", \"骶骨\", \"骶椎\", \"骶髂骨\", \"骶椎\", \"肋骨\", \"颅骨\", \"腰椎\", \"胸椎\", \"脊柱\", \"股骨\", \"肱骨\", \"胫骨\", \"腓骨\", \"尺骨\", \"桡骨\"]) and contains(s.inspection, [\"骨扫描\"]) and contains(s.status, [\"良性病变\", \"退变\", \"退行性变\", \"外伤\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0 or Rneg1 or Rneg2\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`确诊-前列腺肿瘤`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 确诊-前列腺肿瘤:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"前列腺癌\"]) and contains(s.inspection, [\"穿刺\"]) and contains(s.status, [\"腺泡腺癌,腺癌,导管内癌,导管腺癌\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rpos: Rpos0\n" + + "\n" + + " // 确诊-前列腺肿瘤:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"前列腺癌\"]) and contains(s.inspection, [\"穿刺\"]) and contains(s.status, [\"前列腺增生\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-病理组织占比`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺增生手术后病理确诊前列腺癌`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o:`ProfMedV1.Index`/`前列腺-影像学检查`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"TRUE\"\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-侵犯两叶`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-远处淋巴结转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-超过单叶的1/2`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "// TODO \n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-指检异常`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "// TODO \n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o:`ProfMedV1.Index`/`前列腺肿瘤-区域淋巴转移`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "// TODO \n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o:`ProfMedV1.Index`/`确诊-前列腺癌`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"\"\n" + + " Fail(\"未匹配到条件\"): false\n" + + " }\n" + + "}\n" + + "\n" + + "Define (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) {\n" + + " GraphStructure {\n" + + "\t\t(s)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)\n" + + " (o1)-[p21:belongTo]->?(o21: `ProfMedV1.Index`/`确诊-前列腺癌`)\n" + + " \t(o1)-[p22:belongTo]->?(o22: `ProfMedV1.Index`/`前列腺-影像学检查`)\n" + + " \t// T-Stage\n" + + " (o1)-[pT03:belongTo]->?(oT03: `ProfMedV1.Index`/`前列腺增生手术后病理确诊前列腺癌`)\n" + + " (o1)-[pT04:belongTo]->?(oT04: `ProfMedV1.Index`/`前列腺肿瘤-病理组织占比`)\n" + + " (o1)-[pT05:belongTo]->?(oT05: `ProfMedV1.Index`/`前列腺肿瘤-固定或侵犯除精囊外的其他邻近组织`)\n" + + " (o1)-[pT06:belongTo]->?(oT06: `ProfMedV1.Index`/`前列腺肿瘤-侵犯精囊`)\n" + + " (o1)-[pT07:belongTo]->?(oT07: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`)\n" + + " (o1)-[pT08:belongTo]->?(oT08: `ProfMedV1.Index`/`前列腺-影像异常`)\n" + + " (o1)-[pT09:belongTo]->?(oT09: `ProfMedV1.Index`/`前列腺-指检异常`)\n" + + " (o1)-[pT10:belongTo]->?(oT10: `ProfMedV1.Index`/`前列腺-侵犯两叶`)\n" + + " (o1)-[pT11:belongTo]->?(oT11: `ProfMedV1.Index`/`前列腺-超过单叶的1/2`)\n" + + " // N-Stage\n" + + " (o1)-[pN03:belongTo]->?(oN03: `ProfMedV1.Index`/`前列腺肿瘤-区域淋巴转移`)\n" + + " // M-Stage\n" + + " (o1)-[pM03:belongTo]->?(oM03: `ProfMedV1.Index`/`前列腺肿瘤-脑转移`)\n" + + " (o1)-[pM04:belongTo]->?(oM04: `ProfMedV1.Index`/`前列腺肿瘤-肺转移`)\n" + + " (o1)-[pM05:belongTo]->?(oM05: `ProfMedV1.Index`/`前列腺肿瘤-肝转移`)\n" + + " (o1)-[pM06:belongTo]->?(oM06: `ProfMedV1.Index`/`前列腺肿瘤-骨转移`)\n" + + " (o1)-[pM07:belongTo]->?(oM07: `ProfMedV1.Index`/`前列腺肿瘤-远处淋巴结转移`)\n" + + " }\n" + + " Rule {\n" + + " R11(\"是否存在指标-确诊-前列腺癌\"): exists(p21)\n" + + " R12(\"确诊-前列腺癌\"): R11 and p21.indexValue == \"TRUE\"\n" + + "\n" + + " R21(\"是否存在指标-前列腺-影像学检查\"): exists(p22)\n" + + " R22(\"前列腺-影像学检查\"): R21 and p22.indexValue == \"TRUE\"\n" + + "\n" + + " \t// T-Stage\n" + + " RT031(\"是否存在指标-前列腺增生手术后病理确诊前列腺癌\"): exists(pT03)\n" + + " RT032(\"前列腺增生手术后病理确诊前列腺癌\"): RT031 and pT03.indexValue == \"TRUE\"\n" + + " \n" + + " //RT041(\"是否存在指标-前列腺肿瘤-病理组织占比\"): exists(pT04)\n" + + " //RT042(\"前列腺肿瘤-病理组织占比\"): (RT041 and pT04.indexValue)\n" + + " \n" + + " RT051(\"是否存在指标-前列腺肿瘤-固定或侵犯除精囊外的其他邻近组织\"): exists(pT05)\n" + + " RT052(\"前列腺肿瘤-固定或侵犯除精囊外的其他邻近组织\"): RT051 and pT05.indexValue == \"TRUE\"\n" + + " \n" + + " RT061(\"是否存在指标-前列腺肿瘤-侵犯精囊\"): exists(pT06)\n" + + " RT062(\"前列腺肿瘤-侵犯精囊\"): RT061 and pT06.indexValue == \"TRUE\"\n" + + " \n" + + " RT071(\"是否存在指标-前列腺肿瘤-突破前列腺包膜\"): exists(pT07)\n" + + " RT072(\"前列腺肿瘤-突破前列腺包膜\"): RT071 and pT07.indexValue == \"TRUE\"\n" + + " \n" + + " RT081(\"是否存在指标-前列腺-影像异常\"): exists(pT08)\n" + + " RT082(\"前列腺-影像异常\"): RT081 and pT08.indexValue == \"TRUE\"\n" + + " \n" + + " RT091(\"是否存在指标-前列腺-指检异常\"): exists(pT09)\n" + + " RT092(\"前列腺-指检异常\"): RT091 and pT09.indexValue == \"TRUE\"\n" + + " \n" + + " //RT101(\"是否存在指标-前列腺肿瘤-侵犯两叶\"): exists(pT10)\n" + + " //RT102(\"前列腺肿瘤-侵犯两叶\"): RT101 and pT10.indexValue == \"TRUE\"\n" + + " \n" + + " //RT111(\"是否存在指标-前列腺肿瘤-超过单叶的1/2\"): exists(pT11)\n" + + " //RT112(\"前列腺肿瘤-超过单叶的1/2\"): RT111 and pT11.indexValue == \"TRUE\"\n" + + "\n" + + " //T4(\"T4分期\"): not RT051\n" + + " p.tStage = rule_value(not R12, \"未确诊无法判断T分期\", rule_value(RT032, \"T1\", rule_value(not R22, \"Tx\", rule_value(RT052, \"T4\", rule_value(RT062, \"T3b\", rule_value(RT072, \"T3a\", rule_value(RT082 or RT092, \"T2\", \"T0\")))))))\n" + + "\n" + + " // N-Stage\n" + + " RN31(\"是否存在指标-前列腺癌-区域淋巴转移\"): exists(pN03)\n" + + " RN32(\"前列腺癌-区域淋巴转移\"): RN31 and pN03.indexValue == \"TRUE\"\n" + + " \n" + + " //N1(\"N1分期\"): R12 and RN32\n" + + " //N0(\"N0分期\"): R12 and (not RN32) and R22\n" + + " //Nx(\"Nx分期\"): R12 and (not R22)\n" + + " //p.nStage = rule_value(N1, \"N1\", rule_value(N0, \"N0\", rule_value(Nx, \"Nx\", \"无法判断\")))\n" + + " p.nStage = rule_value(not R12, \"未确诊无法判断T分期\", rule_value(not R22, \"Nx\", rule_value(RN32, \"N1\", \"N0\")))\n" + + " \n" + + " \t// M-Stage\n" + + " RM31(\"是否存在指标-前列腺癌-脑转移\"): exists(pM03)\n" + + " RM32(\"前列腺癌-脑转移\"): RM31 and pM03.indexValue == \"TRUE\"\n" + + " \n" + + " RM41(\"是否存在指标-前列腺癌-肺转移\"): exists(pM04)\n" + + " RM42(\"前列腺癌-肺转移\"): RM41 and pM04.indexValue == \"TRUE\"\n" + + " \n" + + " RM51(\"是否存在指标-前列腺癌-肝转移\"): exists(pM05)\n" + + " RM52(\"前列腺癌-肝转移\"): RM51 and pM05.indexValue == \"TRUE\"\n" + + " \n" + + " RM61(\"是否存在指标-前列腺癌-骨转移\"): exists(pM06)\n" + + " RM62(\"前列腺癌-骨转移\"): RM61 and pM06.indexValue == \"TRUE\"\n" + + " \n" + + " RM71(\"是否存在指标-前列腺癌-远处淋巴结转移\"): exists(pM07)\n" + + " RM72(\"前列腺癌-远处淋巴结转移\"): RM71 and pM07.indexValue == \"TRUE\"\n" + + "\n" + + " \n" + + " p.mStage = rule_value(not R12, \"未确诊无法判断T分期\", rule_value(not R22, \"Mx\", rule_value(RM32 or RM42 or RM52, \"M1c\", rule_value(RM62, \"M1b\", rule_value(RM72, \"M1a\", \"M0\")))))\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`)\n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, p.mStage, p.tStage)\n" + + "}\n"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.LocalOptionalTest$MedicalGraphLoader3"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index", "status", "bodypart"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + // schema.put("ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + // Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("indexValue"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + + task.setExecutionRecorder(new DefaultRecorder()); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "未确诊无法判断T分期"); + Assert.assertEquals(result.getRows().get(0)[2], "未确诊无法判断T分期"); + } + + @Test + public void doTestOptional3() { + String dsl = + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " // 前列腺肿瘤突破前列腺包膜:是\n" + + "\t\tR01: contains_any(s.entity, [\"包膜\"]) and contains_any(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) and contains_any(s.status, [\"累及\", \"侵犯\", \"膨隆\", \"突破包膜\"]) and not contains_any(s.status, [\"无\", \"不\", \"未见\"])\n" + + "\t\tR02: contains_any(s.entity, [\"神经血管束\", \"血管神经束\", \"DVC\"]) and contains_any(s.inspection, [\"MRI\", \"PETMRI\"]) and contains_any(s.status, [\"累及\", \"侵犯\"]) and not contains_any(s.status, [\"无\", \"不\", \"未见\"])\n" + + "\t\tR03: R01 or R02\n" + + "\n" + + " // 前列腺肿瘤突破前列腺包膜:否\n" + + " R11: contains_any(s.entity, [\"包膜\"]) and contains_any(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) and contains_any(s.status, [\"完整\"]) and not contains_any(s.status, [\"无\", \"不\", \"未见\"])\n" + + "\t\tR12: contains_any(s.entity, [\"神经血管束\", \"血管神经束\", \"DVC\"]) and contains_any(s.inspection, [\"MRI\", \"PETMRI\"]) and contains_any(s.status, [\"完整\"]) and not contains_any(s.status, [\"无\", \"不\", \"未见\"])\n" + + "\t\tR13: R11 or R12\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(R03, \"TRUE\", rule_value(R13, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + "\t}\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + "\t\t(s1:ProfMedV1.Patient)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)-[pT07: belongTo]->?(o: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`)\n" + + " }\n" + + " Rule {\n" + + "\tRT071(\"是否存在指标-前列腺肿瘤-突破前列腺包膜\"): exists(pT07)\n" + + " RT072(\"前列腺肿瘤-突破前列腺包膜\"): RT071 and pT07.indexValue == \"TRUE\"\n" + + " num = group(s1).countIf(pT07.indexValue == \"TRUE\", pT07)\n" + + " res = rule_value(num > 0, \"T3a\", \"无法判断\")\n" + + " }\n" + + " Action {\n" + + " \tget(s1.id, o1.id, res)\n" + + " }"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.LocalOptionalTest$MedicalGraphLoader3"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index", "status", "bodypart"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + // schema.put("ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + // Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("indexValue"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + + task.setExecutionRecorder(new DefaultRecorder()); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "index1"); + Assert.assertEquals(result.getRows().get(0)[2], "T3a"); + } + + public static class MedicalGraphLoader3 extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("u1", "ProfMedV1.Patient"), + constructionVertex( + "index1", + "ProfMedV1.PatientIndex", + "entity", + "包膜", + "inspection", + "MRI", + "status", + "突破包膜"), + constructionVertex( + "index2", + "ProfMedV1.PatientIndex", + "entity", + "神经血管束", + "inspection", + "MRI", + "status", + "完整"), + constructionVertex("前列腺癌", "ProfMedV1.Disease")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("u1", "inspectionIndex", "index1"), + constructionEdge("u1", "inspectionIndex", "index2")); + } + } + + @Test + public void doTestOptional5() { + String dsl = + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-侵犯精囊`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-侵犯精囊:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"精囊腺\", \"精囊\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"见强化影\", \"异常信号结节影\", \"点状高信号影\", \"分界不清\", \"信号不均\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"累及精囊腺\", \"与精囊腺分界不清\"])\n" + + " Rpos2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱精囊角\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"消失\"])\n" + + " Rpos: Rpos0 or Rpos1 or Rpos2\n" + + "\n" + + " // 前列腺肿瘤-侵犯精囊:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"精囊腺\", \"精囊\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"未见异常\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"])\n" + + " Rneg2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱精囊角\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"存在\"])\n" + + " Rneg: Rneg0 or Rneg1 or Rneg2\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-侵犯精囊`)\n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, o1.id, p.indexValue)\n" + + "}"; + + System.out.println(dsl); + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.LocalOptionalTest$MedicalGraphLoader5"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index", "status", "bodypart"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setExecutionRecorder(new DefaultRecorder()); + task.setExecutorTimeoutMs(99999999999999999L); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("u1", "ProfMedV1.Patient"))); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "index1"); + Assert.assertEquals(result.getRows().get(0)[2], "TRUE"); + } + + @Test + public void doTestOptional7() { + String dsl = + "\n" + + "// TODO \n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o:`ProfMedV1.Index`/`确诊-前列腺癌`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"TRUE\"\n" + + " }\n" + + "}\n" + + "\n" + + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)\n" + + " (s)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)\n" + + " (o1)-[p21:belongTo]->(o21: `ProfMedV1.Index`/`确诊-前列腺癌`)\n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, o1.id,p21.indexValue)\n" + + "}\n"; + + System.out.println(dsl); + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.LocalOptionalTest$MedicalGraphLoader5"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index", "status", "bodypart"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setExecutionRecorder(new DefaultRecorder()); + task.setExecutorTimeoutMs(99999999999999999L); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("u1", "ProfMedV1.Patient"))); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + Assert.assertEquals(result.getRows().size(), 0); + } + + @Test + public void doTestOptional6() { + String dsl = + "\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o:`ProfMedV1.Index`/`确诊-前列腺癌`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " p.indexValue = \"TRUE\"\n" + + " Fail(\"未匹配到条件\"): true\n" + + " }\n" + + "}\n" + + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-侵犯精囊`) {\n" + + " GraphStructure {\n" + + " (s)\n" + + " }\n" + + " Rule {\n" + + " // 前列腺肿瘤-侵犯精囊:是\n" + + " \n" + + " Rpos0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"精囊腺\", \"精囊\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"见强化影\", \"异常信号结节影\", \"点状高信号影\", \"分界不清\", \"信号不均\"])\n" + + " Rpos1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"累及精囊腺\", \"与精囊腺分界不清\"])\n" + + " Rpos2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱精囊角\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"消失\"])\n" + + " Rpos: Rpos0 or Rpos1 or Rpos2\n" + + "\n" + + " // 前列腺肿瘤-侵犯精囊:否\n" + + " \n" + + " Rneg0: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"精囊腺\", \"精囊\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"未见异常\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg1: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"结节\", \"团块\", \"病灶\", \"肿块\", \"异常信号灶\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg2: contains(s.bodypart, [\"前列腺\"]) and contains(s.entity, [\"膀胱精囊角\"]) and contains(s.inspection, [\"MRI\"]) and contains(s.status, [\"存在\"]) and not contains(s.status, [\"无\", \"不\", \"未见\"])\n" + + " Rneg: Rneg0 or Rneg1 or Rneg2\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(Rpos, \"TRUE\", rule_value(Rneg, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.indexValue = value\n" + + " }\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)-[p1:inspectionIndex]->" + + "(o1:ProfMedV1.PatientIndex)-[p3: belongTo]->?(o3:`ProfMedV1.Index`/`确诊-前列腺癌`)\n" + + "(o1:ProfMedV1.PatientIndex)-[p: belongTo]->?(o: `ProfMedV1.Index`/`前列腺肿瘤-侵犯精囊`)\n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, o1.id, p.indexValue,p3.indexValue)\n" + + "}"; + + System.out.println(dsl); + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.LocalOptionalTest$MedicalGraphLoader5"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index", "status", "bodypart"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setExecutionRecorder(new DefaultRecorder()); + task.setExecutorTimeoutMs(99999999999999999L); + task.setStartIdList(Lists.newArrayList(new Tuple2<>("u1", "ProfMedV1.Patient"))); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + Assert.assertEquals(result.getRows().size(), 5); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "index1"); + Assert.assertEquals(result.getRows().get(0)[2], "TRUE"); + Assert.assertEquals(result.getRows().get(0)[3], "TRUE"); + } + + public static class MedicalGraphLoader5 extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("u1", "ProfMedV1.Patient"), + constructionVertex( + "index1", + "ProfMedV1.PatientIndex", + "entity", + "精囊腺", + "bodypart", + "前列腺", + "status", + "信号不均伴局部DWI稍高信号", + "inspection", + "MRI"), + constructionVertex( + "index2", + "ProfMedV1.PatientIndex", + "entity", + "包膜", + "inspection", + "MRI", + "index", + "突破包膜"), + constructionVertex( + "index3", + "ProfMedV1.PatientIndex", + "entity", + "包膜2", + "inspection", + "MRI", + "index", + "突破包膜"), + constructionVertex( + "index4", + "ProfMedV1.PatientIndex", + "entity", + "包膜2", + "inspection", + "MRI", + "index", + "突破包膜"), + constructionVertex( + "index5", + "ProfMedV1.PatientIndex", + "entity", + "包膜2", + "inspection", + "MRI", + "index", + "突破包膜"), + constructionVertex("前列腺癌", "ProfMedV1.Disease"), + constructionVertex("前列腺肿瘤-侵犯精囊", "ProfMedV1.Index")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("u1", "inspectionIndex", "index1"), + constructionEdge("u1", "inspectionIndex", "index2"), + constructionEdge("u1", "inspectionIndex", "index3"), + constructionEdge("u1", "inspectionIndex", "index4"), + constructionEdge("u1", "inspectionIndex", "index5")); + } + } + + public static class MedicalGraphLoader2 extends AbstractLocalGraphLoader { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("u1", "ProfMedV1.Patient"), + constructionVertex("index1", "ProfMedV1.PatientIndex", "entity", "影像学检查"), + constructionVertex( + "index2", + "ProfMedV1.PatientIndex", + "entity", + "包膜", + "inspection", + "MRI", + "index", + "突破包膜"), + constructionVertex("前列腺癌", "ProfMedV1.Disease")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("u1", "inspectionIndex", "index1"), + constructionEdge("u1", "inspectionIndex", "index2")); + } + } + + @Test + public void doTestOptional2() { + String dsl = + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " // 前列腺肿瘤突破前列腺包膜:是\n" + + "\t\tR01: contains(s.entity, [\"包膜\"]) and contains(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) and contains(s.index, [\"累及\", \"侵犯\", \"膨隆\", \"突破包膜\"])\n" + + "\t\tR02: contains(s.entity, [\"神经血管束\", \"血管神经束\", \"DVC\"]) and contains(s.inspection, [\"MRI\", \"PETMRI\"]) and contains(s.index, [\"累及\", \"侵犯\"])\n" + + "\t\tR03: R01 or R02\n" + + "\n" + + " // 前列腺肿瘤突破前列腺包膜:否\n" + + " R11: contains(s.entity, [\"包膜\"]) and contains(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) and contains(s.index, [\"完整\"])\n" + + "\t\tR12: contains(s.entity, [\"神经血管束\", \"血管神经束\", \"DVC\"]) and contains(s.inspection, [\"MRI\", \"PETMRI\"]) and contains(s.index, [\"完整\"])\n" + + "\t\tR13: R11 or R12\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(R03, \"TRUE\", rule_value(R13, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.value = value\n" + + "\t}\n" + + "}"; + dsl += + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-影像学检查`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " // 前列腺肿瘤突破前列腺包膜:是\n" + + "\t\tR01: contains(s.entity, [\"影像学检查\"])\n" + + " \tp.value = \"TRUE\"\n" + + "\t}\n" + + "}"; + + dsl += + "Define (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) {\n" + + " GraphStructure {\n" + + "\t\t(s:ProfMedV1.Patient)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)\n" + + "\t\t(s:ProfMedV1.Patient)-[p2:inspectionIndex]->(o2:ProfMedV1.PatientIndex)\n" + + " (o1)-[p21:belongTo]->?(o21: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`)\n" + + " \t(o2)-[p22:belongTo]->?(o22: `ProfMedV1.Index`/`前列腺-影像学检查`)\n" + + " }\n" + + " Rule {\n" + + " R11(\"是否存在指标-前列腺肿瘤-突破前列腺包膜\"): exists(p21)\n" + + " R12(\"前列腺肿瘤-突破前列腺包膜\"): R11 and p21.value == \"TRUE\"\n" + + "\n" + + " R21(\"是否存在指标-前列腺-影像学检查\"): exists(p22)\n" + + " R22(\"前列腺-影像学检查\"): R21 and p22.value == \"TRUE\"\n" + + "\n" + + " \n" + + " N1(\"N1分期\"): R12\n" + + " N0(\"N0分期\"): R12 and R22\n" + + " Nx(\"Nx分期\"): R12 and (not R22)\n" + + " p.nStage = rule_value(N1, \"N1\", rule_value(N0, \"N0\", rule_value(Nx, \"Nx\", \"无法判断\")))\n" + + " //p.json_value = \"{'nStage': p.nStage}\"\n" + + " //UDF_JSONGet(p.json_value, '$.nStage')\n" + + " }\n" + + "}"; + dsl += + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) \n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, p.nStage)\n" + + "}"; + + System.out.println(dsl); + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.LocalOptionalTest$MedicalGraphLoader2"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index", "status"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setExecutionRecorder(new DefaultRecorder()); + task.setExecutorTimeoutMs(99999999999999999L); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "N1"); + } + + @Test + public void doTestOptional01() { + String dsl = + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " \tp.value = \"\"\n" + + " R1: false" + + "\t}\n" + + "}"; + dsl += + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-影像学检查`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " // 前列腺肿瘤突破前列腺包膜:是\n" + + "\t\tR01: contains(s.entity, \"影像学检查\")\n" + + " \tp.value = \"TRUE\"\n" + + " R1: false" + + "\t}\n" + + "}"; + + dsl += + "Define (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) {\n" + + " GraphStructure {\n" + + "\t\t(s:ProfMedV1.Patient)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)\n" + + " (o1)-[p21:belongTo]->?(o21: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`)\n" + + " \t(o1)-[p22:belongTo]->?(o22: `ProfMedV1.Index`/`前列腺-影像学检查`)\n" + + " }\n" + + " Rule {\n" + + " R11(\"是否存在指标-前列腺肿瘤-突破前列腺包膜\"): exists(p21)\n" + + " R12(\"前列腺肿瘤-突破前列腺包膜\"): R11 and p21.value == \"TRUE\"\n" + + "\n" + + " R21(\"是否存在指标-前列腺-影像学检查\"): exists(p22)\n" + + " R22(\"前列腺-影像学检查\"): R21 and p22.value == \"TRUE\"\n" + + "\n" + + " \n" + + " N1(\"N1分期\"): R12\n" + + " N0(\"N0分期\"): R12 and R22\n" + + " Nx(\"Nx分期\"): R12 and (not R22)\n" + + " p.nStage = rule_value(N1, \"N1\", rule_value(N0, \"N0\", rule_value(Nx, \"Nx\", \"无法判断\")))\n" + + " //p.json_value = \"{'nStage': p.nStage}\"\n" + + " //UDF_JSONGet(p.json_value, '$.nStage')\n" + + " }\n" + + "}"; + dsl += + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) \n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, p.nStage)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass("com.antgroup.openspg.reasoner.runner.local.loader.MedicalGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setExecutionRecorder(new DefaultRecorder()); + task.setExecutorTimeoutMs(99999999999999999L); + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + System.out.println(task.getExecutionRecorder().toReadableString()); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "无法判断"); + } + + @Test + public void doTestOptional() { + String dsl = + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " // 前列腺肿瘤突破前列腺包膜:是\n" + + "\t\tR01: contains(s.entity, \"包膜\") and contains(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) and contains(s.index, [\"累及\", \"侵犯\", \"膨隆\", \"突破包膜\"])\n" + + "\t\tR02: contains(s.entity, [\"神经血管束\", \"血管神经束\", \"DVC\"]) and contains(s.inspection, [\"MRI\", \"PETMRI\"]) and contains(s.index, [\"累及\", \"侵犯\"])\n" + + "\t\tR03: R01 or R02\n" + + "\n" + + " // 前列腺肿瘤突破前列腺包膜:否\n" + + " R11: contains(s.entity, \"包膜\") and contains(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) and contains(s.index, [\"完整\"])\n" + + "\t\tR12: contains(s.entity, [\"神经血管束\", \"血管神经束\", \"DVC\"]) and contains(s.inspection, [\"MRI\", \"PETMRI\"]) and contains(s.index, [\"完整\"])\n" + + "\t\tR13: R11 or R12\n" + + "\n" + + " // rule_value(R, res0, res1)的参数含义:R为判断条件,res0/res1分别为R0为True/False时的输出\n" + + " value = rule_value(R03, \"TRUE\", rule_value(R13, \"FALSE\", \"\"))\n" + + " // 未匹配到条件,则规则不成立(缺省,不连边)\n" + + " Fail(\"未匹配到条件\"): value != \"\"\n" + + " p.value = value\n" + + "\t}\n" + + "}"; + dsl += + "Define (s: ProfMedV1.PatientIndex)-[p: belongTo]->(o: `ProfMedV1.Index`/`前列腺-影像学检查`) {\n" + + "\tGraphStructure {\n" + + " (s)\n" + + "\t}\n" + + "\tRule {\n" + + " // 前列腺肿瘤突破前列腺包膜:是\n" + + "\t\tR01: contains(s.entity, \"影像学检查\")\n" + + " \tp.value = \"TRUE\"\n" + + "\t}\n" + + "}"; + + dsl += + "Define (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) {\n" + + " GraphStructure {\n" + + "\t\t(s:ProfMedV1.Patient)-[p1:inspectionIndex]->(o1:ProfMedV1.PatientIndex)\n" + + " (o1)-[p21:belongTo]->?(o21: `ProfMedV1.Index`/`前列腺肿瘤-突破前列腺包膜`)\n" + + " \t(o1)-[p22:belongTo]->?(o22: `ProfMedV1.Index`/`前列腺-影像学检查`)\n" + + " }\n" + + " Rule {\n" + + " R11(\"是否存在指标-前列腺肿瘤-突破前列腺包膜\"): exists(p21)\n" + + " R12(\"前列腺肿瘤-突破前列腺包膜\"): R11 and p21.value == \"TRUE\"\n" + + "\n" + + " R21(\"是否存在指标-前列腺-影像学检查\"): exists(p22)\n" + + " R22(\"前列腺-影像学检查\"): R21 and p22.value == \"TRUE\"\n" + + "\n" + + " \n" + + " N1(\"N1分期\"): R12\n" + + " N0(\"N0分期\"): R12 and R22\n" + + " Nx(\"Nx分期\"): R12 and (not R22)\n" + + " p.nStage = rule_value(N1, \"N1\", rule_value(N0, \"N0\", rule_value(Nx, \"Nx\", \"无法判断\")))\n" + + " //p.json_value = \"{'nStage': p.nStage}\"\n" + + " //UDF_JSONGet(p.json_value, '$.nStage')\n" + + " }\n" + + "}"; + dsl += + "GraphStructure {\n" + + " (s: ProfMedV1.Patient)-[p: belongTo]->(o: `ProfMedV1.Disease`/`前列腺癌`) \n" + + "}\n" + + "Rule {\n" + + "\n" + + "}\n" + + "Action {\n" + + " get(s.id, p.nStage)\n" + + "}"; + + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + task.setGraphLoadClass("com.antgroup.openspg.reasoner.runner.local.loader.MedicalGraphLoader"); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put( + "ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "entity", "inspection", "index"))); + schema.put("ProfMedV1.Disease", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Patient", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put("ProfMedV1.Index", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "ProfMedV1.Patient_belongTo_ProfMedV1.Disease", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value", "nStage"))); + schema.put( + "ProfMedV1.PatientIndex_belongTo_ProfMedV1.Index", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("value"))); + schema.put( + "ProfMedV1.Patient_inspectionIndex_ProfMedV1.PatientIndex", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + task.getParams().put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.getParams().put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(result); + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "u1"); + Assert.assertEquals(result.getRows().get(0)[1], "无法判断"); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/LocalRunnerTestFactory.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/LocalRunnerTestFactory.java new file mode 100644 index 00000000..1f34cc2d --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/LocalRunnerTestFactory.java @@ -0,0 +1,94 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.generator.AbstractGraphGenerator; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.recorder.DefaultRecorder; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import scala.Tuple2; + +public class LocalRunnerTestFactory { + + public static void runTest( + String dsl, + AbstractLocalGraphLoader graphLoader, + AssertFunction assertFunc, + Map params) { + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + MemGraphState memGraphState = new MemGraphState(); + graphLoader.setGraphState(memGraphState); + graphLoader.load(); + task.setGraphState(memGraphState); + + // catalog + task.setCatalog(getCatalogFromGraphData(graphLoader)); + + // enable subquery + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + task.setParams(params); + task.setExecutorTimeoutMs(30 * 60 * 1000); + + task.setExecutionRecorder(new DefaultRecorder()); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + System.out.println(task.getExecutionRecorder().toReadableString()); + assertFunc.assertResult(result); + } + + private static Catalog getCatalogFromGraphData(AbstractGraphGenerator graph) { + Tuple2>, List>> graphData = + graph.getGraphData(); + Map> schemaMap = new HashMap<>(); + for (IVertex vertex : graphData._1()) { + java.util.Set propertySet = + schemaMap.computeIfAbsent(vertex.getId().getType(), k -> new HashSet<>()); + propertySet.addAll(vertex.getValue().getKeySet()); + } + for (IEdge edge : graphData._2()) { + java.util.Set propertySet = + schemaMap.computeIfAbsent(edge.getType(), k -> new HashSet<>()); + propertySet.addAll(edge.getValue().getKeySet()); + } + Map> schema = new HashMap<>(); + for (Map.Entry> entry : schemaMap.entrySet()) { + schema.put(entry.getKey(), Convert2ScalaUtil.toScalaImmutableSet(entry.getValue())); + } + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + return catalog; + } + + public interface AssertFunction { + void assertResult(LocalReasonerResult result); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/basetest/FilmBaseTestData.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/basetest/FilmBaseTestData.java new file mode 100644 index 00000000..330723e4 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/basetest/FilmBaseTestData.java @@ -0,0 +1,170 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main.basetest; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import scala.Tuple2; + +public class FilmBaseTestData { + public static Catalog getMockCatalogSchema() { + Map> schema = new HashMap<>(); + schema.put( + "Film", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "filmBudget", "filmGross"))); + schema.put( + "FilmDirector", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "age", "name", "gender"))); + schema.put( + "FilmStar", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "age", "name", "gender"))); + schema.put( + "Film_starOfFilm_FilmStar", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("joinTs"))); + schema.put( + "Film_directOfFilm_FilmDirector", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + return catalog; + } + + public static class FilmGraphGenerator extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("root", "Film", "filmBudget", 100), + constructionVertex("f1", "Film"), + constructionVertex("f2", "Film"), + constructionVertex("f3", "Film"), + constructionVertex("L1_1_star", "FilmStar", "age", 60, "gender", "男"), + constructionVertex("L1_2_star", "FilmStar", "age", 30, "gender", "男"), + constructionVertex("L1_3_star", "FilmStar", "age", 18, "gender", "女"), + constructionVertex("L1_1_director", "FilmDirector"), + constructionVertex("L1_2_director", "FilmDirector"), + constructionVertex("L1_3_director", "FilmDirector")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge("root", "starOfFilm", "L1_1_star", 0, "joinTs", 100), + constructionVersionEdge("root", "starOfFilm", "L1_2_star", 0, "joinTs", 200), + constructionVersionEdge("root", "starOfFilm", "L1_3_star", 0, "joinTs", 300), + constructionVersionEdge("root", "starOfFilm", "L1_3_star", 1, "joinTs", 400), + constructionVersionEdge("f1", "starOfFilm", "L1_1_star", 0, "joinTs", 100), + constructionVersionEdge("f1", "starOfFilm", "L1_2_star", 0, "joinTs", 200), + constructionVersionEdge("f1", "starOfFilm", "L1_3_star", 0, "joinTs", 300), + constructionVersionEdge("f2", "starOfFilm", "L1_1_star", 0, "joinTs", 100), + constructionVersionEdge("f2", "starOfFilm", "L1_2_star", 0, "joinTs", 200), + constructionVersionEdge("f2", "starOfFilm", "L1_3_star", 0, "joinTs", 300), + constructionVersionEdge("f3", "starOfFilm", "L1_1_star", 0, "joinTs", 10), + constructionVersionEdge("f3", "starOfFilm", "L1_2_star", 0, "joinTs", 200), + constructionVersionEdge("f3", "starOfFilm", "L1_3_star", 0, "joinTs", 600), + constructionVersionEdge("root", "directOfFilm", "L1_1_director", 0), + constructionVersionEdge("f1", "directOfFilm", "L1_1_director", 0), + constructionVersionEdge("f2", "directOfFilm", "L1_2_director", 0), + constructionVersionEdge("f3", "directOfFilm", "L1_3_director", 0)); + } + } + + public static class FilmGraphGeneratorTopK extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("root", "Film", "filmBudget", 100), + constructionVertex("f1", "Film"), + constructionVertex("f2", "Film"), + constructionVertex("f3", "Film"), + constructionVertex("L1_1_star", "FilmStar", "age", 60, "gender", "男"), + constructionVertex("L1_2_star", "FilmStar", "age", 30, "gender", "男"), + constructionVertex("L1_3_star", "FilmStar", "age", 18, "gender", "男"), + constructionVertex("L1_1_director", "FilmDirector"), + constructionVertex("L1_2_director", "FilmDirector"), + constructionVertex("L1_3_director", "FilmDirector")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge("root", "starOfFilm", "L1_1_star", 0, "joinTs", 100), + constructionVersionEdge("root", "starOfFilm", "L1_2_star", 0, "joinTs", 200), + constructionVersionEdge("root", "starOfFilm", "L1_3_star", 0, "joinTs", 300), + constructionVersionEdge("root", "starOfFilm", "L1_3_star", 1, "joinTs", 400), + constructionVersionEdge("f1", "starOfFilm", "L1_1_star", 0, "joinTs", 100), + constructionVersionEdge("f1", "starOfFilm", "L1_2_star", 0, "joinTs", 200), + constructionVersionEdge("f1", "starOfFilm", "L1_3_star", 0, "joinTs", 300), + constructionVersionEdge("f2", "starOfFilm", "L1_1_star", 0, "joinTs", 101), + constructionVersionEdge("f2", "starOfFilm", "L1_2_star", 0, "joinTs", 201), + constructionVersionEdge("f3", "starOfFilm", "L1_1_star", 0, "joinTs", 10), + constructionVersionEdge("root", "directOfFilm", "L1_1_director", 0), + constructionVersionEdge("f1", "directOfFilm", "L1_1_director", 0), + constructionVersionEdge("f2", "directOfFilm", "L1_2_director", 0), + constructionVersionEdge("f3", "directOfFilm", "L1_3_director", 0)); + } + } + + public static List runTestResult(String dsl, String dataClass) { + Catalog catalog = getMockCatalogSchema(); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // use test catalog + task.setCatalog(catalog); + + task.setGraphLoadClass(dataClass); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + task.setParams(params); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("root", "Film"))); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + List rst = new ArrayList<>(); + for (Object[] r : result.getRows()) { + String[] out = new String[r.length]; + for (int i = 0; i < r.length; i++) { + out[i] = String.valueOf(r[i]); + } + rst.add(out); + } + return rst; + } + + public static List runTestResult(String dsl) { + return runTestResult( + dsl, + "com.antgroup.openspg.reasoner.runner.local.main.basetest.FilmBaseTestData$FilmGraphGenerator"); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/basetest/TransBaseTestData.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/basetest/TransBaseTestData.java new file mode 100644 index 00000000..af5df032 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/basetest/TransBaseTestData.java @@ -0,0 +1,107 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main.basetest; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import scala.Tuple2; + +public class TransBaseTestData { + public static Catalog getMockCatalogSchema() { + Map> schema = new HashMap<>(); + schema.put("User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id"))); + schema.put( + "User_trans_User", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("payDate", "amount"))); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + return catalog; + } + + public static class TransGraphGenerator extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("1", "User"), + constructionVertex("2", "User"), + constructionVertex("3", "User"), + constructionVertex("4", "User"), + constructionVertex("5", "User")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge("1", "trans", "2", 0, "payDate", "20230910", "amount", 150), + constructionVersionEdge("1", "trans", "2", 2, "payDate", "20230910", "amount", 150), + constructionVersionEdge("2", "trans", "3", 0, "payDate", "20230910", "amount", 100), + constructionVersionEdge("2", "trans", "3", 2, "payDate", "20230910", "amount", 100), + constructionVersionEdge("3", "trans", "1", 0, "payDate", "20230910", "amount", 100), + constructionVersionEdge("3", "trans", "1", 2, "payDate", "20230910", "amount", 100), + constructionVersionEdge("4", "trans", "2", 0, "payDate", "20230910", "amount", 200), + constructionVersionEdge("3", "trans", "4", 0, "payDate", "20230910", "amount", 200), + constructionVersionEdge("5", "trans", "2", 0, "payDate", "20230910", "amount", 200), + constructionVersionEdge("3", "trans", "5", 0, "payDate", "20230910", "amount", 200)); + } + } + + public static List runTestResult(String dsl, Map runParams) { + Catalog catalog = getMockCatalogSchema(); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // use test catalog + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.basetest.TransBaseTestData$TransGraphGenerator"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true); + params.putAll(runParams); + task.setParams(params); + + task.setStartIdList(Lists.newArrayList(new Tuple2<>("1", "User"))); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + List rst = new ArrayList<>(); + for (Object[] r : result.getRows()) { + String[] out = new String[r.length]; + for (int i = 0; i < r.length; i++) { + out[i] = String.valueOf(r[i]); + } + rst.add(out); + } + return rst; + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/finbench/FinBenchTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/finbench/FinBenchTest.java new file mode 100644 index 00000000..b8194daf --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/finbench/FinBenchTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main.finbench; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.main.LocalRunnerTestFactory; +import com.antgroup.openspg.reasoner.runner.local.main.LocalRunnerTestFactory.AssertFunction; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class FinBenchTest { + + private AbstractLocalGraphLoader graphLoader; + private Map params; + + @Before + public void init() { + this.graphLoader = + new AbstractLocalGraphLoader() { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A1", "Account"), + constructionVertex("A2", "Account"), + constructionVertex("A3", "Account"), + constructionVertex("A4", "Account"), + constructionVertex("M3", "Medium", "isBlocked", true), + constructionVertex("M4", "Medium", "isBlocked", false)); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("A1", "transfer", "A2", "timestamp", 1, "amount", 100), + constructionEdge("A2", "transfer", "A3", "timestamp", 1, "amount", 100), + constructionEdge("A3", "transfer", "A4", "timestamp", 1, "amount", 50), + constructionEdge("A3", "transfer", "A1", "timestamp", 1, "amount", 50), + constructionEdge("A3", "signIn", "M3", "timestamp", 1), + constructionEdge("A4", "signIn", "M4", "timestamp", 1)); + } + }; + + params = new HashMap<>(); + // start id + List> startIdList = new ArrayList<>(); + startIdList.add(Lists.newArrayList("A1", "Account")); + startIdList.add(Lists.newArrayList("A2", "Account")); + startIdList.add(Lists.newArrayList("A3", "Account")); + params.put(ConfigKey.KG_REASONER_START_ID_LIST, JSON.toJSONString(startIdList)); + + // other params + params.put("startTime", "1"); + params.put("endTime", "1"); + + params.put("trc4id1", "'A1'"); + params.put("trc4id2", "'A2'"); + } + + @Test + public void trc4() { + LocalRunnerTestFactory.runTest( + "GraphStructure {\n" + + " (src:Account) -[edge1:transfer]-> (dst:Account) -[edge2:transfer]-> (other:Account) \n" + + "-[edge3:transfer]-> (src)\n" + + "}\n" + + "Rule {\n" + + " R1(\"src参数\"): src.id == $trc4id1\n" + + " R2(\"dst参数\"): dst.id == $trc4id2\n" + + " R3(\"transfer时间1\"): edge1.timestamp >= $startTime and edge1.timestamp <= $endTime\n" + + " R4(\"transfer时间2\"): edge2.timestamp >= $startTime and edge2.timestamp <= $endTime\n" + + " R5(\"transfer时间3\"): edge3.timestamp >= $startTime and edge3.timestamp <= $endTime\n" + + " numEdge2 = group(src,dst,other).count(edge2)\n" + + " numEdge3 = group(src,dst,other).count(edge3)\n" + + " sumEdge2Amount = group(src,dst,other).sum(edge2.amount)\n" + + " sumEdge3Amount = group(src,dst,other).sum(edge3.amount)\n" + + " maxEdge2Amount = group(src,dst,other).max(edge2.amount)\n" + + " maxEdge3Amount = group(src,dst,other).max(edge3.amount)\n" + + "}\n" + + "Action {\n" + + " get(other.id, numEdge2, sumEdge2Amount, maxEdge2Amount, numEdge3, sumEdge3Amount, maxEdge3Amount)\n" + + "}", + this.graphLoader, + new AssertFunction() { + @Override + public void assertResult(LocalReasonerResult result) { + Assert.assertEquals(result.getRows().size(), 1); + Assert.assertEquals(result.getRows().get(0)[0], "A3"); + Assert.assertEquals(result.getRows().get(0)[1], "1"); + } + }, + this.params); + } +} diff --git a/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/transitive/KgReasonerTransitiveTest.java b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/transitive/KgReasonerTransitiveTest.java new file mode 100644 index 00000000..90aa71c6 --- /dev/null +++ b/reasoner/runner/local-runner/src/test/java/com/antgroup/openspg/reasoner/runner/local/main/transitive/KgReasonerTransitiveTest.java @@ -0,0 +1,121 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local.main.transitive; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner; +import com.antgroup.openspg.reasoner.runner.local.load.graph.AbstractLocalGraphLoader; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult; +import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import scala.collection.immutable.Set; + +public class KgReasonerTransitiveTest { + + @Test + public void testTransitive1() { + String dsl = + "GraphStructure {\n" + + " A [FilmPerson]\n" + + " C,D [FilmDirector]\n" + + " A->C [test] as e1\n" + + " C->D [t1] repeat(0,2) as e2\n" + + "}\n" + + "Rule {\n" + + "}\n" + + "Action {\n" + + " get(A.name,C.name,D.name)\n" + + "}"; + + System.out.println(dsl); + LocalReasonerTask task = new LocalReasonerTask(); + task.setDsl(dsl); + + // add mock catalog + Map> schema = new HashMap<>(); + schema.put("FilmPerson", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "name"))); + schema.put( + "FilmDirector", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "name"))); + schema.put( + "FilmPerson_test_FilmDirector", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + schema.put( + "FilmDirector_t1_FilmDirector", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + task.setCatalog(catalog); + + task.setGraphLoadClass( + "com.antgroup.openspg.reasoner.runner.local.main.transitive.KgReasonerTransitiveTest$GraphLoader"); + + // enable subquery + Map params = new HashMap<>(); + params.put(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true); + params.put(ConfigKey.KG_REASONER_BINARY_PROPERTY, "false"); + params.put(Constants.SPG_REASONER_MULTI_VERSION_ENABLE, "true"); + task.setParams(params); + + KGReasonerLocalRunner runner = new KGReasonerLocalRunner(); + LocalReasonerResult result = runner.run(task); + + // only u1 + // check result + Assert.assertEquals(4, result.getRows().size()); + Assert.assertEquals(3, result.getRows().get(0).length); + java.util.Set dSet = new HashSet<>(); + for (Object[] strings : result.getRows()) { + dSet.add(String.valueOf(strings[2])); + } + Assert.assertTrue(dSet.contains("C1")); + Assert.assertTrue(dSet.contains("C2")); + Assert.assertTrue(dSet.contains("D21")); + Assert.assertTrue(dSet.contains("D22")); + } + + public static class GraphLoader extends AbstractLocalGraphLoader { + + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("A1", "FilmPerson", "name", "A1"), + constructionVertex("C1", "FilmDirector", "name", "C1"), + constructionVertex("A2", "FilmPerson", "name", "A2"), + constructionVertex("C2", "FilmDirector", "name", "C2"), + constructionVertex("D21", "FilmDirector", "name", "D21"), + constructionVertex("D22", "FilmDirector", "name", "D22")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionEdge("A1", "test", "C1"), + constructionEdge("A2", "test", "C2"), + constructionEdge("C2", "t1", "D21"), + constructionEdge("D21", "t1", "D22")); + } + } +} diff --git a/dev/release/docker-compose-no-es.sh b/reasoner/runner/local-runner/src/test/resources/log4j.properties similarity index 69% rename from dev/release/docker-compose-no-es.sh rename to reasoner/runner/local-runner/src/test/resources/log4j.properties index 4f8b9d94..fe06aaf3 100644 --- a/dev/release/docker-compose-no-es.sh +++ b/reasoner/runner/local-runner/src/test/resources/log4j.properties @@ -1,4 +1,3 @@ -# # Copyright 2023 Ant Group CO., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except @@ -9,6 +8,10 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. -# -docker-compose -f docker-compose-no-es.yml up -d +# logger config for test +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout \ No newline at end of file diff --git a/reasoner/runner/local-runner/src/test/scala/com/antgroup/openspg/reasoner/runner/local/LocalRunner.scala b/reasoner/runner/local-runner/src/test/scala/com/antgroup/openspg/reasoner/runner/local/LocalRunner.scala new file mode 100644 index 00000000..eb611e6d --- /dev/null +++ b/reasoner/runner/local-runner/src/test/scala/com/antgroup/openspg/reasoner/runner/local/LocalRunner.scala @@ -0,0 +1,262 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner.local + +import com.antgroup.openspg.reasoner.runner.local.impl.LocalReasonerSession +import com.antgroup.openspg.reasoner.runner.local.rdg.{LocalRow, TypeTags} +import com.antgroup.openspg.reasoner.lube.block.{Block, DDLBlock, MatchBlock, TableResultBlock} +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog +import com.antgroup.openspg.reasoner.lube.common.pattern.{ + GraphPattern, + LinkedPatternConnection, + PatternConnection +} +import com.antgroup.openspg.reasoner.lube.logical.planning.LogicalPlannerContext +import com.antgroup.openspg.reasoner.lube.logical.validate.{Dag, Validator} +import com.antgroup.openspg.reasoner.parser.KgDslParser +import com.google.common.collect.Lists +import org.scalatest.BeforeAndAfter +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, equal} + +class LocalRunner extends AnyFunSpec with BeforeAndAfter { + + it("MockRunner") { + val dsl = + """ + |GraphStructure { + | (Student:Student)-[STEdge:STEdge]->(Teacher:Teacher) + |} + |Rule { + | + |} + |Action { + |get(Student.name, Teacher.name, STEdge.name) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "Student" -> Set.apply("name"), + "Teacher" -> Set.apply("name"), + "Student_STEdge_Teacher" -> Set.apply("name")) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new LocalReasonerSession(new KgDslParser(), catalog, TypeTags.rdgTypeTag) + val plan = session.plan(dsl, Map.empty) + val rst = session.getResult(plan.head) + if (rst.isInstanceOf[LocalRow]) { + rst.asInstanceOf[LocalRow].show(10) + } + } + + // scalastyle:off + it("dependency_analysis") { + val dsl = + """ + |GraphStructure { + | (s:Park)-[e:nearby(s.boundary, o.center, 10.1)]->(o:Subway) + | } + | Rule{} + | Action {get(s)} + |""".stripMargin + val parser = new KgDslParser() + val block = parser.parse(dsl) + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.apply("boundary"), + "Subway" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new LocalReasonerSession(new KgDslParser(), catalog, TypeTags.rdgTypeTag) + implicit val context: LogicalPlannerContext = + LogicalPlannerContext(catalog, parser, Map.empty) + session.plan2UnresolvedLogicalPlan(dsl, Map.empty) + val blockDag: Dag[Block] = Validator.validate(parser, block)(context) + val nodes: Map[String, Block] = blockDag.getNodes + nodes.keySet.size should equal(2) + for (block <- nodes.values) { + println(block.pretty) + } + // define Block + val defineNodeName: String = blockDag.getEdges.apply("result").head + val defineBlock = blockDag.getNode(defineNodeName) + defineBlock.isInstanceOf[DDLBlock] should equal(true) + val defineMatchBlock: MatchBlock = + defineBlock.dependencies.head.dependencies.head.asInstanceOf[MatchBlock] + defineMatchBlock.patterns.size should equal(1) + val partialGraphPattern: GraphPattern = defineMatchBlock.patterns.head._2.graphPattern + partialGraphPattern.nodes.size should equal(3) + val linkedPatternConnection = partialGraphPattern.edges("s").head + linkedPatternConnection.isInstanceOf[LinkedPatternConnection] should equal(true) + linkedPatternConnection.asInstanceOf[LinkedPatternConnection].funcName should equal( + "geo_buffer_and_convert_2_s2CellId") + partialGraphPattern.edges("cell").head.isInstanceOf[PatternConnection] should equal(true) + + // rewrite Block + val originBlock = blockDag.popRoot() + originBlock.isInstanceOf[TableResultBlock] should equal(true) + val originMatchBlock = originBlock.dependencies.head.asInstanceOf[MatchBlock] + val originPartialGraphPattern = originMatchBlock.patterns.head._2.graphPattern + !originPartialGraphPattern.edges("s").head.isInstanceOf[LinkedPatternConnection] should equal( + true) + } + + it("multi_dependency_analysis_error") { + val dsl = + """ + |GraphStructure { + | (s:Park)-[e:nearby(s.boundary, o.center, 10.1)]->(o:Subway) + | (s1:School)-[e2:nearby(s1.center, o2.center, 10)]->(o2:Shop) + | } + | Rule{} + | Action {get(s)} + |""".stripMargin + val parser = new KgDslParser() + val block = parser.parse(dsl) + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.apply("boundary"), + "Subway" -> Set.apply("center"), + "School" -> Set.apply("center"), + "Shop" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty, + "Shop_centerS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new LocalReasonerSession(new KgDslParser(), catalog, TypeTags.rdgTypeTag) + implicit val context: LogicalPlannerContext = + LogicalPlannerContext(catalog, parser, Map.empty) + session.plan2UnresolvedLogicalPlan(dsl, Map.empty) + val blockDag: Dag[Block] = Validator.validate(parser, block)(context) + var blockList: List[Block] = + blockDag.getNodes.filter(entry => !entry._1.equals("result")).values.toList + blockList = blockList :+ blockDag.getNode("result") + blockList.foreach(block => println(block.pretty)) + blockList.size should equal(3) + blockList.head.isInstanceOf[DDLBlock] should equal(true) + blockList(1).isInstanceOf[DDLBlock] should equal(true) + blockList.last.isInstanceOf[TableResultBlock] should equal(true) + val partialGraphPattern = blockList.last + .asInstanceOf[TableResultBlock] + .dependencies + .head + .asInstanceOf[MatchBlock] + .patterns + .head + ._2 + .graphPattern + partialGraphPattern.edges("s1").head.isInstanceOf[LinkedPatternConnection] should equal(false) + partialGraphPattern.edges("s").head.isInstanceOf[LinkedPatternConnection] should equal(false) + } + + it("dependency_analysis_error") { + val dsl = + """ + |GraphStructure { + | (s:Park)-[e:nearby(s, o.center, 10.1)]->(o:Subway) + | } + | Rule{} + | Action {get(s)} + |""".stripMargin + val parser = new KgDslParser() + val block = parser.parse(dsl) + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.apply("boundary"), + "Subway" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new LocalReasonerSession(new KgDslParser(), catalog, TypeTags.rdgTypeTag) + try { + implicit val context: LogicalPlannerContext = + LogicalPlannerContext(catalog, parser, Map.empty) + session.plan2UnresolvedLogicalPlan(dsl, Map.empty) + Validator.validate(parser, block)(context) + false should equal(true) + } catch { + case e: Exception => + e.getMessage.equals("the first parameter in nearby should like A.property") should equal( + true) + } + } + + it("dependency_analysis_error2") { + val dsl = + """ + |GraphStructure { + | (s:Park)-[e:nearby(s, o.center, -10)]->(o:Subway) + | } + | Rule{} + | Action {get(s)} + |""".stripMargin + val parser = new KgDslParser() + val block = parser.parse(dsl) + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.apply("boundary"), + "Subway" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new LocalReasonerSession(new KgDslParser(), catalog, TypeTags.rdgTypeTag) + try { + implicit val context: LogicalPlannerContext = + LogicalPlannerContext(catalog, parser, Map.empty) + session.plan2UnresolvedLogicalPlan(dsl, Map.empty) + Validator.validate(parser, block)(context) + false should equal(true) + } catch { + case e: Exception => + true should equal(true) + } + } + + it("test get graph load config") { + val dsl = + """ + |GraphStructure { + | A, B [FilmDirector] + | C, D [Film] + | E [FilmStar] + | C->A [directFilm] as F1 + | D->B [directFilm] as F2 + | C->E [starOfFilm] as F3 + | D->E [starOfFilm] as F4 + |} + |Rule { + | R1: A.id Set.apply("id", "name"), + "Film" -> Set.empty, + "FilmStar" -> Set.empty, + "Film_directFilm_FilmDirector" -> Set.empty, + "Film_starOfFilm_FilmStar" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new LocalReasonerSession(new KgDslParser(), catalog, TypeTags.rdgTypeTag) + val graphLoaderConfig = session.getGraphLoaderConfig(dsl, Map.empty) + val vertexTypeSet = graphLoaderConfig.allVertexTypes() + val edgeTypeSet = graphLoaderConfig.allEdgeTypes() + vertexTypeSet.containsAll(Lists.newArrayList("FilmDirector", "Film", "FilmStar")) + edgeTypeSet.containsAll( + Lists.newArrayList("Film_starOfFilm_FilmStar", "Film_directFilm_FilmDirector")) + } + // scalastyle:on +} diff --git a/reasoner/runner/runner-common/pom.xml b/reasoner/runner/runner-common/pom.xml new file mode 100644 index 00000000..2f068b8a --- /dev/null +++ b/reasoner/runner/runner-common/pom.xml @@ -0,0 +1,170 @@ + + + + 4.0.0 + + com.antgroup.openspg.reasoner + reasoner-parent + 0.0.1-SNAPSHOT + ../../pom.xml + + + reasoner-runner-common + + + 8 + 8 + + + + + + + org.rocksdb + rocksdbjni + 6.29.5 + provided + + + + + + + + com.antgroup.openspg.reasoner + reasoner-lube-api + + + com.antgroup.openspg.reasoner + reasoner-lube-logical + + + com.antgroup.openspg.reasoner + reasoner-lube-physical + + + com.antgroup.openspg.reasoner + reasoner-common + + + com.antgroup.openspg.reasoner + reasoner-udf + + + com.antgroup.openspg.reasoner + reasoner-kgdsl-parser + + + com.antgroup.openspg.reasoner + reasoner-warehouse-common + + + com.antgroup.openspg.reasoner + reasoner-openspg-catalog + + + + org.apache.commons + commons-lang3 + + + org.projectlombok + lombok + + + org.slf4j + slf4j-api + + + junit + junit + + + org.scalatest + scalatest_${scala.binary.version} + + + org.apache.commons + commons-collections4 + + + org.slf4j + slf4j-log4j12 + + + com.google.guava + guava + + + + com.alibaba + QLExpress + + + + com.alibaba + fastjson + + + com.google.protobuf + protobuf-java + + + + org.rocksdb + rocksdbjni + + + com.opencsv + opencsv + + + + + org.apache.parquet + parquet-hadoop-bundle + + + + + org.apache.hive + hive-jdbc + + + org.apache.arrow + arrow-vector + + + io.netty + netty + + + org.apache.hbase + hbase-mapreduce + + + io.netty + netty-all + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + + + + + diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/batching/DynamicBatchSize.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/batching/DynamicBatchSize.java new file mode 100644 index 00000000..88563dbc --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/batching/DynamicBatchSize.java @@ -0,0 +1,186 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.batching; + +import com.alibaba.fastjson.JSONObject; +import java.util.ArrayList; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DynamicBatchSize { + private static final Logger log = LoggerFactory.getLogger(DynamicBatchSize.class); + + private final List costs = new ArrayList<>(); + private final List batchSizes = new ArrayList<>(); + private long preBatchTime = 0; + private final double step = 1.0; + private long processSize = 0; + private int disturb = 10; + private long minBatchSize = 10; + private long maxBatchSize = 5000000; + + private int nowRound = 0; + private long allSize = 0; + + private final int expectRounds; + + public DynamicBatchSize(long allSize, long minBatchSize, long maxBatchSize, int expectRounds) { + this.allSize = allSize; + this.minBatchSize = minBatchSize; + if (this.minBatchSize > this.allSize) { + this.minBatchSize = this.allSize; + } + this.maxBatchSize = maxBatchSize; + this.expectRounds = expectRounds; + initNewDisturb(); + } + + private int computeProcessTime() { + long curTime = System.currentTimeMillis(); + int cost = (int) (curTime - preBatchTime); + preBatchTime = curTime; + return cost; + } + + public long remainSize() { + long remain = allSize - processSize; + if (remain < 0) { + return 0; + } + return remain; + } + + public long getAllSize() { + return allSize; + } + + public long getProcessSize() { + return processSize; + } + + private void initNewDisturb() { + disturb = (int) (allSize * 0.0005); + if (disturb <= 0) { + disturb = 10; + } + } + + /** + * Retrieve the next batch size. The dynamic batch size calculation formula is N^i = N^(i-1) - p * + * s, where p is the step size, and s is the gradient. + * + * @return + */ + public long getNextBatchSize() { + if (allSize - processSize <= 0) { + return 0; + } + if (nowRound == 0) { + if (expectRounds <= 1) { + processSize = allSize; + return allSize; + } + // Set the first round to the 500th percentile. + preBatchTime = System.currentTimeMillis(); + long curSize = (long) (allSize / (expectRounds * 2.5)); + curSize = uniformBatchSize(curSize); + processSize = processSize + curSize; + batchSizes.add(curSize); + + nowRound++; + return curSize; + } else if (nowRound == 1) { + // Set the second round to the 200th percentile. + long curSize = allSize / expectRounds; + curSize = uniformBatchSize(curSize); + costs.add(computeProcessTime()); + processSize = processSize + curSize; + batchSizes.add(curSize); + + nowRound++; + return curSize; + } else { + int nowCost = computeProcessTime(); + costs.add(nowCost); + int size = costs.size(); + long curSize = 0; + if (batchSizes.get(size - 1) - batchSizes.get(size - 2) == 0) { + // To avoid being restricted to a certain value when the denominator is zero, introduce a + // perturbation to the batchSize. + curSize = batchSizes.get(size - 1) + disturb; + } else { + double tN1 = costs.get(size - 1) * allSize * 1.0 / batchSizes.get(size - 1); + double tN2 = costs.get(size - 2) * allSize * 1.0 / batchSizes.get(size - 2); + double gradient = (tN1 - tN2) / (batchSizes.get(size - 1) - batchSizes.get(size - 2)); + curSize = Double.valueOf(batchSizes.get(size - 1) - step * gradient).intValue(); + if (curSize <= 0) { + // As a fallback, if the calculated value is abnormal, take the size of the previous + // batch. + curSize = batchSizes.get(size - 1); + } + } + curSize = (int) Math.min(allSize - processSize, curSize); + curSize = uniformBatchSize(curSize); + + // Gradient control: an increase of no more than 30% is allowed at a time. + final double gradient = 1.3; + if (batchSizes.size() > 0) { + long lastBatchSize = batchSizes.get(batchSizes.size() - 1); + if (1.0 * curSize / lastBatchSize > gradient) { + curSize = (long) (lastBatchSize * gradient); + } + } + + if (nowCost < 30 * 1000) { + // If each round of calculation is less than 30 seconds, do not allow the batch size to be + // reduced, + // as the gradient is not significant when the load is very low. + long lastSize = batchSizes.get(batchSizes.size() - 1); + if (curSize < lastSize) { + curSize = lastSize + disturb; + } + if (curSize > remainSize()) { + curSize = remainSize(); + } + } + + batchSizes.add(curSize); + processSize = processSize + curSize; + + log.info( + "costList=" + + JSONObject.toJSONString(costs) + + ", batchSizes=" + + JSONObject.toJSONString(batchSizes) + + ", nowRound=" + + nowRound); + nowRound++; + return curSize; + } + } + + private long uniformBatchSize(long batchSize) { + long size = batchSize; + if (batchSize < minBatchSize) { + size = minBatchSize; + } else if (batchSize > maxBatchSize) { + size = maxBatchSize; + } + long remain = remainSize(); + if (size > remain) { + return remain; + } + return size; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/catalog/CatalogFactory.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/catalog/CatalogFactory.java new file mode 100644 index 00000000..a8b55ec9 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/catalog/CatalogFactory.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.catalog; + +import com.antgroup.openspg.reasoner.catalog.impl.KgSchemaConnectionInfo; +import com.antgroup.openspg.reasoner.catalog.impl.OpenSPGCatalog; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface; +import com.antgroup.openspg.reasoner.parser.KgDslParser; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.utils.SimpleObjSerde; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CatalogFactory { + private static final Logger log = LoggerFactory.getLogger(CatalogFactory.class); + + public static Catalog createCatalog(Map params, KgSchemaConnectionInfo connInfo) { + return createCatalog(new KgDslParser(), params, connInfo); + } + + public static Catalog createCatalog( + ParserInterface parser, Map params, KgSchemaConnectionInfo connInfo) { + String catalogStr = (String) params.get(ConfigKey.KG_REASONER_CATALOG); + if (StringUtils.isNotEmpty(catalogStr)) { + return (Catalog) SimpleObjSerde.de(catalogStr); + } + + if (connInfo == null) { + throw new RuntimeException("SchemaConnectionInfo is null"); + } + Catalog catalog = + new OpenSPGCatalog( + Long.parseLong(String.valueOf(params.getOrDefault("projId", 0L))), connInfo, null); + catalog.init(); + return catalog; + } +} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/SuccessReasonerResult.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/BaseContextInitializer.java similarity index 58% rename from server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/SuccessReasonerResult.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/BaseContextInitializer.java index 72843665..843a04db 100644 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/SuccessReasonerResult.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/BaseContextInitializer.java @@ -11,17 +11,19 @@ * or implied. */ -package com.antgroup.openspg.server.core.reasoner.model.service; +package com.antgroup.openspg.reasoner.context; -public class SuccessReasonerResult extends BaseReasonerResult { +import com.antgroup.openspg.reasoner.task.TaskRecord; - private final String tableName; +public abstract class BaseContextInitializer { - public SuccessReasonerResult(String resultTableName) { - this.tableName = resultTableName; + protected TaskRecord taskRecord; + + public void setTaskRecord(TaskRecord taskRecord) { + this.taskRecord = taskRecord; } - public String getTableName() { - return tableName; - } + public abstract T initOnDriver(); + + public abstract void dispatchToWorker(T obj); } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/ContextManagement.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/ContextManagement.java new file mode 100644 index 00000000..66b84b0e --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/ContextManagement.java @@ -0,0 +1,168 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.context; + +import com.antgroup.openspg.reasoner.task.TaskRecord; +import java.lang.reflect.InvocationTargetException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ContextManagement { + private static final Logger log = LoggerFactory.getLogger(ContextManagement.class); + + private TaskRecord taskRecord = null; + + private final Map, Object> contextObjectMap = + new ConcurrentHashMap<>(); + + private ContextManagement() {} + + /** + * get dispatch info + * + * @return + */ + public DispatchContextInfo getDispatchContextInfo() { + return new DispatchContextInfo(this.taskRecord, this.contextObjectMap); + } + + /** get context */ + public T getContext(Class> contextClass) { + Object contextObj = contextObjectMap.get(contextClass); + if (null != contextObj) { + return (T) contextObj; + } + synchronized (ContextManagement.class) { + contextObj = contextObjectMap.get(contextClass); + if (null != contextObj) { + return (T) contextObj; + } + contextObj = createContextObject(contextClass); + contextObjectMap.put(contextClass, contextObj); + return (T) contextObj; + } + } + + /** init context */ + public void initContextOnDriver(TaskRecord taskRecord) { + if (null != this.taskRecord) { + return; + } + synchronized (ContextManagement.class) { + if (null != this.taskRecord) { + return; + } + this.taskRecord = taskRecord; + if (CollectionUtils.isEmpty(this.taskRecord.getInitializerClassList())) { + return; + } + for (String className : this.taskRecord.getInitializerClassList()) { + getContext(getInitializerClass(className)); + } + } + } + + /** dispatch context to worker */ + public void dispatchContextToWorker(DispatchContextInfo dispatchContextInfo) { + if (null != this.taskRecord) { + return; + } + synchronized (ContextManagement.class) { + if (null != this.taskRecord) { + return; + } + this.taskRecord = dispatchContextInfo.getTaskRecord(); + for (Map.Entry, Object> entry : + dispatchContextInfo.getContextObjectMap().entrySet()) { + Object obj = this.contextObjectMap.get(entry.getKey()); + if (null != obj) { + continue; + } + this.contextObjectMap.put(entry.getKey(), entry.getValue()); + callDispatchFunctionOnWorker(entry.getKey(), entry.getValue(), this.taskRecord); + } + } + } + + protected T createContextObject(Class> contextClass) { + try { + BaseContextInitializer initializer = contextClass.getConstructor().newInstance(); + initializer.setTaskRecord(this.taskRecord); + T t = initializer.initOnDriver(); + log.info("ContextManagement,create context, name=" + contextClass.getName()); + return t; + } catch (InstantiationException + | IllegalAccessException + | InvocationTargetException + | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + protected void callDispatchFunctionOnWorker( + Class contextClass, Object obj, TaskRecord taskRecord) { + try { + BaseContextInitializer initializer = contextClass.getConstructor().newInstance(); + initializer.setTaskRecord(this.taskRecord); + initializer.dispatchToWorker(obj); + log.info("ContextManagement,call dispatch context, name=" + contextClass.getName()); + } catch (InstantiationException + | IllegalAccessException + | InvocationTargetException + | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + protected Class> getInitializerClass( + String initializerClassName) { + Class> initializerClass; + try { + initializerClass = + (Class>) Class.forName(initializerClassName); + } catch (Exception e) { + throw new RuntimeException("can not create initializer from name " + initializerClassName, e); + } + return initializerClass; + } + + /** instance */ + private static volatile ContextManagement instance = null; + + /** + * Getter method for property instance. + * + * @return property value of instance + */ + public static ContextManagement getInstance() { + if (null != instance) { + return instance; + } + synchronized (ContextManagement.class) { + if (null != instance) { + return instance; + } + instance = new ContextManagement(); + } + return instance; + } + + /** clear */ + public static void clear() { + instance = null; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/DispatchContextInfo.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/DispatchContextInfo.java new file mode 100644 index 00000000..b21c1409 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/context/DispatchContextInfo.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.context; + +import com.antgroup.openspg.reasoner.task.TaskRecord; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; + +public class DispatchContextInfo implements Serializable { + @Getter private final TaskRecord taskRecord; + + private final Map, Object> contextObjectMap = + new HashMap<>(); + + public DispatchContextInfo( + TaskRecord taskRecord, + Map, Object> contextObjectMap) { + this.taskRecord = taskRecord; + this.contextObjectMap.putAll(contextObjectMap); + } + + public Map, Object> getContextObjectMap() { + return contextObjectMap; + } + + public boolean isEmpty() { + return contextObjectMap.isEmpty(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphState.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphState.java new file mode 100644 index 00000000..58b3bc24 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphState.java @@ -0,0 +1,217 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum; +import com.antgroup.openspg.reasoner.lube.common.rule.Rule; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; + +public interface GraphState { + + /** + * init parameters + * + * @param param + */ + void init(Map param); + + /** + * add vertex with all version properties to graph state + * + * @param vertex + */ + void addVertex(IVertex vertex); + + /** + * with version + * + * @param vertex + */ + void addVertex(IVertex vertex, Long version); + + /** + * merge properties of vertex just write, good performance + * + * @param id + * @param property + * @param mergeType + * @param version + */ + void mergeVertexProperty( + K id, Map property, MergeTypeEnum mergeType, Long version); + + /** + * set the properties of the vertex to be cached if not set, all properties are cached by default + * + * @param properties, key is vertex type, value is the properties need to be cached + */ + void setVertexCacheProperty(Map> properties); + + /** + * get vertex with the specific version properties + * + * @param id + * @param version if null return vertex with the all version properties + * @return + */ + IVertex getVertex(K id, Long version); + + /** + * get vertex with the specific version properties, with filter rule + * + * @param id + * @param version + * @param rule + * @return + */ + IVertex getVertex(K id, Long version, Rule rule); + + /** + * add inEdges and outEdges of a vertex + * + * @param vertexId + * @param inEdges + * @param outEdges + */ + void addEdges(K vertexId, List> inEdges, List> outEdges); + + /** + * update specific edge properties edges are redundantly stored and only one edge has been updated + * need to call (o, p, s, t) to update the other edge + * + * @param s + * @param p + * @param o + * @param version + * @param property + */ + void updateEdgeProperty(K s, String p, K o, Long version, IProperty property); + + /** + * merge specific edge properties edges are redundantly stored and only one edge has been updated + * need to call (o, p, s, t) to update the other edge + * + * @param s + * @param p + * @param o + * @param version should not been null + * @param property + * @param mergeType + */ + void mergeEdgeProperty( + K s, + String p, + K o, + Long version, + Direction direction, + Map property, + MergeTypeEnum mergeType); + + /** + * set the properties of the edge to be cached if not set, all properties are cached by default + * + * @param properties, key is edge type, value is the properties need to be cached + */ + void setEdgeCacheProperty(Map> properties); + + /** + * get the edge without property of the specific type of vertex + * + * @param vertexId + * @param startVersion if null, get default version edge of the vertex + * @param endVersion if null, get default version edge of the vertex + * @param types if null, get all type edges of the vertex + * @param direction + * @return key is edge type, value is direct_o_version + */ + List> getEdgesWithoutProperty( + K vertexId, Long startVersion, Long endVersion, Set types, Direction direction); + + /** + * get specific types edges of vertex + * + * @param vertexId + * @param startVersion if null, get default version edge of the vertex + * @param endVersion if null, get default version edge of the vertex + * @param types not empty + * @param direction + * @return + */ + List> getEdges( + K vertexId, Long startVersion, Long endVersion, Set types, Direction direction); + + /** + * get specific types edges with filter rule + * + * @param vertexId + * @param startVersion + * @param endVersion + * @param direction + * @param typeAndRuleMap, There is OR relationship between List + * @return + */ + List> getEdges( + K vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction, + Map> typeAndRuleMap); + + /** + * get specific type vertex iterator + * + * @param vertexType + * @return + */ + Iterator> getVertexIterator(Set vertexType); + + /** + * get vertex iterator with user-defined filter + * + * @param filter + * @return + */ + Iterator> getVertexIterator(Predicate> filter); + + /** + * get specific type edge iterator + * + * @param edgeType + * @return + */ + Iterator> getEdgeIterator(Set edgeType); + + /** + * get edge iterator with user-defined filter + * + * @param filter + * @return + */ + Iterator> getEdgeIterator(Predicate> filter); + + /** checkPoint */ + void checkPoint(); + + /** close */ + void close(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphStateFactory.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphStateFactory.java new file mode 100644 index 00000000..de21c611 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphStateFactory.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class GraphStateFactory { + private static final Map> GRAPH_STATE_MAP = + new ConcurrentHashMap<>(); + + /** return graph state on worker */ + public static GraphState getGraphState(int index) { + return GRAPH_STATE_MAP.get(index); + } + + /** init graph state */ + public static void putGraphState(int index, GraphState graphState) { + GRAPH_STATE_MAP.put(index, graphState); + } +} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/JobTypeEnum.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphStateTypeEnum.java similarity index 78% rename from server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/JobTypeEnum.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphStateTypeEnum.java index d00c594c..d9c9db56 100644 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/JobTypeEnum.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/GraphStateTypeEnum.java @@ -11,10 +11,11 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model; +package com.antgroup.openspg.reasoner.graphstate; -public enum JobTypeEnum { - BUILDING, - REASONING, - ; +public enum GraphStateTypeEnum { + /** rocksdb */ + ROCKSDB, + /** memory */ + MEMORY } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/generator/AbstractGraphGenerator.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/generator/AbstractGraphGenerator.java new file mode 100644 index 00000000..c5bde7a3 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/generator/AbstractGraphGenerator.java @@ -0,0 +1,255 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate.generator; + +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.common.utils.PropertyUtil; +import com.antgroup.openspg.reasoner.parser.DemoGraphParser; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public abstract class AbstractGraphGenerator implements Serializable { + private static final long serialVersionUID = -5588043366117487019L; + + protected IVertex constructionVertex( + String bizId, String type, Object... kvs) { + int kvsLen = kvs.length; + Object[] property = Arrays.copyOf(kvs, kvsLen + 2); + property[kvsLen] = Constants.CONTEXT_LABEL; + property[kvsLen + 1] = type; + IVertexId vertexId = IVertexId.from(bizId, type); + IVersionProperty versionProperty = + PropertyUtil.buildVertexProperty(vertexId, convert2VersionProperty(bizId, 0, property)); + return new Vertex<>(bizId, versionProperty); + } + + protected IVertex constructionVersionVertex( + String bizId, String type, long version, Object... kvs) { + int kvsLen = kvs.length; + Object[] property = Arrays.copyOf(kvs, kvsLen + 2); + property[kvsLen] = Constants.CONTEXT_LABEL; + property[kvsLen + 1] = type; + IVertexId vertexId = IVertexId.from(bizId, type); + IVersionProperty versionProperty = + PropertyUtil.buildVertexProperty( + vertexId, convert2VersionProperty(bizId, version, property)); + return new Vertex<>(bizId, versionProperty); + } + + protected IEdge constructionEdge(String s, String p, String o, Object... kvs) { + Map rowPropertyMap = convert2Property(kvs); + rowPropertyMap.put(Constants.EDGE_FROM_ID_KEY, s); + rowPropertyMap.put(Constants.EDGE_TO_ID_KEY, o); + IProperty edgeProperty = PropertyUtil.buildEdgeProperty(p, rowPropertyMap); + return new Edge<>(s, o, edgeProperty, 0, Direction.OUT, p); + } + + protected IEdge constructionVersionEdge( + String s, String p, String o, long t, Object... kvs) { + Map rowPropertyMap = convert2Property(kvs); + rowPropertyMap.put(Constants.EDGE_FROM_ID_KEY, s); + rowPropertyMap.put(Constants.EDGE_TO_ID_KEY, o); + IProperty edgeProperty = PropertyUtil.buildEdgeProperty(p, rowPropertyMap); + return new Edge<>(s, o, edgeProperty, t, Direction.OUT, p); + } + + protected Map> convert2VersionProperty( + String bizId, long version, Object... kvs) { + Map rowPropertyMap = convert2Property(kvs); + rowPropertyMap.put(Constants.NODE_ID_KEY, bizId); + Map> result = new HashMap<>(); + for (String key : rowPropertyMap.keySet()) { + Map versionPropertyMap = result.computeIfAbsent(key, k -> new TreeMap<>()); + versionPropertyMap.put(version, rowPropertyMap.get(key)); + } + return result; + } + + protected Map convert2Property(Object... kvs) { + Preconditions.checkArgument(kvs.length % 2 == 0, "The number of config kv should be even."); + Map property = new HashMap<>(); + for (int i = 0; i < kvs.length; i = i + 2) { + property.put(String.valueOf(kvs[i]), kvs[i + 1]); + } + return property; + } + + /** get vertex list than can write to graph state */ + public List> getVertexList() { + return getGraphData()._1(); + } + + /** get edge list that already aggregated by source id */ + public Map< + IVertexId, Tuple2>, List>>> + getEdgeAggregated() { + Map>, List>>> + edgeAggregatedMap = new HashMap<>(); + List> allEdge = getGraphData()._2(); + for (IEdge edge : allEdge) { + Tuple2>, List>> inOutEdgeList = + edgeAggregatedMap.computeIfAbsent( + edge.getSourceId(), k -> new Tuple2<>(new ArrayList<>(), new ArrayList<>())); + if (Direction.OUT.equals(edge.getDirection())) { + inOutEdgeList._2().add(edge); + } else if (Direction.IN.equals(edge.getDirection())) { + inOutEdgeList._1().add(edge); + } + } + return edgeAggregatedMap; + } + + protected Tuple2>, List>> + generateGraphData( + List> vertexList, List> edgeList) { + Map> vertexMap = new HashMap<>(); + vertexList.forEach( + vertex -> + vertexMap.put( + vertex.getId() + "_" + RunnerUtil.getVertexTypeFromProperty(vertex.getValue()), + vertex)); + vertexList.forEach(vertex -> vertexMap.put(vertex.getId(), vertex)); + + List> edgeListWithReasonerId = + edgeList.stream() + .map( + (Function, IEdge>) + edge -> { + IVertex source = vertexMap.get(edge.getSourceId()); + if (null == source) { + throw new RuntimeException( + "source vertex " + edge.getSourceId() + " does not exists"); + } + + IVertex target = vertexMap.get(edge.getTargetId()); + if (null == target) { + throw new RuntimeException( + "target vertex " + edge.getTargetId() + " does not exists"); + } + String sourceType = RunnerUtil.getVertexTypeFromProperty(source.getValue()); + String targetType = RunnerUtil.getVertexTypeFromProperty(target.getValue()); + return new Edge<>( + IVertexId.from(source.getId(), sourceType), + IVertexId.from(target.getId(), targetType), + edge.getValue(), + edge.getVersion(), + edge.getDirection(), + sourceType + "_" + edge.getType() + "_" + targetType); + }) + .collect(Collectors.toList()); + + Tuple2>, List>> result = + new Tuple2<>(new ArrayList<>(), new ArrayList<>()); + + edgeListWithReasonerId.forEach( + edge -> { + result._2().add(edge); + }); + + edgeListWithReasonerId.stream() + .map(this::revertEdge) + .forEach( + edge -> { + result._2().add(edge); + }); + + vertexList.stream() + .map( + (Function, IVertex>) + v -> { + IVertexId id = + IVertexId.from(v.getId(), RunnerUtil.getVertexTypeFromProperty(v.getValue())); + IProperty value = v.getValue(); + value.remove(Constants.CONTEXT_LABEL); + return new Vertex<>(id, value); + }) + .forEach(v -> result._1().add(v)); + return result; + } + + /** + * get demo graph with txt + * + * @param demoGraph + * @return + */ + public Tuple2>, List>> + getGraphData(String demoGraph) { + DemoGraphParser parser = new DemoGraphParser(); + Tuple2< + scala.collection.immutable.List>, + scala.collection.immutable.List>> + data = parser.parse(demoGraph); + return generateGraphData( + Lists.newArrayList(JavaConversions.asJavaCollection(data._1)), + Lists.newArrayList(JavaConversions.asJavaCollection(data._2))); + } + /** get graph data */ + public Tuple2>, List>> + getGraphData() { + String demoGraph = getDemoGraph(); + if (StringUtils.isNotBlank(demoGraph)) { + return getGraphData(demoGraph); + } + List> vertexList = this.genVertexList(); + List> edgeList = this.genEdgeList(); + + return generateGraphData(vertexList, edgeList); + } + + private IEdge revertEdge(IEdge edge) { + return new Edge<>( + edge.getTargetId(), + edge.getSourceId(), + edge.getValue(), + edge.getVersion(), + Direction.OUT == edge.getDirection() ? Direction.IN : Direction.OUT, + edge.getType()); + } + + /** + * use demo graph to parse data + * + * @return + */ + public String getDemoGraph() { + return ""; + } + /** please provide your mock vertex list */ + public abstract List> genVertexList(); + + /** please provide your mock edge list */ + public abstract List> genEdgeList(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/GraphStoreGraphState.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/GraphStoreGraphState.java new file mode 100644 index 00000000..c094d427 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/GraphStoreGraphState.java @@ -0,0 +1,88 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate.impl; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.warehouse.common.AbstractGraphLoader; +import com.antgroup.openspg.reasoner.warehouse.common.VertexSubGraph; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import org.apache.commons.lang3.NotImplementedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Graph State based on Graph Store The multi-version vertex is implemented through the + * multi-version property The multi-version edge is implemented through multiple edges + */ +public class GraphStoreGraphState extends MemGraphState { + private static final Logger log = LoggerFactory.getLogger(GraphStoreGraphState.class); + + private AbstractGraphLoader graphStoreQuery; + + @Override + public void init(Map param) {} + + public void setGraphStoreQuery(AbstractGraphLoader graphStoreQuery) { + this.graphStoreQuery = graphStoreQuery; + } + + @Override + public IVertex getVertex(IVertexId id, Long version) { + if (vertexMap.containsKey(id)) { + return super.getVertex(id, version); + } + log.info("GraphStoreGraphState begin query " + id.toString()); + // query from graph store + VertexSubGraph vertexSubGraph = graphStoreQuery.queryOneHotGraphState(id); + if (vertexSubGraph == null) { + log.warn("GraphStoreGraphState query id " + id.toString() + " not found"); + vertexMap.put(id, null); + return null; + } + addVertex(vertexSubGraph.getVertex()); + addEdges( + vertexSubGraph.getVertex().getId(), + vertexSubGraph.getInEdges(), + vertexSubGraph.getOutEdges()); + return getVertex(id, version); + } + + @Override + public Iterator> getVertexIterator(Set vertexType) { + throw new NotImplementedException("not support getVertexIterator in GraphStoreGraphState"); + } + + @Override + public Iterator> getVertexIterator( + Predicate> filter) { + throw new NotImplementedException("not support getVertexIterator in GraphStoreGraphState"); + } + + @Override + public Iterator> getEdgeIterator(Set edgeType) { + throw new NotImplementedException("not support getEdgeIterator in GraphStoreGraphState"); + } + + @Override + public Iterator> getEdgeIterator( + Predicate> filter) { + throw new NotImplementedException("not support getEdgeIterator in GraphStoreGraphState"); + } +} diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClient.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/IRocksDBGraphStateHelper.java similarity index 57% rename from cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClient.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/IRocksDBGraphStateHelper.java index 5bd796f7..7c5153c3 100644 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClient.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/IRocksDBGraphStateHelper.java @@ -11,12 +11,18 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.tablestore; +package com.antgroup.openspg.reasoner.graphstate.impl; -import com.antgroup.openspg.cloudext.interfaces.tablestore.cmd.TableFileCreateCmd; -import com.antgroup.openspg.common.util.cloudext.CloudExtClient; +import scala.Tuple2; -public interface TableStoreClient extends CloudExtClient { +public interface IRocksDBGraphStateHelper { + Tuple2 mapVersion2WindowRange(Long startVersion, Long endVersion); - TableFileHandler create(TableFileCreateCmd cmd); + Tuple2 mapVersion2WindowRange(Long version); + + long getWriteWindow(long version); + + Object byte2Object(byte[] bytes); + + byte[] object2Byte(Object obj); } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/MemGraphState.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/MemGraphState.java new file mode 100644 index 00000000..8aa7396b --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/MemGraphState.java @@ -0,0 +1,377 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate.impl; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.common.utils.PropertyUtil; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum; +import com.antgroup.openspg.reasoner.lube.common.rule.Rule; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple3; +import scala.Tuple5; + +public class MemGraphState implements GraphState { + private static final Logger log = LoggerFactory.getLogger(MemGraphState.class); + + /** + * vertex map key is s, value is vertex property vertex property is Map> + */ + protected Map vertexMap = new HashMap<>(); + + /** in edges map key is s, value is p_o_t */ + Map>> inEdgeMap = new HashMap<>(); + + /** out edges map key is s, value is p_o_t */ + Map>> outEdgeMap = new HashMap<>(); + + /** edge property map key is spot, value is edge property */ + Map, IProperty> edgePropertyMap = + new HashMap<>(); + + public static final String SEPARATOR = "\u0001"; + + @Override + public void init(Map param) {} + + @Override + public void addVertex(IVertex vertex) { + vertexMap.put(vertex.getId(), vertex.getValue()); + } + + @Override + public void addVertex(IVertex vertex, Long version) { + this.addVertex(vertex); + } + + @Override + public void mergeVertexProperty( + IVertexId id, Map property, MergeTypeEnum mergeType, Long version) { + IVersionProperty iProperty = (IVersionProperty) vertexMap.get(id); + if (iProperty == null) { + log.info("MemGraphState get vertex is null " + id.getInternalId() + " " + id.getType()); + return; + } + for (String key : property.keySet()) { + if (MergeTypeEnum.REPLACE.equals(mergeType)) { + iProperty.put(key, property.get(key), version); + continue; + } + String oldValue = ((String) iProperty.get(key, version)); + String newValue = oldValue == null ? "" : oldValue + SEPARATOR; + iProperty.put(key, (newValue + property.get(key)), version); + } + } + + @Override + public void setVertexCacheProperty(Map> properties) {} + + @Override + public IVertex getVertex(IVertexId id, Long version) { + IVersionProperty iProperty = (IVersionProperty) vertexMap.get(id); + if (iProperty == null) { + log.info("MemGraphState get vertex is null " + id.getInternalId() + " " + id.getType()); + return null; + } + IVersionProperty resultValue = PropertyUtil.buildVertexProperty(id, null); + for (String key : iProperty.getKeySet()) { + Object value = iProperty.get(key, version); + if (value != null) { + resultValue.put(key, value, version); + } + } + return new Vertex<>(id, resultValue); + } + + @Override + public IVertex getVertex(IVertexId id, Long version, Rule rule) { + return getVertex(id, version); + } + + private void addEdgeToMap( + IVertexId vertexId, + List> edges, + Map>> edgeMap) { + Set> edgeNameSet = + edgeMap.computeIfAbsent(vertexId, k -> new HashSet<>()); + for (IEdge edge : edges) { + edgeNameSet.add(new Tuple3<>(edge.getType(), edge.getTargetId(), edge.getVersion())); + Tuple5 spot = + new Tuple5<>( + edge.getSourceId(), + edge.getType(), + edge.getTargetId(), + edge.getDirection(), + edge.getVersion()); + if (edgePropertyMap.containsKey(spot)) { + continue; + } + edgePropertyMap.put(spot, edge.getValue()); + } + } + + @Override + public void addEdges( + IVertexId vertexId, + List> inEdges, + List> outEdges) { + if (CollectionUtils.isNotEmpty(inEdges)) { + addEdgeToMap(vertexId, inEdges, inEdgeMap); + } + if (CollectionUtils.isNotEmpty(outEdges)) { + addEdgeToMap(vertexId, outEdges, outEdgeMap); + } + } + + @Override + public void updateEdgeProperty( + IVertexId s, String p, IVertexId o, Long version, IProperty property) { + List directions = Lists.newArrayList(Direction.IN, Direction.OUT); + Tuple5 spot = null; + for (Direction d : directions) { + spot = new Tuple5<>(s, p, o, d, version); + if (edgePropertyMap.containsKey(spot)) { + break; + } + } + if (spot == null || !edgePropertyMap.containsKey(spot)) { + throw new RuntimeException("edge not found"); + } + edgePropertyMap.put(spot, property); + } + + @Override + public void mergeEdgeProperty( + IVertexId s, + String p, + IVertexId o, + Long version, + Direction direction, + Map property, + MergeTypeEnum mergeType) { + IProperty originProp = null; + Direction finalDirection = direction; + if (finalDirection == null) { + finalDirection = Direction.BOTH; + } + List directions = Lists.newArrayList(finalDirection); + if (Direction.BOTH == finalDirection) { + directions = Lists.newArrayList(Direction.IN, Direction.OUT); + } + for (Direction d : directions) { + Tuple5 spot = + new Tuple5<>(s, p, o, d, version); + originProp = edgePropertyMap.get(spot); + if (null != originProp) { + break; + } + } + + if (null == originProp) { + throw new RuntimeException("edge property not exist"); + } + for (String key : property.keySet()) { + if (MergeTypeEnum.REPLACE.equals(mergeType)) { + originProp.put(key, property.get(key)); + continue; + } + String oldValue = ((String) originProp.get(key)); + String newValue = oldValue == null ? "" : oldValue + SEPARATOR; + originProp.put(key, (newValue + property.get(key))); + } + } + + @Override + public void setEdgeCacheProperty(Map> properties) {} + + private ArrayList> filterEdge( + IVertexId vertexId, + Set> edgeSet, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + ArrayList> result = new ArrayList<>(); + if (null == edgeSet) { + return result; + } + for (Tuple3 pot : edgeSet) { + String type = pot._1(); + IVertexId o = pot._2(); + Long version = pot._3(); + if (endVersion != null && endVersion != 0) { + // Make a judgment only in the presence of a multi-version scenario. + if (!RunnerUtil.between(startVersion, endVersion, version)) { + continue; + } + } + + if (null == types || types.contains(type)) { + result.add(new Edge<>(vertexId, o, null, version, direction, type)); + } + } + return result; + } + + @Override + public List> getEdgesWithoutProperty( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + if (null == direction) { + throw new RuntimeException("direction cant be null!"); + } + + long defaultVersion = 0L; + if (startVersion == null || endVersion == null) { + startVersion = defaultVersion; + endVersion = defaultVersion; + } + + ArrayList> result = new ArrayList<>(); + if (Direction.IN.equals(direction) || Direction.BOTH.equals(direction)) { + result = + filterEdge( + vertexId, inEdgeMap.get(vertexId), startVersion, endVersion, types, Direction.IN); + } + + if (Direction.OUT.equals(direction) || Direction.BOTH.equals(direction)) { + ArrayList> tmp = + filterEdge( + vertexId, outEdgeMap.get(vertexId), startVersion, endVersion, types, Direction.OUT); + result.addAll(tmp); + } + result.trimToSize(); + return result; + } + + @Override + public List> getEdges( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + List> result = + getEdgesWithoutProperty(vertexId, startVersion, endVersion, types, direction); + result.forEach( + edge -> { + List directions = Lists.newArrayList(direction); + if (Direction.BOTH == direction) { + directions = Lists.newArrayList(Direction.IN, Direction.OUT); + } + for (Direction d : directions) { + Tuple5 spot = + new Tuple5<>(vertexId, edge.getType(), edge.getTargetId(), d, edge.getVersion()); + IProperty edgeProperty = edgePropertyMap.get(spot); + if (edgeProperty == null) { + edge.setValue(null); + } else { + edge.setValue(edgeProperty.clone()); + break; + } + } + }); + return result; + } + + @Override + public List> getEdges( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction, + Map> typeAndRuleMap) { + return getEdges(vertexId, startVersion, endVersion, types, direction); + } + + @Override + public Iterator> getVertexIterator(Set vertexType) { + return getVertexIterator( + vertex -> { + String type = RunnerUtil.getVertexType(vertex); + if (null == vertexType) { + return true; + } + return vertexType.contains(type); + }); + } + + @Override + public Iterator> getVertexIterator( + Predicate> filter) { + List> vertexList = new ArrayList<>(); + for (IVertexId vertexId : vertexMap.keySet()) { + IProperty property = vertexMap.get(vertexId); + Vertex vertex = new Vertex<>(vertexId, property); + if (filter.test(vertex)) { + vertexList.add(vertex); + } + } + return vertexList.iterator(); + } + + @Override + public Iterator> getEdgeIterator(Set edgeType) { + return getEdgeIterator( + edge -> { + if (null == edgeType) { + return true; + } + return edgeType.contains(edge.getType()); + }); + } + + @Override + public Iterator> getEdgeIterator( + Predicate> filter) { + List> edgeList = new ArrayList<>(); + for (IVertexId vertexId : vertexMap.keySet()) { + List> edges = + getEdges(vertexId, null, null, null, Direction.BOTH); + edgeList.addAll(edges.stream().filter(filter).collect(Collectors.toList())); + } + return edgeList.iterator(); + } + + @Override + public void checkPoint() {} + + @Override + public void close() {} +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/RocksdbGraphState.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/RocksdbGraphState.java new file mode 100644 index 00000000..b92ce886 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/RocksdbGraphState.java @@ -0,0 +1,741 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate.impl; + +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.common.utils.PropertyUtil; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum; +import com.antgroup.openspg.reasoner.lube.common.rule.Rule; +import com.antgroup.openspg.reasoner.utils.RocksDBUtil; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * GraphState based on Rocksdb vertex key is V_TYPEID_ID_PLACEHOLDER_WINDOWS, value is vertex + * property edge key is E_EDGETYPE_DIRECTION_VERTEXID_WINDOW, value is edges + */ +public class RocksdbGraphState implements GraphState { + private static final Logger log = LoggerFactory.getLogger(RocksdbGraphState.class); + + public final transient RocksDB rocksDB; + private final transient IRocksDBGraphStateHelper helper; + + public RocksdbGraphState(RocksDB rocksDB, IRocksDBGraphStateHelper helper) { + this.rocksDB = rocksDB; + this.helper = helper; + } + + /** + * init parameters + * + * @param param + */ + @Override + public void init(Map param) {} + + /** + * wrap vertex to rocksdb storage + * + * @param vertex + * @return + */ + private Tuple2 wrapVertex(IVertex vertex, long window) { + return new Tuple2<>( + RocksDBUtil.rocksdbKeyAppendWindow( + RocksDBUtil.buildRocksDBVertexKeyWithoutWindow(vertex.getId()), window), + vertex.getValue()); + } + + /** + * add vertex with all version properties to graph state + * + * @param vertex + */ + @Override + public void addVertex(IVertex vertex) { + this.addVertex(vertex, 0L); + } + + /** + * add vertex with all version properties to graph state + * + * @param vertex + */ + @Override + public void addVertex(IVertex vertex, Long version) { + if (vertex.getValue() instanceof IVersionProperty) { + Tuple2 wrapResult = wrapVertex(vertex, version); + writeData(wrapResult._1, wrapResult._2); + return; + } + log.error("Only IVersionProperty is supported."); + throw new RuntimeException("Only IVersionProperty is supported."); + } + + /** + * merge properties of vertex just write, good performance + * + * @param id + * @param property + * @param mergeType + * @param version + */ + @Override + public void mergeVertexProperty( + IVertexId id, Map property, MergeTypeEnum mergeType, Long version) { + Preconditions.checkNotNull(id, "id is null"); + Preconditions.checkNotNull(property, "property is null"); + Preconditions.checkNotNull(mergeType, "mergeType is null"); + + if (MergeTypeEnum.APPEND.equals(mergeType)) { + throw new RuntimeException("APPEND not supported"); + } + IVersionProperty versionProperty = PropertyUtil.buildVertexProperty(id, null); + for (String key : property.keySet()) { + versionProperty.put(key, property.get(key), version); + } + if (versionProperty.getSize() == 0) { + log.warn("no property need to be merged"); + return; + } + IVertex newVertex = new Vertex<>(id, versionProperty); + Tuple2 wrapResult = + wrapVertex(newVertex, this.helper.getWriteWindow(version)); + writeData(wrapResult._1, wrapResult._2); + } + + /** + * set the properties of the vertex to be cached if not set, all properties are cached by default + * + * @param properties + */ + @Override + public void setVertexCacheProperty(Map> properties) { + throw new NotImplementedException("setVertexCacheProperty is not implemented", null); + } + + private static final WriteOptions DEFAULT_WRITE_OPTIONS = new WriteOptions(); + + static { + DEFAULT_WRITE_OPTIONS.setDisableWAL(true); + } + + /** + * write data to rocksdb + * + * @param key + * @param value + */ + private void writeData(byte[] key, Object value) { + if (key == null || key.length == 0) { + log.warn("writeData keys is empty"); + return; + } + try { + byte[] values = helper.object2Byte(value); + rocksDB.put(DEFAULT_WRITE_OPTIONS, key, values); + } catch (RocksDBException e) { + throw new RuntimeException("rocksdb put error, key=" + Arrays.toString(key), e); + } + } + + /** + * @param keyPrefix rocksdb key prefix + * @param windowRange windows + * @return + */ + private List readData(byte[] keyPrefix, Tuple2 windowRange) { + byte[] seekKey; + if (null == windowRange) { + seekKey = RocksDBUtil.rocksdbKeyAppendWindow(keyPrefix, 0L); + } else { + seekKey = RocksDBUtil.rocksdbKeyAppendWindow(keyPrefix, windowRange._1()); + } + List result = new ArrayList<>(); + + // rocksdb seek + ReadOptions readOptions = new ReadOptions(); + readOptions.setPrefixSameAsStart(true); + readOptions.setTotalOrderSeek(false); + RocksIterator it = rocksDB.newIterator(readOptions); + it.seek(seekKey); + + while (it.isValid()) { + byte[] rocksdbKey = it.key(); + boolean prefixEquals = Utils.compareByteArrayPrefix(keyPrefix, rocksdbKey); + if (!prefixEquals) { + break; + } + + if (null != windowRange) { + long window = RocksDBUtil.getRocksdbKeyWindow(rocksdbKey); + if (!RunnerUtil.between(windowRange._1(), windowRange._2(), window)) { + it.next(); + continue; + } + } + result.add(it.value()); + it.next(); + } + it.close(); + return result; + } + + /** + * collapse multiple rocksdb vertex properties to one versionVertexProperty + * + * @param propertyByteList + * @return + */ + private Map> collapseVertexProperty(List propertyByteList) { + List vertexVersionProperties = + propertyByteList.stream() + .map( + bytes -> { + Object obj = helper.byte2Object(bytes); + if (!(obj instanceof IVersionProperty)) { + throw new RuntimeException("vertex property is not IVersionProperty"); + } + return (IVersionProperty) obj; + }) + .collect(Collectors.toList()); + + // merge VertexVersionProperty + Map> resultMap = new HashMap<>(); + for (IVersionProperty property : vertexVersionProperties) { + for (String key : property.getKeySet()) { + TreeMap thisVersionValueMap = + resultMap.computeIfAbsent(key, k -> new TreeMap<>()); + Object otherVersionValueMap = property.getVersionValue(key); + if (null == otherVersionValueMap) { + continue; + } + thisVersionValueMap.putAll(((Map) otherVersionValueMap)); + } + } + return resultMap; + } + + /** + * get vertex with the specific version properties + * + * @param id + * @param version if null return vertex with the all version properties, if Long.MAX_VALUE return + * newest properties + * @return + */ + @Override + public IVertex getVertex(IVertexId id, Long version) { + byte[] rocksdbKeyPrefix = RocksDBUtil.buildRocksDBVertexKeyWithoutWindow(id); + Tuple2 windowRange = this.helper.mapVersion2WindowRange(version); + List propertyList = readData(rocksdbKeyPrefix, windowRange); + if (CollectionUtils.isEmpty(propertyList)) { + return null; + } + Map> tmpProps = collapseVertexProperty(propertyList); + if (null == version) { + return new Vertex<>(id, PropertyUtil.buildVertexProperty(id, tmpProps)); + } + + Map> resultProps = new HashMap<>(); + // filter specific version property + for (String key : tmpProps.keySet()) { + TreeMap versionValue = tmpProps.get(key); + Object value = PropertyUtil.getVersionValue(version, versionValue); + TreeMap tmpMap = new TreeMap<>(); + tmpMap.put(version, value); + resultProps.put(key, tmpMap); + } + return new Vertex<>(id, PropertyUtil.buildVertexProperty(id, resultProps)); + } + + @Override + public IVertex getVertex(IVertexId id, Long version, Rule rule) { + return getVertex(id, version); + } + + /** + * wrap edge to rocksdb storage + * + * @param edges + * @param direction + * @param vertexId + * @return + */ + private List> wrapEdge( + List> edges, Direction direction, IVertexId vertexId) { + List> wrapEdgeList = new ArrayList<>(); + Map>>> type2Version2EdgeMap = + new HashMap<>(); + for (IEdge edge : edges) { + String p = edge.getType(); + Map>> version2EdgeListMap = + type2Version2EdgeMap.computeIfAbsent(p, k -> new HashMap<>()); + List> edgeList = + version2EdgeListMap.computeIfAbsent( + this.helper.getWriteWindow(edge.getVersion()), k -> new ArrayList<>()); + edgeList.add(edge); + } + for (String type : type2Version2EdgeMap.keySet()) { + Map>> version2EdgeListMap = + type2Version2EdgeMap.get(type); + byte[] edgeKeyPrefix = + RocksDBUtil.buildRocksDBEdgeKeyWithoutWindow(type, direction, vertexId); + for (long window : version2EdgeListMap.keySet()) { + byte[] edgeKey = RocksDBUtil.rocksdbKeyAppendWindow(edgeKeyPrefix, window); + IProperty edgeProperty = new EdgeProperty(); + edgeProperty.put("edges", version2EdgeListMap.get(window)); + wrapEdgeList.add(new Tuple2<>(edgeKey, edgeProperty)); + } + } + return wrapEdgeList; + } + + /** + * add inEdges and outEdges of a vertex + * + * @param vertexId + * @param inEdges + * @param outEdges + */ + @Override + public void addEdges( + IVertexId vertexId, + List> inEdges, + List> outEdges) { + if (CollectionUtils.isNotEmpty(outEdges)) { + wrapEdge(outEdges, Direction.OUT, vertexId) + .forEach(tuple -> writeData(tuple._1(), tuple._2())); + } + + if (CollectionUtils.isNotEmpty(inEdges)) { + wrapEdge(inEdges, Direction.IN, vertexId).forEach(tuple -> writeData(tuple._1(), tuple._2())); + } + } + + /** + * update specific edge properties edges are redundantly stored and only one edge has been updated + * need to call (o, p, s, t) to update the other edge + * + * @param s + * @param p + * @param o + * @param version + * @param property + */ + @Override + public void updateEdgeProperty( + IVertexId s, String p, IVertexId o, Long version, IProperty property) { + throw new NotImplementedException("updateEdgeProperty not implement", null); + } + + /** + * merge specific edge properties edges are redundantly stored and only one edge has been updated + * need to call (o, p, s, t) to update the other edge + * + * @param s + * @param p + * @param o + * @param version + * @param property + * @param mergeType + */ + @Override + public void mergeEdgeProperty( + IVertexId s, + String p, + IVertexId o, + Long version, + Direction direction, + Map property, + MergeTypeEnum mergeType) { + Preconditions.checkNotNull(s, "mergeEdgeProperty s is null"); + Preconditions.checkNotNull(p, "mergeEdgeProperty p is null"); + Preconditions.checkNotNull(o, "mergeEdgeProperty o is null"); + Preconditions.checkNotNull(mergeType, "mergeType is null"); + Preconditions.checkNotNull(version, "mergeEdgeProperty version is null"); + Preconditions.checkNotNull(direction, "direction version is null"); + Preconditions.checkNotNull(property, "mergeEdgeProperty property is null"); + if (MergeTypeEnum.APPEND.equals(mergeType)) { + throw new RuntimeException("APPEND not supported"); + } + if (property.isEmpty()) { + log.warn("no property need to be merged"); + return; + } + + IProperty edgeProperty = PropertyUtil.buildEdgeProperty(p, property); + IEdge edge = new Edge<>(s, o, edgeProperty, version, direction, p); + List> inEdges = new ArrayList<>(); + List> outEdges = new ArrayList<>(); + if (Direction.IN.equals(direction) || Direction.BOTH.equals(direction)) { + inEdges.add(edge); + } + if (Direction.OUT.equals(direction) || Direction.BOTH.equals(direction)) { + outEdges.add(edge); + } + addEdges(s, inEdges, outEdges); + } + + /** + * set the properties of the edge to be cached if not set, all properties are cached by default + * + * @param properties + */ + @Override + public void setEdgeCacheProperty(Map> properties) { + throw new NotImplementedException("setEdgeCacheProperty is not implemented", null); + } + + /** + * get the edge without property of the specific type of vertex + * + * @param vertexId + * @param startVersion if null, get default version edge of the vertex + * @param endVersion if null, get default version edge of the vertex + * @param types if null, get all type edges of the vertex + * @param direction + * @return key is edge type, value is direct_o_version + */ + @Override + public List> getEdgesWithoutProperty( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + throw new NotImplementedException("getEdgesWithoutProperty is not implemented", null); + } + + /** + * get specific types edges of vertex + * + * @param vertexId + * @param startVersion if null, get default version edge of the vertex + * @param endVersion if null, get default version edge of the vertex + * @param types not empty + * @param direction + * @return + */ + @Override + public List> getEdges( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + if (CollectionUtils.isEmpty(types)) { + throw new RuntimeException("getEdges input types is empty"); + } + if (null == direction) { + throw new RuntimeException("getEdges input direction is null"); + } + + List> result = new ArrayList<>(); + + if (Direction.OUT.equals(direction) || Direction.BOTH.equals(direction)) { + result.addAll(collectEdges(vertexId, startVersion, endVersion, types, Direction.OUT)); + } + if (Direction.IN.equals(direction) || Direction.BOTH.equals(direction)) { + result.addAll(collectEdges(vertexId, startVersion, endVersion, types, Direction.IN)); + } + return result; + } + + @Override + public List> getEdges( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction, + Map> typeAndRuleMap) { + return getEdges(vertexId, startVersion, endVersion, types, direction); + } + + /** + * collect multiple rocksdb edges of one edge + * + * @param vertexId + * @param startVersion + * @param endVersion + * @param types + * @param direction + * @return + */ + private List> collectEdges( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + List> result = new ArrayList<>(); + for (String type : types) { + byte[] rocksdbKeyPrefix = + RocksDBUtil.buildRocksDBEdgeKeyWithoutWindow(type, direction, vertexId); + List bytesList = + readData(rocksdbKeyPrefix, this.helper.mapVersion2WindowRange(startVersion, endVersion)); + if (CollectionUtils.isEmpty(bytesList)) { + continue; + } + List>> edgeList = + bytesList.stream() + .map( + bytes -> { + Object obj = helper.byte2Object(bytes); + if (!(obj instanceof EdgeProperty)) { + throw new RuntimeException( + "The data that getEdges reads from rocksdb is not EdgeProperty type"); + } + if (!((EdgeProperty) obj).isKeyExist("edges")) { + throw new RuntimeException("getEdges EdgeProperty not contain edges"); + } + List> edges = + ((List>) ((EdgeProperty) obj).get("edges")); + Map> edgeMap = new HashMap<>(); + for (IEdge edge : edges) { + // spot identifies the unique edge + String key = RunnerUtil.getEdgeIdentifier(edge); + IEdge oldEdge = edgeMap.get(key); + if (null == oldEdge) { + edgeMap.put(key, edge); + continue; + } + // Merge the Properties of edges in the same window + for (String propertyName : edge.getValue().getKeySet()) { + oldEdge.getValue().put(propertyName, edge.getValue().get(propertyName)); + } + } + return edgeMap; + }) + .collect(Collectors.toList()); + + // Merge the Properties of edges in the different window + Map> resultEdgeMap = new HashMap<>(); + for (Map> map : edgeList) { + if (resultEdgeMap.isEmpty()) { + resultEdgeMap.putAll(map); + continue; + } + mergeEdgeMap(resultEdgeMap, map); + } + result.addAll(resultEdgeMap.values()); + } + + if (null == startVersion || null == endVersion) { + return result; + } + return result.stream() + .filter(e -> RunnerUtil.between(startVersion, endVersion, e.getVersion())) + .collect(Collectors.toList()); + } + + /** + * merge the multiple edge properties of one edge + * + * @param oldEdgeMap + * @param newEdgeMap + */ + private void mergeEdgeMap( + Map> oldEdgeMap, + Map> newEdgeMap) { + for (String spot : newEdgeMap.keySet()) { + if (!oldEdgeMap.containsKey(spot)) { + oldEdgeMap.put(spot, newEdgeMap.get(spot)); + continue; + } + IEdge oldEdge = oldEdgeMap.get(spot); + IEdge newEdge = newEdgeMap.get(spot); + for (String propertyName : newEdge.getValue().getKeySet()) { + oldEdge.getValue().put(propertyName, newEdge.getValue().get(propertyName)); + } + } + } + + /** + * get specific type vertex iterator + * + * @param vertexType + * @return + */ + @Override + public Iterator> getVertexIterator(Set vertexType) { + return getVertexIterator( + vertex -> { + String type = (vertex.getId()).getType(); + if (null == vertexType) { + return true; + } + return vertexType.contains(type); + }); + } + + private RocksIterator getRocksIterator() { + ReadOptions readOptions = new ReadOptions(); + readOptions.setPrefixSameAsStart(false); + readOptions.setTotalOrderSeek(true); + return rocksDB.newIterator(readOptions); + } + + /** + * get vertex iterator with user-defined filter + * + * @param filter + * @return + */ + @Override + public Iterator> getVertexIterator( + Predicate> filter) { + RocksIterator it = getRocksIterator(); + it.seek(getVertexPrefix()); + if (!it.isValid()) { + log.warn("Rocksdb has not contain vertex prefix key"); + return new Iterator>() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public IVertex next() { + return null; + } + }; + } + + return new Iterator>() { + private IVertex cacheVertex = null; + private byte[] curVertexKey = it.key(); + + @Override + public boolean hasNext() { + if (null == cacheVertex) { + while (true) { + IVertex resultVertex = getNext(); + if (null == resultVertex) { + break; + } else if (filter.test(resultVertex)) { + cacheVertex = resultVertex; + break; + } + } + } + return null != cacheVertex; + } + + @Override + public IVertex next() { + IVertex result = cacheVertex; + cacheVertex = null; + return result; + } + + private IVertex getNext() { + List curVertexPropertyList = new ArrayList<>(); + byte[] nextVertexKey = null; + while (it.isValid()) { + byte[] vertexKey = it.key(); + if (!RocksDBUtil.checkSameVertexKey(curVertexKey, vertexKey)) { + nextVertexKey = vertexKey; + break; + } + curVertexPropertyList.add(it.value()); + it.next(); + } + + if (curVertexPropertyList.isEmpty()) { + return null; + } + IVertexId vertexId = RocksDBUtil.extractVertexId(curVertexKey); + IProperty vertexProperty = + PropertyUtil.buildVertexProperty( + vertexId, collapseVertexProperty(curVertexPropertyList)); + IVertex resultVertex = new Vertex<>(vertexId, vertexProperty); + curVertexKey = nextVertexKey; + return resultVertex; + } + }; + } + + /** + * get specific type edge iterator + * + * @param edgeType + * @return + */ + @Override + public Iterator> getEdgeIterator(Set edgeType) { + throw new NotImplementedException("getEdgeIterator is not implemented", null); + } + + /** + * get edge iterator with user-defined filter + * + * @param filter + * @return + */ + @Override + public Iterator> getEdgeIterator( + Predicate> filter) { + throw new NotImplementedException("getEdgeIterator is not implemented", null); + } + + /** + * get the byte array of vertex prefix + * + * @return + */ + private byte[] getVertexPrefix() { + return RocksDBUtil.VERTEX_FLAG.getBytes(); + } + + /** checkPoint */ + @Override + public void checkPoint() {} + + @Override + public void close() { + this.rocksDB.close(); + } +} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerProgress.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/model/MergeTypeEnum.java similarity index 73% rename from server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerProgress.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/model/MergeTypeEnum.java index d151dd22..95c624c7 100644 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerProgress.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/model/MergeTypeEnum.java @@ -11,8 +11,13 @@ * or implied. */ -package com.antgroup.openspg.server.core.reasoner.model.service; +package com.antgroup.openspg.reasoner.graphstate.model; -import com.antgroup.openspg.server.common.model.base.BaseValObj; +public enum MergeTypeEnum { -public class ReasonerProgress extends BaseValObj {} + /** Replace the existing value */ + REPLACE, + + /** Append to the existing value */ + APPEND +} diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableFileHandler.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/ITableReader.java similarity index 58% rename from cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableFileHandler.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/ITableReader.java index 2b8f9fe2..de9d2c8b 100644 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableFileHandler.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/ITableReader.java @@ -11,18 +11,17 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.tablestore; +package com.antgroup.openspg.reasoner.io; -import com.antgroup.openspg.cloudext.interfaces.tablestore.model.TableRecord; +import com.antgroup.openspg.reasoner.io.model.AbstractTableInfo; +import java.util.Iterator; import java.util.List; -public interface TableFileHandler { - - String getTableName(); - - int write(TableRecord record); - - int batchWrite(List records); +public interface ITableReader extends AutoCloseable, Iterator { + /** init reader for read a list of tables */ + void init( + int index, int parallel, int nowRound, int allRound, List tableInfoList); + /** close reader */ void close() throws Exception; } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/ITableWriter.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/ITableWriter.java new file mode 100644 index 00000000..f102f4bb --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/ITableWriter.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io; + +import com.antgroup.openspg.reasoner.io.model.AbstractTableInfo; + +public interface ITableWriter { + /** open a table writer */ + void open(int taskIndex, int parallel, AbstractTableInfo tableInfo); + + /** write a row data into table */ + void write(Object[] data); + + /** close writer, commit data */ + void close(); + + /** get write count */ + long writeCount(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/IoFactory.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/IoFactory.java new file mode 100644 index 00000000..b0b3500b --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/IoFactory.java @@ -0,0 +1,200 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io; + +import com.antgroup.openspg.reasoner.common.exception.HiveException; +import com.antgroup.openspg.reasoner.common.exception.IllegalArgumentException; +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.io.hive.HiveUtils; +import com.antgroup.openspg.reasoner.io.hive.HiveWriter; +import com.antgroup.openspg.reasoner.io.hive.HiveWriterSession; +import com.antgroup.openspg.reasoner.io.model.AbstractTableInfo; +import com.antgroup.openspg.reasoner.io.model.HiveTableInfo; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.collect.Lists; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +public class IoFactory { + private static final Logger log = LoggerFactory.getLogger(IoFactory.class); + + private static final Map> SESSION_MAP = new HashMap<>(); + + /** create writer session, must call on driver */ + public static String createWriterSession(AbstractTableInfo tableInfo) { + Tuple2 idAndSession = SESSION_MAP.get(tableInfo); + if (null != idAndSession) { + return idAndSession._1(); + } + String sessionId = null; + // create session + if (tableInfo instanceof HiveTableInfo) { + HiveTableInfo hiveTableInfo = (HiveTableInfo) tableInfo; + HiveWriterSession hiveWriterSession = HiveUtils.createHiveWriterSession(hiveTableInfo); + sessionId = hiveWriterSession.getSessionId(); + SESSION_MAP.put(hiveTableInfo, new Tuple2<>(sessionId, hiveWriterSession)); + } + log.info( + "createWriterSession,table_info=" + + tableInfo.getTableInfoKeyString() + + ",sessionId=" + + sessionId); + return sessionId; + } + + /** get writer */ + public static ITableWriter getTableWriter( + String sessionId, int index, int parallel, AbstractTableInfo tableInfo) { + String cacheKey = getCacheKey(sessionId, index); + ITableWriter resultWriter; + if (tableInfo instanceof HiveTableInfo) { + resultWriter = TABLE_WRITER_CACHE.getIfPresent(cacheKey); + if (null != resultWriter) { + return resultWriter; + } + synchronized (TABLE_WRITER_CACHE) { + resultWriter = TABLE_WRITER_CACHE.getIfPresent(cacheKey); + if (null != resultWriter) { + return resultWriter; + } + HiveWriter hiveWriter = new HiveWriter(); + hiveWriter.open(index, parallel, tableInfo); + TABLE_WRITER_CACHE.put(cacheKey, hiveWriter); + } + return TABLE_WRITER_CACHE.getIfPresent(cacheKey); + } + throw new NotImplementedException( + "table type not support," + tableInfo.getClass().getName(), null); + } + + /** + * commit session, call on driver + * + * @param sessionId + */ + public static void commitWriterSession(String sessionId) { + // get session + Object session = null; + Iterator>> it = + SESSION_MAP.entrySet().iterator(); + while (it.hasNext()) { + Tuple2 idAndSession = it.next().getValue(); + if (!sessionId.equals(idAndSession._1())) { + continue; + } + session = idAndSession._2(); + it.remove(); + break; + } + if (null == session) { + return; + } + + log.info("commitWriterSession,sessionId=" + sessionId); + + if (session instanceof HiveWriterSession) { + HiveWriterSession hiveWriterSession = (HiveWriterSession) session; + hiveWriterSession.commit(); + } + } + + /** + * flush writer, call on worker + * + * @param sessionId + * @param index + */ + public static void closeWriter(String sessionId, int index) { + TABLE_WRITER_CACHE.invalidate(getCacheKey(sessionId, index)); + } + + /** get table reader */ + public static ITableReader getTableReader( + int index, int parallel, AbstractTableInfo tableInfo, Object engineContext) { + return getTableReader(index, parallel, 0, 1, Lists.newArrayList(tableInfo), engineContext); + } + + private static final String GEAFLOW_CONTEXT_CLASS_NAME = + "com.antfin.arch.runtime.context.GRuntimeContext"; + private static final String GEAFLOW_TABLER_READER_CLASS_NAME = + "com.antgroup.openspg.reasoner.geaflow.io.hive.GeaflowHiveReader"; + + /** get table reader for multiple round read */ + public static ITableReader getTableReader( + int index, + int parallel, + int nowRound, + int allRound, + List tableInfoList, + Object engineContext) { + if (null == tableInfoList || tableInfoList.isEmpty()) { + throw new IllegalArgumentException( + "tableInfoList", "emptyList", "please input table info list", null); + } + if (tableInfoList.get(0) instanceof HiveTableInfo) { + if (allRound > 1) { + throw new HiveException("hive reader not support multiple round read", null); + } + if (1 != tableInfoList.size()) { + throw new HiveException("hive reader not support multiple table read", null); + } + if (GEAFLOW_CONTEXT_CLASS_NAME.equals(engineContext.getClass().getName())) { + ITableReader tableReader; + String implClassName = GEAFLOW_TABLER_READER_CLASS_NAME; + try { + tableReader = + (ITableReader) + Class.forName(implClassName) + .getConstructor(Object.class) + .newInstance(engineContext); + tableReader.init(index, parallel, nowRound, allRound, tableInfoList); + } catch (Exception e) { + throw new HiveException("can not create ITableReader from name " + implClassName, e); + } + return tableReader; + } + throw new HiveException("unknown engine type, " + engineContext.getClass().getName(), null); + } + throw new NotImplementedException( + "table type not support," + tableInfoList.get(0).getClass().getName(), null); + } + + private static String getCacheKey(String sessionId, int index) { + return sessionId + "_" + index; + } + + private static final Cache TABLE_WRITER_CACHE = + CacheBuilder.newBuilder() + .maximumSize(100) + .expireAfterAccess(3, TimeUnit.HOURS) + .expireAfterWrite(6, TimeUnit.HOURS) + .removalListener( + (RemovalListener) + notification -> { + log.info( + "start_remove_writer, key={}, writeCount={}", + notification.getKey(), + notification.getValue().writeCount()); + notification.getValue().close(); + }) + .build(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveUtils.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveUtils.java new file mode 100644 index 00000000..90db4b40 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveUtils.java @@ -0,0 +1,240 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io.hive; + +import com.antgroup.openspg.reasoner.common.exception.HiveException; +import com.antgroup.openspg.reasoner.common.types.KTBoolean$; +import com.antgroup.openspg.reasoner.common.types.KTDouble$; +import com.antgroup.openspg.reasoner.common.types.KTInteger$; +import com.antgroup.openspg.reasoner.common.types.KTLong$; +import com.antgroup.openspg.reasoner.common.types.KTString$; +import com.antgroup.openspg.reasoner.io.model.HiveTableInfo; +import com.antgroup.openspg.reasoner.lube.catalog.struct.Field; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.util.HadoopOutputFile; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; +import org.apache.parquet.schema.Types.GroupBuilder; +import org.apache.parquet.schema.Types.PrimitiveBuilder; + +public class HiveUtils { + + public static HadoopOutputFile getHadoopOutputFile(HiveTableInfo hiveTableInfo, int index) { + if (StringUtils.isNotEmpty(hiveTableInfo.getHadoopUserName())) { + System.setProperty("HADOOP_USER_NAME", hiveTableInfo.getHadoopUserName()); + } + Triple> outputPathPair = getHdfsOutputPath(hiveTableInfo); + Configuration conf = getHadoopConfiguration(hiveTableInfo, outputPathPair.getLeft()); + String fileName = outputPathPair.getMiddle(); + if (!outputPathPair.getRight().isEmpty()) { + fileName += "/" + String.join("/", outputPathPair.getRight()); + } + fileName += "/" + index; + try { + return HadoopOutputFile.fromPath(new Path(fileName), conf); + } catch (IOException e) { + throw new HiveException("create hadoop output file error, " + fileName, e); + } + } + + public static Configuration getHadoopConfiguration( + HiveTableInfo hiveTableInfo, String defaultFs) { + Configuration conf = new Configuration(); + if (StringUtils.isNotEmpty(hiveTableInfo.getCoreSiteXml())) { + conf.addResource(new Path("file://" + hiveTableInfo.getCoreSiteXml())); + } + if (StringUtils.isNotEmpty(hiveTableInfo.getHdfsSiteXml())) { + conf.addResource(new Path("file://" + hiveTableInfo.getHdfsSiteXml())); + } + conf.set("dfs.client.use.datanode.hostname", "true"); + if (StringUtils.isNotEmpty(defaultFs)) { + conf.set("fs.defaultFS", defaultFs); + } + return conf; + } + + /** get hive output path */ + public static Triple> getHdfsOutputPath( + HiveTableInfo hiveTableInfo) { + List partitionList = new ArrayList<>(); + if (null != hiveTableInfo.getPartition() && !hiveTableInfo.getPartition().isEmpty()) { + // sort partition + List partitionKeyList = Lists.newArrayList(hiveTableInfo.getPartition().keySet()); + partitionKeyList.sort(Comparator.naturalOrder()); + for (String partitionKey : partitionKeyList) { + String partitionValue = hiveTableInfo.getPartition().get(partitionKey); + partitionList.add(partitionKey + "=" + partitionValue); + } + } + if (StringUtils.isEmpty(hiveTableInfo.getOutputParquetPath())) { + // use default hdfs path + return new ImmutableTriple<>(null, "/tmp/holmes/" + hiveTableInfo.getTable(), partitionList); + } + Path hdfsPath = new Path(hiveTableInfo.getOutputParquetPath()); + URI hdfsUri = hdfsPath.toUri(); + if (StringUtils.isEmpty(hdfsUri.getAuthority())) { + return new ImmutableTriple<>(null, hdfsUri.getPath(), partitionList); + } + return new ImmutableTriple<>( + "hdfs://" + hdfsUri.getAuthority(), hdfsUri.getPath(), partitionList); + } + + public static MessageType createParquetMessageType(HiveTableInfo hiveTableInfo) { + GroupBuilder builder = Types.buildMessage(); + for (Field field : hiveTableInfo.getLubeColumns()) { + PrimitiveBuilder> primitiveBuilder; + if (KTInteger$.MODULE$.equals(field.kgType())) { + primitiveBuilder = builder.optional(PrimitiveTypeName.INT32); + } else if (KTLong$.MODULE$.equals(field.kgType())) { + primitiveBuilder = builder.optional(PrimitiveTypeName.INT64); + } else if (KTDouble$.MODULE$.equals(field.kgType())) { + primitiveBuilder = builder.optional(PrimitiveTypeName.DOUBLE); + } else if (KTBoolean$.MODULE$.equals(field.kgType())) { + primitiveBuilder = builder.optional(PrimitiveTypeName.BOOLEAN); + } else if (KTString$.MODULE$.equals(field.kgType())) { + primitiveBuilder = builder.optional(PrimitiveTypeName.BINARY); + } else { + throw new HiveException("unknown schema type, " + field.kgType(), null); + } + builder = primitiveBuilder.named(field.name()); + } + return builder.named(hiveTableInfo.getTable()); + } + + public static void data2Group(Object[] data, Group group, HiveTableInfo hiveTableInfo) { + for (int i = 0; i < data.length; ++i) { + Field field = hiveTableInfo.getLubeColumns().get(i); + Object value = data[i]; + if (null == value) { + continue; + } + + if (KTInteger$.MODULE$.equals(field.kgType())) { + group.append(field.name(), (Integer) value); + } else if (KTLong$.MODULE$.equals(field.kgType())) { + group.append(field.name(), (Long) value); + } else if (KTDouble$.MODULE$.equals(field.kgType())) { + group.append(field.name(), (Double) value); + } else if (KTBoolean$.MODULE$.equals(field.kgType())) { + group.append(field.name(), (Boolean) value); + } else if (KTString$.MODULE$.equals(field.kgType())) { + group.append(field.name(), (String) value); + } else { + throw new HiveException("unknown schema type, " + field.kgType(), null); + } + } + } + + public static Connection initConnection(HiveTableInfo hiveTableInfo) { + return initConnection( + hiveTableInfo.getJdbcUrl(), hiveTableInfo.getJdbcUser(), hiveTableInfo.getJdbcPasswd()); + } + + public static Connection initConnection(String jdbcUrl, String user, String passwd) { + try { + return DriverManager.getConnection(jdbcUrl, user, passwd); + } catch (SQLException e) { + throw new HiveException("connect to " + jdbcUrl + ",error", e); + } + } + + public static Statement createStatement(Connection connection) { + Statement stmt; + try { + stmt = connection.createStatement(); + } catch (SQLException e) { + throw new HiveException("create statement error", e); + } + return stmt; + } + + public static ResultSet runSql(Statement stmt, String sql) { + try { + boolean executeResult = stmt.execute(sql); + if (executeResult) { + return stmt.getResultSet(); + } else { + throw new HiveException("sql has not result set, sql=" + sql, null); + } + } catch (SQLException e) { + throw new HiveException("hive run sql," + sql, e); + } + } + + public static int runSqlUpdate(Statement stmt, String sql) { + try { + boolean executeResult = stmt.execute(sql); + if (executeResult) { + throw new HiveException("sql has not update count, sql=" + sql, null); + } else { + return stmt.getUpdateCount(); + } + } catch (SQLException e) { + throw new HiveException("hive run sql," + sql, e); + } + } + + private static final String HIVE_METASTORE_CONFIG_KEY = "hive.metastore.uris"; + + /** get hive metastore url */ + public static String getHiveMetaStoreUrl(Statement stmt) { + String getHiveMetaStoreUrisSql = "set " + HIVE_METASTORE_CONFIG_KEY; + + String metastoreUris; + try { + ResultSet resultSet = runSql(stmt, getHiveMetaStoreUrisSql); + resultSet.next(); + String configStr = resultSet.getString(1); + if (configStr.startsWith(HIVE_METASTORE_CONFIG_KEY)) { + metastoreUris = configStr.substring(HIVE_METASTORE_CONFIG_KEY.length() + 1); + } else { + metastoreUris = configStr; + } + } catch (SQLException e) { + throw new RuntimeException("hive run sql," + getHiveMetaStoreUrisSql, e); + } + return metastoreUris; + } + + public static HiveWriterSession createHiveWriterSession(HiveTableInfo hiveTableInfo) { + return new HiveWriterSession(hiveTableInfo); + } + + private static final String HIVE_DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver"; + + static { + try { + Class.forName(HIVE_DRIVER_NAME); + } catch (ClassNotFoundException e) { + throw new RuntimeException("init hive driver error", e); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveWriter.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveWriter.java new file mode 100644 index 00000000..b24391f1 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveWriter.java @@ -0,0 +1,89 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io.hive; + +import com.antgroup.openspg.reasoner.common.exception.HiveException; +import com.antgroup.openspg.reasoner.io.ITableWriter; +import com.antgroup.openspg.reasoner.io.model.AbstractTableInfo; +import com.antgroup.openspg.reasoner.io.model.HiveTableInfo; +import java.io.IOException; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileWriter.Mode; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.hadoop.util.HadoopOutputFile; +import org.apache.parquet.schema.MessageType; + +public class HiveWriter implements ITableWriter { + private HiveTableInfo hiveTableInfo; + + private ParquetWriter writer; + private SimpleGroupFactory groupFactory; + + private long writeCount = 0; + + /** open writer */ + @Override + public void open(int taskIndex, int parallel, AbstractTableInfo tableInfo) { + this.hiveTableInfo = (HiveTableInfo) tableInfo; + HadoopOutputFile outputFile = HiveUtils.getHadoopOutputFile(hiveTableInfo, taskIndex); + MessageType messageType = HiveUtils.createParquetMessageType(hiveTableInfo); + try { + this.writer = + KgReasonerBaseParquetWriter.builder(outputFile) + .withWriteMode(Mode.OVERWRITE) + .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_2_0) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .withType(messageType) + .build(); + } catch (IOException e) { + throw new HiveException("build parquet writer error", e); + } + this.groupFactory = new SimpleGroupFactory(messageType); + } + + /** write data */ + @Override + public void write(Object[] data) { + Group group = groupFactory.newGroup(); + HiveUtils.data2Group(data, group, hiveTableInfo); + try { + this.writer.write(group); + } catch (IOException e) { + throw new HiveException("write parquet error", e); + } + writeCount++; + } + + @Override + public void close() { + if (null != writer) { + try { + this.writer.close(); + } catch (IOException e) { + throw new HiveException("close parquet writer error", e); + } finally { + this.writer = null; + this.writeCount = 0L; + } + } + } + + @Override + public long writeCount() { + return this.writeCount; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveWriterSession.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveWriterSession.java new file mode 100644 index 00000000..0101de7b --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/HiveWriterSession.java @@ -0,0 +1,207 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io.hive; + +import com.antgroup.openspg.reasoner.common.exception.HiveException; +import com.antgroup.openspg.reasoner.common.types.KTBoolean$; +import com.antgroup.openspg.reasoner.common.types.KTDouble$; +import com.antgroup.openspg.reasoner.common.types.KTInteger$; +import com.antgroup.openspg.reasoner.common.types.KTLong$; +import com.antgroup.openspg.reasoner.common.types.KTString$; +import com.antgroup.openspg.reasoner.io.model.HiveTableInfo; +import com.antgroup.openspg.reasoner.lube.catalog.struct.Field; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HiveWriterSession { + private static final Logger log = LoggerFactory.getLogger(HiveWriterSession.class); + + private final HiveTableInfo hiveTableInfo; + private Connection connection; + + /** hive writer session */ + public HiveWriterSession(HiveTableInfo hiveTableInfo) { + this.hiveTableInfo = hiveTableInfo; + } + + /** get session id */ + public String getSessionId() { + return String.valueOf(this.hiveTableInfo.hashCode()); + } + + /** commit hive table */ + public void commit() { + initConnection( + this.hiveTableInfo.getJdbcUrl(), + this.hiveTableInfo.getJdbcUser(), + this.hiveTableInfo.getJdbcPasswd()); + createSinkTable(this.hiveTableInfo); + log.info("create hive table done, " + this.hiveTableInfo.getTable()); + } + + private void initConnection(String jdbcUrl, String user, String passwd) { + try { + this.connection = DriverManager.getConnection(jdbcUrl, user, passwd); + } catch (SQLException e) { + throw new HiveException("connect to " + jdbcUrl + ",error", e); + } + } + + /** create hive sink table */ + private void createSinkTable(HiveTableInfo hiveTableInfo) { + String createTableSql = getCreateTableSql(hiveTableInfo); + + String loadDataSql = getLoadDataSql(hiveTableInfo, false); + Statement stmt; + try { + stmt = this.connection.createStatement(); + } catch (SQLException e) { + throw new HiveException("create statement error", e); + } + try { + stmt.execute(createTableSql); + } catch (SQLException e) { + throw new HiveException("hive run sql," + createTableSql, e); + } + if (!this.checkHdfsDataExist(hiveTableInfo)) { + return; + } + try { + stmt.execute(loadDataSql); + } catch (SQLException e) { + log.error("hive load hdfs error, sql=" + loadDataSql, e); + throw new HiveException("hive load hdfs error, sql=" + loadDataSql, e); + } + } + + /** check hive table data is existing */ + private boolean checkHdfsDataExist(HiveTableInfo hiveTableInfo) { + Triple> outputPathPair = + HiveUtils.getHdfsOutputPath(hiveTableInfo); + Configuration conf = HiveUtils.getHadoopConfiguration(hiveTableInfo, outputPathPair.getLeft()); + String dataPath = getHiveDataPath(hiveTableInfo); + try { + FileSystem fs = FileSystem.get(conf); + Path delPath = new Path(dataPath); + boolean isExists = fs.exists(delPath); + if (!isExists) { + log.info("HolmesHiveClient,dataNotExist,path=" + dataPath); + } + return isExists; + } catch (Exception e) { + log.error("HolmesHiveClient,fs_exists,path=" + dataPath, e); + return false; + } + } + + private String getHiveDataPath(HiveTableInfo hiveTableInfo) { + Triple> outputPathTriple = + HiveUtils.getHdfsOutputPath(hiveTableInfo); + StringBuilder sb = new StringBuilder(); + sb.append(outputPathTriple.getMiddle()); + if (!outputPathTriple.getRight().isEmpty()) { + sb.append("/").append(String.join("/", outputPathTriple.getRight())); + } + return sb.toString(); + } + + /** generate a sql for load hive table */ + private String getLoadDataSql(HiveTableInfo hiveTableInfo, boolean withHdfsPrefix) { + Triple> outputPathTriple = + HiveUtils.getHdfsOutputPath(hiveTableInfo); + StringBuilder sb = new StringBuilder(); + sb.append("LOAD DATA INPATH '"); + if (withHdfsPrefix && StringUtils.isNotEmpty(outputPathTriple.getLeft())) { + sb.append(outputPathTriple.getLeft()); + } + sb.append(getHiveDataPath(hiveTableInfo)); + sb.append("' OVERWRITE INTO TABLE `").append(hiveTableInfo.getTable()).append("`"); + if (!outputPathTriple.getRight().isEmpty()) { + // partition + sb.append(" PARTITION("); + boolean first = true; + for (String partition : outputPathTriple.getRight()) { + if (!first) { + sb.append(","); + } + first = false; + sb.append(partition); + } + sb.append(")"); + } + return sb.toString(); + } + + /** generate sql for create hive table */ + private String getCreateTableSql(HiveTableInfo hiveTableInfo) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE IF NOT EXISTS `").append(hiveTableInfo.getTable()).append("`(\n"); + boolean first = true; + for (Field field : hiveTableInfo.getLubeColumns()) { + if (!first) { + sb.append("\n ,"); + } else { + sb.append(" "); + } + first = false; + sb.append("`").append(field.name()).append("` "); + + if (KTInteger$.MODULE$.equals(field.kgType())) { + sb.append("INT"); + } else if (KTLong$.MODULE$.equals(field.kgType())) { + sb.append("BIGINT"); + } else if (KTDouble$.MODULE$.equals(field.kgType())) { + sb.append("DOUBLE"); + } else if (KTBoolean$.MODULE$.equals(field.kgType())) { + sb.append("BOOLEAN"); + } else if (KTString$.MODULE$.equals(field.kgType())) { + sb.append("STRING"); + } else { + throw new HiveException("unknown schema type, " + field.kgType(), null); + } + } + sb.append("\n)\n"); + + if (hiveTableInfo.getPartition().size() > 0) { + sb.append("PARTITIONED BY (\n"); + } + first = true; + for (Map.Entry entry : hiveTableInfo.getPartition().entrySet()) { + if (!first) { + sb.append("\n ,"); + } else { + sb.append(" "); + } + first = false; + sb.append("`").append(entry.getKey()).append("` STRING"); + } + if (hiveTableInfo.getPartition().size() > 0) { + sb.append("\n)\n"); + } + sb.append("STORED AS PARQUET\n"); + sb.append("TBLPROPERTIES ( \n").append(" 'transactional'='false' \n").append(")"); + return sb.toString(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/KgReasonerBaseParquetWriter.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/KgReasonerBaseParquetWriter.java new file mode 100644 index 00000000..1c288954 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/KgReasonerBaseParquetWriter.java @@ -0,0 +1,112 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.io.hive; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.api.WriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.OutputFile; +import org.apache.parquet.schema.MessageType; + +public class KgReasonerBaseParquetWriter extends ParquetWriter { + + /** + * Creates a Builder for configuring ParquetWriter with the example object model. + * + * @param file the output file to create + * @return a {@link Builder} to create a {@link ParquetWriter} + */ + public static Builder builder(OutputFile file) { + return new Builder(file); + } + + /** + * Create a new {@link KgReasonerBaseParquetWriter}. + * + * @param file The file name to write to. + * @param writeSupport The schema to write with. + * @param compressionCodecName Compression code to use, or CompressionCodecName.UNCOMPRESSED + * @param blockSize the block size threshold. + * @param pageSize See parquet write up. Blocks are subdivided into pages for alignment and other + * purposes. + * @param enableDictionary Whether to use a dictionary to compress columns. + * @param conf The Configuration to use. + * @throws IOException + */ + KgReasonerBaseParquetWriter( + Path file, + WriteSupport writeSupport, + CompressionCodecName compressionCodecName, + int blockSize, + int pageSize, + boolean enableDictionary, + boolean enableValidation, + ParquetProperties.WriterVersion writerVersion, + Configuration conf) + throws IOException { + super( + file, + writeSupport, + compressionCodecName, + blockSize, + pageSize, + pageSize, + enableDictionary, + enableValidation, + writerVersion, + conf); + } + + /** Builder implement */ + public static class Builder extends ParquetWriter.Builder { + private MessageType type = null; + private Map extraMetaData = new HashMap(); + + /** + * Builder + * + * @param file + */ + private Builder(OutputFile file) { + super(file); + } + + /** + * withType + * + * @param type + * @return + */ + public Builder withType(MessageType type) { + this.type = type; + return this; + } + + @Override + protected Builder self() { + return this; + } + + @Override + protected WriteSupport getWriteSupport(Configuration conf) { + return new KgReasonerGroupWriteSupport(type, extraMetaData); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/KgReasonerGroupWriteSupport.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/KgReasonerGroupWriteSupport.java new file mode 100644 index 00000000..45a1ea78 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/hive/KgReasonerGroupWriteSupport.java @@ -0,0 +1,59 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.io.hive; + +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.GroupWriter; +import org.apache.parquet.hadoop.api.WriteSupport; +import org.apache.parquet.io.api.RecordConsumer; +import org.apache.parquet.schema.MessageType; + +public class KgReasonerGroupWriteSupport extends WriteSupport { + + private final MessageType schema; + private final Map extraMetaData; + + private GroupWriter groupWriter; + + public KgReasonerGroupWriteSupport() { + this(null, new HashMap()); + } + + public KgReasonerGroupWriteSupport(MessageType schema, Map extraMetaData) { + this.schema = schema; + this.extraMetaData = extraMetaData; + } + + @Override + public String getName() { + return "holmes"; + } + + @Override + public WriteContext init(Configuration configuration) { + return new WriteContext(schema, this.extraMetaData); + } + + @Override + public void prepareForWrite(RecordConsumer recordConsumer) { + groupWriter = new GroupWriter(recordConsumer, schema); + } + + @Override + public void write(Group record) { + groupWriter.write(record); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/AbstractTableInfo.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/AbstractTableInfo.java new file mode 100644 index 00000000..61b653ce --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/AbstractTableInfo.java @@ -0,0 +1,90 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io.model; + +import com.alibaba.fastjson.annotation.JSONField; +import com.antgroup.openspg.reasoner.common.table.Field; +import com.google.common.collect.Lists; +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +@Getter +@Setter +public abstract class AbstractTableInfo implements Serializable { + protected String project; + protected String table; + + protected Map partition; + + protected List columns; + + /** get lube field */ + @JSONField(serialize = false) + public List getLubeColumns() { + return this.columns.stream() + .map( + new Function() { + @Override + public com.antgroup.openspg.reasoner.lube.catalog.struct.Field apply(Field field) { + return new com.antgroup.openspg.reasoner.lube.catalog.struct.Field( + field.getName(), field.getType().getKgType(), true); + } + }) + .collect(Collectors.toList()); + } + + /** hash code */ + @Override + public int hashCode() { + return this.getTableInfoKeyString().hashCode(); + } + + /** get key */ + @JSONField(serialize = false) + public String getTableInfoKeyString() { + String str = "table=" + this.project + "." + this.table; + String partitionString = getPartitionString(); + if (StringUtils.isNotEmpty(partitionString)) { + str += ",partition[" + partitionString + "]"; + } + return str; + } + + /** convert map partition info to string */ + @JSONField(serialize = false) + public String getPartitionString() { + if (null == this.partition) { + return null; + } + + List partitionKeys = Lists.newArrayList(this.partition.keySet()); + partitionKeys.sort(String::compareTo); + + StringBuilder sb = new StringBuilder(); + for (String partitionKey : partitionKeys) { + String partitionValue = this.partition.get(partitionKey); + if (sb.length() > 0) { + sb.append(","); + } + sb.append(partitionKey).append("='").append(partitionValue).append("'"); + } + return sb.toString(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/HiveTableInfo.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/HiveTableInfo.java new file mode 100644 index 00000000..eba4293f --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/HiveTableInfo.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io.model; + +import java.util.Arrays; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class HiveTableInfo extends AbstractTableInfo { + private String hadoopUserName = null; + + // jdbc info + private String jdbcUrl; + private String jdbcUser; + private String jdbcPasswd; + + // output path info + private String outputParquetPath; // can be null + private String coreSiteXml; + private String hdfsSiteXml; + + @Override + public int hashCode() { + Object[] item = new Object[] {table, project, jdbcUrl, getPartitionString()}; + return Arrays.hashCode(item); + } +} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/cmd/SchedulerJobInstQuery.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/ReadRange.java similarity index 54% rename from server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/cmd/SchedulerJobInstQuery.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/ReadRange.java index 120ee5fb..6f188c72 100644 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/cmd/SchedulerJobInstQuery.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/io/model/ReadRange.java @@ -11,21 +11,31 @@ * or implied. */ -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd; +package com.antgroup.openspg.reasoner.io.model; -import com.antgroup.openspg.server.common.model.base.BaseQuery; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import java.util.Set; import lombok.Getter; import lombok.Setter; -import lombok.experimental.Accessors; -@Setter @Getter -@Accessors(chain = true) -public class SchedulerJobInstQuery extends BaseQuery { +@Setter +public class ReadRange implements Comparable { + private long start; + private long end; - private Set status; + /** create read range */ + public ReadRange(long start, long end) { + this.start = start; + this.end = end; + } - private String orderBy; + /** get read count */ + public long getCount() { + return end - start; + } + + /** compare */ + @Override + public int compareTo(ReadRange o) { + return (int) (this.start - o.start); + } } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/AggregationSchemaInfo.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/AggregationSchemaInfo.java new file mode 100644 index 00000000..e471960d --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/AggregationSchemaInfo.java @@ -0,0 +1,64 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import com.antgroup.openspg.reasoner.lube.common.pattern.Connection; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import scala.Tuple2; + +public class AggregationSchemaInfo implements Serializable { + + private final Map>> edgeEndpointMap = new HashMap<>(); + private final Map>> vertexHasEdgeMap = new HashMap<>(); + + /** schema info */ + public AggregationSchemaInfo(Pattern schema) { + for (Connection pc : RunnerUtil.getConnectionSet(schema)) { + edgeEndpointMap.put( + pc.alias(), + Lists.newArrayList(new Tuple2<>(pc.source(), true), new Tuple2<>(pc.target(), false))); + List> sourceEdgeInfoList = + vertexHasEdgeMap.computeIfAbsent(pc.source(), k -> new ArrayList<>()); + sourceEdgeInfoList.add(new Tuple2<>(pc.alias(), true)); + List> targetEdgeInfoList = + vertexHasEdgeMap.computeIfAbsent(pc.target(), k -> new ArrayList<>()); + targetEdgeInfoList.add(new Tuple2<>(pc.alias(), false)); + } + } + + /** + * Getter method for property edgeEndpointMap. + * + * @return property value of edgeEndpointMap + */ + public Map>> getEdgeEndpointMap() { + return edgeEndpointMap; + } + + /** + * Getter method for property vertexHasEdgeMap. + * + * @return property value of vertexHasEdgeMap + */ + public Map>> getVertexHasEdgeMap() { + return vertexHasEdgeMap; + } +} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerResult.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/IVertexId2WorkerIndex.java similarity index 71% rename from server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerResult.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/IVertexId2WorkerIndex.java index 61ca5ea4..baff447b 100644 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerResult.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/IVertexId2WorkerIndex.java @@ -11,8 +11,10 @@ * or implied. */ -package com.antgroup.openspg.server.core.reasoner.model.service; +package com.antgroup.openspg.reasoner.kggraph; -import com.antgroup.openspg.server.common.model.job.BaseJobRunResult; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; -public class BaseReasonerResult extends BaseJobRunResult {} +public interface IVertexId2WorkerIndex { + int workerIndex(IVertexId vertexId); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/KgGraph.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/KgGraph.java new file mode 100644 index 00000000..74339ff8 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/KgGraph.java @@ -0,0 +1,210 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.block.SortItem; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; + +public interface KgGraph { + + /** + * Initialize the tree structure of KgGraph based on the root vertex and adjacent edges + * + * @param root the root of vertex subgraph + * @param alias2EdgeMap the adjacent edges of root in vertex subgraph + * @param schema describe the structure of vertex subgraph Guarantee that the + * source/target/direction in the schema are consistent with the source/target/direction of + * the actual edge + */ + void init( + IVertex root, + Map>> alias2EdgeMap, + Pattern schema); + + /** + * Split out the KgGraphs centered on the actual vertex of the alias from the current KgGraph + * + * @param vertexAliases + * @param schema, the structure of the converted subgraph + * @param filter, Filter function on the split KgGraph, return true will be included + * @param limit, max result limit, null means no constraints are applied + * @return + */ + List> split( + Set vertexAliases, + Pattern schema, + KgGraphSplitStaticParameters staticParameters, + Predicate> filter, + Long limit); + + /** + * Split out the KgGraphs centered on worker index + * + * @param vertexAlias target vertex alias + * @param vertexId2WorkerIndex function, convert vertex id to worker index + * @param schema + * @param filter + * @param limit max result limit, null means no constraints are applied + * @return + */ + Map> splitToWorkerIndex( + String vertexAlias, + IVertexId2WorkerIndex vertexId2WorkerIndex, + Pattern schema, + KgGraphSplitStaticParameters staticParameters, + Predicate> filter, + Long limit); + + /** + * Merge the received messages into the current KgGraph + * + * @param msgs + * @param schema the structure of the merged subgraph + */ + void merge(Collection> msgs, Pattern schema); + + /** + * Add the new KgGraph to the current KgGraph, and the schema of the KgGraph changes + * + * @param subGraph + * @param schema the structure of the expanded subgraph + */ + void expand(KgGraph subGraph, Pattern schema); + + /** + * expend KgGraph and prune by intersection vertex + * + * @return valid or not + */ + int expandAndPrune( + Pattern thisSchema, + KgGraph matchedKgGraph, + Pattern matchedSchema, + Pattern finalSchema, + Set intersectionVertexAliasSet); + + /** + * Get the path from KgGraph, flat all alias vertexes Iterator may return null, which needs to be + * checked when call it. + * + * @param staticParameters + * @param filter, Filter function on path + * @return + */ + Iterator> getPath( + KgGraphSplitStaticParameters staticParameters, Predicate> filter); + + /** aggregate by edge */ + void aggregateEdge( + String edgeAlias, + Set> edgeSet, + AggregationSchemaInfo aggregationSchemaInfo, + Set aggregatedAliasSet); + + /** aggregate by vertex */ + void aggregateVertex( + String vertexAlias, + Set> vertexSet, + AggregationSchemaInfo aggregationSchemaInfo, + Set aggregatedAliasSet); + + /** + * Sort the current KgGraph by sortItems + * + * @param sortItems + * @param limit + * @param schema + */ + void executeSort(List sortItems, int limit, Pattern schema); + + /** + * Adjust the current KgGraph according to scheme including structure and property + * + * @param schema + */ + void adjustGraph(Pattern schema); + + /** + * Get vertex from KgGraph based on vertex alias + * + * @param alias + * @return + */ + List> getVertex(String alias); + + /** + * Get edge from KgGraph based on edge alias + * + * @param alias + * @return + */ + List> getEdge(String alias); + + /** + * get vertex alias set + * + * @return + */ + Set getVertexAlias(); + + /** + * get edge alias set + * + * @return + */ + Set getEdgeAlias(); + + /** Show kgGraph structure */ + void show(); + + /** + * set vertex property + * + * @param alias - vertex alias + * @param propertyMap - key is new property name, value is new property value + * @param version - vertex version, if null add properties to the 0 version + */ + void setVertexProperty(String alias, Map propertyMap, Long version); + + /** + * set edge property + * + * @param alias - edge alias + * @param propertyMap - key is new property name, value is new property value + */ + void setEdgeProperty(String alias, Map propertyMap); + + /** clone vertex set */ + void cloneVertexSet(String alias); + + /** clone edge set */ + void cloneEdgeSet(String alias); + + /** check KgGraph have focus vertex id */ + boolean hasFocusVertexId(String alias, Set focusVertexIdSet); + + /** check duplicate vertex */ + boolean checkDuplicateVertex(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/EdgeCombinationIterator.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/EdgeCombinationIterator.java new file mode 100644 index 00000000..dd1bde7a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/EdgeCombinationIterator.java @@ -0,0 +1,457 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph.impl; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.OptionalEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.lube.common.pattern.Connection; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.Stack; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +public class EdgeCombinationIterator implements Iterator> { + private static final Logger log = LoggerFactory.getLogger(EdgeCombinationIterator.class); + + private final Map>> alias2VertexMap; + private final Map>> alias2EdgeMap; + private final Map>>> aliasEdgeQueryMap; + + private final List edgeIterateInfoList; + private final Map edgeIterateOrderMap; + + private final Stack>> iteratorStack = new Stack<>(); + + private final Stack> resultEdgeStack = new Stack<>(); + + private Long scope = null; + + public EdgeCombinationIterator( + List edgeIterateInfoList, + Map edgeIterateOrderMap, + Map>> alias2VertexMap, + Map>> alias2EdgeMap) { + this.edgeIterateInfoList = edgeIterateInfoList; + this.edgeIterateOrderMap = edgeIterateOrderMap; + this.alias2VertexMap = initVertexQueryMap(alias2VertexMap); + this.alias2EdgeMap = alias2EdgeMap; + this.aliasEdgeQueryMap = initEdgeQueryMap(this.alias2EdgeMap, getEdgeIndexInfo()); + + String firstEdgeAlias = this.edgeIterateInfoList.get(0).getEdgeAlias(); + Iterator> edgeIt = alias2EdgeMap.get(firstEdgeAlias).iterator(); + iteratorStack.push(edgeIt); + } + + private void pushNextIterator() { + EdgeIterateInfo edgeIterateInfo = edgeIterateInfoList.get(iteratorStack.size()); + BuildEdgeIteratorInfo buildEdgeIteratorInfo = edgeIterateInfo.getBuildEdgeIteratorInfo(); + IVertexId queryKey = getQueryNextEdgeKey(buildEdgeIteratorInfo.getQueryKey()); + List> edgeList = + aliasEdgeQueryMap.get(buildEdgeIteratorInfo.getQueryEdgeAlias()).get(queryKey); + if (CollectionUtils.isEmpty(edgeList)) { + resultEdgeStack.pop(); + return; + } + Iterator> edgeIt = edgeList.iterator(); + iteratorStack.push(edgeIt); + } + + private IVertexId getQueryNextEdgeKey(Edge2VertexInfo queryKey) { + int index = this.edgeIterateOrderMap.get(queryKey.getEdgeAlias()); + IEdge edge = this.resultEdgeStack.get(index); + if (queryKey.sourceOrTarget) { + return edge.getSourceId(); + } + return edge.getTargetId(); + } + + public void setScope(Long limit) { + if (null != limit) { + this.scope = limit * 2; + } + } + + private boolean checkIntersectionAndDuplicateVertex( + EdgeIterateInfo edgeIterateInfo, IEdge edge) { + List intersectInfoList = edgeIterateInfo.getIntersectInfoList(); + if (!intersectInfoList.isEmpty()) { + // check intersection + for (IntersectVertexInfo intersectVertexInfo : intersectInfoList) { + IEdge checkEdge = + resultEdgeStack.get(intersectVertexInfo.getCheckVertexInfo().getEdgeIndex()); + IVertexId checkId; + if (intersectVertexInfo.getCheckVertexInfo().isSourceOrTarget()) { + checkId = checkEdge.getSourceId(); + } else { + checkId = checkEdge.getTargetId(); + } + IVertexId nowId; + if (intersectVertexInfo.isIndexSourceOrTarget()) { + nowId = edge.getSourceId(); + } else { + nowId = edge.getTargetId(); + } + if (!nowId.equals(checkId)) { + return true; + } + } + } + + boolean edgeOptional = edge instanceof OptionalEdge; + Map> duplicateVertexCheckMap = + edgeIterateInfo.getDuplicateVertexCheck(); + if (MapUtils.isNotEmpty(duplicateVertexCheckMap)) { + for (Map.Entry> entry : + duplicateVertexCheckMap.entrySet()) { + int edgeIndex = entry.getKey(); + boolean source1 = entry.getValue()._1(); + boolean source2 = entry.getValue()._2(); + IEdge checkEdge = resultEdgeStack.get(edgeIndex); + boolean checkEdgeOptional = checkEdge instanceof OptionalEdge; + IVertexId id1 = source1 ? edge.getSourceId() : edge.getTargetId(); + IVertexId id2 = source2 ? checkEdge.getSourceId() : checkEdge.getTargetId(); + if (checkEdgeOptional && edgeOptional) { + if (id1.equals(id2)) { + IVertexId id3 = source1 ? edge.getTargetId() : edge.getSourceId(); + IVertexId id4 = source2 ? checkEdge.getTargetId() : checkEdge.getSourceId(); + if (id3.equals(id4)) { + continue; + } else { + // invalid + return true; + } + } + } else if ((checkEdgeOptional || edgeOptional)) { + continue; + } + if (id1.equals(id2)) { + return true; + } + } + } + return false; + } + + @Override + public boolean hasNext() { + if (null != this.scope) { + if (nextCount > this.scope) { + return false; + } + } + while (!iteratorStack.isEmpty()) { + Iterator> peekIt = iteratorStack.peek(); + if (peekIt.hasNext()) { + IEdge edge = peekIt.next(); + EdgeIterateInfo edgeIterateInfo = this.edgeIterateInfoList.get(resultEdgeStack.size()); + boolean invalid = checkIntersectionAndDuplicateVertex(edgeIterateInfo, edge); + if (invalid) { + continue; + } + resultEdgeStack.push(edge); + if (resultEdgeStack.size() == this.edgeIterateInfoList.size()) { + return true; + } + pushNextIterator(); + continue; + } + if (1 == iteratorStack.size()) { + return false; + } + iteratorStack.pop(); + resultEdgeStack.pop(); + } + return false; + } + + private long nextCount = 0; + private long logCount = 10 * 10000; + + /** setter */ + public void setLogCount(long logCount) { + this.logCount = logCount; + } + + @Override + public KgGraph next() { + if (resultEdgeStack.size() < this.edgeIterateInfoList.size()) { + if (!hasNext()) { + throw new NoSuchElementException(); + } + } + try { + Map>> alias2VertexMap = new HashMap<>(); + Map>> alias2EdgeMap = new HashMap<>(); + for (int i = 0; i < this.edgeIterateInfoList.size(); ++i) { + EdgeIterateInfo edgeIterateInfo = this.edgeIterateInfoList.get(i); + String edgeAlias = edgeIterateInfo.getEdgeAlias(); + IEdge edge = resultEdgeStack.get(i); + alias2EdgeMap.put(edgeAlias, Sets.newHashSet(edge)); + + String sourceAlias = edgeIterateInfo.getSourceAlias(); + if (!alias2VertexMap.containsKey(sourceAlias)) { + IVertex v = + this.alias2VertexMap.get(sourceAlias).get(edge.getSourceId()); + alias2VertexMap.put(sourceAlias, Sets.newHashSet(v)); + } + + String targetAlias = edgeIterateInfo.getTargetAlias(); + if (!alias2VertexMap.containsKey(targetAlias)) { + IVertex v = + this.alias2VertexMap.get(targetAlias).get(edge.getTargetId()); + alias2VertexMap.put(targetAlias, Sets.newHashSet(v)); + } + } + return new KgGraphImpl(alias2VertexMap, alias2EdgeMap); + } finally { + resultEdgeStack.pop(); + if (nextCount++ > logCount) { + logCount = logCount * 10; + StringBuilder sb = new StringBuilder("EdgeCombinationIterator,count=").append(nextCount); + for (String key : this.alias2EdgeMap.keySet()) { + sb.append(",").append(key).append("=").append(this.alias2EdgeMap.get(key).size()); + } + sb.append("\nEdgeIterateInfoList"); + for (int i = 0; i < this.edgeIterateInfoList.size(); ++i) { + EdgeIterateInfo edgeIterateInfo1 = this.edgeIterateInfoList.get(i); + sb.append(",") + .append(i) + .append(",e=") + .append(edgeIterateInfo1.getEdgeAlias()) + .append(",s=") + .append(edgeIterateInfo1.getSourceAlias()) + .append(",o=") + .append(edgeIterateInfo1.getTargetAlias()); + } + sb.append("\nVertexId"); + for (String key : this.alias2VertexMap.keySet()) { + Map> vertexMap = this.alias2VertexMap.get(key); + if (1 == vertexMap.size()) { + sb.append(",").append(key).append("=").append(vertexMap.keySet().iterator().next()); + } + } + log.info(sb.toString()); + } + } + } + + private Map>>> initEdgeQueryMap( + Map>> alias2EdgeMap, + Map edgeIndexInfo) { + Map>>> result = new HashMap<>(); + for (Map.Entry>> entry : alias2EdgeMap.entrySet()) { + String edgeAlias = entry.getKey(); + if (!edgeIndexInfo.containsKey(edgeAlias)) { + continue; + } + boolean indexSource = edgeIndexInfo.get(edgeAlias); + Map>> edgeMap = + result.computeIfAbsent(edgeAlias, k -> new HashMap<>()); + for (IEdge edge : entry.getValue()) { + IVertexId indexKey; + if (indexSource) { + indexKey = edge.getSourceId(); + } else { + indexKey = edge.getTargetId(); + } + List> edgeList = + edgeMap.computeIfAbsent(indexKey, k -> new ArrayList<>()); + edgeList.add(edge); + } + } + return result; + } + + private Map>> initVertexQueryMap( + Map>> alias2VertexMap) { + Map>> result = new HashMap<>(); + for (Map.Entry>> entry : alias2VertexMap.entrySet()) { + String vertexAlias = entry.getKey(); + Map> vertexMap = + result.computeIfAbsent(vertexAlias, k -> new HashMap<>()); + for (IVertex vertex : entry.getValue()) { + vertexMap.put(vertex.getId(), vertex); + } + } + return result; + } + + private Map getEdgeIndexInfo() { + Map edgeIndexInfo = new HashMap<>(); + for (EdgeIterateInfo edgeIterateInfo : this.edgeIterateInfoList) { + BuildEdgeIteratorInfo buildEdgeIteratorInfo = edgeIterateInfo.getBuildEdgeIteratorInfo(); + if (null == buildEdgeIteratorInfo) { + continue; + } + edgeIndexInfo.put( + buildEdgeIteratorInfo.getQueryEdgeAlias(), buildEdgeIteratorInfo.isIndexSourceOrTarget()); + } + return edgeIndexInfo; + } + + @Data + public static class EdgeIterateInfo implements Serializable { + private static final long serialVersionUID = -7914817333615035385L; + private final String edgeAlias; + private final String sourceAlias; + private final String targetAlias; + + // howto init next edge iterator + private BuildEdgeIteratorInfo buildEdgeIteratorInfo = null; + + // intersection with previous vertex check + private List intersectInfoList = new ArrayList<>(); + + // check duplicate vertex, Map> + private Map> duplicateVertexCheck = new HashMap<>(); + + public EdgeIterateInfo(Connection pc) { + this.edgeAlias = pc.alias(); + this.sourceAlias = pc.source(); + this.targetAlias = pc.target(); + } + } + + public static class IntersectVertexInfo implements Serializable { + // now edge source or target + private static final long serialVersionUID = -9177116827124228267L; + private final boolean indexSourceOrTarget; + + // check vertex info + private final Edge2VertexInfo checkVertexInfo; + + public IntersectVertexInfo(boolean indexSourceOrTarget, Edge2VertexInfo checkVertexInfo) { + this.indexSourceOrTarget = indexSourceOrTarget; + this.checkVertexInfo = checkVertexInfo; + } + + /** + * Getter method for property indexSourceOrTarget. + * + * @return property value of indexSourceOrTarget + */ + public boolean isIndexSourceOrTarget() { + return indexSourceOrTarget; + } + + /** + * Getter method for property checkVertexInfo. + * + * @return property value of checkVertexInfo + */ + public Edge2VertexInfo getCheckVertexInfo() { + return checkVertexInfo; + } + } + + public static class BuildEdgeIteratorInfo implements Serializable { + // query edge alias + private static final long serialVersionUID = -301765123145077701L; + private final String queryEdgeAlias; + private final boolean indexSourceOrTarget; + + // how to get query vertex id from previous kg graph + private final Edge2VertexInfo queryKey; + + public BuildEdgeIteratorInfo( + String queryEdgeAlias, boolean indexSourceOrTarget, Edge2VertexInfo edge2VertexInfo) { + this.queryEdgeAlias = queryEdgeAlias; + this.indexSourceOrTarget = indexSourceOrTarget; + this.queryKey = edge2VertexInfo; + } + + /** + * Getter method for property queryEdgeAlias. + * + * @return property value of queryEdgeAlias + */ + public String getQueryEdgeAlias() { + return queryEdgeAlias; + } + + /** + * Getter method for property queryKey. + * + * @return property value of queryKey + */ + public Edge2VertexInfo getQueryKey() { + return queryKey; + } + + /** + * Getter method for property indexSourceOrTarget. + * + * @return property value of indexSourceOrTarget + */ + public boolean isIndexSourceOrTarget() { + return indexSourceOrTarget; + } + } + + public static class Edge2VertexInfo implements Serializable { + private static final long serialVersionUID = -8606574641924896889L; + private final int edgeIndex; + private final String edgeAlias; + private final boolean sourceOrTarget; + + public Edge2VertexInfo(int edgeIndex, String edgeAlias, boolean sourceOrTarget) { + this.edgeIndex = edgeIndex; + this.edgeAlias = edgeAlias; + this.sourceOrTarget = sourceOrTarget; + } + + /** + * Getter method for property edgeAlias. + * + * @return property value of edgeAlias + */ + public String getEdgeAlias() { + return edgeAlias; + } + + /** + * Getter method for property sourceOrTarget. + * + * @return property value of sourceOrTarget + */ + public boolean isSourceOrTarget() { + return sourceOrTarget; + } + + /** + * Getter method for property edgeIndex. + * + * @return property value of edgeIndex + */ + public int getEdgeIndex() { + return edgeIndex; + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphImpl.java new file mode 100644 index 00000000..82c863af --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphImpl.java @@ -0,0 +1,1239 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph.impl; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.common.exception.InvalidGraphException; +import com.antgroup.openspg.reasoner.common.exception.KGValueException; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.MirrorVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.common.primitives.Bytes; +import com.antgroup.openspg.reasoner.common.utils.PropertyUtil; +import com.antgroup.openspg.reasoner.kggraph.AggregationSchemaInfo; +import com.antgroup.openspg.reasoner.kggraph.IVertexId2WorkerIndex; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.lube.block.SortItem; +import com.antgroup.openspg.reasoner.lube.common.pattern.Connection; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.util.KgGraphSchema; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public class KgGraphImpl implements KgGraph, Serializable { + private static final Logger log = LoggerFactory.getLogger(KgGraphImpl.class); + /** Vertex aliases to vertex mappings */ + private static final long serialVersionUID = 4129310228727972996L; + + private Map>> alias2VertexMap; + + /** Edge aliases to edge mappings */ + private Map>> alias2EdgeMap; + + /** default */ + public KgGraphImpl() { + this.alias2VertexMap = new HashMap<>(); + this.alias2EdgeMap = new HashMap<>(); + } + + /** from map */ + public KgGraphImpl( + Map>> alias2VertexMap, + Map>> alias2EdgeMap) { + this.alias2VertexMap = alias2VertexMap; + this.alias2EdgeMap = alias2EdgeMap; + } + + /** copy */ + public KgGraphImpl(KgGraphImpl kgGraph) { + this.alias2VertexMap = new HashMap<>(kgGraph.getAlias2VertexMap()); + this.alias2EdgeMap = new HashMap<>(kgGraph.getAlias2EdgeMap()); + } + + /** + * Initialize the tree structure of KgGraph based on the root vertex and adjacent edges + * + * @param root the root of vertex subgraph + * @param adjEdges the adjacent edges of root in vertex subgraph + * @param schema describe the structure of vertex subgraph + */ + @Override + public void init( + IVertex root, + Map>> adjEdges, + Pattern schema) { + if (null == root) { + throw new KGValueException("Root and adjEdges cant be null", null); + } + + // Data that does not conform to the schema + Set edgeSchemaSet = RunnerUtil.getConnectionSet(schema); + for (Connection connection : edgeSchemaSet) { + List> edgeList = adjEdges.get(connection.alias()); + if (CollectionUtils.isEmpty(edgeList)) { + log.warn("[KgGraphImpl.init] Data that does not conform to the schema"); + return; + } + } + + String vertexAlias = schema.root().alias(); + Set> vertexSet = + alias2VertexMap.computeIfAbsent(vertexAlias, k -> new HashSet<>()); + vertexSet.add(root); + + if (null == adjEdges || adjEdges.isEmpty()) { + // log.info("[KgGraphImpl.init] adjEdges is empty, just scan root."); + return; + } + for (String edgeAlias : adjEdges.keySet()) { + Set> edgeSet = + alias2EdgeMap.computeIfAbsent(edgeAlias, k -> new HashSet<>()); + edgeSet.addAll(adjEdges.get(edgeAlias)); + + // add virtual vertex + Set connectionSet = + JavaConversions.setAsJavaSet(schema.topology().get(schema.root().alias()).get()); + String targetVertexAlias = + connectionSet.stream() + .filter(connection -> connection.alias().equals(edgeAlias)) + .map(connection -> connection.target()) + .collect(Collectors.toList()) + .get(0); + Set> targetVertexSet = + alias2VertexMap.computeIfAbsent(targetVertexAlias, k -> new HashSet<>()); + adjEdges.get(edgeAlias).stream() + .map(e -> e.getTargetId()) + .map(targetId -> new Vertex(targetId)) + .forEach(v -> targetVertexSet.add(v)); + } + } + + private Bytes getKgGraphKeyBySplitVertex(List vertexAliases, KgGraph kgGraph) { + ByteBuffer byteBuffer = ByteBuffer.allocate(vertexAliases.size() * 16); + for (String vertexAlias : vertexAliases) { + IVertexId vid = kgGraph.getVertex(vertexAlias).get(0).getId(); + byteBuffer.put(vid.getBytes()); + } + return new Bytes(byteBuffer.array()); + } + + /** + * Split out the KgGraphs centered on the actual vertex of the alias from the current KgGraph + * + * @param inputVertexAliases + * @param schema + * @param filter + * @param limit + * @return + */ + @Override + public List> split( + Set inputVertexAliases, + Pattern schema, + KgGraphSplitStaticParameters staticParameters, + Predicate> filter, + Long limit) { + Set vertexAliases = Sets.newHashSet(inputVertexAliases); + + // remove alias that vertex set size is 1 + vertexAliases.removeIf(vertexAlias -> 1 == this.alias2VertexMap.get(vertexAlias).size()); + if (vertexAliases.isEmpty()) { + // not need split + if (null != filter && !filter.test(this)) { + return new ArrayList<>(); + } + return Lists.newArrayList(this); + } + + // can do simple copy split + if (staticParameters.canDoSampleSplit(this.alias2VertexMap)) { + List> resultList = + doSimpleSplit(vertexAliases, schema, staticParameters, limit); + if (null == filter) { + return resultList; + } + List> tmpResultList = resultList; + resultList = new ArrayList<>(); + for (KgGraph kgGraph : tmpResultList) { + if (!filter.test(kgGraph)) { + continue; + } + resultList.add(kgGraph); + } + return resultList; + } + + Map> resultMap = new HashMap<>(); + List vertexAliasList = Lists.newArrayList(vertexAliases); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + this.alias2VertexMap, + this.alias2EdgeMap); + it.setScope(limit); + while (it.hasNext()) { + KgGraph path = it.next(); + if (null == path) { + continue; + } + if (null != filter && !filter.test(path)) { + continue; + } + Bytes key = getKgGraphKeyBySplitVertex(vertexAliasList, path); + KgGraph kgGraph = resultMap.computeIfAbsent(key, k -> new KgGraphImpl()); + kgGraph.merge(Lists.newArrayList(path), schema); + if (null != limit && resultMap.size() >= limit) { + // reach max path limit + break; + } + } + return Lists.newArrayList(resultMap.values()); + } + + /** + * @param vertexAlias target vertex alias + * @param vertexId2WorkerIndex function, convert vertex id to worker index + * @param schema + * @param filter + * @param limit max result limit, null means no constraints are applied + * @return + */ + @Override + public Map> splitToWorkerIndex( + String vertexAlias, + IVertexId2WorkerIndex vertexId2WorkerIndex, + Pattern schema, + KgGraphSplitStaticParameters staticParameters, + Predicate> filter, + Long limit) { + Map> result = new HashMap<>(); + Set vertexAliases = Sets.newHashSet(vertexAlias); + // remove alias that vertex set size is 1 + vertexAliases.removeIf(alias -> 1 == this.alias2VertexMap.get(alias).size()); + if (vertexAliases.isEmpty()) { + // not need split + int workerIndex = + vertexId2WorkerIndex.workerIndex(this.getVertex(vertexAlias).get(0).getId()); + if (workerIndex < 0) { + return result; + } + result.put(workerIndex, this); + return result; + } + + // can do simple copy split + if (null == filter && staticParameters.canDoSampleSplit(this.alias2VertexMap)) { + return doSimpleSplitToWorkerIndex(vertexAlias, vertexId2WorkerIndex, staticParameters); + } + + List> splitList = + this.split(vertexAliases, schema, staticParameters, filter, limit); + for (KgGraph kgGraph : splitList) { + int workerIndex = + vertexId2WorkerIndex.workerIndex(kgGraph.getVertex(vertexAlias).get(0).getId()); + if (workerIndex < 0) { + continue; + } + KgGraph kgGraphOnWorker = + result.computeIfAbsent(workerIndex, k -> new KgGraphImpl()); + kgGraphOnWorker.merge(Lists.newArrayList(kgGraph), schema); + } + return result; + } + + private Map getVertexId2IndexMap( + IVertexId2WorkerIndex vertexId2WorkerIndex, + Set> needSplitVertexSet) { + Map result = new HashMap<>(); + for (IVertex vertex : needSplitVertexSet) { + int index = vertexId2WorkerIndex.workerIndex(vertex.getId()); + result.put(vertex.getId(), index); + } + return result; + } + + private Map> doSimpleSplitToWorkerIndex( + String vertexAlias, + IVertexId2WorkerIndex vertexId2WorkerIndex, + KgGraphSplitStaticParameters staticParameters) { + // base graph, for copy + Map>> baseAlias2VertexMap = + new HashMap<>(this.alias2VertexMap); + Map>> baseAlias2EdgeMap = + new HashMap<>(this.alias2EdgeMap); + + Set> needSplitVertexSet = baseAlias2VertexMap.remove(vertexAlias); + Map, Set>> needSplitEdgeMap = + new HashMap<>(); + Set needSplitConnectionSet = staticParameters.getNeedSplitEdgeSet(); + for (Connection pc : needSplitConnectionSet) { + Tuple2 edgeTypeAndIndexSource = + new Tuple2<>(pc.alias(), pc.source().equals(vertexAlias)); + needSplitEdgeMap.put(edgeTypeAndIndexSource, baseAlias2EdgeMap.remove(pc.alias())); + } + + Map id2IndexMap = + getVertexId2IndexMap(vertexId2WorkerIndex, needSplitVertexSet); + Set workerIndexSet = new HashSet<>(); + + Map>> splitVertexMap = new HashMap<>(); + for (IVertex vertex : needSplitVertexSet) { + Integer index = id2IndexMap.get(vertex.getId()); + if (null == index || index < 0) { + continue; + } + workerIndexSet.add(index); + List> vertexList = + splitVertexMap.computeIfAbsent(index, k -> new ArrayList<>()); + vertexList.add(vertex); + } + Map, List>> splitEdgeMap = new HashMap<>(); + for (Map.Entry, Set>> entry : + needSplitEdgeMap.entrySet()) { + String edgeAlias = entry.getKey()._1(); + boolean indexSource = entry.getKey()._2(); + for (IEdge edge : entry.getValue()) { + IVertexId vertexId; + if (indexSource) { + vertexId = edge.getSourceId(); + } else { + vertexId = edge.getTargetId(); + } + Integer index = id2IndexMap.get(vertexId); + if (null == index || index < 0) { + continue; + } + workerIndexSet.add(index); + List> edgeList = + splitEdgeMap.computeIfAbsent(new Tuple2<>(edgeAlias, index), k -> new ArrayList<>()); + edgeList.add(edge); + } + } + + Map> result = new HashMap<>(); + + for (int index : workerIndexSet) { + List> vertexList = splitVertexMap.get(index); + if (CollectionUtils.isEmpty(vertexList)) { + continue; + } + boolean hasEmptyEdgeList = false; + Map>> copyAlias2EdgeMap = new HashMap<>(); + for (Connection pc : needSplitConnectionSet) { + List> edgeList = + splitEdgeMap.get(new Tuple2<>(pc.alias(), index)); + if (CollectionUtils.isEmpty(edgeList)) { + hasEmptyEdgeList = true; + break; + } + copyAlias2EdgeMap.put(pc.alias(), Sets.newHashSet(edgeList)); + } + if (hasEmptyEdgeList) { + continue; + } + + KgGraphImpl kgGraph = new KgGraphImpl(); + kgGraph.alias2VertexMap.putAll(baseAlias2VertexMap); + kgGraph.alias2EdgeMap.putAll(baseAlias2EdgeMap); + kgGraph.alias2VertexMap.put(vertexAlias, Sets.newHashSet(vertexList)); + kgGraph.alias2EdgeMap.putAll(copyAlias2EdgeMap); + result.put(index, kgGraph); + } + return result; + } + + public List> doSimpleSplit( + Set vertexAliases, + Pattern schema, + KgGraphSplitStaticParameters staticParameters, + Long limit) { + // base graph, for copy + Map>> baseAlias2VertexMap = + new HashMap<>(this.alias2VertexMap); + Map>> baseAlias2EdgeMap = + new HashMap<>(this.alias2EdgeMap); + for (String alias : vertexAliases) { + baseAlias2VertexMap.remove(alias); + } + Set needSplitEdgeSet = staticParameters.getNeedSplitEdgeSet(); + for (Connection pc : needSplitEdgeSet) { + baseAlias2EdgeMap.remove(pc.alias()); + } + + List> edgeCombinationList = staticParameters.getSplitConnectedSubgraph(); + if (edgeCombinationList.size() > 1) { + throw new InvalidGraphException("not support unconnected graph", null); + } + Map>> alias2VertexMap = new HashMap<>(); + for (String vertexAlias : staticParameters.getConnectedSubgraphVertexAliasSet()) { + alias2VertexMap.put(vertexAlias, this.alias2VertexMap.get(vertexAlias)); + } + Map>> alias2EdgeMap = new HashMap<>(); + for (String edgeAlias : staticParameters.getConnectedSubgraphEdgeAliasSet()) { + alias2EdgeMap.put(edgeAlias, this.alias2EdgeMap.get(edgeAlias)); + } + // use sub edge iterate info list + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getSubEdgeIterateInfoList(), + staticParameters.getSubEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + it.setScope(limit); + List sortedVertexAliasList = Lists.newArrayList(vertexAliases); + Map> result = new HashMap<>(); + long count = 0; + while (it.hasNext()) { + KgGraph kgg = it.next(); + if (null == kgg) { + continue; + } + count++; + + KgGraphImpl kgGraph = (KgGraphImpl) kgg; + kgGraph.getAlias2VertexMap().putAll(baseAlias2VertexMap); + kgGraph.getAlias2EdgeMap().putAll(baseAlias2EdgeMap); + + if (kgGraph.checkDuplicateVertex()) { + continue; + } + + KgGraphKey kgGraphKey = getKgGraphKey(kgGraph, sortedVertexAliasList); + KgGraph oldKgGraph = result.get(kgGraphKey); + if (null == oldKgGraph) { + result.put(kgGraphKey, kgGraph); + } else { + oldKgGraph.merge(Lists.newArrayList(kgGraph), schema); + } + if (null != limit && count >= limit) { + break; + } + } + return Lists.newArrayList(result.values()); + } + + private KgGraphKey getKgGraphKey(KgGraph kgGraph, List vertexAliases) { + IVertexId[] vertexIds = new IVertexId[vertexAliases.size()]; + for (int i = 0; i < vertexAliases.size(); ++i) { + String alias = vertexAliases.get(i); + IVertexId vertexId = kgGraph.getVertex(alias).get(0).getId(); + vertexIds[i] = vertexId; + } + return new KgGraphKey(vertexIds); + } + + /** + * Merge the received messages into the current KgGraph + * + * @param msgs + * @param schema the structure of the merged subgraph + */ + @Override + public void merge(Collection> msgs, Pattern schema) { + if (CollectionUtils.isEmpty(msgs)) { + return; + } + + for (KgGraph msg : msgs) { + // merge vertex + KgGraphImpl graph = (KgGraphImpl) msg; + for (String vertexAlias : graph.getAlias2VertexMap().keySet()) { + Set> otherVertexSet = + graph.getAlias2VertexMap().get(vertexAlias); + if (!this.alias2VertexMap.containsKey(vertexAlias)) { + this.alias2VertexMap.put(vertexAlias, otherVertexSet); + } else { + Set> thisVertexSet = this.alias2VertexMap.get(vertexAlias); + if (thisVertexSet != otherVertexSet) { + Set> newVertexSet = new HashSet<>(thisVertexSet); + newVertexSet.addAll(otherVertexSet); + this.alias2VertexMap.put(vertexAlias, newVertexSet); + // this.alias2VertexMap.get(vertexAlias).addAll(otherVertexSet); + } + } + } + + // merge edge + for (String edgeAlias : graph.getAlias2EdgeMap().keySet()) { + Set> otherEdgeSet = graph.getAlias2EdgeMap().get(edgeAlias); + if (!this.alias2EdgeMap.containsKey(edgeAlias)) { + this.alias2EdgeMap.put(edgeAlias, otherEdgeSet); + } else { + Set> thisEdgeSet = this.alias2EdgeMap.get(edgeAlias); + if (thisEdgeSet != otherEdgeSet) { + Set> newEdgeSet = new HashSet<>(thisEdgeSet); + newEdgeSet.addAll(otherEdgeSet); + this.alias2EdgeMap.put(edgeAlias, newEdgeSet); + // this.alias2EdgeMap.get(edgeAlias).addAll(otherEdgeSet); + } + } + } + } + } + + /** + * VirtualVertex is a point in graph matching that only has an ID without any attributes. As the + * matching process progresses, attributes will be added to it, and here we merge the attributes + * into it. + */ + private void mergeAndOverwriteVirtualVertex(List> msgs) { + Map>> aliasVirtualVertexMap = + new HashMap<>(); + + for (KgGraph msg : msgs) { + // merge vertex + KgGraphImpl graph = (KgGraphImpl) msg; + for (String vertexAlias : graph.getAlias2VertexMap().keySet()) { + + Set> thisVertexSet = + this.alias2VertexMap.computeIfAbsent(vertexAlias, k -> new HashSet<>()); + Set> otherVertexSet = + graph.getAlias2VertexMap().getOrDefault(vertexAlias, new HashSet<>()); + + Map> virtualVertexMap = + aliasVirtualVertexMap.get(vertexAlias); + if (null == virtualVertexMap) { + // init virtual vertex map + virtualVertexMap = new HashMap<>(); + for (IVertex v : thisVertexSet) { + if (v.getValue() != null) { + continue; + } + virtualVertexMap.put(v.getId(), v); + } + aliasVirtualVertexMap.put(vertexAlias, virtualVertexMap); + } + + // If otherVertexSet has real vertex, delete the virtual vertex in thisVertexSet + if (!virtualVertexMap.isEmpty()) { + for (IVertex otherVertex : otherVertexSet) { + if (virtualVertexMap.containsKey(otherVertex.getId()) + && null != otherVertex.getValue()) { + thisVertexSet.remove(virtualVertexMap.get(otherVertex.getId())); + } + } + } + thisVertexSet.addAll(otherVertexSet); + } + + // merge edge + for (String edgeAlias : graph.getAlias2EdgeMap().keySet()) { + Set> thisEdgeSet = + this.getAlias2EdgeMap().computeIfAbsent(edgeAlias, k -> new HashSet<>()); + Set> otherEdgeSet = graph.getAlias2EdgeMap().get(edgeAlias); + if (CollectionUtils.isNotEmpty(otherEdgeSet)) { + thisEdgeSet.addAll(otherEdgeSet); + } + } + } + } + + /** + * Add the new KgGraph to the current KgGraph, and the schema of the KgGraph changes + * + * @param subGraph + * @param schema the structure of the expanded subgraph + */ + @Override + public void expand(KgGraph subGraph, Pattern schema) { + String rootAlias = schema.root().alias(); + Set> rootVertexSet = this.alias2VertexMap.remove(rootAlias); + IVertex rootVertex = rootVertexSet.iterator().next(); + if (rootVertex instanceof MirrorVertex) { + KgGraphImpl subKgGraph = (KgGraphImpl) subGraph; + IVertex matchedRootVertex = + subKgGraph.getAlias2VertexMap().get(rootAlias).iterator().next(); + subKgGraph + .getAlias2VertexMap() + .put(rootAlias, Sets.newHashSet(new MirrorVertex<>(matchedRootVertex))); + } + this.merge(Lists.newArrayList(subGraph), schema); + } + + @Override + public int expandAndPrune( + Pattern thisSchema, + KgGraph matchedKgGraph, + Pattern matchedSchema, + Pattern finalSchema, + Set intersectionVertexAliasSet) { + KgGraphImpl tmpMatchedKgGraph = new KgGraphImpl((KgGraphImpl) matchedKgGraph); + int minMatchCount = Integer.MAX_VALUE; + for (String vertexAlias : intersectionVertexAliasSet) { + Set> matchedVertexSet = + tmpMatchedKgGraph.alias2VertexMap.remove(vertexAlias); + Set> thisVertexSet = this.alias2VertexMap.get(vertexAlias); + if (thisVertexSet instanceof HashSet) { + thisVertexSet = + (Set>) + ((HashSet>) thisVertexSet).clone(); + } else { + thisVertexSet = new HashSet<>(thisVertexSet); + } + boolean change = thisVertexSet.retainAll(matchedVertexSet); + if (thisVertexSet.isEmpty()) { + // no intersection, return empty KgGraph + return 0; + } + + if (thisVertexSet.size() < minMatchCount) { + minMatchCount = thisVertexSet.size(); + } + + if (change) { + this.alias2VertexMap.put(vertexAlias, thisVertexSet); + Map pruneEdgeAliasAndDirection = + getPruneEdgeAliasAndDirection(thisSchema, vertexAlias); + this.pruningEdgesByVertexSet(thisVertexSet, pruneEdgeAliasAndDirection); + } + + if (matchedVertexSet.size() > thisVertexSet.size()) { + tmpMatchedKgGraph.alias2VertexMap.put(vertexAlias, thisVertexSet); + Map pruneEdgeAliasAndDirection = + getPruneEdgeAliasAndDirection(matchedSchema, vertexAlias); + tmpMatchedKgGraph.pruningEdgesByVertexSet(thisVertexSet, pruneEdgeAliasAndDirection); + } + } + this.alias2VertexMap.remove(matchedSchema.root().alias()); + this.merge(Lists.newArrayList(tmpMatchedKgGraph), finalSchema); + return minMatchCount; + } + + private Map getPruneEdgeAliasAndDirection(Pattern schema, String vertexAlias) { + Set pcSet = + JavaConversions.setAsJavaSet(KgGraphSchema.getNeighborEdges(schema, vertexAlias)); + Map pruneEdgeAliasAndDirection = new HashMap<>(); + for (Connection pc : pcSet) { + if (pc.source().equals(vertexAlias)) { + pruneEdgeAliasAndDirection.put(pc.alias(), true); + } else if (pc.target().equals(vertexAlias)) { + pruneEdgeAliasAndDirection.put(pc.alias(), false); + } + } + return pruneEdgeAliasAndDirection; + } + + private void pruningEdgesByVertexSet( + Set> validVertexSet, + Map pruneEdgeAliasAndDirection) { + IVertex useForCheckVertex = new Vertex<>(null, null); + for (Map.Entry entry : pruneEdgeAliasAndDirection.entrySet()) { + String edgeAlias = entry.getKey(); + boolean checkSource = entry.getValue(); + Set> newEdgeSet = new HashSet<>(); + Set> edgeSet = this.alias2EdgeMap.get(edgeAlias); + for (IEdge edge : edgeSet) { + if (checkSource) { + useForCheckVertex.setId(edge.getSourceId()); + } else { + useForCheckVertex.setId(edge.getTargetId()); + } + if (!validVertexSet.contains(useForCheckVertex)) { + continue; + } + newEdgeSet.add(edge); + } + this.alias2EdgeMap.put(edgeAlias, newEdgeSet); + } + } + + /** + * Get the path from KgGraph, flat all alias vertexes + * + * @return + */ + @Override + public Iterator> getPath( + KgGraphSplitStaticParameters staticParameters, Predicate> filter) { + // single vertex path + if (this.alias2EdgeMap.isEmpty() && 1 == this.alias2VertexMap.size()) { + List> result = new ArrayList<>(); + this.alias2VertexMap.forEach( + new BiConsumer>>() { + @Override + public void accept(String vertexAlias, Set> vertexSet) { + for (IVertex vertex : vertexSet) { + Map>> alias2VertexMap = new HashMap<>(); + alias2VertexMap.put(vertexAlias, Sets.newHashSet(vertex)); + KgGraph kgGraph = new KgGraphImpl(alias2VertexMap, alias2EdgeMap); + if (null != filter && !filter.test(kgGraph)) { + continue; + } + result.add(kgGraph); + } + } + }); + return result.iterator(); + } + // getPathSize(); + return new KgGraphIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + this.alias2VertexMap, + this.alias2EdgeMap, + filter); + } + + private static final long LOG_PATH_SIZE_THRESHOLD = 10 * 10000 * 10000; + + /** + * Different versions of edges may exist between two vertexes, so the Cartesian product of edges + * is used as the total number of paths + * + * @return + */ + private Long getPathSize() { + long size = 1; + if (alias2EdgeMap.isEmpty()) { + // only have one vertex alias + int vertexAliasNum = alias2VertexMap.keySet().size(); + if (vertexAliasNum != 1) { + throw new RuntimeException( + String.format( + "Schema has only one vertex, but KgGraph has %d vertex alias", vertexAliasNum)); + } + for (String vertexAlias : alias2VertexMap.keySet()) { + size = alias2VertexMap.get(vertexAlias).size(); + return size; + } + } + Map edgeCountMap = new HashMap<>(); + for (Map.Entry>> entry : + this.alias2EdgeMap.entrySet()) { + String type = entry.getKey(); + int edgeSize = entry.getValue().size(); + edgeCountMap.put(type, edgeSize); + size *= edgeSize; + } + if (size > LOG_PATH_SIZE_THRESHOLD) { + List idList = new ArrayList<>(); + for (Map.Entry>> entry : + this.alias2VertexMap.entrySet()) { + if (1 == entry.getValue().size()) { + idList.add(entry.getValue().iterator().next().getId()); + } + edgeCountMap.put(entry.getKey(), entry.getValue().size()); + } + log.info( + "KgGraphPathSize=" + + size + + ",aliasCountMap=" + + JSON.toJSONString(edgeCountMap) + + ",idList=" + + idList); + } + return size; + } + + @Override + public void aggregateEdge( + String edgeAlias, + Set> edgeSet, + AggregationSchemaInfo aggregationSchemaInfo, + Set aggregatedAliasSet) { + if (aggregatedAliasSet.contains(edgeAlias)) { + return; + } + aggregatedAliasSet.add(edgeAlias); + + Set> oldEdgeSet = this.alias2EdgeMap.get(edgeAlias); + + // Prune the edge + this.alias2EdgeMap.put(edgeAlias, edgeSet); + + if (edgeSet.size() == oldEdgeSet.size()) { + return; + } + + List> endpointVertexes = + aggregationSchemaInfo.getEdgeEndpointMap().get(edgeAlias); + for (Tuple2 tuple2 : endpointVertexes) { + String vertexAlias = tuple2._1(); + if (aggregatedAliasSet.contains(vertexAlias)) { + continue; + } + boolean checkSource = tuple2._2(); + + Set validVertexIdSet = new HashSet<>(); + for (IEdge edge : edgeSet) { + if (checkSource) { + validVertexIdSet.add(edge.getSourceId()); + } else { + validVertexIdSet.add(edge.getTargetId()); + } + } + Set> newVertexSet = new HashSet<>(); + for (IVertex vertex : this.alias2VertexMap.get(vertexAlias)) { + if (validVertexIdSet.contains(vertex.getId())) { + newVertexSet.add(vertex); + } + } + this.aggregateVertex(vertexAlias, newVertexSet, aggregationSchemaInfo, aggregatedAliasSet); + } + } + + @Override + public void aggregateVertex( + String vertexAlias, + Set> vertexSet, + AggregationSchemaInfo aggregationSchemaInfo, + Set aggregatedAliasSet) { + if (aggregatedAliasSet.contains(vertexAlias)) { + return; + } + aggregatedAliasSet.add(vertexAlias); + + Set> oldVertexSet = this.alias2VertexMap.get(vertexAlias); + + // Prune the vertex + this.alias2VertexMap.put(vertexAlias, vertexSet); + + if (vertexSet.size() == oldVertexSet.size()) { + return; + } + + IVertex tmpCheckVertex = new Vertex<>(); + List> edgeInfoList = + aggregationSchemaInfo.getVertexHasEdgeMap().get(vertexAlias); + for (Tuple2 tuple2 : edgeInfoList) { + String edgeAlias = tuple2._1(); + if (aggregatedAliasSet.contains(edgeAlias)) { + continue; + } + boolean checkSource = tuple2._2(); + + Set> newEdgeSet = new HashSet<>(); + for (IEdge edge : this.alias2EdgeMap.get(edgeAlias)) { + if (checkSource) { + tmpCheckVertex.setId(edge.getSourceId()); + } else { + tmpCheckVertex.setId(edge.getTargetId()); + } + if (vertexSet.contains(tmpCheckVertex)) { + newEdgeSet.add(edge); + } + } + this.aggregateEdge(edgeAlias, newEdgeSet, aggregationSchemaInfo, aggregatedAliasSet); + } + } + + /** + * Sort the current KgGraph by sortItems + * + * @param sortItems + * @param limit + * @param schema + */ + @Override + public void executeSort(List sortItems, int limit, Pattern schema) {} + + /** + * Adjust the current KgGraph according to scheme including structure and property + * + * @param schema + */ + @Override + public void adjustGraph(Pattern schema) {} + + /** + * Get vertex from KgGraph based on vertex alias + * + * @param alias + * @return + */ + @Override + public List> getVertex(String alias) { + List> result = new ArrayList<>(); + Set> vertexSet = alias2VertexMap.get(alias); + if (CollectionUtils.isNotEmpty(vertexSet)) { + result.addAll(vertexSet); + } + return result; + } + + /** + * Get edge from KgGraph based on edge alias + * + * @param alias + * @return + */ + @Override + public List> getEdge(String alias) { + List> result = new ArrayList<>(); + Set> edgeSet = alias2EdgeMap.get(alias); + if (CollectionUtils.isNotEmpty(edgeSet)) { + result.addAll(edgeSet); + } + return result; + } + + @Override + public Set getVertexAlias() { + return alias2VertexMap.keySet(); + } + + @Override + public Set getEdgeAlias() { + return alias2EdgeMap.keySet(); + } + + @Override + public void show() { + log.info("--------------------Vertex---------------------------"); + StringBuffer sb = new StringBuffer(); + for (String vertexAlias : alias2VertexMap.keySet()) { + sb.setLength(0); + sb.append(vertexAlias).append(": "); + alias2VertexMap.get(vertexAlias).stream() + .forEach(vertex -> sb.append(vertex.getId()).append(", ")); + log.info(sb.toString()); + } + log.info("--------------------Edge-----------------------------"); + for (String edgeAlias : alias2EdgeMap.keySet()) { + sb.setLength(0); + sb.append(edgeAlias).append(": "); + alias2EdgeMap.get(edgeAlias).stream() + .forEach( + edge -> { + sb.append(edge.getSourceId()); + if (edge.getDirection().equals(Direction.OUT)) { + sb.append(" -> "); + } + if (edge.getDirection().equals(Direction.IN)) { + sb.append(" <- "); + } + if (edge.getDirection().equals(Direction.BOTH)) { + sb.append(" <-> "); + } + sb.append(edge.getTargetId()); + sb.append(", "); + }); + log.info(sb.toString()); + } + } + + @Override + public boolean hasFocusVertexId(String alias, Set focusVertexIdSet) { + Set> vertexSet = alias2VertexMap.get(alias); + if (null == vertexSet) { + return false; + } + for (IVertex v : vertexSet) { + if (focusVertexIdSet.contains(v.getId())) { + return true; + } + } + return false; + } + + @Override + public boolean checkDuplicateVertex() { + Set idSet = new HashSet<>(); + Set mirrorIdSet = new HashSet<>(); + for (Set> vertexSet : this.alias2VertexMap.values()) { + if (1 == vertexSet.size()) { + IVertex vertex = vertexSet.iterator().next(); + IVertexId id = vertex.getId(); + if (vertex instanceof MirrorVertex) { + if (mirrorIdSet.contains(id)) { + return true; + } + mirrorIdSet.add(vertex.getId()); + continue; + } + if (idSet.contains(id)) { + return true; + } else { + idSet.add(id); + } + } + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry>> entry : alias2VertexMap.entrySet()) { + String alias = entry.getKey(); + Set> vertexSet = entry.getValue(); + sb.append("\nvertex=").append(alias).append(",size=").append(vertexSet.size()).append(",{"); + int count = 0; + for (IVertex v : vertexSet) { + if (count++ > 100) { + break; + } + sb.append(v); + } + sb.append("}"); + } + + for (Map.Entry>> entry : alias2EdgeMap.entrySet()) { + String alias = entry.getKey(); + Set> edgeSet = entry.getValue(); + sb.append("\nedge=").append(alias).append(",c=").append(edgeSet.size()).append("{"); + int count = 0; + for (IEdge e : edgeSet) { + if (count++ > 100) { + break; + } + sb.append(e); + } + sb.append("}"); + } + return sb.toString(); + } + + /** + * Getter method for property alias2VertexMap. + * + * @return property value of alias2VertexMap + */ + public Map>> getAlias2VertexMap() { + return alias2VertexMap; + } + + /** + * Setter method for property alias2VertexMap. + * + * @param alias2VertexMap value to be assigned to property alias2VertexMap + */ + public void setAlias2VertexMap(Map>> alias2VertexMap) { + this.alias2VertexMap = alias2VertexMap; + } + + /** + * Getter method for property alias2EdgeMap. + * + * @return property value of alias2EdgeMap + */ + public Map>> getAlias2EdgeMap() { + return alias2EdgeMap; + } + + /** + * Setter method for property alias2EdgeMap. + * + * @param alias2EdgeMap value to be assigned to property alias2EdgeMap + */ + public void setAlias2EdgeMap(Map>> alias2EdgeMap) { + this.alias2EdgeMap = alias2EdgeMap; + } + + private static class KgGraphIterator extends EdgeCombinationIterator { + private final Predicate> filter; + + public KgGraphIterator( + List edgeIterateInfoList, + Map edgeIterateOrderMap, + Map>> alias2VertexMap, + Map>> alias2EdgeMap, + Predicate> filter) { + super(edgeIterateInfoList, edgeIterateOrderMap, alias2VertexMap, alias2EdgeMap); + this.filter = filter; + } + + @Override + public KgGraph next() { + KgGraph kgGraph = super.next(); + if (null == kgGraph) { + return null; + } + if (null != filter && !filter.test(kgGraph)) { + return null; + } + return kgGraph; + } + } + + /** + * set vertex property + * + * @param alias - vertex alias + * @param propertyMap - key is new property name, value is new property value + * @param version - vertex version, if null add properties to the 0 version + */ + @Override + public void setVertexProperty(String alias, Map propertyMap, Long version) { + if (StringUtils.isBlank(alias)) { + log.warn("[KgGraphImpl.setVertexProperty] alias is empty"); + return; + } + + if (!alias2VertexMap.containsKey(alias)) { + throw new RuntimeException( + String.format("[KgGraphImpl.setVertexProperty] no vertex alias with alias = %s", alias)); + } + + Set> vertexSet = alias2VertexMap.get(alias); + Set> updateVertexSet = new HashSet<>(); + vertexSet.forEach( + vertex -> { + if (null == vertex.getValue()) { + // without expendInto, vertex property can be null + IVersionProperty versionProperty = + PropertyUtil.buildVertexProperty(vertex.getId(), new HashMap<>()); + vertex.setValue(versionProperty); + } + if (!(vertex.getValue() instanceof IVersionProperty)) { + throw new RuntimeException( + "[KgGraphImpl.setVertexProperty] vertex property is not multi-version"); + } + if (null == propertyMap || propertyMap.isEmpty()) { + log.debug( + "[KgGraphImpl.setVertexProperty] propertyMap is empty, delete alias all properties"); + vertex.setValue(PropertyUtil.buildVertexProperty(vertex.getId(), null)); + updateVertexSet.add(vertex); + } else { + IVersionProperty versionProperty; + if (vertex.getValue() == null) { + versionProperty = PropertyUtil.buildVertexProperty(vertex.getId(), null); + } else { + versionProperty = (IVersionProperty) vertex.getValue().clone(); + } + for (Map.Entry entry : propertyMap.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (null == value) { + versionProperty.remove(key); + continue; + } + versionProperty.put(key, propertyMap.get(key), version); + } + IVertex newVertex = vertex.clone(); + newVertex.setValue(versionProperty); + updateVertexSet.add(newVertex); + } + }); + alias2VertexMap.put(alias, updateVertexSet); + } + + /** + * set edge property + * + * @param alias - edge alias + * @param propertyMap - key is new property name, value is new property value + */ + @Override + public void setEdgeProperty(String alias, Map propertyMap) { + if (StringUtils.isBlank(alias)) { + log.warn("[KgGraphImpl.setEdgeProperty] alias is empty"); + return; + } + + if (!alias2EdgeMap.containsKey(alias)) { + throw new RuntimeException( + String.format("[KgGraphImpl.setEdgeProperty] no edge alias with alias = %s", alias)); + } + Set> edgeSet = alias2EdgeMap.get(alias); + Set> updateEdgeSet = new HashSet<>(); + edgeSet.forEach( + edge -> { + if (null == propertyMap || propertyMap.isEmpty()) { + log.debug( + "[KgGraphImpl.setEdgeProperty] propertyMap is empty, delete alias all properties"); + edge.setValue(PropertyUtil.buildEdgeProperty(edge.getType(), null)); + updateEdgeSet.add(edge); + } else { + IProperty edgeValue; + if (edge.getValue() == null) { + edgeValue = PropertyUtil.buildEdgeProperty(edge.getType(), null); + } else { + edgeValue = edge.getValue().clone(); + } + for (Map.Entry entry : propertyMap.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (null == value) { + edgeValue.remove(key); + continue; + } + edgeValue.put(key, propertyMap.get(key)); + } + IEdge newEdge = edge.clone(); + newEdge.setValue(edgeValue); + updateEdgeSet.add(newEdge); + } + }); + alias2EdgeMap.put(alias, updateEdgeSet); + } + + @Override + public void cloneVertexSet(String alias) { + Set> vertexSet = alias2VertexMap.get(alias); + Set> newVertexSet = new HashSet<>(); + for (IVertex vertex : vertexSet) { + Vertex newVertex = new Vertex<>(vertex.getId()); + if (null == vertex.getValue()) { + // without expendInto, vertex value can be null + newVertex.setValue(null); + } else { + newVertex.setValue(vertex.getValue().clone()); + } + newVertexSet.add(newVertex); + } + this.alias2VertexMap.put(alias, newVertexSet); + } + + @Override + public void cloneEdgeSet(String alias) { + Set> edgeSet = alias2EdgeMap.get(alias); + Set> newEdgeSet = new HashSet<>(); + for (IEdge edge : edgeSet) { + Edge newEdge = + new Edge<>( + edge.getSourceId(), + edge.getTargetId(), + null, + edge.getVersion(), + edge.getDirection(), + edge.getType()); + if (null == edge.getValue()) { + // without expendInto, vertex value can be null + newEdge.setValue(null); + } else { + newEdge.setValue(edge.getValue().clone()); + } + newEdgeSet.add(newEdge); + } + alias2EdgeMap.put(alias, newEdgeSet); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphKey.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphKey.java new file mode 100644 index 00000000..6bdaa52b --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphKey.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph.impl; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import java.util.Arrays; + +public class KgGraphKey { + private final IVertexId[] vertexIds; + + public KgGraphKey(IVertexId[] vertexIds) { + this.vertexIds = vertexIds; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof KgGraphKey)) { + return false; + } + KgGraphKey other = (KgGraphKey) obj; + return Arrays.equals(this.vertexIds, other.vertexIds); + } + + @Override + public int hashCode() { + return Arrays.hashCode(vertexIds); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphSplitStaticParameters.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphSplitStaticParameters.java new file mode 100644 index 00000000..e852368a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/kggraph/impl/KgGraphSplitStaticParameters.java @@ -0,0 +1,438 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph.impl; + +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.impl.EdgeCombinationIterator.BuildEdgeIteratorInfo; +import com.antgroup.openspg.reasoner.kggraph.impl.EdgeCombinationIterator.Edge2VertexInfo; +import com.antgroup.openspg.reasoner.kggraph.impl.EdgeCombinationIterator.EdgeIterateInfo; +import com.antgroup.openspg.reasoner.kggraph.impl.EdgeCombinationIterator.IntersectVertexInfo; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public class KgGraphSplitStaticParameters implements Serializable { + private static final long serialVersionUID = 7532256372761127085L; + private final List edgeIterateInfoList; + private final Map edgeIterateOrderMap; + + private final Set needSplitEdgeSet; + private final List> splitConnectedSubgraph; + private final Set neighborAliasSet; + private final Set connectedSubgraphVertexAliasSet; + private final Set connectedSubgraphEdgeAliasSet; + + // split subgraph edge iterator info + private final List subEdgeIterateInfoList; + private final Map subEdgeIterateOrderMap; + + /** + * Extract the logic that is repeatedly invoked in KgGraph, and complete it once during + * initialization. + */ + public KgGraphSplitStaticParameters(Set splitVertexAliases, Pattern schema) { + this.edgeIterateInfoList = initEdgeIterateInfo(schema); + this.edgeIterateOrderMap = initEdgeIterateOrderMap(this.edgeIterateInfoList); + + if (CollectionUtils.isNotEmpty(splitVertexAliases)) { + this.needSplitEdgeSet = getNeedSplitEdgeSet(splitVertexAliases, schema); + this.splitConnectedSubgraph = getSplitConnectedSubgraph(this.needSplitEdgeSet); + if (this.splitConnectedSubgraph.size() > 1) { + this.neighborAliasSet = null; + } else { + this.neighborAliasSet = getNeighborAliasSet(splitVertexAliases, schema); + } + Tuple2, Set> setTuple2 = + getVertexAndEdgeAliasSet(this.splitConnectedSubgraph.get(0)); + this.connectedSubgraphVertexAliasSet = setTuple2._1(); + this.connectedSubgraphEdgeAliasSet = setTuple2._2(); + this.subEdgeIterateInfoList = + getSubEdgeIterateInfoList( + this.splitConnectedSubgraph.get(0), this.connectedSubgraphVertexAliasSet, schema); + this.subEdgeIterateOrderMap = initEdgeIterateOrderMap(this.subEdgeIterateInfoList); + } else { + this.needSplitEdgeSet = null; + this.splitConnectedSubgraph = null; + this.neighborAliasSet = null; + this.connectedSubgraphVertexAliasSet = null; + this.connectedSubgraphEdgeAliasSet = null; + this.subEdgeIterateInfoList = null; + this.subEdgeIterateOrderMap = null; + } + } + + /** + * Getter method for property needSplitEdgeSet. + * + * @return property value of needSplitEdgeSet + */ + public Set getNeedSplitEdgeSet() { + return needSplitEdgeSet; + } + + /** + * Getter method for property splitConnectedSubgraph. + * + * @return property value of splitConnectedSubgraph + */ + public List> getSplitConnectedSubgraph() { + return splitConnectedSubgraph; + } + + /** + * Getter method for property edgeIterateInfoList. + * + * @return property value of edgeIterateInfoList + */ + public List getEdgeIterateInfoList() { + return edgeIterateInfoList; + } + + /** + * Getter method for property edgeIterateOrderMap. + * + * @return property value of edgeIterateOrderMap + */ + public Map getEdgeIterateOrderMap() { + return edgeIterateOrderMap; + } + + /** + * Getter method for property vertexAliasSet. + * + * @return property value of vertexAliasSet + */ + public Set getConnectedSubgraphVertexAliasSet() { + return connectedSubgraphVertexAliasSet; + } + + /** + * Getter method for property edgeAliasSet. + * + * @return property value of edgeAliasSet + */ + public Set getConnectedSubgraphEdgeAliasSet() { + return connectedSubgraphEdgeAliasSet; + } + + /** + * Getter method for property subEdgeIterateInfoList. + * + * @return property value of subEdgeIterateInfoList + */ + public List getSubEdgeIterateInfoList() { + return subEdgeIterateInfoList; + } + + /** + * Getter method for property subEdgeIterateOrderMap. + * + * @return property value of subEdgeIterateOrderMap + */ + public Map getSubEdgeIterateOrderMap() { + return subEdgeIterateOrderMap; + } + + public boolean canDoSampleSplit(Map>> alias2VertexMap) { + if (null == this.neighborAliasSet) { + return false; + } + for (String alias : this.neighborAliasSet) { + if (alias2VertexMap.get(alias).size() > 1) { + return false; + } + } + return true; + } + + private List getSubEdgeIterateInfoList( + Set subgraphEdgeSet, + Set connectedSubgraphVertexAliasSet, + Pattern schema) { + Map nodes = new HashMap<>(); + Map> topology = new HashMap<>(); + for (String vertexAlias : connectedSubgraphVertexAliasSet) { + nodes.put(vertexAlias, schema.getNode(vertexAlias)); + for (Connection pc : subgraphEdgeSet) { + if (pc.source().equals(vertexAlias)) { + Set sets = topology.computeIfAbsent(vertexAlias, k -> new HashSet<>()); + sets.add(pc); + } + } + } + Map> topologyImmutableSet = new HashMap<>(); + for (Map.Entry> entry : topology.entrySet()) { + topologyImmutableSet.put( + entry.getKey(), JavaConversions.asScalaSet(entry.getValue()).toSet()); + } + + Pattern subSchema = + new PartialGraphPattern( + "", + JavaConversions.mapAsScalaMap(nodes).toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topologyImmutableSet)); + return initEdgeIterateInfo(subSchema); + } + + private static Set getNeighborAliasSet(Set vertexAliases, Pattern schema) { + Set neighborAliasSet = new HashSet<>(); + for (Connection pc : RunnerUtil.getConnectionSet(schema)) { + for (String alias : vertexAliases) { + String neighborAlias = RunnerUtil.getNeighborAlias(alias, pc); + if (StringUtils.isNotEmpty(neighborAlias)) { + if (vertexAliases.contains(neighborAlias)) { + return null; + } + neighborAliasSet.add(neighborAlias); + } + } + } + return neighborAliasSet; + } + + /** Obtain the aliases of the nodes and edges covered by the edge list. */ + private static Tuple2, Set> getVertexAndEdgeAliasSet( + Set connectedEdgeSet) { + Set vertexAlias = new HashSet<>(); + Set edgeAlias = new HashSet<>(); + for (Connection pc : connectedEdgeSet) { + vertexAlias.add(pc.source()); + vertexAlias.add(pc.target()); + edgeAlias.add(pc.alias()); + } + return new Tuple2<>(vertexAlias, edgeAlias); + } + + /** + * Divide the PatternConnectionSet that needs to be partitioned into several connected + * PatternConnectionSets. + */ + private List> getSplitConnectedSubgraph(Set needSplitEdgeSet) { + needSplitEdgeSet = Sets.newHashSet(needSplitEdgeSet); + List> result = new ArrayList<>(); + while (!needSplitEdgeSet.isEmpty()) { + Set connectedEdgeSet = new HashSet<>(); + Set touchedVertexSet = new HashSet<>(); + + boolean hasConnectedEdge = true; + + while (hasConnectedEdge) { + boolean findOneConnectedEdge = false; + Iterator it = needSplitEdgeSet.iterator(); + while (it.hasNext()) { + Connection pc = it.next(); + if ((touchedVertexSet.isEmpty()) + || (touchedVertexSet.contains(pc.source()) + || touchedVertexSet.contains(pc.target()))) { + findOneConnectedEdge = true; + connectedEdgeSet.add(pc); + touchedVertexSet.add(pc.source()); + touchedVertexSet.add(pc.target()); + it.remove(); + } + } + if (!findOneConnectedEdge) { + hasConnectedEdge = false; + } + } + result.add(connectedEdgeSet); + } + return result; + } + + private static Set getNeedSplitEdgeSet(Set vertexAliases, Pattern schema) { + Set needSplitEdgeSet = new HashSet<>(); + for (Connection pc : RunnerUtil.getConnectionSet(schema)) { + if (vertexAliases.contains(pc.source()) || vertexAliases.contains(pc.target())) { + needSplitEdgeSet.add(pc); + } + } + return needSplitEdgeSet; + } + + private static Map initEdgeIterateOrderMap( + List edgeIterateInfoList) { + Map edgeIterateOrderMap = new HashMap<>(); + for (int i = 0; i < edgeIterateInfoList.size(); ++i) { + EdgeIterateInfo edgeIterateInfo = edgeIterateInfoList.get(i); + edgeIterateOrderMap.put(edgeIterateInfo.getEdgeAlias(), i); + } + return edgeIterateOrderMap; + } + + private static List initEdgeIterateInfo(Pattern schema) { + List edgeIterateInfoList = new ArrayList<>(); + List patternConnectionList = + Lists.newArrayList(RunnerUtil.getConnectionSet(schema)); + Map patternConnectionMap = new HashMap<>(); + for (Connection pc : patternConnectionList) { + patternConnectionMap.put(pc.alias(), pc); + } + // First, determine the order in which to iterate over the Edges. + // Ensure that the current Edge shares a common Vertex with the Edges that have been iterated + // over previously. + Set touchedVertexAliasSet = new HashSet<>(); + while (!patternConnectionList.isEmpty()) { + boolean findOne = false; + Iterator it = patternConnectionList.iterator(); + while (it.hasNext()) { + Connection pc = it.next(); + if (touchedVertexAliasSet.contains(pc.source()) + || touchedVertexAliasSet.contains(pc.target())) { + EdgeIterateInfo edgeIterateInfo = new EdgeIterateInfo(pc); + edgeIterateInfoList.add(edgeIterateInfo); + + touchedVertexAliasSet.add(pc.source()); + touchedVertexAliasSet.add(pc.target()); + + it.remove(); + findOne = true; + break; + } + } + if (!findOne) { + Connection target = null; + if (CollectionUtils.isEmpty(edgeIterateInfoList)) { + String edgeAlias = patternConnectionList.get(0).alias(); + for (Connection pc : patternConnectionList) { + if (pc.alias().equals(edgeAlias)) { + target = pc; + break; + } + } + patternConnectionList.remove(target); + } else { + target = patternConnectionList.remove(0); + } + EdgeIterateInfo edgeIterateInfo = new EdgeIterateInfo(target); + edgeIterateInfoList.add(edgeIterateInfo); + + touchedVertexAliasSet.add(target.source()); + touchedVertexAliasSet.add(target.target()); + } + } + + // Calculate the intersection of edges. + Map> touchedVertexAlias2EdgeMap = new HashMap<>(); + for (int i = 0; i < edgeIterateInfoList.size(); ++i) { + EdgeIterateInfo edgeIterateInfo = edgeIterateInfoList.get(i); + List sourceVertexInfoList = + touchedVertexAlias2EdgeMap.getOrDefault( + edgeIterateInfo.getSourceAlias(), new ArrayList<>()); + List targetVertexInfoList = + touchedVertexAlias2EdgeMap.getOrDefault( + edgeIterateInfo.getTargetAlias(), new ArrayList<>()); + + if (i > 0) { + Edge2VertexInfo queryKey; + boolean indexSource; + if (!sourceVertexInfoList.isEmpty()) { + queryKey = sourceVertexInfoList.remove(0); + indexSource = true; + } else { + queryKey = targetVertexInfoList.remove(0); + indexSource = false; + } + + BuildEdgeIteratorInfo buildEdgeIteratorInfo = + new BuildEdgeIteratorInfo(edgeIterateInfo.getEdgeAlias(), indexSource, queryKey); + edgeIterateInfo.setBuildEdgeIteratorInfo(buildEdgeIteratorInfo); + } + + List intersectInfoList = new ArrayList<>(); + for (Edge2VertexInfo edge2VertexInfo : sourceVertexInfoList) { + intersectInfoList.add(new IntersectVertexInfo(true, edge2VertexInfo)); + } + for (Edge2VertexInfo edge2VertexInfo : targetVertexInfoList) { + intersectInfoList.add(new IntersectVertexInfo(false, edge2VertexInfo)); + } + edgeIterateInfo.setIntersectInfoList(intersectInfoList); + + List sourceVertexEdgeList = + touchedVertexAlias2EdgeMap.computeIfAbsent( + edgeIterateInfo.getSourceAlias(), k -> new ArrayList<>()); + sourceVertexEdgeList.add(new Edge2VertexInfo(i, edgeIterateInfo.getEdgeAlias(), true)); + List targetVertexEdgeList = + touchedVertexAlias2EdgeMap.computeIfAbsent( + edgeIterateInfo.getTargetAlias(), k -> new ArrayList<>()); + targetVertexEdgeList.add(new Edge2VertexInfo(i, edgeIterateInfo.getEdgeAlias(), false)); + } + + // init duplicate vertex check map + for (int i = 1; i < edgeIterateInfoList.size(); ++i) { + EdgeIterateInfo nowEdgeIterateInfo = edgeIterateInfoList.get(i); + Connection nowPc = patternConnectionMap.get(nowEdgeIterateInfo.getEdgeAlias()); + String nowPcSource = nowPc.source(); + String nowPcTarget = nowPc.target(); + Set sourceCheckAliasSet = new HashSet<>(); + Set targetCheckAliasSet = new HashSet<>(); + for (int j = 0; j < i; ++j) { + EdgeIterateInfo checkEdgeIterateInfo = edgeIterateInfoList.get(j); + Connection checkPc = patternConnectionMap.get(checkEdgeIterateInfo.getEdgeAlias()); + String checkPcSource = checkPc.source(); + String checkPcTarget = checkPc.target(); + if (!sourceCheckAliasSet.contains(checkPcSource) + && hasSameVertexType(schema, nowPcSource, checkPcSource)) { + nowEdgeIterateInfo.getDuplicateVertexCheck().put(j, new Tuple2<>(true, true)); + } + sourceCheckAliasSet.add(checkPcSource); + if (!sourceCheckAliasSet.contains(checkPcTarget) + && hasSameVertexType(schema, nowPcSource, checkPcTarget)) { + nowEdgeIterateInfo.getDuplicateVertexCheck().put(j, new Tuple2<>(true, false)); + } + sourceCheckAliasSet.add(checkPcTarget); + if (!targetCheckAliasSet.contains(checkPcSource) + && hasSameVertexType(schema, nowPcTarget, checkPcSource)) { + nowEdgeIterateInfo.getDuplicateVertexCheck().put(j, new Tuple2<>(false, true)); + } + targetCheckAliasSet.add(checkPcSource); + if (!targetCheckAliasSet.contains(checkPcTarget) + && hasSameVertexType(schema, nowPcTarget, checkPcTarget)) { + nowEdgeIterateInfo.getDuplicateVertexCheck().put(j, new Tuple2<>(false, false)); + } + targetCheckAliasSet.add(checkPcTarget); + } + } + return edgeIterateInfoList; + } + + private static boolean hasSameVertexType(Pattern schema, String alias1, String alias2) { + if (alias1.equals(alias2)) { + return false; + } + Set nowTypeSet = + Sets.newHashSet(JavaConversions.setAsJavaSet(schema.getNode(alias1).typeNames())); + Set checkTypeSet = + Sets.newHashSet(JavaConversions.setAsJavaSet(schema.getNode(alias2).typeNames())); + nowTypeSet.retainAll(checkTypeSet); + // same type,need check + return !nowTypeSet.isEmpty(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/BaseStartIdRecoder.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/BaseStartIdRecoder.java new file mode 100644 index 00000000..3e683a92 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/BaseStartIdRecoder.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.loader; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public abstract class BaseStartIdRecoder implements Iterator { + private static final Map START_ID_MAP = new ConcurrentHashMap<>(); + + /** base recoder */ + public static BaseStartIdRecoder get(String contextId, int index, boolean init, boolean mem) { + if (init) { + BaseStartIdRecoder recoder; + if (mem) { + recoder = new MemStartIdRecoder(); + } else { + recoder = new DiskStartIdRecorder(getKey(contextId, index)); + } + START_ID_MAP.put(getKey(contextId, index), recoder); + } + return START_ID_MAP.get(getKey(contextId, index)); + } + + /** get start id recoder */ + public static BaseStartIdRecoder get(String contextId, int index) { + return START_ID_MAP.get(getKey(contextId, index)); + } + + /** remove recoder */ + public static void remove(String contextId, int index) { + START_ID_MAP.remove(getKey(contextId, index)); + } + + private static String getKey(String contextId, int index) { + return contextId + index; + } + + public abstract void addStartId(IVertexId id); + + public abstract void flush(); + + public abstract long getStartIdCount(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/DiskStartIdRecorder.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/DiskStartIdRecorder.java new file mode 100644 index 00000000..9b698171 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/DiskStartIdRecorder.java @@ -0,0 +1,203 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.loader; + +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.common.exception.SystemError; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.google.common.collect.Lists; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Predicate; + +public class DiskStartIdRecorder extends MemStartIdRecoder { + + private final String id; + + /** start id recorder on disk */ + public DiskStartIdRecorder(String id) { + this.id = id; + Utils.deletePath(getFileName(id)); + internalIt = null; + } + + @Override + public void flush() { + try { + write(getFileName(id), Lists.newArrayList(startIdSet)); + } catch (Exception e) { + throw new SystemError("write start id error", e); + } finally { + startIdSet = null; + } + internalIt = getStartIdIterator(id, null); + } + + @Override + public long getStartIdCount() { + long count = 0; + String filePath = getFileName(id); + try { + DataInputStream dataInputStream = + new DataInputStream(Files.newInputStream(Paths.get(filePath))); + long size = dataInputStream.readLong(); + dataInputStream.close(); + count += size; + } catch (Throwable e) { + throw new SystemError("load start id file=" + filePath, e); + } + return count; + } + + @Override + public boolean hasNext() { + if (null == internalIt) { + return false; + } + return internalIt.hasNext(); + } + + @Override + public IVertexId next() { + return internalIt.next(); + } + + private static final String START_ID_PATH_PREFIX = "/tmp/holmes/start_id/"; + + private String getFileName(String id) { + return START_ID_PATH_PREFIX + id + "/start_id.bin"; + } + + private void write(String fileName, List idList) throws Exception { + File file = new File(fileName); + if (file.exists()) { + Utils.deletePath(fileName); + } + file.getParentFile().mkdirs(); + + FileOutputStream outputStream = new FileOutputStream(fileName, true); + + ByteBuffer bb = ByteBuffer.allocate(Long.BYTES); + bb.putLong(idList.size()); + outputStream.write(bb.array()); + + for (IVertexId id : idList) { + byte[] idBytes = id.getBytes(); + ByteBuffer byteBuffer = ByteBuffer.allocate(idBytes.length + Integer.BYTES); + byteBuffer.putInt(idBytes.length); + byteBuffer.put(idBytes); + outputStream.write(byteBuffer.array()); + } + outputStream.close(); + } + + private Iterator getStartIdIterator(String id, Predicate idPredicate) { + List idInputStreamList = new ArrayList<>(); + + String filePath = getFileName(id); + try { + DataInputStream dataInputStream = + new DataInputStream(Files.newInputStream(Paths.get(filePath))); + idInputStreamList.add(dataInputStream); + } catch (Exception e) { + throw new SystemError("can not found start id file=" + filePath, e); + } + + return new Iterator() { + private final List inputStreamList = idInputStreamList; + + private int nowStreamIndex = 0; + private long nowSize = -1; + + private IVertexId cachedId = null; + + @Override + public boolean hasNext() { + if (null == cachedId) { + cachedId = getNextId(); + } + return null != cachedId; + } + + @Override + public IVertexId next() { + if (null == cachedId) { + cachedId = getNextId(); + } + IVertexId rst = cachedId; + cachedId = null; + return rst; + } + + private IVertexId getNextId() { + while (true) { + DataInputStream dataInputStream = getNextDataInputStream(); + if (null == dataInputStream) { + return null; + } + + byte[] idBytes; + try { + int idSize = dataInputStream.readInt(); + idBytes = new byte[idSize]; + int r = dataInputStream.read(idBytes); + if (r < 0) { + return null; + } + } catch (Exception e) { + throw new SystemError("read error", e); + } + IVertexId id = IVertexId.from(idBytes); + nowSize--; + if (null != idPredicate && !idPredicate.test(id)) { + continue; + } + return id; + } + } + + private DataInputStream getNextDataInputStream() { + while (nowStreamIndex < inputStreamList.size()) { + if (nowSize > 0) { + return inputStreamList.get(nowStreamIndex); + } + + if (nowSize < 0) { + DataInputStream dataInputStream = inputStreamList.get(nowStreamIndex); + try { + nowSize = dataInputStream.readLong(); + } catch (IOException e) { + throw new SystemError("read error", e); + } + } + if (nowSize == 0) { + nowStreamIndex++; + nowSize = -1; + continue; + } + return inputStreamList.get(nowStreamIndex); + } + return null; + } + }; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/IStartIdRecoder.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/IStartIdRecoder.java new file mode 100644 index 00000000..2c3f694f --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/IStartIdRecoder.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.loader; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; + +public interface IStartIdRecoder { + + void addStartId(IVertexId id); + + void flush(); + + long getStartIdCount(); +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/MemStartIdRecoder.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/MemStartIdRecoder.java new file mode 100644 index 00000000..ab97c0fe --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/loader/MemStartIdRecoder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.loader; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +public class MemStartIdRecoder extends BaseStartIdRecoder { + // use TreeSet, sort vertex id + protected Set startIdSet = new TreeSet<>(); + + protected Iterator internalIt = null; + + @Override + public void addStartId(IVertexId id) { + startIdSet.add(id); + } + + @Override + public void flush() { + this.internalIt = this.startIdSet.iterator(); + } + + @Override + public long getStartIdCount() { + return startIdSet.size(); + } + + @Override + public boolean hasNext() { + return this.internalIt.hasNext(); + } + + @Override + public IVertexId next() { + IVertexId id = this.internalIt.next(); + this.internalIt.remove(); + return id; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/pattern/PatternMatcher.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/pattern/PatternMatcher.java new file mode 100644 index 00000000..13c52da4 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/pattern/PatternMatcher.java @@ -0,0 +1,316 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.pattern; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.SPO; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.type.MapType2IdFactory; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.common.pattern.Connection; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.lube.common.pattern.PatternElement; +import com.antgroup.openspg.reasoner.lube.common.rule.Rule; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Predicate; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Option; +import scala.collection.JavaConversions; + +public class PatternMatcher implements Serializable { + private static final Logger log = LoggerFactory.getLogger(PatternMatcher.class); + private static final long serialVersionUID = -484913653122295682L; + private final String taskId; + private final GraphState graphState; + private long initTime; + + public PatternMatcher(String taskId, GraphState graphState) { + this.taskId = taskId; + this.graphState = graphState; + this.initTime = System.currentTimeMillis(); + } + + public void resetInitTime() { + this.initTime = System.currentTimeMillis(); + } + + /** + * vertex center pattern match + * + * @param id vertex id + * @param pattern pattern + * @param limit, max result limit, null means no constraints are applied + * @return + */ + public KgGraph patternMatch( + IVertexId id, + Long startVersion, + Long endVersion, + Pattern pattern, + List rootVertexRuleList, + Map> dstVertexRuleMap, + Map> edgeRuleMap, + Map> edgeValidTargetIdSet, + Rule vertexRule, + Map> edgeTypeRuleMap, + Long limit, + boolean enableStarPathLimit, + long timeoutMillis) { + + if (timeoutMillis != 0 && (System.currentTimeMillis() - this.initTime) > timeoutMillis) { + log.warn("PatternMatcher patternMatch timeout id=" + id.toString()); + return null; + } + IVertex vertex = graphState.getVertex(id, endVersion, vertexRule); + if (null == vertex) { + return null; + } + + KgGraph kgGraph = new KgGraphImpl(); + Map>> adjEdges = new HashMap<>(); + + // check root types + PatternElement patternElement = pattern.root(); + if (!patternElement.typeNames().contains(RunnerUtil.getVertexType(vertex))) { + return null; + } + + // check root vertex rules + Map vertexContext = RunnerUtil.vertexContext(vertex, pattern.root().alias()); + if (!rootVertexRuleList.isEmpty()) { + if (!RuleRunner.getInstance().check(vertexContext, rootVertexRuleList, this.taskId)) { + return null; + } + } + + // start match edges + Option> patternConnectionSet = + pattern.topology().get(pattern.root().alias()); + if (patternConnectionSet.isEmpty()) { + // no edges need match + kgGraph.init(vertex, adjEdges, pattern); + return kgGraph; + } + Set patternConnections = JavaConversions.setAsJavaSet(patternConnectionSet.get()); + + Map>>> edgeTypeDirectionMap = + new TreeMap<>(); + for (Connection patternConnection : patternConnections) { + Direction direction = patternConnection.direction(); + List edgeSpoTypeList = getEdgeSpoTypeList(patternConnection, pattern); + for (String edgeType : edgeSpoTypeList) { + Map>> directionArrayListMap = + edgeTypeDirectionMap.computeIfAbsent(edgeType, k -> new TreeMap<>()); + if (Direction.BOTH.equals(direction)) { + directionArrayListMap.computeIfAbsent(Direction.OUT, k -> new ArrayList<>()); + directionArrayListMap.computeIfAbsent(Direction.IN, k -> new ArrayList<>()); + } else { + directionArrayListMap.computeIfAbsent(direction, k -> new ArrayList<>()); + } + } + } + + // query edges from graph state + for (String edgeType : edgeTypeDirectionMap.keySet()) { + Map>> directionArrayListMap = + edgeTypeDirectionMap.get(edgeType); + for (Direction direction : directionArrayListMap.keySet()) { + ArrayList> edgeList = directionArrayListMap.get(direction); + List> edges = + graphState.getEdges( + id, + startVersion, + endVersion, + Sets.newHashSet(edgeType), + direction, + edgeTypeRuleMap); + edgeList.addAll(edges); + } + } + + // do edge match + for (Connection patternConnection : patternConnections) { + String edgeAlias = patternConnection.alias(); + Direction direction = patternConnection.direction(); + ArrayList> willMatchEdgeList = new ArrayList<>(); + for (String edgeType : getEdgeSpoTypeList(patternConnection, pattern)) { + Map>> directionArrayListMap = + edgeTypeDirectionMap.get(edgeType); + if (Direction.BOTH.equals(direction)) { + directionArrayListMap.values().forEach(willMatchEdgeList::addAll); + } else { + willMatchEdgeList.addAll(directionArrayListMap.get(direction)); + } + } + + Set validVertexIdSet = edgeValidTargetIdSet.get(edgeAlias); + if (null != validVertexIdSet) { + willMatchEdgeList.removeIf( + new Predicate>() { + @Override + public boolean test(IEdge edge) { + return !validVertexIdSet.contains(edge.getTargetId()); + } + }); + } + + // check dst vertex rule list + List dstVertexRuleList = dstVertexRuleMap.get(patternConnection.target()); + if (CollectionUtils.isNotEmpty(dstVertexRuleList)) { + willMatchEdgeList.removeIf( + new Predicate>() { + @Override + public boolean test(IEdge e) { + Map vertexPropertyMap = RunnerUtil.dstVertexContext(e); + Map context = new HashMap<>(); + context.put(patternConnection.target(), vertexPropertyMap); + return !RuleRunner.getInstance().check(context, dstVertexRuleList, taskId); + } + }); + } + + List> validEdges = + matchEdges( + vertexContext, willMatchEdgeList, patternConnection, pattern, edgeRuleMap, limit); + if (CollectionUtils.isEmpty(validEdges)) { + // one edge pattern connection no match + return null; + } + adjEdges.put(edgeAlias, validEdges); + } + + // do path limit + if (null != limit && enableStarPathLimit && adjEdges.size() > 1) { + RunnerUtil.doStarPathLimit(adjEdges, limit, 10); + } + + kgGraph.init(vertex, adjEdges, pattern); + return kgGraph; + } + + private List> matchEdges( + Map vertexContext, + ArrayList> edgeList, + Connection patternConnection, + Pattern pattern, + Map> edgeRuleMap, + Long limit) { + ArrayList> result = new ArrayList<>(); + long oneTypeEdgeCount = 0; + for (IEdge edge : edgeList) { + if (!isEdgeMatch( + vertexContext, edge, patternConnection, pattern, edgeRuleMap.get(patternConnection))) { + continue; + } + oneTypeEdgeCount++; + if (null != limit && oneTypeEdgeCount > limit) { + // reach max path limit + break; + } + result.add(edge); + } + result.trimToSize(); + return result; + } + + private boolean isEdgeMatch( + Map vertexContext, + IEdge edge, + Connection patternConnection, + Pattern pattern, + List edgeRuleList) { + SPO spo = new SPO(edge.getType()); + // edge type match + if (!patternConnection.relTypes().contains(spo.getP())) { + return false; + } + + // edge target type match + if (!pattern + .getNode(patternConnection.target()) + .typeNames() + .contains(RunnerUtil.getEdgeTargetType(edge))) { + return false; + } + + // edge source type match + if (!pattern + .getNode(patternConnection.source()) + .typeNames() + .contains(RunnerUtil.getEdgeSourceType(edge))) { + return false; + } + + // edge rule match + if (null != edgeRuleList && !edgeRuleList.isEmpty()) { + Map edgeContext = + RunnerUtil.edgeContext(edge, spo, patternConnection.alias()); + edgeContext.putAll(vertexContext); + if (!RuleRunner.getInstance().check(edgeContext, edgeRuleList, this.taskId)) { + return false; + } + } + + return true; + } + + private List getEdgeSpoTypeList(Connection patternConnection, Pattern pattern) { + List edgeSpoTypeList = new ArrayList<>(); + Set pSet = + Sets.newHashSet(JavaConversions.asJavaCollection(patternConnection.relTypes())); + String sourceAlias = patternConnection.source(); + String targetAlias = patternConnection.target(); + if (Direction.IN.equals(patternConnection.direction())) { + sourceAlias = patternConnection.target(); + targetAlias = patternConnection.source(); + } + Set sSet = + Sets.newHashSet(JavaConversions.asJavaCollection(pattern.getNode(sourceAlias).typeNames())); + Set oSet = + Sets.newHashSet(JavaConversions.asJavaCollection(pattern.getNode(targetAlias).typeNames())); + if (Direction.BOTH.equals(patternConnection.direction())) { + sSet.addAll(oSet); + oSet = sSet; + } + + for (String s : sSet) { + for (String p : pSet) { + for (String o : oSet) { + SPO spo = new SPO(s, p, o); + String spoStr = spo.toString(); + Long id = MapType2IdFactory.getMapType2Id().getIdByType(spoStr); + if (null != id) { + edgeSpoTypeList.add(spoStr); + } + } + } + } + return edgeSpoTypeList; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/DecryptUtils.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/DecryptUtils.java new file mode 100644 index 00000000..d8e040c7 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/DecryptUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.progress; + +import java.io.Serializable; +import java.security.Key; +import java.util.Base64; +import javax.crypto.Cipher; +import javax.crypto.spec.SecretKeySpec; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DecryptUtils implements Serializable { + private static final Logger log = LoggerFactory.getLogger(DecryptUtils.class); + + private static final String key = "triedthisfunctio"; + + public static String encryptAccessInfo(String info) { + try { + Key aesKey = new SecretKeySpec(key.getBytes(), "AES"); + Cipher cipher = Cipher.getInstance("AES"); + // encrypt the text + cipher.init(Cipher.ENCRYPT_MODE, aesKey); + byte[] encrypted = cipher.doFinal(info.getBytes()); + return Base64.getEncoder().encodeToString(encrypted); + } catch (Exception e) { + log.error("encrypt error", e); + } + return null; + } + + public static String decryptAccessInfo(String code) { + // Create key and cipher + try { + byte[] codeBytes = Base64.getDecoder().decode(code); + Key aesKey = new SecretKeySpec(key.getBytes(), "AES"); + Cipher cipher = Cipher.getInstance("AES"); + // decrypt the text + cipher.init(Cipher.DECRYPT_MODE, aesKey); + return new String(cipher.doFinal(codeBytes)); + } catch (Exception e) { + log.error("decrypt error", e); + } + return null; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/ProgressReport.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/ProgressReport.java new file mode 100644 index 00000000..55769ee8 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/ProgressReport.java @@ -0,0 +1,108 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.progress; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.progress.ProgressStatus.JobStatus; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +public class ProgressReport { + private static ProgressStatus progressStatus = null; + + /** report error */ + public static void reportError(Map config, Throwable e) { + if (!config.containsKey(ConfigKey.KG_REASONER_PARAMS)) { + return; + } + if (null == progressStatus) { + init(getProgressPath(config), 1); + } + updateStatus(JobStatus.error, e.getMessage()); + } + + /** init progress status report */ + public static void init(Object path, long totalSteps) { + if (null == path) { + progressStatus = null; + return; + } + if (null == progressStatus) { + throw new RuntimeException(); + } + progressStatus.init(totalSteps); + } + + /** set total num */ + public static void setStepTotal(long step, long total) { + if (null != progressStatus) { + progressStatus.setStepTotal(step, total); + } + } + + /** update progress */ + public static void updateProgress(long batchId, long readOffset, long processOffset) { + if (null != progressStatus) { + progressStatus.updateProgress(batchId, readOffset, processOffset); + } + } + + /** finish task */ + public static void finishedProgress() { + if (null != progressStatus) { + progressStatus.finishedProgress(); + } + } + + /** report status */ + public static void updateStatus(JobStatus jobStatus, String errMsg) { + if (null != progressStatus) { + progressStatus.updateStatus(jobStatus, errMsg); + } + } + + /** clear */ + public static void clear() { + progressStatus = null; + } + + /** set time consume type */ + public static void setTimeConsumeType(TimeConsumeType type) { + if (null != progressStatus) { + progressStatus.setTimeConsumeType(type); + } + } + + /** update */ + public static void persistenceProgressStatus() { + if (null != progressStatus) { + progressStatus.persistenceProgressStatus(); + } + } + + private static String getProgressPath(Map config) { + String paramStringEncoded = String.valueOf(config.get(ConfigKey.KG_REASONER_PARAMS)); + String paramsJsonString = + new String(Base64.getDecoder().decode(paramStringEncoded), StandardCharsets.UTF_8); + Map params = new HashMap<>(JSON.parseObject(paramsJsonString)); + return (String) params.get(ConfigKey.KG_REASONER_PROGRESS_PATH); + } + + public static void setProgressStatus(ProgressStatus progressStatus) { + ProgressReport.progressStatus = progressStatus; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/ProgressStatus.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/ProgressStatus.java new file mode 100644 index 00000000..95b245aa --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/ProgressStatus.java @@ -0,0 +1,246 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.progress; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.annotation.JSONField; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class ProgressStatus implements Serializable { + private static final Logger log = LoggerFactory.getLogger(ProgressStatus.class); + + @Getter @Setter private JobStatus status; + @Getter @Setter private ProgressInfo progressInfo; + @Getter private String errMsg; + private final String instanceId; + private final String taskKey; + private final String persistenceWay; + @Getter private final Map timeConsumeMap = new HashMap<>(); + private TimeConsumeType nowType; + private long nowTypeStartMs; + + public ProgressStatus(JSONObject context) { + this.instanceId = context.getString("instanceId"); + this.taskKey = context.getString("taskKey"); + this.progressInfo = new ProgressInfo(); + this.errMsg = context.getString("errMsg"); + this.status = str2Status(context.getString("status")); + this.persistenceWay = "oss"; + } + + public ProgressStatus(String instanceId, String taskKey, String persistenceWay) { + this.status = JobStatus.pending; + this.progressInfo = new ProgressInfo(); + this.instanceId = instanceId; + this.taskKey = taskKey; + this.persistenceWay = persistenceWay; + } + + public void init(long totalSteps) { + this.status = JobStatus.pending; + this.progressInfo.setTotalSteps(totalSteps); + } + + /** Persistent progress status files */ + public abstract void persistenceProgressStatus(); + + /** Retrieve task status information from storage. */ + public abstract void refresh(); + + /** Reset the exists progress status file */ + public abstract void reset(); + + private JobStatus str2Status(String s) { + if ("ERROR".equals(s)) { + return JobStatus.error; + } else if ("FINISH".equals(s)) { + return JobStatus.finished; + } else if ("RUNNING".equals(s)) { + return JobStatus.running; + } else { + return JobStatus.pending; + } + } + + private String status2Str(JobStatus status) { + if (status == JobStatus.error) { + return "ERROR"; + } else if (status == JobStatus.finished) { + return "FINISH"; + } else if (status == JobStatus.running) { + return "RUNNING"; + } else { + // pending + return "WAIT"; + } + } + + public String toJson() { + Map context = new HashMap<>(); + context.put("instanceId", this.instanceId); + context.put("taskKey", this.taskKey); + context.put("errMsg", this.errMsg); + context.put("progressInfo", this.progressInfo); + context.put("status", status2Str(this.status)); + context.put("timeConsumeMap", this.timeConsumeMap); + context.put("ts", System.currentTimeMillis() / 1000); + return JSON.toJSONString(context, true); + } + + public void updateStatus(JobStatus jobStatus, String errMsg) { + this.status = jobStatus; + this.errMsg = errMsg; + persistenceProgressStatus(); + } + + public void finishedProgress() { + this.status = JobStatus.finished; + this.progressInfo.setProcessOffset(this.progressInfo.getRealTotal()); + this.progressInfo.setReadOffset(this.progressInfo.getRealTotal()); + persistenceProgressStatus(); + } + + public boolean isFinished() { + return JobStatus.finished == this.status; + } + + public boolean isError() { + return JobStatus.error == this.status; + } + + public void setStepTotal(long step, long total) { + if (isFinished()) { + throw new RuntimeException( + "task is finished, can not change status, now status=" + this.status); + } + if (step == this.progressInfo.getCurStep() && total == this.progressInfo.getTotal()) { + // The same steps do not need to be reset. + return; + } + this.status = JobStatus.running; + this.progressInfo.setTotal(total); + this.progressInfo.setCurStep(step); + if (this.progressInfo.getTotalSteps() < this.progressInfo.getCurStep()) { + this.progressInfo.setTotalSteps(step); + } + this.progressInfo.setProcessOffset(0); + this.progressInfo.setReadOffset(0); + + persistenceProgressStatus(); + } + + public void updateProgress(long batchId, long readOffset, long processOffset) { + if (isFinished()) { + throw new RuntimeException( + "task is finished, can not change status, now status=" + this.status); + } + this.status = JobStatus.running; + this.progressInfo.setBatchId(batchId); + this.progressInfo.setReadOffset(readOffset); + this.progressInfo.setProcessOffset(processOffset); + if (this.progressInfo.getProcessOffset() > this.progressInfo.getTotal()) { + log.warn( + "ProgressStatus::updateProgress ProcessOffset(" + + this.progressInfo.getProcessOffset() + + ") > Total(" + + this.progressInfo.getTotal() + + ")"); + this.progressInfo.setTotal(this.progressInfo.getRealProcessOffset()); + } + + persistenceProgressStatus(); + } + + public void setTimeConsumeType(TimeConsumeType type) { + long nowMs = System.currentTimeMillis(); + + if (null == nowType) { + nowType = type; + nowTypeStartMs = nowMs; + } else if (!nowType.equals(type)) { + Long oldV = timeConsumeMap.getOrDefault(nowType, 0L); + timeConsumeMap.put(nowType, oldV + (nowMs - nowTypeStartMs)); + + nowType = type; + nowTypeStartMs = nowMs; + } + } + + public enum JobStatus implements Serializable { + running, + pending, + error, + finished + } + + @Getter + @Setter + public static class ProgressInfo implements Serializable { + private long batchId; + private long totalSteps; + private long curStep; + private long total; + private long readOffset; + private long processOffset; + + private long shrinkFactor = 1; + + public long getTotal() { + return total / this.shrinkFactor; + } + + @JSONField(serialize = false) + public long getRealTotal() { + return total; + } + + public void setTotal(long total) { + if (total <= Integer.MAX_VALUE) { + shrinkFactor = 1; + } else { + long tmpTotal = total; + while (tmpTotal > Integer.MAX_VALUE) { + tmpTotal = tmpTotal / 10; + shrinkFactor = shrinkFactor * 10; + } + } + this.total = total; + } + + @JSONField(serialize = false) + public long getRealReadOffset() { + return readOffset; + } + + public long getReadOffset() { + return readOffset / this.shrinkFactor; + } + + public long getProcessOffset() { + return processOffset / this.shrinkFactor; + } + + @JSONField(serialize = false) + public long getRealProcessOffset() { + return processOffset; + } + } +} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/BaseScheduler.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/TimeConsumeType.java similarity index 72% rename from server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/BaseScheduler.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/TimeConsumeType.java index 7b26cf68..2b02d916 100644 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/BaseScheduler.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/progress/TimeConsumeType.java @@ -10,12 +10,12 @@ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. */ +package com.antgroup.openspg.reasoner.progress; -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.scheduler; +import java.io.Serializable; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Getter -public abstract class BaseScheduler {} +public enum TimeConsumeType implements Serializable { + WAIT_RESOURCE, + LOAD_GRAPH, + COMPUTE, +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/ExtractRelationImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/ExtractRelationImpl.java new file mode 100644 index 00000000..058b6313 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/ExtractRelationImpl.java @@ -0,0 +1,163 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.SPO; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.block.AddPredicate; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.common.pattern.Element; +import com.antgroup.openspg.reasoner.lube.common.pattern.EntityElement; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.lube.common.pattern.PatternElement; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.antgroup.openspg.reasoner.warehouse.utils.WareHouseUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import scala.collection.JavaConversions; + +public class ExtractRelationImpl implements Serializable { + + private static final long serialVersionUID = 3442064493302533370L; + private final AddPredicate addPredicate; + + @Getter private final String predicate; + private final Direction direction; + private final boolean withReverseEdge; + + private final Pattern kgGraphSchema; + private final Map initRuleContext; + private final long version; + + private final Map> propertyRuleMap = new HashMap<>(); + + private final String taskId; + + private final PatternElement sourceElement; + private final EntityElement targetEntityElement; + private final PatternElement targetPatternElement; + + /** ddl add edge implement */ + public ExtractRelationImpl( + AddPredicate addPredicate, Pattern kgGraphSchema, long version, String taskId) { + this.addPredicate = addPredicate; + this.kgGraphSchema = kgGraphSchema; + this.initRuleContext = RunnerUtil.getKgGraphInitContext(this.kgGraphSchema); + this.version = version; + this.taskId = taskId; + + this.predicate = this.addPredicate.predicate().label(); + this.direction = this.addPredicate.predicate().direction(); + this.withReverseEdge = !(addPredicate.predicate().target() instanceof EntityElement); + + Map propertyExprMap = + JavaConversions.mapAsJavaMap(addPredicate.predicate().fields()); + for (String propertyName : propertyExprMap.keySet()) { + List rule = WareHouseUtils.getRuleList(propertyExprMap.get(propertyName)); + this.propertyRuleMap.put(propertyName, rule); + } + + PatternElement sourceElement = (PatternElement) addPredicate.predicate().source(); + Element te = addPredicate.predicate().target(); + EntityElement targetEntityElement = null; + PatternElement targetPatternElement = null; + if (te instanceof EntityElement) { + targetEntityElement = (EntityElement) te; + this.propertyRuleMap.put( + Constants.EDGE_TO_ID_KEY, Lists.newArrayList("'" + targetEntityElement.id() + "'")); + } else { + targetPatternElement = (PatternElement) te; + this.propertyRuleMap.put( + Constants.EDGE_TO_ID_KEY, Lists.newArrayList(targetPatternElement.alias() + ".id")); + } + this.propertyRuleMap.put( + Constants.EDGE_FROM_ID_KEY, Lists.newArrayList(sourceElement.alias() + ".id")); + + this.sourceElement = sourceElement; + this.targetEntityElement = targetEntityElement; + this.targetPatternElement = targetPatternElement; + } + + /** get ddl edges */ + public IEdge extractEdge(KgGraph kgGraph) { + IVertexId s = kgGraph.getVertex(sourceElement.alias()).get(0).getId(); + IVertexId o = getTargetVertexId(targetEntityElement, targetPatternElement, kgGraph); + + IEdge willAddedEdge = + new Edge<>(s, o, getEdgeProperty(kgGraph), version, direction, getEdgeType(s, o)); + + Map>> alias2EdgeMap = new HashMap<>(); + alias2EdgeMap.put(this.addPredicate.predicate().alias(), Sets.newHashSet(willAddedEdge)); + KgGraph newKgGraph = new KgGraphImpl(new HashMap<>(), alias2EdgeMap); + kgGraph.merge(Lists.newArrayList(newKgGraph), null); + + return willAddedEdge; + } + + private IProperty getEdgeProperty(KgGraph kgGraph) { + Map edgeProps = new HashMap<>(); + Map context = RunnerUtil.kgGraph2Context(this.initRuleContext, kgGraph); + for (String propertyName : this.propertyRuleMap.keySet()) { + List ruleList = this.propertyRuleMap.get(propertyName); + Object value = RuleRunner.getInstance().executeExpression(context, ruleList, this.taskId); + edgeProps.put(propertyName, value); + } + return new EdgeProperty(edgeProps); + } + + private IVertexId getTargetVertexId( + EntityElement entityElement, PatternElement patternElement, KgGraph path) { + if (null != entityElement) { + return IVertexId.from(String.valueOf(entityElement.id()), entityElement.label()); + } + return path.getVertex(patternElement.alias()).get(0).getId(); + } + + private String getEdgeType(IVertexId s, IVertexId o) { + SPO spo = new SPO(s.getType(), predicate, o.getType()); + return spo.toString(); + } + + /** reverse edge */ + public IEdge createReverseEdge(IEdge edge) { + return new Edge<>( + edge.getTargetId(), + edge.getSourceId(), + edge.getValue(), + edge.getVersion(), + Utils.reverseDirection(edge.getDirection()), + edge.getType()); + } + + /** need add reverse edge */ + public boolean withReverseEdge() { + return this.withReverseEdge; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/ExtractVertexImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/ExtractVertexImpl.java new file mode 100644 index 00000000..b1de356a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/ExtractVertexImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.common.utils.PropertyUtil; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.block.AddVertex; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.antgroup.openspg.reasoner.warehouse.utils.WareHouseUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import scala.collection.JavaConversions; + +public class ExtractVertexImpl implements Serializable { + + private static final long serialVersionUID = 7788704894791547885L; + private final Pattern kgGraphSchema; + private final Map initRuleContext; + private final long version; + + private final Map> propertyRuleMap = new HashMap<>(); + private final String type; + private final String alias; + private final String taskId; + + /** ddl add edge implement */ + public ExtractVertexImpl( + AddVertex addVertex, Pattern kgGraphSchema, long version, String taskId) { + this.kgGraphSchema = kgGraphSchema; + this.initRuleContext = RunnerUtil.getKgGraphInitContext(this.kgGraphSchema); + this.version = version; + this.taskId = taskId; + + Map propertyExprMap = JavaConversions.mapAsJavaMap(addVertex.props()); + for (String propertyName : propertyExprMap.keySet()) { + List rule = WareHouseUtils.getRuleList(propertyExprMap.get(propertyName)); + propertyRuleMap.put(propertyName, rule); + } + this.type = addVertex.s().typeNames().iterator().next(); + this.alias = addVertex.s().alias(); + } + + /** get ddl vertex */ + public IVertex extractVertex(KgGraph kgGraph) { + Map vertexProps = new HashMap<>(); + Map context = RunnerUtil.kgGraph2Context(this.initRuleContext, kgGraph); + for (String propertyName : this.propertyRuleMap.keySet()) { + List ruleList = this.propertyRuleMap.get(propertyName); + Object value = RuleRunner.getInstance().executeExpression(context, ruleList, this.taskId); + vertexProps.put(propertyName, value); + } + IVertexId vertexId; + if (vertexProps.containsKey("id")) { + vertexId = IVertexId.from(String.valueOf(vertexProps.get("id")), this.type); + } else { + StringBuilder vertexIdSb = new StringBuilder(); + List aliasList = Lists.newArrayList(kgGraph.getVertexAlias()); + aliasList.sort(String::compareTo); + for (String vertexAlias : aliasList) { + vertexIdSb.append(kgGraph.getVertex(vertexAlias).get(0).getId()); + } + vertexId = IVertexId.from(vertexIdSb.toString(), this.type); + } + IVertex willAddedVertex = + new Vertex<>(vertexId, getVertexProperty(vertexId, this.version, context)); + + Map>> alias2VertexMap = new HashMap<>(); + alias2VertexMap.put(this.alias, Sets.newHashSet(willAddedVertex)); + KgGraph newKgGraph = new KgGraphImpl(alias2VertexMap, new HashMap<>()); + kgGraph.merge(Lists.newArrayList(newKgGraph), null); + return willAddedVertex; + } + + private IProperty getVertexProperty( + IVertexId vertexId, long version, Map context) { + Map> property = new HashMap<>(); + for (String propertyName : this.propertyRuleMap.keySet()) { + List ruleList = this.propertyRuleMap.get(propertyName); + Object value = RuleRunner.getInstance().executeExpression(context, ruleList, this.taskId); + TreeMap versionValueMap = new TreeMap<>(); + versionValueMap.put(version, value); + property.put(propertyName, versionValueMap); + } + return PropertyUtil.buildVertexProperty(vertexId, property); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/FoldEdgeImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/FoldEdgeImpl.java new file mode 100644 index 00000000..0c970ac2 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/FoldEdgeImpl.java @@ -0,0 +1,180 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.OptionalEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.PathEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.MirrorVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.common.pattern.Connection; +import com.antgroup.openspg.reasoner.lube.common.pattern.PartialGraphPattern; +import com.antgroup.openspg.reasoner.util.PathConnection; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class FoldEdgeImpl implements Serializable { + + private static final long serialVersionUID = -3632032496948942544L; + private final FoldRepeatEdgeInfo foldRepeatEdgeInfo; + + private FoldRepeatEdgeInfo noneMatchRepeatFoldInfo = null; + + public FoldEdgeImpl(PartialGraphPattern kgGraphSchema, FoldRepeatEdgeInfo foldRepeatEdgeInfo) { + for (Connection connection : RunnerUtil.getConnectionSet(kgGraphSchema)) { + if (!(connection instanceof PathConnection)) { + continue; + } + PathConnection pathConnection = (PathConnection) connection; + noneMatchRepeatFoldInfo = + new FoldRepeatEdgeInfo( + pathConnection.alias(), + pathConnection.alias(), + pathConnection.source(), + pathConnection.target()); + } + this.foldRepeatEdgeInfo = foldRepeatEdgeInfo; + } + + private void foldNone(KgGraphImpl kgGraph) { + Set> vertexSet = + kgGraph.getAlias2VertexMap().get(this.noneMatchRepeatFoldInfo.getFromVertexAlias()); + Set> newVertexSet = new HashSet<>(); + Set> newEdgeSet = new HashSet<>(); + for (IVertex vertex : vertexSet) { + IVertex newVertex = new MirrorVertex<>(vertex); + newVertexSet.add(newVertex); + + IEdge newEdge = new OptionalEdge<>(vertex.getId(), vertex.getId()); + newEdgeSet.add(newEdge); + } + kgGraph.getAlias2VertexMap().put(this.noneMatchRepeatFoldInfo.getToVertexAlias(), newVertexSet); + kgGraph.getAlias2EdgeMap().put(this.noneMatchRepeatFoldInfo.getToEdgeAlias(), newEdgeSet); + } + + public List> fold(KgGraphImpl kgGraph) { + List> result = new ArrayList<>(); + Set> pathEdgeSet = + kgGraph.getAlias2EdgeMap().get(this.foldRepeatEdgeInfo.getToEdgeAlias()); + + Set> fromEdgeSet = + kgGraph.getAlias2EdgeMap().remove(this.foldRepeatEdgeInfo.getFromEdgeAlias()); + if (null == fromEdgeSet || fromEdgeSet.isEmpty()) { + // repeat match nothing + Set> toVertexSet = + kgGraph.getAlias2VertexMap().get(this.foldRepeatEdgeInfo.getToVertexAlias()); + if (null == toVertexSet) { + foldNone(kgGraph); + } + result.add(kgGraph); + return result; + } else { + IEdge edge = fromEdgeSet.iterator().next(); + if (edge instanceof OptionalEdge) { + if (null != pathEdgeSet) { + kgGraph.getAlias2VertexMap().remove(this.foldRepeatEdgeInfo.getFromVertexAlias()); + result.add(kgGraph); + return result; + } + Set> fromVertexSet = + kgGraph.getAlias2VertexMap().remove(this.foldRepeatEdgeInfo.getFromVertexAlias()); + + Set> newVertexSet = new HashSet<>(); + for (IVertex vertex : fromVertexSet) { + newVertexSet.add(new MirrorVertex<>(vertex)); + } + kgGraph.getAlias2VertexMap().put(this.foldRepeatEdgeInfo.getToVertexAlias(), newVertexSet); + kgGraph.getAlias2EdgeMap().put(this.foldRepeatEdgeInfo.getToEdgeAlias(), fromEdgeSet); + result.add(kgGraph); + return result; + } + } + + if (null == pathEdgeSet) { + // Construct the pathEdge in the first wind + Set> toEdgeSet = new HashSet<>(); + for (IEdge edge : fromEdgeSet) { + PathEdge pathEdge = + new PathEdge<>((Edge) edge); + toEdgeSet.add(pathEdge); + } + kgGraph.getAlias2EdgeMap().put(this.foldRepeatEdgeInfo.getToEdgeAlias(), toEdgeSet); + + // Rename node + kgGraph + .getAlias2VertexMap() + .put( + this.foldRepeatEdgeInfo.getToVertexAlias(), + kgGraph.getAlias2VertexMap().remove(this.foldRepeatEdgeInfo.getFromVertexAlias())); + result.add(kgGraph); + return result; + } + + // An existing PathEdge is present; increase the length of the PathEdge. + + // This map has only one value. + Set> toVertexSet = + kgGraph.getAlias2VertexMap().get(this.foldRepeatEdgeInfo.getToVertexAlias()); + Map> toVertexMap = new HashMap<>(); + for (IVertex v : toVertexSet) { + toVertexMap.put(v.getId(), v); + } + + Map>> fromEdgeMap = new HashMap<>(); + for (IEdge edge : fromEdgeSet) { + fromEdgeMap.computeIfAbsent(edge.getSourceId(), k -> new ArrayList<>()).add(edge); + } + + // Construct a new PathEdge. + Set> newPathEdgeSet = new HashSet<>(); + for (IEdge edge : pathEdgeSet) { + PathEdge pathEdge = + (PathEdge) edge; + IVertexId pathEdgeSearchId = pathEdge.getTargetId(); + Vertex nextVertex = + (Vertex) toVertexMap.get(pathEdgeSearchId); + List> nextEdgeList = fromEdgeMap.get(pathEdgeSearchId); + for (IEdge nextIEdge : nextEdgeList) { + Edge nextEdge = (Edge) nextIEdge; + PathEdge newPathEdge = + new PathEdge<>(pathEdge, nextVertex, nextEdge); + if (newPathEdge.haveLoop()) { + continue; + } + newPathEdgeSet.add(newPathEdge); + } + } + + kgGraph.getAlias2EdgeMap().put(this.foldRepeatEdgeInfo.getToEdgeAlias(), newPathEdgeSet); + kgGraph + .getAlias2VertexMap() + .put( + this.foldRepeatEdgeInfo.getToVertexAlias(), + kgGraph.getAlias2VertexMap().remove(this.foldRepeatEdgeInfo.getFromVertexAlias())); + result.add(kgGraph); + return result; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/FoldRepeatEdgeInfo.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/FoldRepeatEdgeInfo.java new file mode 100644 index 00000000..986797c8 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/FoldRepeatEdgeInfo.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.exception.IllegalArgumentException; +import java.io.Serializable; +import org.apache.commons.lang3.StringUtils; + +public class FoldRepeatEdgeInfo implements Serializable { + private final String fromEdgeAlias; + private final String toEdgeAlias; + private final String fromVertexAlias; + private final String toVertexAlias; + + public FoldRepeatEdgeInfo( + String fromEdgeAlias, String toEdgeAlias, String fromVertexAlias, String toVertexAlias) { + if (StringUtils.isEmpty(fromEdgeAlias) + || StringUtils.isEmpty(toEdgeAlias) + || StringUtils.isEmpty(fromVertexAlias) + || StringUtils.isEmpty(toEdgeAlias)) { + throw new IllegalArgumentException("no empty string", "", "", null); + } + this.fromEdgeAlias = fromEdgeAlias; + this.toEdgeAlias = toEdgeAlias; + this.fromVertexAlias = fromVertexAlias; + this.toVertexAlias = toVertexAlias; + } + + /** + * Getter method for property fromEdgeAlias. + * + * @return property value of fromEdgeAlias + */ + public String getFromEdgeAlias() { + return fromEdgeAlias; + } + + /** + * Getter method for property toEdgeAlias. + * + * @return property value of toEdgeAlias + */ + public String getToEdgeAlias() { + return toEdgeAlias; + } + + /** + * Getter method for property fromVertexAlias. + * + * @return property value of fromVertexAlias + */ + public String getFromVertexAlias() { + return fromVertexAlias; + } + + /** + * Getter method for property toVertexAlias. + * + * @return property value of toVertexAlias + */ + public String getToVertexAlias() { + return toVertexAlias; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/GroupByKeyItem.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/GroupByKeyItem.java new file mode 100644 index 00000000..b2aa58bb --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/GroupByKeyItem.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import java.util.Arrays; + +public class GroupByKeyItem { + private final Object[] keys; + + public GroupByKeyItem(Object[] keys) { + this.keys = keys; + } + + @Override + public int hashCode() { + return Arrays.hashCode(keys); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof GroupByKeyItem)) { + return false; + } + GroupByKeyItem other = (GroupByKeyItem) obj; + return Arrays.equals(this.keys, other.keys); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/GroupByKgGraphImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/GroupByKgGraphImpl.java new file mode 100644 index 00000000..20a67779 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/GroupByKgGraphImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.collections4.CollectionUtils; + +public class GroupByKgGraphImpl implements Serializable { + + private static final long serialVersionUID = 5982445397315165348L; + protected final List byAliasList; + protected final Set byAliasSet; + protected final Pattern schema; + protected final Long limit; + + private final KgGraphSplitStaticParameters staticParameters; + + private final KgGraphListProcess kgGraphListProcess; + + /** impl */ + public GroupByKgGraphImpl( + List byAliasList, KgGraphListProcess kgGraphListProcess, Pattern schema, Long limit) { + this.byAliasList = byAliasList; + this.byAliasSet = Sets.newHashSet(byAliasList); + this.schema = schema; + this.limit = limit; + this.staticParameters = new KgGraphSplitStaticParameters(this.byAliasSet, this.schema); + this.kgGraphListProcess = kgGraphListProcess; + } + + /** return list */ + public List> groupReduce(Collection> sameRootKgGraphList) { + if (CollectionUtils.isEmpty(this.byAliasList)) { + return doReduce(sameRootKgGraphList); + } + return doSplitThenReduce(sameRootKgGraphList); + } + + private List> doReduce(Collection> sameRootKgGraphList) { + return new ArrayList<>(this.kgGraphListProcess.reduce(sameRootKgGraphList)); + } + + private List> doSplitThenReduce(Collection> values) { + Map>> kgGraphMap = new HashMap<>(); + for (KgGraph value : values) { + List> kgGraphList = + value.split(byAliasSet, schema, staticParameters, null, limit); + for (KgGraph kgGraph : kgGraphList) { + ByKey byKey = getByKey(kgGraph); + List> existKgGraphList = + kgGraphMap.computeIfAbsent(byKey, k -> new ArrayList<>()); + existKgGraphList.add(kgGraph); + } + } + List> result = new ArrayList<>(kgGraphMap.size()); + for (List> kgGraphList : kgGraphMap.values()) { + result.addAll(this.kgGraphListProcess.reduce(kgGraphList)); + } + return result; + } + + protected ByKey getByKey(KgGraph kgGraph) { + IVertexId[] vertexIds = new IVertexId[this.byAliasList.size()]; + for (int i = 0; i < this.byAliasList.size(); ++i) { + String alias = this.byAliasList.get(i); + vertexIds[i] = kgGraph.getVertex(alias).get(0).getId(); + } + return new ByKey(vertexIds); + } + + /** vertex id array */ + public static class ByKey { + protected final IVertexId[] vertexIds; + + /** init */ + public ByKey(IVertexId[] vertexIds) { + this.vertexIds = vertexIds; + } + + @Override + public int hashCode() { + return Arrays.hashCode(vertexIds); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ByKey)) { + return false; + } + ByKey other = (ByKey) obj; + return Arrays.equals(this.vertexIds, other.vertexIds); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphAddFieldsImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphAddFieldsImpl.java new file mode 100644 index 00000000..bdd28b1d --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphAddFieldsImpl.java @@ -0,0 +1,145 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.exception.IllegalArgumentException; +import com.antgroup.openspg.reasoner.common.graph.type.GraphItemType; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.PartialGraphPattern; +import com.antgroup.openspg.reasoner.lube.logical.EdgeVar; +import com.antgroup.openspg.reasoner.lube.logical.NodeVar; +import com.antgroup.openspg.reasoner.lube.logical.PropertyVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.util.KgGraphSchema; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import lombok.Builder; +import lombok.Data; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public class KgGraphAddFieldsImpl implements Serializable { + private static final long serialVersionUID = -4500069867630323486L; + private final PartialGraphPattern kgGraphSchema; + private final Map initRuleContext; + private final KgGraphSplitStaticParameters staticParameters; + private final long version; + private final String taskId; + private final List addFieldsInfoList = new ArrayList<>(); + + /** add fields implement */ + public KgGraphAddFieldsImpl( + Map> addFieldsInfo, + PartialGraphPattern kgGraphSchema, + long version, + String taskId) { + this.taskId = taskId; + this.kgGraphSchema = kgGraphSchema; + this.initRuleContext = RunnerUtil.getKgGraphInitContext(this.kgGraphSchema); + this.staticParameters = new KgGraphSplitStaticParameters(null, this.kgGraphSchema); + this.version = version; + Map alias2TypeMap = + new HashMap<>(JavaConversions.mapAsJavaMap(KgGraphSchema.alias2Type(this.kgGraphSchema))); + + for (Map.Entry> entry : addFieldsInfo.entrySet()) { + Var field = entry.getKey(); + String alias; + List propertyNameList = new ArrayList<>(); + + if (field instanceof NodeVar) { + NodeVar nodeField = (NodeVar) field; + alias = nodeField.name(); + JavaConversions.setAsJavaSet(nodeField.fields()) + .forEach(f -> propertyNameList.add(f.name())); + } else if (field instanceof EdgeVar) { + EdgeVar edgeField = (EdgeVar) field; + alias = edgeField.name(); + JavaConversions.setAsJavaSet(edgeField.fields()) + .forEach(f -> propertyNameList.add(f.name())); + } else if (field instanceof PropertyVar) { + PropertyVar propertyVar = (PropertyVar) field; + alias = propertyVar.name(); + propertyNameList.add(propertyVar.field().name()); + } else { + throw new IllegalArgumentException( + "NodeVar or EdgeVar", field.getClass().getName(), "", null); + } + this.addFieldsInfoList.add( + AddFieldsInfo.builder() + .alias(alias) + .type(alias2TypeMap.get(alias)) + .propertyNameList(propertyNameList) + .expressionList(entry.getValue()) + .build()); + } + } + + /** implement */ + public List> map(KgGraph value) { + List> result = new ArrayList<>(); + Iterator> it = value.getPath(staticParameters, null); + while (it.hasNext()) { + KgGraph path = it.next(); + if (path == null) { + continue; + } + Map context = RunnerUtil.kgGraph2Context(this.initRuleContext, path); + Map, Map> alias2UpdatePropertyMap = + new HashMap<>(); + for (AddFieldsInfo addFieldsInfo : this.addFieldsInfoList) { + Tuple2 key = + new Tuple2<>(addFieldsInfo.getAlias(), addFieldsInfo.getType()); + Object expressionResult = + RuleRunner.getInstance() + .executeExpression(context, addFieldsInfo.getExpressionList(), this.taskId); + Map propertyMap = + alias2UpdatePropertyMap.computeIfAbsent(key, k -> new HashMap<>()); + for (String propertyName : addFieldsInfo.getPropertyNameList()) { + propertyMap.put(propertyName, expressionResult); + } + } + + for (Map.Entry, Map> entry : + alias2UpdatePropertyMap.entrySet()) { + String alias = entry.getKey()._1(); + GraphItemType type = entry.getKey()._2(); + Map propertyMap = entry.getValue(); + if (GraphItemType.VERTEX.equals(type)) { + path.setVertexProperty(alias, propertyMap, version); + } else { + path.setEdgeProperty(alias, propertyMap); + } + } + result.add(path); + } + return result; + } + + @Data + @Builder + private static class AddFieldsInfo implements Serializable { + private String alias; + private GraphItemType type; + private List propertyNameList; + private List expressionList; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphAggregateImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphAggregateImpl.java new file mode 100644 index 00000000..03eff7bb --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphAggregateImpl.java @@ -0,0 +1,372 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.AggregationSchemaInfo; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.expr.AggIfOpExpr; +import com.antgroup.openspg.reasoner.lube.common.expr.AggOpExpr; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.common.expr.GetField; +import com.antgroup.openspg.reasoner.lube.common.expr.Ref; +import com.antgroup.openspg.reasoner.lube.common.expr.UnaryOpExpr; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.lube.logical.EdgeVar; +import com.antgroup.openspg.reasoner.lube.logical.NodeVar; +import com.antgroup.openspg.reasoner.lube.logical.PropertyVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.rdg.common.groupProcess.AggIfOpProcessBaseGroupProcess; +import com.antgroup.openspg.reasoner.rdg.common.groupProcess.AggOpProcessBaseGroupProcess; +import com.antgroup.openspg.reasoner.rdg.common.groupProcess.BaseGroupProcess; +import com.antgroup.openspg.reasoner.udf.model.BaseUdaf; +import com.antgroup.openspg.reasoner.udf.model.UdafMeta; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.apache.commons.collections4.CollectionUtils; +import scala.Tuple2; + +public class KgGraphAggregateImpl implements Serializable { + + private static final long serialVersionUID = -4440174716170402565L; + private final Pattern kgGraphSchema; + private final KgGraphSplitStaticParameters staticParameters; + private final Set vertexAliasSet; + private final Map aggregatorMap; + private final Long maxPathLimit; + private final String taskId; + private final Set byAliasSet; + + private final AggregationSchemaInfo aggregationSchemaInfo; + + // Property aggregator is given priority processing. + private transient Map> propertyAggregatorParsedMap; + + // Process after aggregation by 'first'. + private transient Map> aggregatorParsedMap; + + /** aggregate implement */ + public KgGraphAggregateImpl( + String taskId, + String rootAlias, + List byAliasList, + Pattern kgGraphSchema, + Map aggregatorMap, + Long maxPathLimit) { + this.kgGraphSchema = kgGraphSchema; + this.aggregationSchemaInfo = new AggregationSchemaInfo(this.kgGraphSchema); + this.staticParameters = new KgGraphSplitStaticParameters(null, this.kgGraphSchema); + this.vertexAliasSet = RunnerUtil.getVertexAliasSet(kgGraphSchema); + this.aggregatorMap = new HashMap<>(aggregatorMap); + this.maxPathLimit = maxPathLimit; + this.taskId = taskId; + this.byAliasSet = Sets.newHashSet(byAliasList); + this.byAliasSet.add(rootAlias); + } + + private BaseGroupProcess getGroupProcess(Var var, Aggregator aggOpExpr) { + if (aggOpExpr instanceof AggOpExpr) { + return new AggOpProcessBaseGroupProcess(this.taskId, var, aggOpExpr); + } + + if (aggOpExpr instanceof AggIfOpExpr) { + return new AggIfOpProcessBaseGroupProcess(this.taskId, var, aggOpExpr); + } + throw new NotImplementedException( + "unsupported aggregator, type=" + aggOpExpr.getClass().getName(), null); + } + + /** init implement */ + public void init() { + this.aggregatorParsedMap = new HashMap<>(); + this.propertyAggregatorParsedMap = new HashMap<>(); + for (Map.Entry entry : this.aggregatorMap.entrySet()) { + Var var = entry.getKey(); + String aliasKey = var.name(); + Aggregator aggregator = entry.getValue(); + + BaseGroupProcess aggOpProcess = getGroupProcess(var, aggregator); + List list; + if (aggOpProcess.isFirstAgg()) { + list = this.aggregatorParsedMap.computeIfAbsent(aliasKey, k -> new ArrayList<>()); + } else { + list = this.propertyAggregatorParsedMap.computeIfAbsent(aliasKey, k -> new ArrayList<>()); + } + list.add(aggOpProcess); + } + } + + /** process property var */ + public Map propertyVarMap( + Collection> values, List aggInfoList) { + + Map propertyMap = new HashMap<>(); + for (BaseGroupProcess aggInfo : aggInfoList) { + PropertyVar var = (PropertyVar) aggInfo.getVar(); + + // Perform aggregation calculation. + UdafMeta udafMeta = aggInfo.getUdafMeta(); + Object[] udafInitParams = aggInfo.getUdfInitParams(); + List ruleList = aggInfo.getRuleList(); + List> valueFilteredList = getValueFilteredList(values, ruleList); + Object aggValue = + doAggregation(valueFilteredList, udafMeta, udafInitParams, aggInfo.getAggEle()); + String targetPropertyName = var.field().name(); + propertyMap.put(targetPropertyName, aggValue); + } + return propertyMap; + } + + private List> getValueFilteredList( + Collection> values, List ruleList) { + List> valueFilteredList = new ArrayList<>(); + for (KgGraph value : values) { + if (CollectionUtils.isNotEmpty(ruleList)) { + KgGraph kgGraph = + RunnerUtil.filterKgGraph( + value, kgGraphSchema, staticParameters, ruleList, maxPathLimit); + if (null != kgGraph) { + valueFilteredList.add(kgGraph); + } + } else { + valueFilteredList.add(value); + } + } + return valueFilteredList; + } + + /** do aggregate */ + public KgGraph map(Collection> values) { + // Aggregating should according to graph, so we need remove duplicate vertexes and edges + // according to ID + KgGraphImpl kgGraph = new KgGraphImpl(); + kgGraph.merge(values, this.kgGraphSchema); + + // use merged kg graph as input + values = Lists.newArrayList(kgGraph); + Map> alias2PropertyMap = new HashMap<>(); + for (Map.Entry> entry : + this.propertyAggregatorParsedMap.entrySet()) { + String alias = entry.getKey(); + alias2PropertyMap.put(alias, propertyVarMap(values, entry.getValue())); + } + + KgGraph value = values.iterator().next(); + + for (Map.Entry> entry : alias2PropertyMap.entrySet()) { + String alias = entry.getKey(); + Map propertyMap = entry.getValue(); + if (vertexAliasSet.contains(alias)) { + IVertex vertex = value.getVertex(alias).get(0); + value.aggregateVertex( + alias, Sets.newHashSet(vertex), this.aggregationSchemaInfo, new HashSet<>()); + aggToVirtualVertexId((KgGraphImpl) value, alias, null); + value.setVertexProperty(alias, propertyMap, 0L); + } else { + IEdge edge = value.getEdge(alias).get(0); + value.aggregateEdge( + alias, Sets.newHashSet(edge), this.aggregationSchemaInfo, new HashSet<>()); + aggToVirtualEdgeId((KgGraphImpl) value, alias); + value.setEdgeProperty(alias, propertyMap); + } + } + + for (Map.Entry> entry : this.aggregatorParsedMap.entrySet()) { + String alias = entry.getKey(); + for (BaseGroupProcess aggInfo : entry.getValue()) { + Var var = aggInfo.getVar(); + + // Perform aggregate calculations. + UdafMeta udafMeta = aggInfo.getUdafMeta(); + Object[] udafInitParams = aggInfo.getUdfInitParams(); + List ruleList = aggInfo.getRuleList(); + List> valueFilteredList = getValueFilteredList(values, ruleList); + Object aggValue = + doAggregation(valueFilteredList, udafMeta, udafInitParams, aggInfo.getAggEle()); + + // Assign the aggregation result. + if (var instanceof NodeVar) { + IVertex vertex = (IVertex) aggValue; + value.aggregateVertex( + alias, Sets.newHashSet(vertex), this.aggregationSchemaInfo, new HashSet<>()); + } else if (var instanceof EdgeVar) { + IEdge edge = (IEdge) aggValue; + value.aggregateEdge( + alias, Sets.newHashSet(edge), this.aggregationSchemaInfo, new HashSet<>()); + } else { + throw new RuntimeException("will never run this code"); + } + } + } + + return value; + } + + private IVertex aggToVirtualVertexId( + KgGraphImpl value, String vertexAlias, String expectEdgeAlias) { + Set> vertexSet = value.getAlias2VertexMap().get(vertexAlias); + IVertex vertex = vertexSet.iterator().next(); + IVertexId newId = + IVertexId.from(UUID.randomUUID().getMostSignificantBits(), vertex.getId().getType()); + List> edgeInfoList = + aggregationSchemaInfo.getVertexHasEdgeMap().get(vertexAlias); + for (Tuple2 edgeInfo : edgeInfoList) { + String edgeAlias = edgeInfo._1(); + if (edgeAlias.equals(expectEdgeAlias)) { + continue; + } + boolean checkSource = edgeInfo._2(); + Set> edgeSet = value.getAlias2EdgeMap().get(edgeAlias); + Iterator> edgeIt = edgeSet.iterator(); + + List> newEdgeList = new ArrayList<>(); + while (edgeIt.hasNext()) { + IEdge edge = edgeIt.next(); + if (checkSource) { + if (edge.getSourceId().equals(vertex.getId())) { + edgeIt.remove(); + IEdge newEdge = + new Edge<>( + newId, + edge.getTargetId(), + edge.getValue(), + edge.getVersion(), + edge.getDirection(), + edge.getType()); + newEdgeList.add(newEdge); + } + } else { + if (edge.getTargetId().equals(vertex.getId())) { + edgeIt.remove(); + IEdge newEdge = + new Edge<>( + edge.getSourceId(), + newId, + edge.getValue(), + edge.getVersion(), + edge.getDirection(), + edge.getType()); + newEdgeList.add(newEdge); + } + } + } + edgeSet.addAll(newEdgeList); + } + IVertex newVertex = vertex.clone(); + newVertex.setId(newId); + vertexSet.remove(vertex); + vertexSet.add(newVertex); + return newVertex; + } + + private IEdge aggToVirtualEdgeId(KgGraphImpl value, String edgeAlias) { + Set> edgeSet = value.getAlias2EdgeMap().get(edgeAlias); + IEdge edge = edgeSet.iterator().next(); + + IVertexId sourceId = edge.getSourceId(); + IVertexId targetId = edge.getTargetId(); + List> edgeInfoList = + aggregationSchemaInfo.getEdgeEndpointMap().get(edgeAlias); + for (Tuple2 tuple2 : edgeInfoList) { + String vertexAlias = tuple2._1(); + if (byAliasSet.contains(vertexAlias)) { + continue; + } + IVertex newVertex = aggToVirtualVertexId(value, vertexAlias, edgeAlias); + boolean checkSource = tuple2._2(); + if (checkSource) { + sourceId = newVertex.getId(); + } else { + targetId = newVertex.getId(); + } + } + IEdge newEdge = edge.clone(); + newEdge.setSourceId(sourceId); + newEdge.setTargetId(targetId); + edgeSet.remove(edge); + edgeSet.add(newEdge); + return newEdge; + } + + private void updateUdafDataFromProperty(BaseUdaf udaf, IProperty property, String propertyName) { + if (property.isKeyExist(propertyName)) { + udaf.update(property.get(propertyName)); + } + } + + private Object doAggregation( + List> valueFilteredList, + UdafMeta udafMeta, + Object[] udafInitParams, + Expr sourceExpr) { + BaseUdaf udaf = udafMeta.createAggregateFunction(); + if (null != udafInitParams) { + udaf.initialize(udafInitParams); + } + + String sourceAlias = null; + String sourcePropertyName = null; + if (sourceExpr instanceof Ref) { + Ref sourceRef = (Ref) sourceExpr; + sourceAlias = sourceRef.refName(); + } else if (sourceExpr instanceof UnaryOpExpr) { + UnaryOpExpr expr = (UnaryOpExpr) sourceExpr; + GetField getField = (GetField) expr.name(); + sourceAlias = ((Ref) expr.arg()).refName(); + sourcePropertyName = getField.fieldName(); + } + + String finalSourcePropertyName = sourcePropertyName; + for (KgGraph valueFiltered : valueFilteredList) { + if (valueFiltered.getVertexAlias().contains(sourceAlias)) { + List> vertexList = valueFiltered.getVertex(sourceAlias); + if (sourcePropertyName == null) { + vertexList.forEach(udaf::update); + } else { + vertexList.forEach( + v -> updateUdafDataFromProperty(udaf, v.getValue(), finalSourcePropertyName)); + } + } else { + List> edgeList = valueFiltered.getEdge(sourceAlias); + if (sourcePropertyName == null) { + edgeList.forEach(udaf::update); + } else { + edgeList.forEach( + e -> updateUdafDataFromProperty(udaf, e.getValue(), finalSourcePropertyName)); + } + } + } + + return udaf.evaluate(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphDropFieldsImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphDropFieldsImpl.java new file mode 100644 index 00000000..41c2d487 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphDropFieldsImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.lube.catalog.struct.Field; +import com.antgroup.openspg.reasoner.lube.logical.*; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.collections4.MapUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public class KgGraphDropFieldsImpl implements Serializable { + + private static final long serialVersionUID = 3662110639574654863L; + private final Map> dropVertexAlias2PropertyMap = new HashMap<>(); + private final Map> dropEdgeAlias2PropertyMap = new HashMap<>(); + + private final Map>> dropPathEdgeAlias2PropertyMap = + new HashMap<>(); + + /** init */ + public KgGraphDropFieldsImpl(Set dropFieldSet) { + for (Var field : dropFieldSet) { + if (field instanceof NodeVar) { + Tuple2> tuple2 = getDropInfo(field); + dropVertex(tuple2._1(), tuple2._2()); + } else if (field instanceof EdgeVar) { + Tuple2> tuple2 = getDropInfo(field); + dropEdge(tuple2._1(), tuple2._2()); + } else if (field instanceof RepeatPathVar) { + RepeatPathVar arrayVar = (RepeatPathVar) field; + PathVar pathVar = arrayVar.pathVar(); + for (int i = 0; i < pathVar.elements().size(); ++i) { + Var subVar = pathVar.elements().apply(i); + Tuple2> tuple2 = getDropInfo(subVar); + Map> subMap = + dropPathEdgeAlias2PropertyMap.computeIfAbsent(pathVar.name(), k -> new HashMap<>()); + Map propertyMap = + subMap.computeIfAbsent(tuple2._1(), k -> new HashMap<>()); + for (String key : tuple2._2()) { + propertyMap.put(key, null); + } + } + } + } + clearEmptyDropInfo(); + } + + private void clearEmptyDropInfo() { + this.dropVertexAlias2PropertyMap + .entrySet() + .removeIf(entry -> MapUtils.isEmpty(entry.getValue())); + this.dropEdgeAlias2PropertyMap.entrySet().removeIf(entry -> MapUtils.isEmpty(entry.getValue())); + this.dropPathEdgeAlias2PropertyMap + .entrySet() + .removeIf( + entry -> { + entry.getValue().entrySet().removeIf(entry2 -> MapUtils.isEmpty(entry2.getValue())); + return MapUtils.isEmpty(entry.getValue()); + }); + } + + /** check need to drop op */ + public boolean needDrop() { + if (!dropVertexAlias2PropertyMap.isEmpty()) { + return true; + } + if (!dropEdgeAlias2PropertyMap.isEmpty()) { + return true; + } + if (!dropPathEdgeAlias2PropertyMap.isEmpty()) { + return true; + } + return false; + } + + private Tuple2> getDropInfo(Var var) { + String alias = null; + Set propertyNameSet = null; + if (var instanceof NodeVar) { + NodeVar nodeField = (NodeVar) var; + alias = nodeField.name(); + propertyNameSet = + JavaConversions.setAsJavaSet(nodeField.fields()).stream() + .map(Field::name) + .collect(Collectors.toSet()); + } else if (var instanceof EdgeVar) { + EdgeVar edgeField = (EdgeVar) var; + alias = edgeField.name(); + propertyNameSet = + JavaConversions.setAsJavaSet(edgeField.fields()).stream() + .map(Field::name) + .collect(Collectors.toSet()); + dropEdge(alias, propertyNameSet); + } + return new Tuple2<>(alias, propertyNameSet); + } + + private void dropVertex(String alias, Set propertySet) { + Map propertyMap = + this.dropVertexAlias2PropertyMap.computeIfAbsent(alias, k -> new HashMap<>()); + for (String key : propertySet) { + propertyMap.put(key, null); + } + } + + private void dropEdge(String alias, Set propertySet) { + Map propertyMap = + this.dropEdgeAlias2PropertyMap.computeIfAbsent(alias, k -> new HashMap<>()); + for (String key : propertySet) { + propertyMap.put(key, null); + } + } + + /** + * drop fields + * + * @param kgGraph + */ + public void doDropFields(KgGraph kgGraph) { + for (Map.Entry> entry : + this.dropVertexAlias2PropertyMap.entrySet()) { + kgGraph.setVertexProperty(entry.getKey(), entry.getValue(), 0L); + } + for (Map.Entry> entry : this.dropEdgeAlias2PropertyMap.entrySet()) { + kgGraph.setEdgeProperty(entry.getKey(), entry.getValue()); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphFirstEdgeAggImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphFirstEdgeAggImpl.java new file mode 100644 index 00000000..d007fe58 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphFirstEdgeAggImpl.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import java.io.Serializable; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class KgGraphFirstEdgeAggImpl implements Serializable { + + private final List firstEdgeAliasList; + + /** init */ + public KgGraphFirstEdgeAggImpl(List firstEdgeAliasList) { + this.firstEdgeAliasList = firstEdgeAliasList; + } + + /** first edge */ + public KgGraph map(KgGraph value) { + for (String edgeAlias : firstEdgeAliasList) { + KgGraphImpl kgGraph = (KgGraphImpl) value; + Set> edgeSet = kgGraph.getAlias2EdgeMap().get(edgeAlias); + Set keySet = new HashSet<>(); + Set> newEdgeSet = new HashSet<>(); + for (IEdge edge : edgeSet) { + FirstEdgeKey key = new FirstEdgeKey(edge.getSourceId(), edge.getTargetId()); + if (keySet.contains(key)) { + continue; + } + keySet.add(key); + newEdgeSet.add(edge); + } + kgGraph.getAlias2EdgeMap().put(edgeAlias, newEdgeSet); + } + return value; + } + + protected static class FirstEdgeKey { + private final byte[] keyBytes = new byte[Long.BYTES * 4]; + + /** init first edge key */ + public FirstEdgeKey(IVertexId id1, IVertexId id2) { + System.arraycopy(id1.getBytes(), 0, keyBytes, 0, id1.getBytes().length); + System.arraycopy(id2.getBytes(), 0, keyBytes, Long.BYTES * 2, id2.getBytes().length); + } + + @Override + public int hashCode() { + return Arrays.hashCode(this.keyBytes); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof FirstEdgeKey)) { + return false; + } + FirstEdgeKey that = (FirstEdgeKey) obj; + return Arrays.equals(this.keyBytes, that.keyBytes); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphJoinImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphJoinImpl.java new file mode 100644 index 00000000..42b10c9a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphJoinImpl.java @@ -0,0 +1,196 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.OptionalEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.NoneVertex; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.common.pattern.Connection; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.lube.logical.EdgeVar; +import com.antgroup.openspg.reasoner.lube.logical.NodeVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.lube.physical.planning.InnerJoin$; +import com.antgroup.openspg.reasoner.lube.physical.planning.JoinType; +import com.antgroup.openspg.reasoner.lube.physical.planning.LeftOuterJoin$; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public class KgGraphJoinImpl implements Serializable { + private static final long serialVersionUID = -5771663962539687553L; + private final JoinType joinType; + private final String leftAlias; + private final String rightAlias; + private final Map otherJoinAliasMap = new HashMap<>(); + private final Map otherJoinAliasReverseMap = new HashMap<>(); + private final Long pathLimit; + + private final scala.collection.immutable.Map lhsSchemaMapping; + private final scala.collection.immutable.Map rhsSchemaMapping; + private final Pattern rightSchema; + + private final KgGraphRenameImpl rhsRenameImpl; + + private static final Set SUPPORTED_JOIN_TYPE = new HashSet<>(); + + static { + SUPPORTED_JOIN_TYPE.add(LeftOuterJoin$.MODULE$); + SUPPORTED_JOIN_TYPE.add(InnerJoin$.MODULE$); + } + + public KgGraphJoinImpl( + JoinType joinType, + scala.collection.immutable.List> onAlias, + scala.collection.immutable.Map lhsSchemaMapping, + scala.collection.immutable.Map rhsSchemaMapping, + Pattern rightSchema, + Long pathLimit) { + this.joinType = joinType; + if (!SUPPORTED_JOIN_TYPE.contains(this.joinType)) { + throw new NotImplementedException("not supported join type", null); + } + this.leftAlias = onAlias.head()._1(); + this.rightAlias = onAlias.head()._2(); + for (int i = 1; i < onAlias.size(); ++i) { + Tuple2 tuple2 = onAlias.apply(i); + otherJoinAliasMap.put(tuple2._1(), tuple2._2()); + otherJoinAliasReverseMap.put(tuple2._2(), tuple2._1()); + } + this.rightSchema = rightSchema; + this.pathLimit = pathLimit; + this.lhsSchemaMapping = lhsSchemaMapping; + this.rhsSchemaMapping = rhsSchemaMapping; + this.rhsRenameImpl = new KgGraphRenameImpl(rhsSchemaMapping); + } + + public List> join( + Collection> left, Collection> right) { + long count = 0; + List> result = new ArrayList<>(); + for (KgGraph leftKgGraph : left) { + if (CollectionUtils.isEmpty(right)) { + if (LeftOuterJoin$.MODULE$.equals(joinType)) { + leftJoinNone((KgGraphImpl) leftKgGraph); + result.add(leftKgGraph); + count++; + } + continue; + } + if (null != this.pathLimit && count > this.pathLimit) { + continue; + } + for (KgGraph rightKgGraph : right) { + if (null != this.pathLimit && count > this.pathLimit) { + break; + } + if (!leftKgGraph + .getVertex(leftAlias) + .get(0) + .getId() + .equals(rightKgGraph.getVertex(rightAlias).get(0).getId())) { + // id not equals + continue; + } + boolean otherVertexIdEquals = true; + for (Map.Entry entry : this.otherJoinAliasMap.entrySet()) { + if (!leftKgGraph + .getVertex(entry.getKey()) + .get(0) + .getId() + .equals(rightKgGraph.getVertex(entry.getValue()).get(0).getId())) { + // id not equals + otherVertexIdEquals = false; + break; + } + } + if (!otherVertexIdEquals) { + continue; + } + rightKgGraph = this.rhsRenameImpl.renameAndRemoveRoot(rightKgGraph, rightAlias); + KgGraph newKgGraph = new KgGraphImpl((KgGraphImpl) leftKgGraph); + newKgGraph.merge(Lists.newArrayList(rightKgGraph), null); + result.add(newKgGraph); + count++; + } + } + return result; + } + + private void leftJoinNone(KgGraphImpl kgGraph) { + Map rightVertexAliasMap = new HashMap<>(); + Map rightEdgeAliasMap = new HashMap<>(); + for (Map.Entry entry : + JavaConversions.mapAsJavaMap(this.rhsSchemaMapping).entrySet()) { + if (entry.getKey() instanceof NodeVar) { + rightVertexAliasMap.put(entry.getKey().name(), entry.getValue().name()); + } else if (entry.getKey() instanceof EdgeVar) { + rightEdgeAliasMap.put(entry.getKey().name(), entry.getValue().name()); + } else { + throw new RuntimeException( + "unsupported alias mapping type, " + entry.getKey().getClass().getName()); + } + } + for (Connection connection : RunnerUtil.getConnectionSet(this.rightSchema)) { + Tuple2> sourceTuple2 = + getNoneVertex(connection.source(), rightVertexAliasMap, kgGraph); + Tuple2> targetTuple2 = + getNoneVertex(connection.target(), rightVertexAliasMap, kgGraph); + if (StringUtils.isNotEmpty(sourceTuple2._1())) { + kgGraph.getAlias2VertexMap().put(sourceTuple2._1(), Sets.newHashSet(sourceTuple2._2())); + } + if (StringUtils.isNotEmpty(targetTuple2._1())) { + kgGraph.getAlias2VertexMap().put(targetTuple2._1(), Sets.newHashSet(targetTuple2._2())); + } + kgGraph + .getAlias2EdgeMap() + .put( + rightEdgeAliasMap.get(connection.alias()), + Sets.newHashSet( + new OptionalEdge<>(sourceTuple2._2().getId(), targetTuple2._2().getId()))); + } + } + + private Tuple2> getNoneVertex( + String rightRowAlias, Map rightVertexAliasMap, KgGraphImpl kgGraph) { + if (rightRowAlias.equals(this.rightAlias)) { + return new Tuple2<>(null, new NoneVertex<>(kgGraph.getVertex(this.leftAlias).get(0))); + } + if (otherJoinAliasReverseMap.containsKey(rightRowAlias)) { + return new Tuple2<>( + null, + new NoneVertex<>(kgGraph.getVertex(otherJoinAliasReverseMap.get(rightRowAlias)).get(0))); + } + return new Tuple2<>( + rightVertexAliasMap.get(rightRowAlias), + new NoneVertex<>(kgGraph.getVertex(this.leftAlias).get(0))); + } +} diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/TableRecord.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphListProcess.java similarity index 56% rename from cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/TableRecord.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphListProcess.java index fcea7058..377c7f29 100644 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/TableRecord.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphListProcess.java @@ -11,15 +11,14 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.tablestore.model; +package com.antgroup.openspg.reasoner.rdg.common; -import com.antgroup.openspg.server.common.model.base.BaseValObj; -import lombok.AllArgsConstructor; -import lombok.Getter; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import java.io.Serializable; +import java.util.Collection; +import java.util.List; -@Getter -@AllArgsConstructor -public class TableRecord extends BaseValObj { - - private final Object[] values; +public interface KgGraphListProcess extends Serializable { + List> reduce(Collection> kgGraphs); } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphRenameImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphRenameImpl.java new file mode 100644 index 00000000..fa77c480 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphRenameImpl.java @@ -0,0 +1,70 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.lube.logical.EdgeVar; +import com.antgroup.openspg.reasoner.lube.logical.NodeVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import scala.collection.JavaConversions; + +public class KgGraphRenameImpl implements Serializable { + private final Map vertexAliasMap; + private final Map edgeAliasMap; + + public KgGraphRenameImpl(scala.collection.immutable.Map schemaMapping) { + this.vertexAliasMap = new HashMap<>(); + this.edgeAliasMap = new HashMap<>(); + for (Map.Entry entry : JavaConversions.mapAsJavaMap(schemaMapping).entrySet()) { + if (entry.getKey() instanceof NodeVar + && entry.getValue() instanceof NodeVar + && !entry.getKey().name().equals(entry.getValue().name())) { + this.vertexAliasMap.put(entry.getKey().name(), entry.getValue().name()); + } else if (entry.getKey() instanceof EdgeVar + && entry.getValue() instanceof EdgeVar + && !entry.getKey().name().equals(entry.getValue().name())) { + this.edgeAliasMap.put(entry.getKey().name(), entry.getValue().name()); + } + } + } + + public KgGraph rename(KgGraph value) { + if (this.vertexAliasMap.isEmpty() && this.edgeAliasMap.isEmpty()) { + return value; + } + KgGraphImpl kgGraph = (KgGraphImpl) value; + for (Map.Entry entry : this.vertexAliasMap.entrySet()) { + kgGraph + .getAlias2VertexMap() + .put(entry.getValue(), kgGraph.getAlias2VertexMap().remove(entry.getKey())); + } + for (Map.Entry entry : this.edgeAliasMap.entrySet()) { + kgGraph + .getAlias2EdgeMap() + .put(entry.getValue(), kgGraph.getAlias2EdgeMap().remove(entry.getKey())); + } + return value; + } + + public KgGraph renameAndRemoveRoot(KgGraph value, String rootVertexAlias) { + KgGraphImpl kgGraph = (KgGraphImpl) rename(value); + kgGraph.getAlias2VertexMap().remove(this.vertexAliasMap.get(rootVertexAlias)); + return kgGraph; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphSortImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphSortImpl.java new file mode 100644 index 00000000..bff70e31 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphSortImpl.java @@ -0,0 +1,129 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.block.SortItem; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.lube.logical.NodeVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +public class KgGraphSortImpl implements KgGraphListProcess { + private final scala.collection.immutable.List groupKey; + private final scala.collection.immutable.List sortItems; + private final int limit; + private final KgGraphSplitStaticParameters staticParameters; + + public KgGraphSortImpl( + scala.collection.immutable.List groupKey, + scala.collection.immutable.List sortItems, + Pattern schema, + int limit) { + this.groupKey = groupKey; + this.sortItems = sortItems; + this.limit = limit; + Set splitVertexAliases = new HashSet<>(); + for (int i = 0; i < groupKey.size(); ++i) { + Var var = groupKey.apply(i); + if (var instanceof NodeVar) { + splitVertexAliases.add(var.name()); + } + } + this.staticParameters = new KgGraphSplitStaticParameters(splitVertexAliases, schema); + } + + @Override + public List> reduce(Collection> values) { + Map>>> groupByMap = + new HashMap<>(); + Iterator> valuesIt = values.iterator(); + while (valuesIt.hasNext()) { + KgGraph graph = valuesIt.next(); + valuesIt.remove(); + Iterator> it = graph.getPath(this.staticParameters, null); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + Object[] keys = RunnerUtil.getVarFromKgGraph(kgGraph, groupKey); + TreeMap>> sortedTreeMap = + groupByMap.computeIfAbsent(new GroupByKeyItem(keys), k -> new TreeMap<>()); + KgGraphSortItem newItem = new KgGraphSortItem(this.sortItems, kgGraph); + int allSize = 0; + for (List> kgGraphList : sortedTreeMap.values()) { + allSize += kgGraphList.size(); + } + if (allSize < limit) { + List> kgGraphList = + sortedTreeMap.computeIfAbsent(newItem, k -> new ArrayList<>()); + kgGraphList.add(kgGraph); + } else { + // Exceeding the threshold, truncate or retain the minimum value + Map.Entry>> lastEntry = + sortedTreeMap.lastEntry(); + int compareValue = newItem.compareTo(lastEntry.getKey()); + if (compareValue > 0) { + continue; + } else if (0 == compareValue) { + lastEntry.getValue().add(kgGraph); + continue; + } + List> kgGraphList = + sortedTreeMap.computeIfAbsent(newItem, k -> new ArrayList<>()); + kgGraphList.add(kgGraph); + sortedTreeMap.remove(sortedTreeMap.lastKey()); + } + } + } + + List> result = new ArrayList<>(); + for (TreeMap>> mapValues : groupByMap.values()) { + int count = 0; + for (List> kgGraphList : mapValues.values()) { + if (count >= limit) { + break; + } else if (kgGraphList.size() + count > limit) { + TreeSet sortItemTreeSet = new TreeSet<>(); + for (KgGraph kgGraph : kgGraphList) { + sortItemTreeSet.add(new KgGraphSortItem(this.sortItems, kgGraph)); + } + for (int i = sortItemTreeSet.size(); i > (limit - count); --i) { + sortItemTreeSet.remove(sortItemTreeSet.last()); + } + kgGraphList = new ArrayList<>(sortItemTreeSet.size()); + for (KgGraphSortItem item : sortItemTreeSet) { + kgGraphList.add(item.getKgGraph()); + } + } + count += kgGraphList.size(); + result.addAll(kgGraphList); + } + } + return result; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphSortItem.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphSortItem.java new file mode 100644 index 00000000..e68ef439 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/KgGraphSortItem.java @@ -0,0 +1,81 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.lube.block.Desc; +import com.antgroup.openspg.reasoner.lube.block.SortItem; +import com.antgroup.openspg.reasoner.lube.common.expr.GetField; +import com.antgroup.openspg.reasoner.lube.common.expr.Ref; +import com.antgroup.openspg.reasoner.lube.common.expr.UnaryOpExpr; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; + +public class KgGraphSortItem implements Comparable { + private final scala.collection.immutable.List sortItems; + private final KgGraph kgGraph; + + public KgGraphSortItem( + scala.collection.immutable.List sortItems, KgGraph kgGraph) { + this.sortItems = sortItems; + this.kgGraph = kgGraph; + } + + /** + * Getter method for property kgGraph. + * + * @return property value of kgGraph + */ + public KgGraph getKgGraph() { + return kgGraph; + } + + @Override + public int compareTo(KgGraphSortItem o) { + Object[] thisValues = getValues(); + Object[] otherValues = o.getValues(); + + for (int i = 0; i < this.sortItems.size(); ++i) { + SortItem sortItem = sortItems.apply(i); + boolean desc = sortItem instanceof Desc; + + Object v1 = thisValues[i]; + Object v2 = otherValues[i]; + + int r = RunnerUtil.compareTwoObject(v1, v2); + if (0 == r) { + r = this.hashCode() - o.hashCode(); + } + if (desc) { + return -r; + } + return r; + } + return 0; + } + + private Object[] getValues() { + Object[] values = new Object[this.sortItems.size()]; + for (int i = 0; i < this.sortItems.size(); ++i) { + SortItem sortItem = this.sortItems.apply(i); + UnaryOpExpr unaryOpExpr = (UnaryOpExpr) sortItem.expr(); + GetField name = (GetField) unaryOpExpr.name(); + Ref ref = (Ref) unaryOpExpr.arg(); + String alias = ref.refName(); + String propertyName = name.fieldName(); + values[i] = RunnerUtil.getProperty(this.kgGraph, alias, propertyName); + } + return values; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/SinkRelationImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/SinkRelationImpl.java new file mode 100644 index 00000000..fd70530d --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/SinkRelationImpl.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.GraphState; +import com.google.common.collect.Lists; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +public class SinkRelationImpl { + private static final Logger log = LoggerFactory.getLogger(SinkRelationImpl.class); + private final GraphState graphState; + private final int taskIndex; + + /** sink edge implement */ + public SinkRelationImpl(GraphState graphState, int taskIndex) { + this.graphState = graphState; + this.taskIndex = taskIndex; + } + + /** write to graph state and return number */ + public long sink(Collection> values) { + long sinkCount = 0; + log.info("SinkRelation,start,index=" + taskIndex + ",size=" + values.size()); + Map>, Set>>> + edgeMap = new HashMap<>(); + for (IEdge edge : values) { + Tuple2>, Set>> edgeListTuple2 = + edgeMap.computeIfAbsent( + edge.getSourceId(), k -> new Tuple2<>(new HashSet<>(), new HashSet<>())); + if (Direction.IN.equals(edge.getDirection())) { + edgeListTuple2._1().add(edge); + } else { + edgeListTuple2._2().add(edge); + } + } + + for (Map.Entry< + IVertexId, Tuple2>, Set>>> + entry : edgeMap.entrySet()) { + IVertexId vertexId = entry.getKey(); + Set> inEdgeList = entry.getValue()._1(); + Set> outEdgeList = entry.getValue()._2(); + this.graphState.addEdges( + vertexId, Lists.newArrayList(inEdgeList), Lists.newArrayList(outEdgeList)); + sinkCount++; + if (Utils.randomLog()) { + log.info( + "SinkRelation,v=" + + vertexId + + ",inEdge=" + + inEdgeList.size() + + ",outEdge=" + + outEdgeList + + ",sinkCount=" + + sinkCount); + } + } + return sinkCount; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/UnfoldEdgeImpl.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/UnfoldEdgeImpl.java new file mode 100644 index 00000000..fca13731 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/UnfoldEdgeImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.OptionalEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.PathEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.MirrorVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.google.common.collect.Sets; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import scala.Tuple2; + +public class UnfoldEdgeImpl implements Serializable { + private static final long serialVersionUID = 8752841991024282878L; + private final UnfoldRepeatEdgeInfo unfoldRepeatEdgeInfo; + + public UnfoldEdgeImpl(UnfoldRepeatEdgeInfo unfoldRepeatEdgeInfo) { + this.unfoldRepeatEdgeInfo = unfoldRepeatEdgeInfo; + } + + public List> unfold(KgGraphImpl kgGraph) { + List> result = new ArrayList<>(); + result.add(kgGraph); + Set> edgeSet = + kgGraph.getAlias2EdgeMap().get(this.unfoldRepeatEdgeInfo.getEdgeAlias()); + for (IEdge iEdge : edgeSet) { + if (iEdge instanceof OptionalEdge) { + continue; + } + PathEdge pathEdge = + ((PathEdge) iEdge).clone(); + while (pathEdge.getEdgeList().size() > this.unfoldRepeatEdgeInfo.getLower()) { + if (1 == pathEdge.getEdgeList().size()) { + // unfold to optional edge + KgGraphImpl newKgGraph = new KgGraphImpl(kgGraph); + Edge edge = pathEdge.getEdgeList().get(0); + IVertexId anchorVertexId = edge.getSourceId(); + IVertex newAnchorVertex = null; + for (IVertex v : + kgGraph.getAlias2VertexMap().get(this.unfoldRepeatEdgeInfo.getAnchorVertexAlias())) { + if (v.getId().equals(anchorVertexId)) { + newAnchorVertex = new MirrorVertex<>(v); + break; + } + } + if (null == newAnchorVertex) { + throw new RuntimeException( + "unfold_error, kgGraph=" + kgGraph + ",info=" + this.unfoldRepeatEdgeInfo); + } + newKgGraph + .getAlias2EdgeMap() + .put( + this.unfoldRepeatEdgeInfo.getEdgeAlias(), + Sets.newHashSet(new OptionalEdge<>(anchorVertexId, anchorVertexId))); + newKgGraph + .getAlias2VertexMap() + .put( + this.unfoldRepeatEdgeInfo.getFoldVertexAlias(), Sets.newHashSet(newAnchorVertex)); + result.add(newKgGraph); + break; + } + Tuple2, Vertex> tuple2 = + pathEdge.seversTail(); + KgGraphImpl newKgGraph = new KgGraphImpl(kgGraph); + newKgGraph + .getAlias2EdgeMap() + .put(this.unfoldRepeatEdgeInfo.getEdgeAlias(), Sets.newHashSet(pathEdge)); + newKgGraph + .getAlias2VertexMap() + .put(this.unfoldRepeatEdgeInfo.getFoldVertexAlias(), Sets.newHashSet(tuple2._2())); + result.add(newKgGraph); + pathEdge = pathEdge.clone(); + } + } + return result; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/UnfoldRepeatEdgeInfo.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/UnfoldRepeatEdgeInfo.java new file mode 100644 index 00000000..89f06e27 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/UnfoldRepeatEdgeInfo.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rdg.common; + +import com.alibaba.fastjson.JSON; +import java.io.Serializable; +import lombok.Getter; + +@Getter +public class UnfoldRepeatEdgeInfo implements Serializable { + private static final long serialVersionUID = 7558197193435125339L; + private final String edgeAlias; + private final String foldVertexAlias; + private final String anchorVertexAlias; + private final int lower; + + public UnfoldRepeatEdgeInfo( + String edgeAlias, String foldVertexAlias, String anchorVertexAlias, int lower) { + this.edgeAlias = edgeAlias; + this.foldVertexAlias = foldVertexAlias; + this.anchorVertexAlias = anchorVertexAlias; + this.lower = lower; + } + + @Override + public String toString() { + return JSON.toJSONString(this); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/AggIfOpProcessBaseGroupProcess.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/AggIfOpProcessBaseGroupProcess.java new file mode 100644 index 00000000..81cafbcc --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/AggIfOpProcessBaseGroupProcess.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.rdg.common.groupProcess; + +import com.antgroup.openspg.reasoner.lube.common.expr.AggIfOpExpr; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.AggregatorOpSet; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.warehouse.utils.WareHouseUtils; +import java.io.Serializable; +import java.util.List; + +public class AggIfOpProcessBaseGroupProcess extends BaseGroupProcess implements Serializable { + + /** + * constructor + * + * @param taskId + * @param var + * @param aggregator + */ + public AggIfOpProcessBaseGroupProcess(String taskId, Var var, Aggregator aggregator) { + super(taskId, var, aggregator); + } + + /** + * get op + * + * @return + */ + public AggIfOpExpr getAggIfOpExpr() { + return (AggIfOpExpr) this.aggOp; + } + + @Override + protected List parseRuleList() { + return WareHouseUtils.getRuleList(getAggIfOpExpr().condition()); + } + + @Override + public AggregatorOpSet getAggOpSet() { + return getAggIfOpExpr().aggOpExpr().name(); + } + + @Override + public Expr getAggEle() { + return getAggIfOpExpr().aggOpExpr().aggEleExpr(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/AggOpProcessBaseGroupProcess.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/AggOpProcessBaseGroupProcess.java new file mode 100644 index 00000000..1e424c8a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/AggOpProcessBaseGroupProcess.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.rdg.common.groupProcess; + +import com.antgroup.openspg.reasoner.lube.common.expr.AggOpExpr; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.AggregatorOpSet; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import java.io.Serializable; +import java.util.List; + +public class AggOpProcessBaseGroupProcess extends BaseGroupProcess implements Serializable { + + public AggOpProcessBaseGroupProcess(String taskId, Var var, Aggregator aggregator) { + super(taskId, var, aggregator); + } + + public AggOpExpr getAggOpExpr() { + return (AggOpExpr) this.aggOp; + } + + @Override + protected List parseRuleList() { + return null; + } + + @Override + public AggregatorOpSet getAggOpSet() { + return getAggOpExpr().name(); + } + + @Override + public Expr getAggEle() { + return getAggOpExpr().aggEleExpr(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/BaseGroupProcess.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/BaseGroupProcess.java new file mode 100644 index 00000000..a6fe40e9 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rdg/common/groupProcess/BaseGroupProcess.java @@ -0,0 +1,167 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.rdg.common.groupProcess; + +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.common.types.KTString$; +import com.antgroup.openspg.reasoner.lube.common.expr.AggUdf; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.AggregatorOpSet; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.logical.PropertyVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.udf.UdfMngFactory; +import com.antgroup.openspg.reasoner.udf.model.UdafMeta; +import com.antgroup.openspg.reasoner.warehouse.utils.WareHouseUtils; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import scala.collection.JavaConversions; + +public abstract class BaseGroupProcess implements Serializable { + protected Var var; + protected UdafMeta udafMeta; + protected Object[] udfInitParams; + protected List ruleList; + protected Aggregator aggOp; + protected String taskId; + + /** + * Construct from var and aggregator + * + * @param taskId + * @param var + * @param aggregator + */ + public BaseGroupProcess(String taskId, Var var, Aggregator aggregator) { + this.taskId = taskId; + this.var = var; + this.aggOp = aggregator; + this.ruleList = parseRuleList(); + this.udfInitParams = parseUdfInitParams(); + this.udafMeta = parseUdafMeta(); + } + + /** + * judge is first agg function + * + * @return + */ + public boolean isFirstAgg() { + return (!(var instanceof PropertyVar)); + } + + /** + * get udaf str name from op + * + * @param op + * @return + */ + public String getUdafStrName(AggregatorOpSet op) { + if (op instanceof AggUdf) { + AggUdf aggUdf = (AggUdf) op; + return aggUdf.name(); + } else { + return op.toString(); + } + } + + private Object[] getUdafInitializeParams(List exprList) { + Object[] params = new Object[exprList.size()]; + for (int i = 0; i < exprList.size(); ++i) { + Expr expr = exprList.get(i); + List paramRuleList = WareHouseUtils.getRuleList(expr); + Object value = + RuleRunner.getInstance().executeExpression(new HashMap<>(), paramRuleList, this.taskId); + params[i] = value; + } + return params; + } + + protected Object[] parseUdfInitParams() { + Object[] udfInitParams = null; + AggregatorOpSet aggregatorOpSet = getAggOpSet(); + if (aggregatorOpSet instanceof AggUdf) { + AggUdf aggUdf = (AggUdf) aggregatorOpSet; + udfInitParams = getUdafInitializeParams(JavaConversions.seqAsJavaList(aggUdf.funcArgs())); + } + return udfInitParams; + } + + protected UdafMeta parseUdafMeta() { + String udafName = getUdafStrName(getAggOpSet()); + UdafMeta udafMeta = UdfMngFactory.getUdfMng().getUdafMeta(udafName, KTString$.MODULE$); + if (udafMeta == null) { + throw new NotImplementedException("unsupported aggregator function, type=" + udafName, null); + } + return udafMeta; + } + + /** + * parse rule list by op + * + * @return + */ + protected abstract List parseRuleList(); + + /** + * get agg op set by op + * + * @return + */ + public abstract AggregatorOpSet getAggOpSet(); + + /** + * get agg ele by op + * + * @return + */ + public abstract Expr getAggEle(); + + /** + * getter + * + * @return + */ + public Var getVar() { + return var; + } + + /** + * getter + * + * @return + */ + public UdafMeta getUdafMeta() { + return udafMeta; + } + + /** + * getter + * + * @return + */ + public Object[] getUdfInitParams() { + return udfInitParams; + } + + /** + * getter + * + * @return + */ + public List getRuleList() { + return ruleList; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/DefaultRecorder.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/DefaultRecorder.java new file mode 100644 index 00000000..99b671d0 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/DefaultRecorder.java @@ -0,0 +1,162 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.recorder; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Stack; + +public class DefaultRecorder implements IExecutionRecorder { + private final Stack actionStack = new Stack<>(); + + public DefaultRecorder() {} + + @Override + public String toReadableString() { + return actionStack.get(0).toString(); + } + + @Override + public void entryRDG(String rdg) { + SubAction nowAction = null; + if (!actionStack.isEmpty()) { + nowAction = actionStack.peek(); + } + SubAction newAction = new SubAction(rdg); + if (null != nowAction) { + nowAction.subActionList.add(newAction); + } + actionStack.push(newAction); + } + + @Override + public void leaveRDG() { + actionStack.pop(); + } + + @Override + public void stageResult(String stage, long result) { + SubAction nowAction = actionStack.peek(); + nowAction.subActionList.add(new SampleAction(stage, result)); + } + + @Override + public void stageResultWithDesc(String stage, long result, String finishDescribe) { + SubAction nowAction = actionStack.peek(); + nowAction.subActionList.add(new SampleAction(stage, result, finishDescribe)); + } + + public abstract static class AbstractAction { + protected final long time; + + protected AbstractAction(long time) { + this.time = time; + } + } + + public static class SampleAction extends AbstractAction { + + private final String describe; + private final Long num; + + private final String finishDescribe; + + public SampleAction(String describe, long num) { + super(System.currentTimeMillis()); + this.describe = describe; + this.num = num; + this.finishDescribe = null; + } + + public SampleAction(String describe, long num, String finishDescribe) { + super(System.currentTimeMillis()); + this.describe = describe; + this.num = num; + this.finishDescribe = finishDescribe; + } + + public SampleAction(String describe, Long num, long time) { + super(time); + this.describe = describe; + this.num = num; + this.finishDescribe = null; + } + + @Override + public String toString() { + SimpleDateFormat formatter = new SimpleDateFormat("HH:mm:ss.SSS"); + String prefix = null == num ? "SUBQUERY" : "NUM=" + num; + return prefix + ", " + describe + " @" + formatter.format(new Date(this.time)); + } + } + + public static class SubAction extends AbstractAction { + private final String describe; + private final List subActionList = new ArrayList<>(); + + public SubAction(String describe) { + super(System.currentTimeMillis()); + this.describe = describe; + } + + @Override + public String toString() { + List printLineList = new ArrayList<>(); + getPrettyLines(printLineList, "", this, true); + + StringBuilder sb = new StringBuilder(); + for (String line : printLineList) { + sb.append(line).append("\n"); + } + return sb.toString(); + } + + private void getPrettyLines( + List printLines, String prefix, AbstractAction action, boolean last) { + if (action instanceof SampleAction) { + if (last) { + printLines.add(prefix + "└─" + action); + } else { + printLines.add(prefix + "├─" + action); + } + } else { + SubAction subAction = (SubAction) action; + String finishDescribe = ""; + if (!subAction.subActionList.isEmpty()) { + AbstractAction finish = subAction.subActionList.get(subAction.subActionList.size() - 1); + if (finish instanceof SampleAction) { + SampleAction finishSampleAction = (SampleAction) finish; + finishDescribe = finishSampleAction.finishDescribe; + } + } + getPrettyLines( + printLines, prefix, new SampleAction(finishDescribe, null, subAction.time), last); + + String newPrefix; + if (last) { + newPrefix = prefix + " "; + } else { + newPrefix = prefix + "│ "; + } + for (int i = 0; i < subAction.subActionList.size(); ++i) { + boolean isLast = i + 1 == subAction.subActionList.size(); + AbstractAction aa = subAction.subActionList.get(i); + getPrettyLines(printLines, newPrefix, aa, isLast); + } + } + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/EmptyRecorder.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/EmptyRecorder.java new file mode 100644 index 00000000..3a22952a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/EmptyRecorder.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.recorder; + +/** This is an empty recorder */ +public class EmptyRecorder implements IExecutionRecorder { + @Override + public String toReadableString() { + return ""; + } + + @Override + public void entryRDG(String rdg) {} + + @Override + public void leaveRDG() {} + + @Override + public void stageResult(String stage, long result) {} + + @Override + public void stageResultWithDesc(String stage, long result, String finishDescribe) {} +} diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/cmd/TableFileCreateCmd.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/IExecutionRecorder.java similarity index 52% rename from cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/cmd/TableFileCreateCmd.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/IExecutionRecorder.java index 6eb13440..5f8a8a40 100644 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/cmd/TableFileCreateCmd.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/recorder/IExecutionRecorder.java @@ -11,18 +11,21 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.tablestore.cmd; +package com.antgroup.openspg.reasoner.recorder; -import com.antgroup.openspg.cloudext.interfaces.tablestore.model.ColumnMeta; -import com.antgroup.openspg.server.common.model.base.BaseCmd; -import lombok.AllArgsConstructor; -import lombok.Getter; +public interface IExecutionRecorder { + /** get readable string */ + String toReadableString(); -@Getter -@AllArgsConstructor -public class TableFileCreateCmd extends BaseCmd { + /** call when entry a new rdg */ + void entryRDG(String rdg); - private final String name; + /** call when leave a rdg */ + void leaveRDG(); - private final ColumnMeta[] columns; + /** record result num, like filer, expendInto */ + void stageResult(String stage, long result); + + /** finish */ + void stageResultWithDesc(String stage, long result, String finishDescribe); } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/RuleRunner.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/RuleRunner.java new file mode 100644 index 00000000..0ba579e5 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/RuleRunner.java @@ -0,0 +1,189 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rule; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.rule.op.OperatorEqualsLessMore; +import com.antgroup.openspg.reasoner.rule.op.OperatorIn; +import com.antgroup.openspg.reasoner.rule.op.OperatorLike; +import com.antgroup.openspg.reasoner.rule.op.OperatorMultiDiv; +import com.antgroup.openspg.reasoner.rule.udf.UdfWrapper; +import com.antgroup.openspg.reasoner.udf.UdfMng; +import com.antgroup.openspg.reasoner.udf.UdfMngFactory; +import com.antgroup.openspg.reasoner.udf.model.RuntimeUdfMeta; +import com.antgroup.openspg.reasoner.udf.model.UdfOperatorTypeEnum; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Lists; +import com.ql.util.express.DefaultContext; +import com.ql.util.express.ExpressRunner; +import com.ql.util.express.Operator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +public class RuleRunner { + private static final Logger log = LoggerFactory.getLogger(RuleRunner.class); + + private static Cache> contextCache = + CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(24, TimeUnit.HOURS).build(); + + private final ExpressRunner EXPRESS_RUNNER = new ExpressRunner(); + + /** + * set running context + * + * @param taskId + * @param context + */ + public void putRuleRunningContext(String taskId, Map context) { + contextCache.put(taskId, context); + } + + /** + * get running context by id + * + * @param taskId + * @return + */ + public Map getRuleRunningContext(String taskId) { + Map result = contextCache.getIfPresent(taskId); + if (result == null) { + result = new HashMap<>(); + } + return result; + } + /** + * last rule as filter + * + * @param context + * @param ruleList + * @param taskId + * @return + */ + public boolean check(Map context, List ruleList, String taskId) { + DefaultContext ctx = new DefaultContext<>(); + ctx.putAll(context); + ctx.putAll(getRuleRunningContext(taskId)); + for (int i = 0; i < ruleList.size(); ++i) { + String rule = ruleList.get(i); + try { + Object tmpRet = EXPRESS_RUNNER.execute(rule, ctx, null, true, false); + if ((1 + i) == ruleList.size()) { + return (Boolean) tmpRet; + } + } catch (Exception e) { + if (Utils.randomLog()) { + log.warn("RuleRunner error, rule=" + rule + ",ctx=" + JSON.toJSONString(context), e); + } + return false; + } + } + return true; + } + + public Object executeExpression( + Map context, List expressionList, String taskId) { + DefaultContext ctx = new DefaultContext<>(); + ctx.putAll(context); + ctx.putAll(getRuleRunningContext(taskId)); + for (int i = 0; i < expressionList.size(); ++i) { + String rule = expressionList.get(i); + try { + Object rst = EXPRESS_RUNNER.execute(rule, ctx, null, true, false); + if ((1 + i) == expressionList.size()) { + return rst; + } + } catch (Exception e) { + log.warn("RuleRunner error, rule=" + rule + ",ctx=" + JSON.toJSONString(context), e); + return null; + } + } + return null; + } + + private RuleRunner() {} + + private static volatile RuleRunner instance = null; + + public static RuleRunner getInstance() { + if (null != instance) { + return instance; + } + synchronized (RuleRunner.class) { + if (null == instance) { + RuleRunner runner = new RuleRunner(); + runner.init(); + instance = runner; + } + } + return instance; + } + + private void init() { + // disable print error + // InstructionSet.printInstructionError = false; + // use short circuit + EXPRESS_RUNNER.setShortCircuit(true); + registerUdf(); + overrideOperator(); + } + + /** register all udfs */ + private void registerUdf() { + UdfMng udfMng = UdfMngFactory.getUdfMng(); + List runtimeUdfMetaList = udfMng.getAllRuntimeUdfMeta(); + for (RuntimeUdfMeta runtimeUdfMeta : runtimeUdfMetaList) { + try { + if (UdfOperatorTypeEnum.OPERATOR.equals(runtimeUdfMeta.getUdfType())) { + log.debug("EXPRESS_RUNNER.addOperator,name=" + runtimeUdfMeta.getName()); + EXPRESS_RUNNER.addOperator(runtimeUdfMeta.getName(), new UdfWrapper(runtimeUdfMeta)); + } else { + log.debug("EXPRESS_RUNNER.addFunction,name=" + runtimeUdfMeta.getName()); + EXPRESS_RUNNER.addFunction(runtimeUdfMeta.getName(), new UdfWrapper(runtimeUdfMeta)); + } + } catch (Exception e) { + if (e.getMessage().contains("重复定义操作符")) { + log.warn("rule runner replace operator, name=" + runtimeUdfMeta.getName()); + EXPRESS_RUNNER.replaceOperator(runtimeUdfMeta.getName(), new UdfWrapper(runtimeUdfMeta)); + continue; + } + throw new RuntimeException(e); + } + } + } + + private void overrideOperator() { + Lists.newArrayList( + new Tuple2("<", new OperatorEqualsLessMore("<")), + new Tuple2(">", new OperatorEqualsLessMore(">")), + new Tuple2("<=", new OperatorEqualsLessMore("<=")), + new Tuple2(">=", new OperatorEqualsLessMore(">=")), + new Tuple2("==", new OperatorEqualsLessMore("==")), + new Tuple2("!=", new OperatorEqualsLessMore("!=")), + new Tuple2("<>", new OperatorEqualsLessMore("<>")), + new Tuple2("*", new OperatorMultiDiv("*")), + new Tuple2("/", new OperatorMultiDiv("/")), + new Tuple2("%", new OperatorMultiDiv("%")), + new Tuple2("mod", new OperatorMultiDiv("mod")), + new Tuple2("like", new OperatorLike("like")), + new Tuple2("in", new OperatorIn("in"))) + .forEach(udfTuple -> EXPRESS_RUNNER.replaceOperator(udfTuple._1(), udfTuple._2())); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorEqualsLessMore.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorEqualsLessMore.java new file mode 100644 index 00000000..35463624 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorEqualsLessMore.java @@ -0,0 +1,175 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rule.op; + +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.ql.util.express.Operator; +import java.math.BigDecimal; +import java.util.Date; + +/** Optimize qlexpress code, return false when exception ==,>,>=,<,<=,!= */ +public class OperatorEqualsLessMore extends Operator { + public OperatorEqualsLessMore(String aName) { + this.name = aName; + } + + public OperatorEqualsLessMore(String aAliasName, String aName, String aErrorInfo) { + this.name = aName; + this.aliasName = aAliasName; + this.errorInfo = aErrorInfo; + } + + public Object executeInner(Object[] list) throws Exception { + return executeInner(list[0], list[1]); + } + + public Object executeInner(Object op1, Object op2) { + return executeInner(this.name, op1, op2); + } + + public static boolean executeInner(String opStr, Object obj1, Object obj2) { + if (obj1 == null && obj2 == null) { + return "==".equals(opStr); + } else if (obj1 == null || obj2 == null) { + if ("==".equals(opStr)) { + return false; + } else if ("!=".equals(opStr) || "<>".equals(opStr)) { + return true; + } else if (">".equals(opStr) + || "<".equals(opStr) + || ">=".equals(opStr) + || "<=".equals(opStr)) { + return false; + } + } + + Integer i = compareDataWithoutException(obj1, obj2); + if (i == null) { + return false; + } + boolean result; + if (i > 0) { + result = ">".equals(opStr) || ">=".equals(opStr) || "!=".equals(opStr) || "<>".equals(opStr); + } else if (i == 0) { + result = "==".equals(opStr) || ">=".equals(opStr) || "<=".equals(opStr); + } else { + result = "<".equals(opStr) || "<=".equals(opStr) || "!=".equals(opStr) || "<>".equals(opStr); + } + return result; + } + + public static Integer compareDataWithoutException(Object op1, Object op2) { + Integer compareResult = -1; + if (op1 instanceof String) { + compareResult = ((String) op1).compareTo(op2.toString()); + } else if (op2 instanceof String) { + compareResult = op1.toString().compareTo((String) op2); + } else if (op1 instanceof Character && op2 instanceof Character) { + compareResult = ((Character) op1).compareTo((Character) op2); + } else if (op1 instanceof Number && op2 instanceof Number) { + // Make a numerical comparison + compareResult = compareNumber((Number) op1, (Number) op2); + } else if ((op1 instanceof Boolean) && (op2 instanceof Boolean)) { + if (((Boolean) op1).booleanValue() == ((Boolean) op2).booleanValue()) { + compareResult = 0; + } else { + compareResult = -1; + } + } else if ((op1 instanceof Date) && (op2 instanceof Date)) { + compareResult = ((Date) op1).compareTo((Date) op2); + } else if ((op1 instanceof Comparable)) { + compareResult = ((Comparable) op1).compareTo(op2); + } else if ((op2 instanceof Comparable)) { + compareResult = -((Comparable) op2).compareTo(op1); + } else { + return null; + } + return compareResult; + } + + private static final int NUMBER_TYPE_BYTE = 1; + private static final int NUMBER_TYPE_SHORT = 2; + private static final int NUMBER_TYPE_INT = 3; + private static final int NUMBER_TYPE_LONG = 4; + private static final int NUMBER_TYPE_FLOAT = 5; + private static final int NUMBER_TYPE_DOUBLE = 6; + private static final int NUMBER_TYPE_DECIMAL = 7; + + public static int getSeq(Class aClass) { + if (aClass == Byte.class || aClass == byte.class) { + return NUMBER_TYPE_BYTE; + } + if (aClass == Short.class || aClass == short.class) { + return NUMBER_TYPE_SHORT; + } + if (aClass == Integer.class || aClass == int.class) { + return NUMBER_TYPE_INT; + } + if (aClass == Long.class || aClass == long.class) { + return NUMBER_TYPE_LONG; + } + if (aClass == Float.class || aClass == float.class) { + return NUMBER_TYPE_FLOAT; + } + if (aClass == Double.class || aClass == double.class) { + return NUMBER_TYPE_DOUBLE; + } + if (aClass == BigDecimal.class) { + return NUMBER_TYPE_DECIMAL; + } + throw new NotImplementedException("unsupported data type:" + aClass.getName(), null); + } + + public static Integer compareNumber(Number op1, Number op2) { + int type1 = getSeq(op1.getClass()); + int type2 = getSeq(op2.getClass()); + int type = Math.max(type1, type2); + if (NUMBER_TYPE_BYTE == type) { + byte o1 = op1.byteValue(); + byte o2 = op2.byteValue(); + return Byte.compare(o1, o2); + } + if (NUMBER_TYPE_SHORT == type) { + short o1 = op1.shortValue(); + short o2 = op2.shortValue(); + return Short.compare(o1, o2); + } + if (NUMBER_TYPE_INT == type) { + int o1 = op1.intValue(); + int o2 = op2.intValue(); + return Integer.compare(o1, o2); + } + if (NUMBER_TYPE_LONG == type) { + long o1 = op1.longValue(); + long o2 = op2.longValue(); + return Long.compare(o1, o2); + } + if (NUMBER_TYPE_FLOAT == type) { + float o1 = op1.floatValue(); + float o2 = op2.floatValue(); + return Float.compare(o1, o2); + } + if (NUMBER_TYPE_DOUBLE == type) { + double o1 = op1.doubleValue(); + double o2 = op2.doubleValue(); + return Double.compare(o1, o2); + } + if (NUMBER_TYPE_DECIMAL == type) { + BigDecimal o1 = new BigDecimal(op1.toString()); + BigDecimal o2 = new BigDecimal(op2.toString()); + return o1.compareTo(o2); + } + return null; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorIn.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorIn.java new file mode 100644 index 00000000..c0071408 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorIn.java @@ -0,0 +1,72 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.rule.op; + +import com.ql.util.express.Operator; +import java.lang.reflect.Array; +import java.util.List; + +public class OperatorIn extends Operator { + public OperatorIn(String aName) { + this.name = aName; + } + + public OperatorIn(String aAliasName, String aName, String aErrorInfo) { + this.name = aName; + this.aliasName = aAliasName; + this.errorInfo = aErrorInfo; + } + + @Override + public Object executeInner(Object[] list) throws Exception { + Object obj = list[0]; + if (obj == null) { + // object is null, can not call method + return false; + } else if (!((obj instanceof Number) || (obj instanceof String))) { + return false; + } else if (list.length == 2 && (list[1].getClass().isArray() || list[1] instanceof List)) { + if (obj.equals(list[1])) { + return true; + } else if (list[1].getClass().isArray()) { + int len = Array.getLength(list[1]); + for (int i = 0; i < len; i++) { + boolean f = OperatorEqualsLessMore.executeInner("==", obj, Array.get(list[1], i)); + if (f) { + return Boolean.TRUE; + } + } + } else { + @SuppressWarnings("unchecked") + List array = (List) list[1]; + for (Object o : array) { + boolean f = OperatorEqualsLessMore.executeInner("==", obj, o); + if (f) { + return Boolean.TRUE; + } + } + } + return false; + } else if (list.length == 2 && obj instanceof String && list[1] instanceof String) { + return ((String) list[1]).contains(String.valueOf(obj)); + } else { + for (int i = 1; i < list.length; i++) { + boolean f = OperatorEqualsLessMore.executeInner("==", obj, list[i]); + if (f) { + return Boolean.TRUE; + } + } + return Boolean.FALSE; + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorLike.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorLike.java new file mode 100644 index 00000000..4bab9548 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorLike.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.rule.op; + +public class OperatorLike extends com.ql.util.express.instruction.op.OperatorLike { + public OperatorLike(String name) { + super(name); + } + + @Override + public Object executeInner(Object[] list) throws Exception { + if (list[0] == null || list[1] == null) { + return false; + } else { + return executeInner(list[0], list[1]); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorMultiDiv.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorMultiDiv.java new file mode 100644 index 00000000..bf1b8a0a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/op/OperatorMultiDiv.java @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rule.op; + +/** div op without exception return null when exception */ +public class OperatorMultiDiv extends com.ql.util.express.instruction.op.OperatorMultiplyDivide { + public OperatorMultiDiv(String name) { + super(name); + } + + public Object executeInner(Object[] list) throws Exception { + if (!(list[0] instanceof Number) || !(list[1] instanceof Number)) { + return null; + } else { + return executeInner(list[0], list[1]); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/udf/UdfWrapper.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/udf/UdfWrapper.java new file mode 100644 index 00000000..e2ffb949 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/rule/udf/UdfWrapper.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rule.udf; + +import com.antgroup.openspg.reasoner.udf.model.RuntimeUdfMeta; +import com.ql.util.express.Operator; + +public class UdfWrapper extends Operator { + + private final RuntimeUdfMeta runtimeUdfMeta; + + /** + * udf wrapper for qlexpress + * + * @param runtimeUdfMeta + */ + public UdfWrapper(RuntimeUdfMeta runtimeUdfMeta) { + this.runtimeUdfMeta = runtimeUdfMeta; + } + + @Override + public Object executeInner(Object[] objects) throws Exception { + return this.runtimeUdfMeta.invoke(objects); + } + + @Override + public String toString() { + return "qlexpress udf " + this.runtimeUdfMeta.getName(); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/runner/ConfigKey.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/runner/ConfigKey.java new file mode 100644 index 00000000..741da66a --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/runner/ConfigKey.java @@ -0,0 +1,139 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.runner; + +public class ConfigKey { + + /** dsl content */ + public static final String KG_REASONER_DSL = "kg.reasoner.dsl"; + + /** dsl task parameters */ + public static final String KG_REASONER_PARAMS = "kg.reasoner.params"; + + /** rayag worker num */ + public static final String RAYAG_WORKER_NUM = "rayag.worker.num"; + + /** rayag driver num */ + public static final String KG_REASONER_DRIVER_NUM = "kg.reasoner.driver.num"; + + /** rayag source num */ + public static final String KG_REASONER_SOURCE_NUM = "kg.reasoner.source.num"; + + /** source function */ + public static final String KG_REASONER_STREAMING_SOURCE_FUNCTION = + "kg.reasoner.streaming.source.function"; + + /** driver class name, only for olap */ + public static final String KG_REASONER_DRIVER_CLASS_NAME = "kg.reasoner.driver.class.name"; + + /** kg reasoner msg broker config */ + public static final String KG_REASONER_MSG_BROKER_CONFIG = "kg.reasoner.msg.broker.config"; + + /** kg reasoner output table config */ + public static final String KG_REASONER_OUTPUT_TABLE_CONFIG = "output.table.config"; + + /** sink to file */ + public static final String KG_REASONER_SINK_FILE = "kg.reasoner.sink.file"; + + /** sink table info */ + public static final String KG_REASONER_SINK_TABLE_INFO = "kg.reasoner.sink.table.info"; + + /** progress path */ + public static final String KG_REASONER_PROGRESS_PATH = "engine.progress.path"; + + /** start id list */ + public static final String KG_REASONER_START_ID_LIST = "kg.reasoner.start.id.list"; + + /** kg reasoner result server config */ + public static final String KG_REASONER_RESULT_SERVER = "kg.reasoner.result.server"; + + /** kg reasoner data namespace server config */ + public static final String KG_REASONER_NAMESPACE_SERVER = "kg.reasoner.namespace.server"; + + /** kg reasoner schema server token config */ + public static final String KG_REASONER_SCHEMA_SERVER_TOKEN = "kg.reasoner.schema.server.token"; + + /** preload enable */ + public static final String KG_REASONER_PRE_LOAD_ENABLE = "kg.reasoner.preload.enable"; + + /** query dfs config */ + public static final String KG_REASONER_QUERY_DFS_CONFIG = "kg.reasoner.query.dfs.config"; + + /** kg reasoner olap env config */ + public static final String KG_REASONER_OLAP_ENV = "kg.reasoner.olap.env"; + + /** kg reasoner olap init project */ + public static final String KG_REASONER_OLAP_INIT_PROJECT = "kg.reasoner.olap.init.project"; + + /** kg reasoner olap exec topic config */ + public static final String KG_REASONER_OLAP_EXEC_TOPIC = "kg.reasoner.olap.exec.topic"; + + /** kg reasoner olap task handler config */ + public static final String KG_REASONER_OLAP_TASK_HANDLER = "kg.reasoner.olap.task.handler"; + + /** set catalog */ + public static final String KG_REASONER_CATALOG = "kg.reasoner.catalog"; + + /** set graph load config */ + public static final String KG_REASONER_GRAPH_LOAD_CONFIG = "kg.reasoner.graph.load.config"; + + /** enable cache project */ + public static final String KG_REASONER_PLAN_CACHE_PROJECT_ENABLE = + "kg.reasoner.plan_cache_project_enable"; + + /** set graph data */ + public static final String KG_REASONER_GRAPH_DATA = "kg.reasoner.graph.data"; + + /** set binary property, default is true */ + public static final String KG_REASONER_BINARY_PROPERTY = "kg.reasoner.binary.property"; + + /** mock graph data */ + public static final String KG_REASONER_MOCK_GRAPH_DATA = "kg.reasoner.mock.graph.data"; + + /** expect batch num */ + public static final String KG_REASONER_EXPECT_BATCH_NUM = "kg.reasoner.expect.batch.num"; + + /** output graph config */ + public static final String KG_REASONER_OUTPUT_GRAPH = "kg.reasoner.output.graph"; + + /** max path limit use __max_path_size__ compatible with 1.0 */ + public static final String KG_REASONER_MAX_PATH_LIMIT = "__max_path_size__"; + + /** max path limit, exceeding threshold will throw an exception only for local runner */ + public static final String KG_REASONER_STRICT_MAX_PATH_THRESHOLD = + "kg.reasoner.strict.path.threshold"; + + /** edge limit for one vertex use __max_edge_size__ compatible with 1.0 */ + public static final String KG_REASONER_MAX_EDGE_LIMIT = "__max_edge_size__"; + + /** edge limit for one vertex and one type */ + public static final String KG_REASONER_MAX_EDGE_EACH_TYPE_LIMIT = "__max_edge_size_each_type__"; + + /** dsl start id info */ + public static final String KG_REASONER_START_ID_INFO = "start.id.info"; + + /** initializer class list */ + public static final String KG_REASONER_INITIALIZER_CLASS_LIST = + "kg.reasoner.initializer.class.list"; + + /** add extra schema to catalog */ + public static final String KG_REASONER_EXTRA_SCHEMA_JSON = "kg.reasoner.extra.schema.json"; + + /** extra graph load config */ + public static final String KG_REASONER_EXTRA_GRAPH_LOAD_CONFIG = + "kg.reasoner.extra.graph.load.config"; + + /** split KgGraph to worker */ + public static final String KG_REASONER_SPLIT_TO_WORKER = "kg.reasoner.enable.split.to.worker"; +} diff --git a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/ColumnMeta.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/shuffle/Shuffler.java similarity index 62% rename from cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/ColumnMeta.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/shuffle/Shuffler.java index bd3e8a2f..3183371c 100644 --- a/cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/ColumnMeta.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/shuffle/Shuffler.java @@ -11,15 +11,13 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.tablestore.model; +package com.antgroup.openspg.reasoner.shuffle; -import com.antgroup.openspg.server.common.model.base.BaseValObj; -import lombok.AllArgsConstructor; -import lombok.Getter; +import com.antgroup.openspg.reasoner.warehouse.common.partition.PartitionerFactory; -@Getter -@AllArgsConstructor -public class ColumnMeta extends BaseValObj { +public class Shuffler { - private final String name; + public static int getPartitionIndex(Object obj, int parallel) { + return PartitionerFactory.getPartition(parallel).partition(obj); + } } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerCsvWriter.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerCsvWriter.java new file mode 100644 index 00000000..7e961e70 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerCsvWriter.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.sink; + +import com.antgroup.openspg.reasoner.common.Utils; +import com.opencsv.CSVWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class KgReasonerCsvWriter { + + private static final Map CSV_WRITER_MAP = new ConcurrentHashMap<>(); + + public static void write(String file, Object[] data) { + CSVWriter writer = + CSV_WRITER_MAP.computeIfAbsent( + file, + k -> { + Utils.deletePath(file); + Utils.createFile(file); + try { + return new CSVWriter(new FileWriter(file)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + String[] strData = new String[data.length]; + for (int i = 0; i < data.length; ++i) { + strData[i] = String.valueOf(data[i]); + } + + List listStrData = new ArrayList<>(); + listStrData.add(strData); + writer.writeAll(listStrData); + } + + public static void flush(String file) { + CSVWriter writer = CSV_WRITER_MAP.remove(file); + if (null == writer) { + return; + } + try { + writer.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerSinkType.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerSinkType.java new file mode 100644 index 00000000..54731852 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerSinkType.java @@ -0,0 +1,25 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.sink; + +import java.io.Serializable; + +public enum KgReasonerSinkType implements Serializable { + /** log */ + LOG, + /** loca csv file */ + FILE, + /** hive table */ + HIVE +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerSinkUtils.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerSinkUtils.java new file mode 100644 index 00000000..b3ee71a4 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/KgReasonerSinkUtils.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.sink; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.antgroup.openspg.reasoner.io.model.AbstractTableInfo; +import com.antgroup.openspg.reasoner.io.model.HiveTableInfo; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import java.util.Map; + +public class KgReasonerSinkUtils { + + /** get sink type from config params */ + public static KgReasonerSinkType getKgReasonerSinkType(Map params) { + JSONObject outputTableConfig = getOutputTableConfig(params); + if (null == outputTableConfig) { + return KgReasonerSinkType.LOG; + } + String outputType = outputTableConfig.getString("type"); + return KgReasonerSinkType.valueOf(outputType); + } + + /** get sink table info from config */ + public static AbstractTableInfo getSinkTableInfo(Map params) { + KgReasonerSinkType sinkType = getKgReasonerSinkType(params); + if (KgReasonerSinkType.HIVE.equals(sinkType)) { + String tableInfoStr = String.valueOf(params.get(ConfigKey.KG_REASONER_SINK_TABLE_INFO)); + return JSON.parseObject(tableInfoStr, HiveTableInfo.class); + } + return null; + } + + /** get write file */ + public static String getKgReasonerSinkFile(Map params) { + return String.valueOf(params.get(ConfigKey.KG_REASONER_SINK_FILE)); + } + + private static JSONObject getOutputTableConfig(Map params) { + JSONObject outputTableConfig; + Object outputTableConfigObj = params.get(ConfigKey.KG_REASONER_OUTPUT_TABLE_CONFIG); + if (null == outputTableConfigObj) { + return null; + } + if (outputTableConfigObj instanceof String) { + outputTableConfig = JSON.parseObject(String.valueOf(outputTableConfigObj)); + } else { + outputTableConfig = (JSONObject) outputTableConfigObj; + } + return outputTableConfig; + } +} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/CallbackResult.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/QueryGraphStateInfo.java similarity index 52% rename from server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/CallbackResult.java rename to reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/QueryGraphStateInfo.java index 1f9ce46a..cf41242d 100644 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/CallbackResult.java +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/sink/QueryGraphStateInfo.java @@ -11,19 +11,21 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model; +package com.antgroup.openspg.reasoner.sink; -import com.antgroup.openspg.server.common.model.job.BaseJobRunResult; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; +import com.antgroup.openspg.reasoner.common.graph.type.GraphItemType; import java.io.Serializable; -import lombok.AllArgsConstructor; -import lombok.Getter; +import java.util.List; +import lombok.Data; -@Getter -@AllArgsConstructor -public class CallbackResult implements Serializable { +@Data +public class QueryGraphStateInfo implements Serializable { + private GraphItemType sourceGraphItemType; + private List sourceTypeList; - private final JobInstStatusEnum status; + private GraphItemType targetGraphItemType; + private List targetPropertyNameList; - private final BaseJobRunResult result; + private List targetEdgeTypeList; + private String targetVertexIdString; } diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/task/TaskRecord.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/task/TaskRecord.java new file mode 100644 index 00000000..c431aae3 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/task/TaskRecord.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.task; + +import com.antgroup.openspg.reasoner.graphstate.GraphStateTypeEnum; +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import lombok.Builder; +import lombok.Data; +import scala.Tuple2; + +@Data +@Builder +public class TaskRecord implements Serializable { + /** task id */ + private String taskId; + + /** task parallel */ + private int parallel; + + /** graph state type */ + private GraphStateTypeEnum graphStateType; + + /** class name of graph loader */ + private String graphLoaderJobClassName; + + /** start id from input */ + private List> startIdList; + + /** expect batch number, batching control */ + private int expectBatchNum; + + /** dsl */ + private String dsl; + + /** initializer class list */ + private List initializerClassList; + + /** task params */ + private Map params; +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/HttpInvoker.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/HttpInvoker.java new file mode 100644 index 00000000..2f377795 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/HttpInvoker.java @@ -0,0 +1,157 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.utils; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.common.http.AkgHttpClient; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; + +public class HttpInvoker { + /** + * do post content + * + * @param url + * @param encodedContentByte + * @param encoding + * @param connectTimeoutMs + * @param readTimeoutMs + * @param isJson + * @return + * @throws IOException + */ + public static AkgHttpClient.HttpResult doPostWithContent( + String url, + byte encodedContentByte[], + String encoding, + int connectTimeoutMs, + int readTimeoutMs, + boolean isJson) + throws IOException { + + HttpURLConnection conn = null; + try { + // http config + conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setConnectTimeout(connectTimeoutMs > 3000 ? connectTimeoutMs : 3000); + conn.setReadTimeout(readTimeoutMs); + conn.setDoOutput(true); + conn.setDoInput(true); + + // header config + conn.setRequestProperty("Accept", "application/json"); + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json"); + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + } + + conn.getOutputStream().write(encodedContentByte); + + int respCode = conn.getResponseCode(); + String resp = null; + + // result check + if (HttpURLConnection.HTTP_OK == respCode) { + resp = IOUtils.toString(conn.getInputStream(), encoding); + } else { + if (conn.getErrorStream() != null) { + resp = IOUtils.toString(conn.getErrorStream(), encoding); + } else { + resp = IOUtils.toString(conn.getInputStream(), encoding); + } + } + return new AkgHttpClient.HttpResult(respCode, resp); + } finally { + if (null != conn) { + conn.disconnect(); + } + } + } + + /** + * post with json format + * + * @param url + * @param paramValues + * @param encoding + * @param connectTimeoutMs + * @param readTimeoutMs + * @return + * @throws IOException + */ + public static AkgHttpClient.HttpResult doPostWithJSON( + String url, + Map paramValues, + String encoding, + int connectTimeoutMs, + int readTimeoutMs) + throws IOException { + + String encodedContent = JSON.toJSONString(paramValues); + return doPostWithContent( + url, encodedContent.getBytes(), encoding, connectTimeoutMs, readTimeoutMs, true); + } + + /** + * post with url encode format + * + * @param url + * @param paramValues + * @param encoding + * @param connectTimeoutMs + * @param readTimeoutMs + * @return + * @throws IOException + */ + public static AkgHttpClient.HttpResult doPostWithParameters( + String url, + Map paramValues, + String encoding, + int connectTimeoutMs, + int readTimeoutMs) + throws IOException { + + String encodedContent = encodingUrlParameters(paramValues); + return doPostWithContent( + url, + encodedContent.getBytes(StandardCharsets.UTF_8), + encoding, + connectTimeoutMs, + readTimeoutMs, + false); + } + + /** + * encoding url + * + * @param params + * @return + */ + private static String encodingUrlParameters(Map params) { + List kvs = new ArrayList<>(params.size()); + for (String k : params.keySet()) { + kvs.add(k + "=" + URLEncoder.encode(params.get(k))); + } + return StringUtils.join(kvs, "&"); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/PredicateKgGraph.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/PredicateKgGraph.java new file mode 100644 index 00000000..c0c6d8da --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/PredicateKgGraph.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.utils; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +public class PredicateKgGraph implements Predicate> { + + private final Pattern kgGraphSchema; + private final Map initRuleContext; + private final List ruleList; + + /** implement a predicate for KgGraph */ + public PredicateKgGraph(Pattern kgGraphSchema, List ruleList) { + this.kgGraphSchema = kgGraphSchema; + this.initRuleContext = RunnerUtil.getKgGraphInitContext(this.kgGraphSchema); + this.ruleList = ruleList; + } + + @Override + public boolean test(KgGraph kgGraph) { + Map context = RunnerUtil.kgGraph2Context(this.initRuleContext, kgGraph); + return RuleRunner.getInstance().check(context, ruleList, ""); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/RocksDBUtil.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/RocksDBUtil.java new file mode 100644 index 00000000..4d3b66d6 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/RocksDBUtil.java @@ -0,0 +1,148 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.utils; + +import com.antgroup.openspg.reasoner.common.exception.IllegalArgumentException; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.type.MapType2IdFactory; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import java.nio.ByteBuffer; +import scala.Tuple2; + +public class RocksDBUtil { + + private static final String PLACEHOLDER = "\01"; + + public static final String VERTEX_FLAG = "V"; + + public static final String EDGE_FLAG = "E"; + + public static final Integer ROCKSDB_KEY_PREFIX_LENGTH = 18; + + /** + * append window to rocksdb key prefix + * + * @param keyPrefix + * @param window + * @return + */ + public static byte[] rocksdbKeyAppendWindow(byte[] keyPrefix, long window) { + ByteBuffer buffer = ByteBuffer.allocate(keyPrefix.length + Long.BYTES); + buffer.put(keyPrefix); + buffer.putLong(window); + return buffer.array(); + } + + /** + * get window from rocksdb key + * + * @param key + * @return + */ + public static long getRocksdbKeyWindow(byte[] key) { + ByteBuffer buffer = ByteBuffer.wrap(key); + buffer.position(key.length - Long.BYTES); + return buffer.getLong(); + } + + /** + * construct vertex key on rocksdb V_w_id_fill_W + * + * @param vertexId + * @return + */ + public static byte[] buildRocksDBVertexKeyWithoutWindow(IVertexId vertexId) { + ByteBuffer buffer = ByteBuffer.allocate(RocksDBUtil.ROCKSDB_KEY_PREFIX_LENGTH); + buffer.put(VERTEX_FLAG.getBytes()); + buffer.putLong(vertexId.getTypeId()); + buffer.putLong(vertexId.getInternalId()); + buffer.put(PLACEHOLDER.getBytes()); + return buffer.array(); + } + + /** + * construct edge key on rocksdb E_T_IN_vid_W + * + * @param edgeType + * @param direction + * @param vertexId + * @return + */ + public static byte[] buildRocksDBEdgeKeyWithoutWindow( + String edgeType, Direction direction, IVertexId vertexId) { + if (Direction.BOTH.equals(direction)) { + throw new RuntimeException("Not allowed BOTH edge in RocksDB"); + } + ByteBuffer buffer = ByteBuffer.allocate(RocksDBUtil.ROCKSDB_KEY_PREFIX_LENGTH); + buffer.put(EDGE_FLAG.getBytes()); + Long edgeTypeId = MapType2IdFactory.getMapType2Id().getIdByType(edgeType); + if (null == edgeTypeId) { + throw new IllegalArgumentException("valid edge type", edgeType, "", null); + } + buffer.putLong(edgeTypeId); + buffer.put(String.valueOf(direction.getValue()).getBytes()); + buffer.putLong(vertexId.getInternalId()); + return buffer.array(); + } + + public static Tuple2 splitVertexKey(byte[] vertexKey) { + ByteBuffer vertex = ByteBuffer.wrap(vertexKey); + byte[] prefix = new byte[1]; + vertex.get(prefix); + String vertexFlag1 = String.valueOf((char) prefix[0]); + byte[] typeAndId = new byte[16]; + vertex.get(typeAndId); + + return new Tuple2(vertexFlag1, IVertexId.from(typeAndId)); + } + + /** + * Check whether the two vertex keys are the same point + * + * @param vertexKey1 + * @param vertexKey2 + * @return + */ + public static Boolean checkSameVertexKey(byte[] vertexKey1, byte[] vertexKey2) { + if (!checkIsVertex(vertexKey1) || !checkIsVertex(vertexKey2)) { + return false; + } + IVertexId id1 = extractVertexId(vertexKey1); + IVertexId id2 = extractVertexId(vertexKey2); + return id1.equals(id2); + } + + /** + * extract vertexId from vertex key + * + * @param vertexKey + * @return + */ + public static IVertexId extractVertexId(byte[] vertexKey) { + Tuple2 splitResult = splitVertexKey(vertexKey); + return splitResult._2; + } + + /** + * check whether the vertexKey is a vertex + * + * @param vertexKey + * @return + */ + public static Boolean checkIsVertex(byte[] vertexKey) { + Tuple2 splitResult = splitVertexKey(vertexKey); + String prefix = splitResult._1; + return VERTEX_FLAG.equals(prefix); + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/RunnerUtil.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/RunnerUtil.java new file mode 100644 index 00000000..1b383ca7 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/RunnerUtil.java @@ -0,0 +1,1067 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.utils; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.exception.NotImplementedException; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.SPO; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.OptionalEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.PathEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.type.MapType2IdFactory; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.MirrorVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.NoneVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.block.AddPredicate; +import com.antgroup.openspg.reasoner.lube.block.AddVertex; +import com.antgroup.openspg.reasoner.lube.catalog.struct.Field; +import com.antgroup.openspg.reasoner.lube.common.expr.AggOpExpr; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.common.expr.First$; +import com.antgroup.openspg.reasoner.lube.common.graph.IRField; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.lube.common.rule.Rule; +import com.antgroup.openspg.reasoner.lube.logical.EdgeVar; +import com.antgroup.openspg.reasoner.lube.logical.NodeVar; +import com.antgroup.openspg.reasoner.lube.logical.PropertyVar; +import com.antgroup.openspg.reasoner.lube.logical.RepeatPathVar; +import com.antgroup.openspg.reasoner.lube.logical.RichVar; +import com.antgroup.openspg.reasoner.lube.logical.Var; +import com.antgroup.openspg.reasoner.lube.utils.RuleUtils; +import com.antgroup.openspg.reasoner.rdg.common.FoldRepeatEdgeInfo; +import com.antgroup.openspg.reasoner.rdg.common.UnfoldRepeatEdgeInfo; +import com.antgroup.openspg.reasoner.rule.RuleRunner; +import com.antgroup.openspg.reasoner.runner.ConfigKey; +import com.antgroup.openspg.reasoner.session.KGReasonerSession; +import com.antgroup.openspg.reasoner.udf.UdfMng; +import com.antgroup.openspg.reasoner.udf.UdfMngFactory; +import com.antgroup.openspg.reasoner.udf.model.BaseUdtf; +import com.antgroup.openspg.reasoner.udf.model.LinkedUdtfResult; +import com.antgroup.openspg.reasoner.udf.model.UdtfMeta; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.antgroup.openspg.reasoner.util.KgGraphSchema; +import com.antgroup.openspg.reasoner.warehouse.common.config.EdgeLoaderConfig; +import com.antgroup.openspg.reasoner.warehouse.common.config.GraphLoaderConfig; +import com.antgroup.openspg.reasoner.warehouse.common.config.VertexLoaderConfig; +import com.antgroup.openspg.reasoner.warehouse.common.partition.BasePartitioner; +import com.antgroup.openspg.reasoner.warehouse.utils.WareHouseUtils; +import com.google.common.collect.Lists; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; +import scala.collection.JavaConversions; + +public class RunnerUtil { + private static final Logger log = LoggerFactory.getLogger(RunnerUtil.class); + + /** kg id is global id property */ + public static final String KG_REASONER_PROPERTY_GLOBAL_ID = "kgId"; + + /** + * Check if cur is between start and end + * + * @param start + * @param end + * @param cur + * @return + */ + public static Boolean between(Long start, Long end, Long cur) { + boolean isInInterval = true; + if (start != null && start != 0) { + isInInterval = cur >= start; + } + if (end != null && end != 0) { + isInInterval = isInInterval && (cur <= end); + } + return isInInterval; + } + + /** get vertex type */ + public static String getVertexType(IVertex vertex) { + return vertex.getId().getType(); + } + + /** get edge source type */ + public static String getEdgeSourceType(IEdge edge) { + return edge.getSourceId().getType(); + } + + /** get edge source biz id */ + public static String getEdgeSourceBizId(IEdge edge) { + throw new NotImplementedException("", null); + } + + /** get edge target type */ + public static String getEdgeTargetType(IEdge edge) { + return edge.getTargetId().getType(); + } + + /** get edge target biz id */ + public static String getEdgeTargetBizId(IEdge edge) { + throw new NotImplementedException("", null); + } + + /** expend KgGraph to path, filter than merge */ + public static KgGraph filterKgGraph( + KgGraph value, + Pattern kgGraphSchema, + KgGraphSplitStaticParameters staticParameters, + List ruleList, + Long maxPathLimit) { + KgGraph resultMsg = new KgGraphImpl(); + List> mergeMsgList = new ArrayList<>(); + + Predicate> predicate = new PredicateKgGraph(kgGraphSchema, ruleList); + Iterator> pathIt = value.getPath(staticParameters, predicate); + long count = 0; + while (pathIt.hasNext()) { + KgGraph path = pathIt.next(); + if (null == path) { + continue; + } + + mergeMsgList.add(path); + count++; + if (null != maxPathLimit && count >= maxPathLimit) { + break; + } + } + + if (CollectionUtils.isEmpty(mergeMsgList)) { + return null; + } + + resultMsg.merge(mergeMsgList, kgGraphSchema); + return resultMsg; + } + + /** filter kgGraph by rule, will not expend kgGraph to path, just split by vertex alias set */ + public static List> filterKgGraph( + KgGraph value, + Set splitAliasSet, + Pattern kgGraphSchema, + KgGraphSplitStaticParameters staticParameters, + List ruleList, + Long maxPathLimit) { + Predicate> predicate = new PredicateKgGraph(kgGraphSchema, ruleList); + return value.split(splitAliasSet, kgGraphSchema, staticParameters, predicate, maxPathLimit); + } + + public static void doStarPathLimit( + Map>> adjEdges, long limit, int minEdgeNum) { + long resultCount = 1; + int shrinkFactorValidNum = 0; + final double shrinkFactorThreshold = 1.1; + for (Map.Entry>> entry : adjEdges.entrySet()) { + int size = entry.getValue().size(); + if (!(size <= minEdgeNum)) { + shrinkFactorValidNum++; + } + resultCount = resultCount * size; + } + if (resultCount <= limit || shrinkFactorValidNum <= 0) { + return; + } + double shrinkFactor = Math.pow(1.0 * resultCount / limit, 1.0 / shrinkFactorValidNum); + if (shrinkFactor <= shrinkFactorThreshold) { + return; + } + for (Map.Entry>> entry : adjEdges.entrySet()) { + List> edgeList = entry.getValue(); + if (edgeList.size() <= minEdgeNum) { + continue; + } + int newListSize = (int) (edgeList.size() / shrinkFactor) + 1; + if (newListSize < edgeList.size()) { + entry.setValue(new ArrayList<>(edgeList.subList(0, newListSize))); + } + } + } + + /** get vertex type from property */ + public static String getVertexTypeFromProperty(IProperty property) { + return String.valueOf(property.get(Constants.CONTEXT_LABEL)); + } + + /** + * KgGraph path 2 context + * + * @param kgGraph + * @return + */ + public static Map kgGraph2Context( + Map initContext, KgGraph kgGraph) { + Map context = new HashMap<>(); + for (String alias : kgGraph.getVertexAlias()) { + List> vertexList = kgGraph.getVertex(alias); + if (CollectionUtils.isEmpty(vertexList)) { + continue; + } + context.put(alias, vertexContext(vertexList.get(0))); + } + + for (String alias : kgGraph.getEdgeAlias()) { + List> edgeList = kgGraph.getEdge(alias); + if (CollectionUtils.isEmpty(edgeList)) { + continue; + } + context.put(alias, edgeContext(edgeList.get(0), null, kgGraph)); + } + for (Map.Entry entry : initContext.entrySet()) { + context.putIfAbsent(entry.getKey(), entry.getValue()); + } + return context; + } + + /** init context */ + public static Map getKgGraphInitContext(Pattern kgGraphSchema) { + Map context = new HashMap<>(); + for (Connection connection : RunnerUtil.getConnectionSet(kgGraphSchema)) { + context.put(connection.source(), new HashMap<>()); + context.put(connection.alias(), new HashMap<>()); + context.put(connection.target(), new HashMap<>()); + } + return context; + } + + /** get vertex context in alias */ + public static Map vertexContext( + IVertex vertex, String alias) { + Map result = new HashMap<>(); + result.put(alias, vertexContext(vertex)); + return result; + } + + /** vertex context */ + public static Map vertexContext(IVertex vertex) { + Map vertexProperty = new HashMap<>(); + if (vertex instanceof MirrorVertex) { + vertexProperty.put(Constants.MIRROR_VERTEX_FLAG, true); + } else if (vertex instanceof NoneVertex) { + return noneVertexContext((NoneVertex) vertex); + } + IProperty property = vertex.getValue(); + if (null != property) { + for (String key : property.getKeySet()) { + vertexProperty.put(key, property.get(key)); + } + } + vertexProperty.put(Constants.CONTEXT_LABEL, getVertexType(vertex)); + vertexProperty.put(Constants.VERTEX_INTERNAL_ID_KEY, vertex.getId().getInternalId()); + return vertexProperty; + } + + private static Map noneVertexContext(NoneVertex vertex) { + Map vertexProperty = new HashMap<>(); + IProperty property = vertex.getValue(); + if (null != property) { + for (String key : property.getKeySet()) { + vertexProperty.put(key, property.get(key)); + } + } + vertexProperty.put(Constants.NONE_VERTEX_FLAG, true); + return vertexProperty; + } + + /** get edge context in alias */ + public static Map edgeContext( + IEdge edge, SPO spo, String alias) { + if (null == spo) { + spo = new SPO(edge.getType()); + } + Map result = new HashMap<>(); + result.put(alias, edgeContext(edge, spo.getP(), null)); + return result; + } + + /** get edge context */ + public static Map edgeContext(IEdge edge, String edgeType) { + return edgeContext(edge, edgeType, null); + } + + /** get edge context */ + public static Map edgeContext( + IEdge edge, String edgeType, KgGraph kgGraph) { + if (edge instanceof PathEdge) { + return pathEdgeContext((PathEdge) edge, edgeType, kgGraph); + } else if (edge instanceof OptionalEdge) { + return optionalEdgeContext((OptionalEdge) edge); + } + Map edgeProperty = new HashMap<>(); + if (StringUtils.isEmpty(edgeType)) { + SPO spo = new SPO(edge.getType()); + edgeType = spo.getP(); + } + IProperty property = edge.getValue(); + if (null != property) { + for (String key : property.getKeySet()) { + edgeProperty.put(key, property.get(key)); + } + } + edgeProperty.put(Constants.CONTEXT_LABEL, edgeType); + IVertexId fromId = edge.getSourceId(); + IVertexId toId = edge.getSourceId(); + if (Direction.IN.equals(edge.getDirection())) { + IVertexId tmp = fromId; + fromId = toId; + toId = tmp; + } + edgeProperty.put(Constants.EDGE_FROM_INTERNAL_ID_KEY, fromId.getInternalId()); + edgeProperty.put(Constants.EDGE_FROM_ID_TYPE_KEY, fromId.getType()); + edgeProperty.put(Constants.EDGE_TO_INTERNAL_ID_KEY, toId.getInternalId()); + edgeProperty.put(Constants.EDGE_TO_ID_TYPE_KEY, toId.getType()); + return edgeProperty; + } + + public static Map pathEdgeContext( + PathEdge edge, String edgeType, KgGraph kgGraph) { + Map edgeProperty = new HashMap<>(); + edgeProperty.put(Constants.CONTEXT_LABEL, edgeType); + return edgeProperty; + } + + private static Map optionalEdgeContext( + OptionalEdge optionalEdge) { + Map edgeProperty = new HashMap<>(); + IProperty property = optionalEdge.getValue(); + if (null != property) { + for (String key : property.getKeySet()) { + edgeProperty.put(key, property.get(key)); + } + } + edgeProperty.put(Constants.OPTIONAL_EDGE_FLAG, true); + return edgeProperty; + } + + /** get dst vertex context */ + public static Map dstVertexContext(IEdge edge) { + Map edgeContext = edgeContext(edge, null); + Map vertexProperty = new HashMap<>(); + if (Direction.OUT.equals(edge.getDirection())) { + vertexProperty.put(Constants.CONTEXT_LABEL, edgeContext.get(Constants.EDGE_TO_ID_TYPE_KEY)); + vertexProperty.put(Constants.NODE_ID_KEY, edgeContext.get(Constants.EDGE_TO_ID_KEY)); + vertexProperty.put( + Constants.VERTEX_INTERNAL_ID_KEY, edgeContext.get(Constants.EDGE_TO_INTERNAL_ID_KEY)); + } else { + vertexProperty.put(Constants.CONTEXT_LABEL, edgeContext.get(Constants.EDGE_FROM_ID_TYPE_KEY)); + vertexProperty.put(Constants.NODE_ID_KEY, edgeContext.get(Constants.EDGE_FROM_ID_KEY)); + vertexProperty.put( + Constants.VERTEX_INTERNAL_ID_KEY, edgeContext.get(Constants.EDGE_FROM_INTERNAL_ID_KEY)); + } + return vertexProperty; + } + + public static final String FLATTEN_SEPARATOR = "."; + + /** + * make context flatten + * + * @param context + * @return + */ + public static Map flattenContext(Map context) { + Map result = new HashMap<>(); + for (Map.Entry entry : context.entrySet()) { + if (!(entry.getValue() instanceof Map)) { + continue; + } + Map propertyMap = (Map) entry.getValue(); + for (Map.Entry propertyEntry : propertyMap.entrySet()) { + result.put( + entry.getKey() + FLATTEN_SEPARATOR + propertyEntry.getKey(), propertyEntry.getValue()); + } + } + return result; + } + + /** load csv file to list */ + public static List loadCsvFile(String file) { + List result = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + String line; + while ((line = br.readLine()) != null) { + String[] values = line.split(","); + for (int i = 0; i < values.length; ++i) { + values[i] = values[i].substring(1, values[i].length() - 1); + } + result.add(values); + } + } catch (IOException e) { + throw new RuntimeException("load csv file error", e); + } + return result; + } + + /** get connection set from pattern */ + public static Set getConnectionSet(Pattern schema) { + Set connectionSet = new HashSet<>(); + for (String pe : JavaConversions.mapAsJavaMap(schema.topology()).keySet()) { + Set partConnectionSet = + JavaConversions.setAsJavaSet(schema.topology().get(pe).get()); + connectionSet.addAll(partConnectionSet); + } + return connectionSet; + } + + /** get neighbor alias */ + public static String getNeighborAlias(String alias, Connection pc) { + if (alias.equals(pc.source())) { + return pc.target(); + } else if (alias.equals(pc.target())) { + return pc.source(); + } + return null; + } + + /** + * get start id info from params + * + * @param params + * @return + */ + public static List> getStartIdFromParams(Map params) { + String startIdListStr = (String) params.get(ConfigKey.KG_REASONER_START_ID_LIST); + if (StringUtils.isEmpty(startIdListStr)) { + return null; + } + List> startIdList = new ArrayList<>(); + JSONArray idListJson = JSON.parseArray(startIdListStr); + for (int i = 0; i < idListJson.size(); ++i) { + String idTupleStr = idListJson.getString(i); + JSONArray idTupleJson = JSON.parseArray(idTupleStr); + startIdList.add(new Tuple2<>(idTupleJson.getString(0), idTupleJson.getString(1))); + } + return startIdList; + } + + /** get dsl config map */ + public static Map getOfflineDslParams(Map configMap, boolean base64Decode) { + String rowStr = RunnerUtil.getStringOrDefault(configMap, ConfigKey.KG_REASONER_PARAMS, ""); + if (StringUtils.isEmpty(rowStr)) { + return new HashMap<>(); + } + String paramsJsonStr = rowStr; + if (base64Decode) { + paramsJsonStr = new String(Base64.getDecoder().decode(rowStr), StandardCharsets.UTF_8); + } + if (StringUtils.isEmpty(paramsJsonStr)) { + return new HashMap<>(); + } + return new HashMap<>(JSON.parseObject(paramsJsonStr)); + } + + /** + * get string from map or return default value + * + * @param config + * @param configKey + * @param defaultValue + * @return + */ + public static String getStringOrDefault(Map config, String configKey, String defaultValue) { + if (config.containsKey(configKey)) { + return String.valueOf(config.get(configKey)); + } else { + return defaultValue; + } + } + + /** + * compute the linked edge + * + * @return + */ + public static List> linkEdge( + String taskId, + KgGraph kgGraph, + PartialGraphPattern kgGraphSchema, + KgGraphSplitStaticParameters staticParameters, + EdgePattern linkedEdgePattern, + UdtfMeta udtfMeta, + BasePartitioner partitioner) { + Iterator> it = kgGraph.getPath(staticParameters, null); + List> mergeList = new ArrayList<>(); + + while (it.hasNext()) { + KgGraph path = it.next(); + + Map context = + RunnerUtil.kgGraph2Context(RunnerUtil.getKgGraphInitContext(kgGraphSchema), path); + List exprList = JavaConversions.seqAsJavaList(linkedEdgePattern.edge().params()); + List paramList = new ArrayList<>(); + for (Expr expr : exprList) { + List exprStr = WareHouseUtils.getRuleList(expr); + Object parameter = RuleRunner.getInstance().executeExpression(context, exprStr, taskId); + paramList.add(parameter); + } + + BaseUdtf tableFunction = udtfMeta.createTableFunction(); + tableFunction.process(paramList); + List> udtfResult = tableFunction.getCollector(); + List linkedUdtfResultList = + udtfResult.stream() + .flatMap(List::stream) + .filter(Objects::nonNull) + .map( + obj -> { + if (!(obj instanceof LinkedUdtfResult)) { + throw new RuntimeException("linked udtf must return LinkedUdtfResult"); + } + return ((LinkedUdtfResult) obj); + }) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(linkedUdtfResultList)) { + continue; + } + String sourceAlias = linkedEdgePattern.src().alias(); + List> sourceList = path.getVertex(sourceAlias); + if (null == sourceList || sourceList.size() != 1) { + throw new RuntimeException("There is more than one start vertex in kgGraph path"); + } + IVertexId sourceId = sourceList.get(0).getId(); + Connection pc = linkedEdgePattern.edge(); + + Map>> newAliasVertexMap = new HashMap<>(); + Map>> newAliasEdgeMap = new HashMap<>(); + for (LinkedUdtfResult linkedUdtfResult : linkedUdtfResultList) { + for (String targetIdStr : linkedUdtfResult.getTargetVertexIdList()) { + // add target vertex + String targetAlias = pc.target(); + PatternElement targetVertexMeta = linkedEdgePattern.dst(); + List targetVertexTypes = + new ArrayList<>(JavaConversions.setAsJavaSet(targetVertexMeta.typeNames())); + if (targetVertexTypes.size() == 0) { + throw new RuntimeException( + "Linked edge target vertex type must contains at least one type"); + } + for (String targetVertexType : targetVertexTypes) { + IVertexId targetId = IVertexId.from(targetIdStr, targetVertexType); + if (partitioner != null && !partitioner.canPartition(targetId)) { + continue; + } + // need add property with id + Set> newVertexSet = + newAliasVertexMap.computeIfAbsent(targetAlias, k -> new HashSet<>()); + newVertexSet.add(new Vertex(targetId)); + + // construct new edge + IEdge linkedEdge = new Edge<>(sourceId, targetId, null); + linkedEdge.setType( + sourceId.getType() + + "_" + + linkedEdgePattern.edge().funcName() + + "_" + + targetVertexType); + String edgeAlias = pc.alias(); + + Set> newEdgeSet = + newAliasEdgeMap.computeIfAbsent(edgeAlias, k -> new HashSet<>()); + newEdgeSet.add(linkedEdge); + } + } + } + if (!(newAliasVertexMap.isEmpty() && newAliasEdgeMap.isEmpty())) { + KgGraph newKgGraph = new KgGraphImpl(newAliasVertexMap, newAliasEdgeMap); + path.merge(Lists.newArrayList(newKgGraph), null); + mergeList.add(path); + } + } + return mergeList; + } + + /** + * choose the udtfMeta corresponding to linkedEdgePattern + * + * @param pattern + * @return + */ + public static UdtfMeta chooseUdtfMeta(EdgePattern pattern) { + UdfMng udfMng = UdfMngFactory.getUdfMng(); + List udtfMetaList = udfMng.getAllUdtfMeta(); + String funcName = pattern.edge().funcName(); + UdtfMeta resultUdtfMeta = null; + for (UdtfMeta udtfMeta : udtfMetaList) { + if (funcName.equals(udtfMeta.getName())) { + resultUdtfMeta = udtfMeta; + break; + } + } + if (null == resultUdtfMeta) { + throw new RuntimeException("No found udtf=" + funcName); + } + return resultUdtfMeta; + } + + /** get rule use vertex and edge */ + public static Tuple2, Set> getRuleUseVertexAndEdgeSet( + Rule rule, PartialGraphPattern kgGraphSchema) { + Set vertexAliasSet = new HashSet<>(); + Set edgePatternSet = new HashSet<>(); + scala.collection.immutable.Set nodesAlias = KgGraphSchema.getNodesAlias(kgGraphSchema); + scala.collection.immutable.Set edgesAlias = KgGraphSchema.getEdgesAlias(kgGraphSchema); + List irFieldList = + JavaConversions.seqAsJavaList( + RuleUtils.getAllInputFieldInRule(rule, nodesAlias, edgesAlias)); + for (IRField irField : irFieldList) { + if (nodesAlias.contains(irField.name())) { + vertexAliasSet.add(irField.name()); + } + if (edgesAlias.contains(irField.name())) { + Connection pc = KgGraphSchema.getPatternConnection(kgGraphSchema, irField.name()); + edgePatternSet.add(pc); + } + } + for (Connection pc : edgePatternSet) { + vertexAliasSet.remove(pc.source()); + vertexAliasSet.remove(pc.target()); + } + return new Tuple2<>(vertexAliasSet, edgePatternSet); + } + + /** if match pattern contains edges in kgGraphSchema, there has intersection alias */ + public static Set getIntersectionAliasSet(Pattern kgGraphSchema, Pattern matchPattern) { + Set matchRootNeighborSet = new HashSet<>(); + String matchRootAlias = matchPattern.root().alias(); + for (Connection pc : RunnerUtil.getConnectionSet(matchPattern)) { + String neighborAlias = RunnerUtil.getNeighborAlias(matchRootAlias, pc); + if (StringUtils.isNotEmpty(neighborAlias)) { + matchRootNeighborSet.add(neighborAlias); + } + } + + Set kgGraphAliasSet = new HashSet<>(); + for (Connection pc : RunnerUtil.getConnectionSet(kgGraphSchema)) { + kgGraphAliasSet.add(pc.source()); + kgGraphAliasSet.add(pc.target()); + } + + matchRootNeighborSet.retainAll(kgGraphAliasSet); + if (matchRootNeighborSet.isEmpty()) { + return null; + } + return matchRootNeighborSet; + } + + /** get intersection vertex's edge target */ + public static Map> getEdgeAlias2ValidTargetIdMap( + Set intersectionAliasSet, + Collection> kgGraphList, + Pattern schema) { + Map> edgeAlias2ValidTargetIdMap = new HashMap<>(); + Set pcSet = new HashSet<>(); + for (String alias : intersectionAliasSet) { + pcSet.addAll(JavaConversions.setAsJavaSet(KgGraphSchema.getNeighborEdges(schema, alias))); + } + for (Connection pc : pcSet) { + Set idSet = + edgeAlias2ValidTargetIdMap.computeIfAbsent(pc.alias(), k -> new HashSet<>()); + for (KgGraph kgGraph : kgGraphList) { + List> vList = kgGraph.getVertex(pc.target()); + for (IVertex v : vList) { + idSet.add(v.getId()); + } + } + } + return edgeAlias2ValidTargetIdMap; + } + + /** + * @param matchedKgGraph + * @param rootAlias + * @return + */ + public static int getMinVertexCount(KgGraph matchedKgGraph, String rootAlias) { + int minMatchCount = Integer.MAX_VALUE; + for (String alias : matchedKgGraph.getVertexAlias()) { + if (alias.equals(rootAlias)) { + continue; + } + int size = matchedKgGraph.getVertex(alias).size(); + if (size < minMatchCount) { + minMatchCount = size; + } + } + if (minMatchCount == Integer.MAX_VALUE) { + return 1; + } + return minMatchCount; + } + + /** is vertex alias */ + public static Boolean isVertexAlias(String alias, Pattern pattern) { + for (Connection pc : getConnectionSet(pattern)) { + if (pc.alias().equals(alias)) { + return false; + } else if (pc.source().equals(alias)) { + return true; + } else if (pc.target().equals(alias)) { + return true; + } + } + return null; + } + + /** get vertex alias set */ + public static Set getVertexAliasSet(Pattern pattern) { + Set result = new HashSet<>(); + for (Connection pc : getConnectionSet(pattern)) { + result.add(pc.source()); + result.add(pc.target()); + } + return result; + } + + /** check is all first aggregator */ + public static List getFirstEdgeAliasList( + scala.collection.immutable.Map aggregations) { + List result = new ArrayList<>(); + if (null == aggregations || aggregations.isEmpty()) { + return null; + } + for (Map.Entry entry : JavaConversions.mapAsJavaMap(aggregations).entrySet()) { + Var var = entry.getKey(); + if (!(var instanceof EdgeVar)) { + return null; + } + Aggregator aggregator = entry.getValue(); + if (!(aggregator instanceof AggOpExpr)) { + return null; + } + AggOpExpr aggOpExpr = (AggOpExpr) aggregator; + if (aggOpExpr.name() != First$.MODULE$) { + return null; + } + EdgeVar edgeVar = (EdgeVar) var; + result.add(edgeVar.name()); + } + return result; + } + + private static final Map>>> + JOIN_RIGHT_DATA_STATIC = new ConcurrentHashMap<>(); + + public static Map>> getRightJoinData(int index) { + return JOIN_RIGHT_DATA_STATIC.computeIfAbsent(index, k -> new HashMap<>()); + } + + public static void clearRightJoinData(int index) { + JOIN_RIGHT_DATA_STATIC.remove(index); + } + + /** get join schema */ + public static PartialGraphPattern getAfterJoinSchema( + PartialGraphPattern thisSchema, + PartialGraphPattern otherSchema, + scala.collection.immutable.List> onAlias, + scala.collection.immutable.Map rhsSchemaMapping) { + + Map aliasNameMapBack = new HashMap<>(); + Map javaRhsSchemaMapping = + new HashMap<>(JavaConversions.mapAsJavaMap(rhsSchemaMapping)); + scala.collection.immutable.Set scalaEmptySet = + JavaConversions.asScalaSet(new HashSet<>()).toSet(); + for (Tuple2 joinOnAlias : JavaConversions.seqAsJavaList(onAlias)) { + for (Map.Entry entry : javaRhsSchemaMapping.entrySet()) { + if (entry.getKey().name().equals(joinOnAlias._2())) { + aliasNameMapBack.put( + new NodeVar(entry.getValue().name(), scalaEmptySet), + new NodeVar(joinOnAlias._1(), scalaEmptySet)); + } + } + } + PartialGraphPattern otherRdgSchemaAfterRename = + KgGraphSchema.schemaAliasMapping(otherSchema, rhsSchemaMapping); + if (!aliasNameMapBack.isEmpty()) { + otherRdgSchemaAfterRename = + KgGraphSchema.schemaAliasMapping( + otherRdgSchemaAfterRename, Convert2ScalaUtil.toScalaImmutableMap(aliasNameMapBack)); + } + return KgGraphSchema.expandSchema(thisSchema, otherRdgSchemaAfterRename); + } + + /** get fold repeat info */ + public static FoldRepeatEdgeInfo getFoldRepeatEdgeInfo( + scala.collection.immutable.List, RichVar>> + windMapping, + PartialGraphPattern kgGraphSchema) { + String fromEdgeAlias = windMapping.head()._1.apply(1).name(); + String toEdgeAlias = windMapping.head()._2.name(); + String fromVertexAlias = windMapping.head()._1.apply(2).name(); + String toVertexAlias = + ((RepeatPathVar) (windMapping.head()._2)).pathVar().elements().apply(2).name(); + return new FoldRepeatEdgeInfo(fromEdgeAlias, toEdgeAlias, fromVertexAlias, toVertexAlias); + } + + /** get unfold repeat edge info */ + public static UnfoldRepeatEdgeInfo getUnfoldEdgeInfo( + scala.collection.immutable.List>> + mapping, + PartialGraphPattern kgGraphSchema) { + String edgeAlias = ((RepeatPathVar) mapping.head()._1).pathVar().name(); + String foldVertexAlias = mapping.head()._2().apply(2).name(); + String anchorVertexAlias = mapping.head()._2().apply(0).name(); + int lower = ((RepeatPathVar) mapping.head()._1).lower(); + return new UnfoldRepeatEdgeInfo(edgeAlias, foldVertexAlias, anchorVertexAlias, lower); + } + + /** readable pattern */ + public static String getReadablePattern(Pattern pattern) { + StringBuilder sb = new StringBuilder(); + sb.append("root=").append(pattern.root().alias()); + Set connectionSet = RunnerUtil.getConnectionSet(pattern); + if (connectionSet.isEmpty()) { + return sb.toString(); + } + sb.append(",edge="); + boolean first = true; + for (Connection connection : connectionSet) { + if (!first) { + sb.append(","); + } else { + first = false; + } + sb.append(connection.alias()); + } + return sb.toString(); + } + + /** readable */ + public static String getReadableAsList(scala.collection.immutable.List as) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < as.size(); ++i) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(as.apply(i)); + } + return sb.toString(); + } + + public static String getReadableByKey(scala.collection.immutable.List byKey) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < byKey.size(); ++i) { + if (sb.length() > 0) { + sb.append(","); + } + Var var = byKey.apply(i); + sb.append(var.name()); + } + return sb.toString(); + } + + public static String getReadableAddVertex(AddVertex addVertex) { + StringBuilder sb = new StringBuilder(); + sb.append("alias=") + .append(addVertex.s().alias()) + .append(",type=") + .append(addVertex.s().typeNames().iterator().next()); + return sb.toString(); + } + + public static String getReadableAddPredicate(AddPredicate addPredicate) { + StringBuilder sb = new StringBuilder(); + sb.append("alias=").append(addPredicate.predicate().alias()); + sb.append(",type=").append(addPredicate.predicate().label()); + sb.append(",from=").append(addPredicate.predicate().source()); + sb.append(",to=").append(addPredicate.predicate().target()); + return sb.toString(); + } + + public static String getReadableAddFields(Map> addFieldsInfo) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry> entry : addFieldsInfo.entrySet()) { + if (sb.length() > 0) { + sb.append(","); + } + Var var = entry.getKey(); + String str = var.toString(); + if (var instanceof PropertyVar) { + PropertyVar propertyVar = (PropertyVar) var; + str = propertyVar.name() + "." + propertyVar.field().name(); + } + sb.append(str).append("=").append(Arrays.toString(entry.getValue().toArray(new String[0]))); + } + return sb.toString(); + } + + public static String getReadableJoinString( + scala.collection.immutable.List> onAlias, + scala.collection.immutable.Map rhsSchemaMapping) { + StringBuilder sb = new StringBuilder(); + sb.append(onAlias); + Map javaRhsSchemaMapping = + new HashMap<>(JavaConversions.mapAsJavaMap(rhsSchemaMapping)); + sb.append(",right="); + for (Map.Entry entry : javaRhsSchemaMapping.entrySet()) { + sb.append(","); + Var var = entry.getKey(); + if (var instanceof NodeVar) { + sb.append("v=").append(var.name()); + } else if (var instanceof EdgeVar) { + sb.append("e=").append(var.name()); + } + } + return sb.toString(); + } + + /** get group by var */ + public static Object[] getVarFromKgGraph( + KgGraph kgGraph, scala.collection.immutable.List varList) { + Object[] result = new Object[varList.size()]; + for (int i = 0; i < varList.size(); ++i) { + Var var = varList.apply(i); + if (var instanceof NodeVar) { + result[i] = getVertexProperty(kgGraph, var.name(), Constants.VERTEX_INTERNAL_ID_KEY); + } else if (var instanceof PropertyVar) { + PropertyVar propertyVar = (PropertyVar) var; + result[i] = getProperty(kgGraph, propertyVar.name(), propertyVar.field().name()); + } + } + return result; + } + + /** get property */ + public static Object getProperty(KgGraph kgGraph, String alias, String propertyName) { + IProperty propertyMap; + List> vertexList = kgGraph.getVertex(alias); + if (CollectionUtils.isEmpty(vertexList)) { + List> edgeList = kgGraph.getEdge(alias); + if (CollectionUtils.isEmpty(edgeList)) { + throw new RuntimeException("unknown alias " + alias + "," + kgGraph); + } + propertyMap = edgeList.get(0).getValue(); + } else { + propertyMap = vertexList.get(0).getValue(); + } + return propertyMap.get(propertyName); + } + + /** get vertex property from KgGraph */ + public static Object getVertexProperty( + KgGraph kgGraph, String alias, String propertyName) { + IProperty propertyMap; + List> vertexList = kgGraph.getVertex(alias); + if (CollectionUtils.isEmpty(vertexList)) { + throw new RuntimeException("unknown alias " + alias + "," + kgGraph); + } + if (Constants.VERTEX_INTERNAL_ID_KEY.equals(propertyName)) { + return vertexList.get(0).getId(); + } + propertyMap = vertexList.get(0).getValue(); + return propertyMap.get(propertyName); + } + + /** compare two object */ + public static int compareTwoObject(Object v1, Object v2) { + if (null == v1) { + if (null == v2) { + return 0; + } + return -1; + } + if (!(v1 instanceof Comparable)) { + throw new RuntimeException("value can not comparable, " + v1.getClass().getName()); + } + if (null == v2) { + return 1; + } + Comparable c1 = (Comparable) v1; + return c1.compareTo(v2); + } + + /** get edge string id */ + public static String getEdgeIdentifier(IEdge edge) { + return edge.getSourceId().getInternalId() + + edge.getType() + + edge.getTargetId().getInternalId() + + edge.getVersion() + + edge.getDirection(); + } + + /** init type id mapping */ + public static void initTypeMapping(GraphLoaderConfig graphLoaderConfig) { + for (VertexLoaderConfig vertexLoaderConfig : graphLoaderConfig.getVertexLoaderConfigs()) { + MapType2IdFactory.getMapType2Id().getIdByType(vertexLoaderConfig.getVertexType()); + } + for (EdgeLoaderConfig edgeLoaderConfig : graphLoaderConfig.getEdgeLoaderConfigs()) { + MapType2IdFactory.getMapType2Id().getIdByType(edgeLoaderConfig.getEdgeType()); + } + } + + /** get running context */ + public static Map getTaskRunningContext( + KGReasonerSession session, Map params) { + Map taskRunningContext = new HashMap<>(); + Map idFilterMaps = + JavaConversions.mapAsJavaMap(session.getIdFilterParameters()); + Set variableMap = JavaConversions.setAsJavaSet(session.getParameterVariable()); + if (variableMap != null && variableMap.size() != 0) { + for (String var : variableMap) { + if (!params.containsKey(var)) { + throw new RuntimeException("parameter " + var + " must input"); + } + Expr expr = session.parser().parseExpr(params.get(var).toString()); + List rule = WareHouseUtils.getRuleList(expr); + Object obj = RuleRunner.getInstance().executeExpression(new HashMap<>(), rule, ""); + + if (idFilterMaps.containsValue(var)) { + List originIds = new ArrayList<>(); + // convert 2 internal id + if (obj instanceof Object[]) { + Object[] ids = (Object[]) obj; + for (Object id : ids) { + originIds.add(id.toString()); + } + } else { + originIds.add(obj); + } + obj = originIds.toArray(); + } + taskRunningContext.put(var, obj); + } + } + return taskRunningContext; + } +} diff --git a/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/SimpleObjSerde.java b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/SimpleObjSerde.java new file mode 100644 index 00000000..3a192359 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/utils/SimpleObjSerde.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Base64; + +public class SimpleObjSerde { + + /** serialize obj */ + public static String ser(Object obj) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ObjectOutputStream objectOutputStream = null; + try { + objectOutputStream = new ObjectOutputStream(byteArrayOutputStream); + objectOutputStream.writeObject(obj); + } catch (IOException e) { + throw new RuntimeException(e); + } + byte[] bytes = byteArrayOutputStream.toByteArray(); + return Base64.getEncoder().encodeToString(bytes); + } + + /** deserialize obj */ + public static Object de(String encode) { + byte[] bytes = Base64.getDecoder().decode(encode); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + ObjectInputStream objectInputStream; + try { + objectInputStream = new ObjectInputStream(byteArrayInputStream); + return objectInputStream.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } +} diff --git a/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/session/KGReasonerSession.scala b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/session/KGReasonerSession.scala new file mode 100644 index 00000000..a8d7627d --- /dev/null +++ b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/session/KGReasonerSession.scala @@ -0,0 +1,451 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.session + +import scala.collection.mutable +import scala.collection.mutable.ListBuffer +import scala.reflect.runtime.universe.TypeTag + +import com.antgroup.openspg.reasoner.common.constants.Constants +import com.antgroup.openspg.reasoner.common.exception.UnsupportedOperationException +import com.antgroup.openspg.reasoner.common.types.KTObject +import com.antgroup.openspg.reasoner.common.utils.ParameterUtils +import com.antgroup.openspg.reasoner.lube.Logging +import com.antgroup.openspg.reasoner.lube.block._ +import com.antgroup.openspg.reasoner.lube.catalog.Catalog +import com.antgroup.openspg.reasoner.lube.catalog.struct.{Field, NodeType} +import com.antgroup.openspg.reasoner.lube.common.pattern._ +import com.antgroup.openspg.reasoner.lube.logical.{SolvedModel, _} +import com.antgroup.openspg.reasoner.lube.logical.operators.LogicalOperator +import com.antgroup.openspg.reasoner.lube.logical.optimizer.LogicalOptimizer +import com.antgroup.openspg.reasoner.lube.logical.planning.{ + LogicalPlanner, + LogicalPlannerContext, + SubQueryMerger +} +import com.antgroup.openspg.reasoner.lube.logical.validate.Validator +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface +import com.antgroup.openspg.reasoner.lube.physical.{GraphSession, PropertyGraph} +import com.antgroup.openspg.reasoner.lube.physical.operators.{PhysicalOperator, Select} +import com.antgroup.openspg.reasoner.lube.physical.planning.{ + PhysicalPlanner, + PhysicalPlannerContext +} +import com.antgroup.openspg.reasoner.lube.physical.rdg.{RDG, Result} +import com.antgroup.openspg.reasoner.util.LoaderUtil +import com.antgroup.openspg.reasoner.warehouse.common.config.GraphLoaderConfig + +/** + * Base class for KGReasoner pipeline. + * The class provides a generic implementation of the necessary steps to + * execute a KGDSL/GQL query on Knowledge Graph through tabular based engine, including parsing, + * unresolved logical planning, logical planning an physical planning + * + * @param parser + * @param catalog + * @param typeTag$T$0 + * @tparam T + */ +abstract class KGReasonerSession[T <: RDG[T]: TypeTag]( + val parser: ParserInterface, + val catalog: Catalog) + extends Logging { + private val graphSession: GraphSession[T] = new GraphSession() + private var loaderConfig: GraphLoaderConfig = new GraphLoaderConfig() + private var variableParameters: Set[String] = Set.empty + private var idFilterParameters: Map[String, String] = Map.empty + + /** + * Init KGReasonerSession. + */ + def init(): Unit = { + catalog.init() + } + + /** + * Execute a KGDSL/GQL query, return [[Result]] + * step1: Parse query to unresolved logical plan, which is represented by [[Block]]. + * step2: Use [[LogicalPlanner]] to plan unresolved logical plan to logical plan, + * which is represented by [[LogicalOperator]] + * step3: Use [[PhysicalPlanner]] to plan logical plan to physical plan, + * which is represented by [[PhysicalOperator]] + * + * @param query KGDSL/GQL query + * @param params runtime config + * @return + */ + def getResult(query: String, params: Map[String, Object]): Result = { + val physicalPlan = plan(query, params) + var result: Result = null + for (physicalOp <- physicalPlan) { + result = getResult(physicalOp) + } + result + } + + def getResult(physicalPlan: PhysicalOperator[T]): Result = { + physicalPlan match { + case select: Select[T] => + select.row + case _ => + physicalPlan.rdg + } + } + + /** + * get variable parameter name + * @return + */ + def getParameterVariable(): Set[String] = variableParameters + + /** + * get id filter parameters + * @return + */ + def getIdFilterParameters(): Map[String, String] = idFilterParameters + + /** + * get load config from parse plan + */ + def getLoaderConfig(): GraphLoaderConfig = loaderConfig + + /** + * Generate the optimization physical plan of giving KGDSL/GQL query. + * step1: Parse query to unresolved logical plan, which is represented by [[Block]]. + * step2: Use [[LogicalPlanner]] to plan unresolved logical plan to logical plan, + * which is represented by [[LogicalOperator]] + * step3: Use [[PhysicalPlanner]] to plan logical plan to physical plan, + * which is represented by [[PhysicalOperator]] + * + * @param query + * @param params + * @return + */ + def plan(query: String, params: Map[String, Object]): List[PhysicalOperator[T]] = { + var start = System.currentTimeMillis() + val blocks = plan2UnresolvedLogicalPlan(query, params) + if (ParameterUtils.isEnableSPGPlanPrettyPrint(params)) { + for (block: Block <- blocks) { + logger.info(block.pretty) + } + } + + logger.info( + "benchmark main plan plan2UnresolvedLogicalPlan cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + val optimizedLogicalPlan = plan2LogicalPlan(blocks, params) + logger.info( + "benchmark main plan plan2LogicalPlan cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + val physicalPlan = plan2PhysicalPlan(optimizedLogicalPlan, params) + logger.info( + "benchmark main plan plan2PhysicalPlan cost = " + + (System.currentTimeMillis() - start)) + if (!graphSession.hasGraph(Catalog.defaultGraphName)) { + logger.info(s"begin to load graph data for ${Catalog.defaultGraphName}") + val loaderConfig = LoaderUtil.getLoaderConfig(optimizedLogicalPlan, catalog) + this.loaderConfig = loaderConfig + val rdg = loadGraph(loaderConfig) + graphSession.register(Catalog.defaultGraphName, rdg) + } + physicalPlan + } + + /** + * Load a graph from Knowledge Graph to [[KGReasonerSession]] + * + * @param graphLoaderConfig + * @return + */ + def loadGraph(graphLoaderConfig: GraphLoaderConfig): PropertyGraph[T] + + /** + * Return the specific [[PropertyGraph]] + * + * @param graphName + * @return + */ + def getGraph(graphName: String): PropertyGraph[T] = { + graphSession.getGraph(graphName) + } + + /** + * Return if has load KG, which is the default graph. + * @return + */ + def hasLoadGraph: Boolean = { + graphSession.hasGraph(Catalog.defaultGraphName) + } + + /** + * get the new added property in node or edge + * + * @param physicalOperator + * @return + */ + protected def getNewProperty( + query: String, + params: Map[String, Object]): mutable.Map[String, mutable.HashSet[Field]] = { + val unresolvedLogicalPlan = plan2UnresolvedLogicalPlan(query, params) + val logicalPlanList = plan2LogicalPlan(unresolvedLogicalPlan, params) + val entityType2FiledMap: mutable.HashMap[String, mutable.HashSet[Field]] = + new mutable.HashMap() + + for (logicalPlan: LogicalOperator <- logicalPlanList) { + val partEntityType2FiledMap = + logicalPlan.transform[mutable.HashMap[String, mutable.HashSet[Field]]] { + case (logicalOp: LogicalOperator, mapList) => + var type2FiledMap: mutable.HashMap[String, mutable.HashSet[Field]] = null + if (mapList.isEmpty) { + type2FiledMap = new mutable.HashMap[String, mutable.HashSet[Field]]() + } else { + type2FiledMap = mapList.head + } + + val varList: List[Var] = logicalOp.fields + val solvedModel: SolvedModel = logicalOp.solved + varList.foreach(v => + v match { + case NodeVar(name, fields) => + type2FiledMap = addFieldToMap(type2FiledMap, name, fields, solvedModel) + case EdgeVar(name, fields) => + type2FiledMap = addFieldToMap(type2FiledMap, name, fields, solvedModel) + case PropertyVar(name, field) => + type2FiledMap = + addFieldToMap(type2FiledMap, name, Set.apply(field), solvedModel) + case _ => type2FiledMap + }) + type2FiledMap + case (_, mapList) => mapList.head + } + for (entityType: String <- partEntityType2FiledMap.keySet) { + val fieldSet = partEntityType2FiledMap(entityType) + if (entityType2FiledMap.contains(entityType)) { + val existFiledSet = entityType2FiledMap(entityType) + existFiledSet ++= fieldSet + entityType2FiledMap += (entityType -> existFiledSet) + } else { + entityType2FiledMap += (entityType -> fieldSet) + } + } + } + entityType2FiledMap + } + + private def addFieldToMap( + type2FieldMap: mutable.HashMap[String, mutable.HashSet[Field]], + alias: String, + newFields: Set[Field], + solvedModel: SolvedModel): mutable.HashMap[String, mutable.HashSet[Field]] = { + val entityTypeSet: Set[String] = solvedModel.alias2Types(alias) + for (entityType: String <- entityTypeSet) { + val existFiledSet: mutable.HashSet[Field] = new mutable.HashSet[Field]() + if (type2FieldMap.contains(entityType)) { + existFiledSet ++= type2FieldMap(entityType) + } + existFiledSet ++= newFields + type2FieldMap += (entityType -> existFiledSet) + } + type2FieldMap + } + + private def plan2LogicalPlan( + blockList: List[Block], + params: Map[String, Object]): List[LogicalOperator] = { + var start = System.currentTimeMillis() + // plan to logical plan + implicit val context: LogicalPlannerContext = LogicalPlannerContext(catalog, parser, params) + logger.info( + "benchmark LogicalPlannerContext cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + val dag = if (blockList.size > 1) { + Validator.validate(blockList) + } else { + Validator.validate(parser, blockList.head) + } + logger.info( + "benchmark validate cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + val logicalPlans = LogicalPlanner.plan(dag) + logger.info( + "benchmark LogicalPlanner.plan cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + val optimizedLogicalDag = logicalPlans.map(LogicalOptimizer.optimize(_)) + logger.info( + "benchmark logicalPlans.map cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + var optimizedLogicalPlans: List[LogicalOperator] = null + val subQueryEnable = context.params + .getOrElse(Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, "false") + .toString + logger.info("subQueryEnable " + subQueryEnable) + if ("true".equals(subQueryEnable)) { + val subQueryMerger = new SubQueryMerger(optimizedLogicalDag) + optimizedLogicalPlans = List.apply(subQueryMerger.plan) + } else { + optimizedLogicalPlans = + optimizedLogicalDag.order().reverse.map(optimizedLogicalDag.popNode(_)) + } + logger.info( + "benchmark optimizedLogicalPlans cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + if (ParameterUtils.isEnableSPGPlanPrettyPrint(params)) { + for (optimizedLogicalPlan <- optimizedLogicalPlans) { + logger.info(s"optimized logical plan:\n${optimizedLogicalPlan.pretty}\n") + } + } + logger.info( + "benchmark logical show cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + optimizedLogicalPlans + } + + def plan2UnresolvedLogicalPlan(query: String, params: Map[String, Object]): List[Block] = { + // parser query to unresolved logical plan + var start = System.currentTimeMillis() + val blocks = planToBlock(query, params) + logger.info( + "benchmark planToBlock cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + registerSchema(blocks) + logger.info( + "benchmark registerSchema cost = " + + (System.currentTimeMillis() - start)) + start = System.currentTimeMillis() + blocks + } + + private def planToBlock(query: String, param: Map[String, Object] = Map.empty): List[Block] = { + // parser query to unresolved logical plan + // check + val blocks = parser.parseMultipleStatement(query, param) + // parameter parse + variableParameters = parser.getAllParameters() + idFilterParameters = parser.getIdFilterParameters() + if (blocks.size == 1) { + blocks + } else { + legalMultiStatement(blocks) + blocks + } + } + + /** + * Only the last block can be result block + * + * @param blocks + * @return + */ + private def legalMultiStatement(blocks: List[Block]) = { + for (i <- 0 until blocks.size) { + val block = blocks(i) + if (i < blocks.size - 1 && !block.isInstanceOf[DDLBlock]) { + throw UnsupportedOperationException( + "multiple statement requires the only the last statement is get action") + } else if (i == blocks.size - 1 && !block.isInstanceOf[TableResultBlock]) { + throw UnsupportedOperationException( + "multiple statement requires the only the last statement is get action") + } + } + } + + private def registerSchema(blocks: List[Block]) = { + for (block <- blocks) { + block match { + case DDLBlock(ddlOps, _) => + val ddlOp = ddlOps.head + ddlOp match { + case AddProperty(s, propertyName, propertyType) => + val graph = catalog.getGraph(block.graph.graphName) + for (label <- s.typeNames) { + graph.addProperty(label, propertyName, propertyType, false) + } + case AddPredicate(predicate) => + val graph = catalog.getGraph(block.graph.graphName) + predicate match { + case PredicateElement(label, _, src: Element, dst: Element, fields, direction) => + val srcTypes = getType(src) + val dstTypes = getType(dst) + for (s <- srcTypes) { + for (o <- dstTypes) { + graph.addEdge( + s, + label, + o, + direction, + fields.keySet.map(name => new Field(name, KTObject, false)), + false) + } + } + if (src.isInstanceOf[EntityElement]) { + srcTypes.foreach(graph.addNode(_, NodeType.CONCEPT, Set.empty)) + } + if (dst.isInstanceOf[EntityElement]) { + dstTypes.foreach(graph.addNode(_, NodeType.CONCEPT, Set.empty)) + } + } + case _ => + } + case _ => + } + } + } + + private def getType(element: Element): Set[String] = { + element match { + case EntityElement(id, label, _) => Set.apply(label + "/" + id, label) + case PatternElement(_, typeNames, _) => typeNames + } + } + + private def plan2PhysicalPlan( + logicalPlanList: List[LogicalOperator], + params: Map[String, Object]): List[PhysicalOperator[T]] = { + val physicalPlanList = new ListBuffer[PhysicalOperator[T]]() + for (logicalPlan: LogicalOperator <- logicalPlanList) { + val physicalPlan = + PhysicalPlanner + .plan[T](logicalPlan)( + implicitly[TypeTag[T]], + PhysicalPlannerContext(catalog, graphSession, params)) + physicalPlanList.+=(physicalPlan) + if (ParameterUtils.isEnableSPGPlanPrettyPrint(params)) { + logger.info(s"kgreasoner physical plan:\n${physicalPlan.pretty}\n") + } + } + physicalPlanList.toList + } + + /** + * get the graph loader config from query dsl + * @param query + * @param params + * @return + */ + def getGraphLoaderConfig(query: String, params: Map[String, Object]): GraphLoaderConfig = { + val blocks = plan2UnresolvedLogicalPlan(query, params) + val optimizedLogicalPlan = plan2LogicalPlan(blocks, params) + LoaderUtil.getLoaderConfig(optimizedLogicalPlan, catalog) + } + +} diff --git a/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/Convert2ScalaUtil.scala b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/Convert2ScalaUtil.scala new file mode 100644 index 00000000..46810430 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/Convert2ScalaUtil.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.util + +import scala.collection.JavaConverters._ + +object Convert2ScalaUtil { + + def toScalaImmutableMap[T1, T2](jMap: java.util.Map[T1, T2]): Map[T1, T2] = { + jMap.asScala.toMap; + } + + def toScalaSeq[T](jList: java.util.List[T]): Seq[T] = { + jList.asScala + } + + def toScalaList[T](jList: java.util.List[T]): List[T] = { + jList.asScala.toList + } + + def toScalaImmutableSet[T](jSet: java.util.Set[T]): Set[T] = { + jSet.asScala.toSet + } + +} diff --git a/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/KgGraphSchema.scala b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/KgGraphSchema.scala new file mode 100644 index 00000000..230905c8 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/KgGraphSchema.scala @@ -0,0 +1,361 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.util + +import com.antgroup.openspg.reasoner.batching.DynamicBatchSize +import com.antgroup.openspg.reasoner.common.graph.`type`.GraphItemType +import com.antgroup.openspg.reasoner.lube.common.pattern._ +import com.antgroup.openspg.reasoner.lube.logical.{EdgeVar, NodeVar, Var} +import com.antgroup.openspg.reasoner.rdg.common.FoldRepeatEdgeInfo +import org.slf4j.LoggerFactory +import scala.collection.mutable + +object KgGraphSchema { + private val logger = LoggerFactory.getLogger(classOf[DynamicBatchSize]) + + /** + * convert pattern to KgGraph schema + * + * @param pattern + * @return + */ + def convert2KgGraphSchema(pattern: Pattern): PartialGraphPattern = { + val patternConnectionList = pattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .map(pc => { PatternConnection.insureDirection(pattern.root.alias, pc) }) + .toSet + + val topology = + convert2Topology(patternConnectionList, Map.apply((pattern.root.alias -> pattern.root))) + + val nodes = topology.values + .flatMap(pcs => { pcs.map(pc => (pc.source, pc.target)) }) + .flatMap(x => List(x._1, x._2)) + .map(x => (x, pattern.getNode(x))) + .toMap ++ Map.apply((pattern.root.alias -> pattern.root)) + + PartialGraphPattern(pattern.root.alias, nodes, topology) + } + + /** + * change KgGraph schema root + * + * @param pattern + * @param target + * @return + */ + def schemaChangeRoot(pattern: Pattern, targetAlias: String): PartialGraphPattern = { + if (null == targetAlias) { + val vertexMap = new mutable.HashMap[String, PatternElement]() + pattern.topology.values + .flatMap(pcs => pcs.map(pc => (pc.source, pc.target))) + .flatMap(x => List(x._1, x._2)) + .foreach(k => vertexMap.put(k, pattern.getNode(k))) + return PartialGraphPattern(null, vertexMap.toMap, pattern.topology) + } + expandSchema( + pattern, + PartialGraphPattern( + targetAlias, + Map.apply((targetAlias -> pattern.getNode(targetAlias))), + Map.empty)) + } + + /** + * change schema vertex alias + */ + def schemaAliasMapping(pattern: Pattern, schemaMapping: Map[Var, Var]): PartialGraphPattern = { + val vertexAliasMapping = schemaMapping + .filter(kv => { + kv._1.isInstanceOf[NodeVar] && kv._2.isInstanceOf[NodeVar] + }) + .map(kv => { (kv._1.asInstanceOf[NodeVar].name, kv._2.asInstanceOf[NodeVar].name) }) + .filter(kv => !kv._1.equals(kv._2)) + + val edgeAliasMapping = schemaMapping + .filter(kv => { + kv._1.isInstanceOf[EdgeVar] && kv._2.isInstanceOf[EdgeVar] + }) + .map(kv => { (kv._1.asInstanceOf[EdgeVar].name, kv._2.asInstanceOf[EdgeVar].name) }) + .filter(kv => !kv._1.equals(kv._2)) + + val patternConnectionSet = pattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .map(pc => { + var patternConnection = pc.asInstanceOf[PatternConnection] + val renameSource = vertexAliasMapping.get(patternConnection.source) + if (renameSource.isDefined) { + patternConnection = new PatternConnection( + patternConnection.alias, + renameSource.get, + patternConnection.relTypes, + patternConnection.target, + patternConnection.direction, + patternConnection.rule, + patternConnection.limit, + patternConnection.exists, + patternConnection.optional) + } + val renameTarget = vertexAliasMapping.get(patternConnection.target) + if (renameTarget.isDefined) { + patternConnection = new PatternConnection( + patternConnection.alias, + patternConnection.source, + patternConnection.relTypes, + renameTarget.get, + patternConnection.direction, + patternConnection.rule, + patternConnection.limit, + patternConnection.exists, + patternConnection.optional) + } + val renameEdge = edgeAliasMapping.get(patternConnection.alias) + if (renameEdge.isDefined) { + patternConnection = new PatternConnection( + renameEdge.get, + patternConnection.source, + patternConnection.relTypes, + patternConnection.target, + patternConnection.direction, + patternConnection.rule, + patternConnection.limit, + patternConnection.exists, + patternConnection.optional) + } + patternConnection + }) + .map(pc => pc.asInstanceOf[Connection]) + .toSet + + val vertexMap = new mutable.HashMap[String, PatternElement]() + pattern.topology.values + .flatMap(pcs => pcs.map(pc => (pc.source, pc.target))) + .flatMap(x => List(x._1, x._2)) + .foreach(k => { + val oldPatternElement = pattern.getNode(k) + val renameAlias = vertexAliasMapping.get(k) + if (renameAlias.isEmpty) { + vertexMap.put(k, oldPatternElement) + } else { + val newPatternElement = + PatternElement(renameAlias.get, oldPatternElement.typeNames, oldPatternElement.rule) + vertexMap.put(renameAlias.get, newPatternElement) + } + }) + + val topology = convert2Topology(patternConnectionSet, vertexMap.toMap) + + var rootAlias = pattern.root.alias + val renameAlias = vertexAliasMapping.get(rootAlias) + if (renameAlias.isDefined) { + rootAlias = renameAlias.get + } + + PartialGraphPattern(rootAlias, vertexMap.toMap, topology) + } + + /** + * After executing expandInto in RDG, expand the schema + * @param existingPattern + * @param addedPattern + * @return + */ + def expandSchema(existingPattern: Pattern, addedPattern: Pattern): PartialGraphPattern = { + val patternConnectionSet = addedPattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .map(pc => { PatternConnection.insureDirection(addedPattern.root.alias, pc) }) + .toSet ++ + existingPattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .toSet + + val vertexMap = new mutable.HashMap[String, PatternElement]() + addedPattern.topology.values + .flatMap(pcs => pcs.map(pc => (pc.source, pc.target))) + .flatMap(x => List(x._1, x._2)) + .foreach(k => vertexMap.put(k, addedPattern.getNode(k))) + existingPattern.topology.values + .flatMap(pcs => pcs.map(pc => (pc.source, pc.target))) + .flatMap(x => List(x._1, x._2)) + .foreach(k => vertexMap.put(k, existingPattern.getNode(k))) + vertexMap.put(addedPattern.root.alias, addedPattern.root) + + val topology = convert2Topology(patternConnectionSet, vertexMap.toMap) + + PartialGraphPattern(addedPattern.root.alias, vertexMap.toMap, topology) + } + + def foldPathEdgeSchema( + existingPattern: Pattern, + windEdge: FoldRepeatEdgeInfo): PartialGraphPattern = { + + val foldConnection = existingPattern.topology.values + .flatMap(pcs => { pcs.map(pc => pc) }) + .filter(p => p.alias.equals(windEdge.getFromEdgeAlias)) + .toList + .head + .asInstanceOf[PatternConnection] + + val existingPathConnection = existingPattern.topology.values + .flatMap(pcs => { pcs.map(pc => pc) }) + .filter(p => p.alias.equals(windEdge.getToEdgeAlias)) + .toList + .headOption + + var newPathConnection: PathConnection = null + var rootAlias: String = null + if (existingPathConnection.isEmpty) { + val sourceAlias = foldConnection.source + val targetAlias = windEdge.getToVertexAlias + newPathConnection = PathConnection( + windEdge.getToEdgeAlias, + sourceAlias, + foldConnection.relTypes, + targetAlias, + foldConnection.direction, + null, + List.empty, + List.apply(foldConnection)) + rootAlias = existingPattern.root.alias + if (rootAlias.equals(windEdge.getFromVertexAlias)) { + rootAlias = windEdge.getToVertexAlias + } + } else { + val opc = existingPathConnection.get.asInstanceOf[PathConnection] + val oldFoldEdgeSchema = opc.edgeSchemaList.last + val nextFoldEdgeSchema = PatternConnection( + foldConnection.alias, + oldFoldEdgeSchema.target, + foldConnection.relTypes, + foldConnection.target, + foldConnection.direction, + foldConnection.rule) + + val foldVertexSchema = existingPattern.getNode(windEdge.getToVertexAlias) + val nextFoldVertexSchema = PatternElement( + nextFoldEdgeSchema.source, + foldVertexSchema.typeNames, + foldVertexSchema.rule) + newPathConnection = PathConnection( + opc.alias, + opc.source, + opc.relTypes, + opc.target, + opc.direction, + opc.rule, + opc.vertexSchemaList :+ nextFoldVertexSchema, + opc.edgeSchemaList :+ nextFoldEdgeSchema) + rootAlias = newPathConnection.alias + "." + newPathConnection.edgeSchemaList.last.source + } + + val existVertexPattern = existingPattern.getNode(windEdge.getFromVertexAlias) + val newVertexPattern = + PatternElement( + windEdge.getToVertexAlias, + existVertexPattern.typeNames, + existVertexPattern.rule) + + val nodes = existingPattern.topology.values + .flatMap(pcs => { pcs.map(pc => (pc.source, pc.target)) }) + .flatMap(x => List(x._1, x._2)) + .filter(p => !p.equals(windEdge.getFromVertexAlias)) + .map(x => (x, existingPattern.getNode(x))) + .toMap ++ Map.apply((newVertexPattern.alias, newVertexPattern)) + + val patternConnectionSet = existingPattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .filter(pc => !pc.alias.equals(windEdge.getFromEdgeAlias)) + .filter(pc => !pc.alias.equals(windEdge.getToEdgeAlias)) + .toSet ++ Set.apply(newPathConnection) + + val topology = convert2Topology(patternConnectionSet, nodes) + + PartialGraphPattern(rootAlias, nodes, topology) + } + + /** + * get alias is edge or vertex + */ + def alias2Type(pattern: PartialGraphPattern): Map[String, GraphItemType] = { + pattern.nodes.values + .map(pe => (pe.alias, GraphItemType.VERTEX)) + .toMap ++ + pattern.topology.values + .flatMap(pcs => pcs.map(pc => pc)) + .map(pc => (pc.alias, GraphItemType.EDGE)) + .toMap + } + + private def convert2Topology( + patternConnectionSet: Set[Connection], + vertexMap: Map[String, PatternElement]): Map[String, Set[Connection]] = { + val topologyOut = patternConnectionSet + .groupBy(pc => pc.source) + .mapValues(list => list) + + val topologyIn = patternConnectionSet + .groupBy(pc => pc.target) + .mapValues(list => list) + + val topology = topologyOut.foldLeft(topologyIn) { case (m, (key, value)) => + m + (key -> (m.getOrElse(key, Set.empty[PatternConnection]) ++ value)) + } + + topology.filterKeys(key => vertexMap.contains(key)).view.force + } + + def getNodesAlias(pattern: Pattern): Set[String] = { + pattern.topology.keySet ++ + pattern.topology.values + .flatMap(pcs => pcs.map(pc => (pc.source, pc.target))) + .flatMap(x => List(x._1, x._2)) + .toSet + } + + def getEdgesAlias(pattern: Pattern): Set[String] = { + pattern.topology.values + .flatMap(pcs => pcs.map(pc => pc.alias)) + .toSet + } + + def getPatternConnection(pattern: Pattern, edgeAlias: String): Connection = { + val result: Option[Connection] = pattern.topology.values + .flatMap(pcs => pcs.toList) + .find(_.alias.equals(edgeAlias)) + result match { + case Some(value) => value + case None => null + } + } + + def getNeighborEdges(pattern: Pattern, vertexAlias: String): Set[Connection] = { + pattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .filter(pc => { pc.source.equals(vertexAlias) || pc.target.equals(vertexAlias) }) + .toSet + } + +} diff --git a/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/LoaderUtil.scala b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/LoaderUtil.scala new file mode 100644 index 00000000..77b7fb6e --- /dev/null +++ b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/LoaderUtil.scala @@ -0,0 +1,403 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.util + +import com.antgroup.openspg.reasoner.common.graph.edge.{Direction, SPO} +import com.antgroup.openspg.reasoner.lube.catalog.{Catalog, SemanticPropertyGraph} +import com.antgroup.openspg.reasoner.lube.common.pattern.{ + PartialGraphPattern, + Pattern, + PatternConnection +} +import com.antgroup.openspg.reasoner.lube.common.rule.Rule +import com.antgroup.openspg.reasoner.lube.logical.{ + EdgeVar, + NodeVar, + PathVar, + RepeatPathVar, + SolvedModel, + Var +} +import com.antgroup.openspg.reasoner.lube.logical.PatternOps.PatternOps +import com.antgroup.openspg.reasoner.lube.logical.operators._ +import com.antgroup.openspg.reasoner.warehouse.common.config.{ + EdgeLoaderConfig, + GraphLoaderConfig, + VertexLoaderConfig +} +import scala.collection.JavaConverters._ +import scala.collection.mutable + +object LoaderUtil { + + def getLoaderConfig( + logicalPlans: List[LogicalOperator], + catalog: Catalog): GraphLoaderConfig = { + val loaderConfig = new GraphLoaderConfig() + for (logicalPlan <- logicalPlans) { + loaderConfig.merge(getLoaderConfig(logicalPlan, catalog)) + } + if (logicalPlans.size == 1) { + val ruleMap = getFilterRule(logicalPlans.head) + if (!ruleMap.isEmpty) { + for (v <- loaderConfig.getVertexLoaderConfigs().asScala) { + if (ruleMap.contains(v.getVertexType)) { + v.setPropertiesFilterRules(List.apply(ruleMap(v.getVertexType)).asJava) + } + } + for (e <- loaderConfig.getEdgeLoaderConfigs.asScala) { + if (ruleMap.contains(e.getEdgeType)) { + e.setPropertiesFilterRules(List.apply(ruleMap(e.getEdgeType)).asJava) + } + } + } + } + loaderConfig.verify() + } + + private def findAllTypesAllowIsolateVertex(logicalPlan: LogicalOperator): Set[String] = { + logicalPlan.transform[Set[String]] { + case (LinkedExpand(_, linkedEdgePattern), typeSets) => + typeSets.flatMap(_.headOption).toSet ++ linkedEdgePattern.src.typeNames + case (_, typeSets) => typeSets.flatMap(_.headOption).toSet + } + } + + def merge(cur: SolvedModel, other: SolvedModel): SolvedModel = { + val alias2Types = new mutable.HashMap[String, Set[String]]() + val fields = new mutable.HashMap[String, Var]() + for (pair <- cur.alias2Types) { + alias2Types.put(pair._1, pair._2) + } + for (pair <- other.alias2Types) { + if (alias2Types.contains(pair._1)) { + alias2Types.put(pair._1, alias2Types(pair._1) ++ pair._2) + } else { + alias2Types.put(pair._1, pair._2) + } + } + for (pair <- cur.fields) { + fields.put(pair._1, pair._2) + } + for (pair <- other.fields) { + if (fields.contains(pair._1)) { + fields.put(pair._1, pair._2.merge(fields.get(pair._1))) + } else { + fields.put(pair._1, pair._2) + } + } + cur.copy(alias2Types = alias2Types.toMap, fields = fields.toMap) + } + + private def mergeDirection(d1: Direction, d2: Direction): Direction = { + if (Direction.BOTH.equals(d1) || Direction.BOTH.equals(d2)) { + Direction.BOTH + } else if (Direction.IN.equals(d1) && Direction.OUT.equals(d2)) { + Direction.BOTH + } else if (Direction.OUT.equals(d1) && Direction.IN.equals(d2)) { + Direction.BOTH + } else if (null == d1) { + d2 + } else { + d1 + } + } + + private def mergeDirectionMaps( + directionMaps: List[Map[String, Direction]]): Map[String, Direction] = { + directionMaps.foldLeft(Map.empty[String, Direction]) { (acc, map) => + acc ++ map.map { case (key, value) => + key -> mergeDirection(acc.getOrElse(key, null), value) + } + } + } + + private def mergeVertexAlias2TypeMap( + alis2TypesMapList: List[Map[String, Set[String]]]): Map[String, Set[String]] = { + alis2TypesMapList.foldLeft(Map.empty[String, Set[String]]) { (acc, map) => + acc ++ map.map { case (key, value) => + key -> (acc.getOrElse(key, Set.empty[String]) ++ value) + } + } + } + + private def getVertexAlias2TypeMapFromPattern(pattern: Pattern): Map[String, Set[String]] = { + var alias2TypesMap = Map.apply(pattern.root.alias -> pattern.root.typeNames) + pattern match { + case partialGraphPattern: PartialGraphPattern => + val alias2TypesMap2 = + partialGraphPattern.nodes.values.map(pe => (pe.alias, pe.typeNames)).toMap + alias2TypesMap = mergeVertexAlias2TypeMap(List.apply(alias2TypesMap, alias2TypesMap2)) + case _ => + } + alias2TypesMap + } + + private def getVertexAlias2TypeMap(logicalPlan: LogicalOperator): Map[String, Set[String]] = { + logicalPlan.transform[Map[String, Set[String]]] { + case (PatternScan(_, pattern), alis2TypesMapList) => + mergeVertexAlias2TypeMap(alis2TypesMapList :+ getVertexAlias2TypeMapFromPattern(pattern)) + case (ExpandInto(_, _, pattern), alis2TypesMapList) => + mergeVertexAlias2TypeMap(alis2TypesMapList :+ getVertexAlias2TypeMapFromPattern(pattern)) + case (LinkedExpand(_, linkedEdgePattern), alis2TypesMapList) => + val srcAlias2TypesMap = + Map.apply(linkedEdgePattern.src.alias -> linkedEdgePattern.src.typeNames) + val dstAlias2TypesMap = + Map.apply(linkedEdgePattern.dst.alias -> linkedEdgePattern.dst.typeNames) + mergeVertexAlias2TypeMap(alis2TypesMapList :+ srcAlias2TypesMap :+ dstAlias2TypesMap) + case (_, alis2TypesMapList) => mergeVertexAlias2TypeMap(alis2TypesMapList) + } + } + + private def getEdgeLoadDirectionMapFromPattern( + pattern: Pattern, + alias2TypesMap: Map[String, Set[String]], + graph: SemanticPropertyGraph): Map[String, Direction] = { + + val resultMap = new mutable.HashMap[String, Direction]() + val rootAlias = pattern.root.alias + pattern.topology.values + .flatMap(pcs => { + pcs.map(pc => pc) + }) + .map(pc => PatternConnection.insureDirection(rootAlias, pc)) + .foreach(pc => { + val srcTypeSet = alias2TypesMap.getOrElse(pc.source, Set.empty) + val dstTypeSet = alias2TypesMap.getOrElse(pc.target, Set.empty) + + srcTypeSet + .flatMap(s => + dstTypeSet.flatMap(o => + pc.relTypes.flatMap(p => { + if (Direction.IN.equals(pc.direction)) { List((o, p, s, Direction.IN)) } + else if (Direction.OUT.equals(pc.direction)) { List((s, p, o, Direction.OUT)) } + else { + val spo = new SPO(s, p, o) + scala.util.control.Exception.ignoring(classOf[NoSuchElementException]) { + val edge = graph.getEdge(spo.toString) + if (null != edge && edge.resolved) { + List((s, p, o, Direction.OUT), (s, p, o, Direction.IN)) + } + } + scala.util.control.Exception.ignoring(classOf[NoSuchElementException]) { + val ops = new SPO(o, p, s) + val edgeR = graph.getEdge(ops.toString) + if (null != edgeR && edgeR.resolved) { + List((o, p, s, Direction.OUT), (o, p, s, Direction.IN)) + } + } + List.empty + } + }))) + .foreach(tuple => { + val spo = new SPO(tuple._1, tuple._2, tuple._3) + resultMap.put( + spo.toString, + mergeDirection(resultMap.getOrElse(spo.toString, null), tuple._4)) + }) + }) + + resultMap.toMap + } + + private def getEdgeLoadDirectionMap(logicalPlan: LogicalOperator): Map[String, Direction] = { + val alias2TypesMap = getVertexAlias2TypeMap(logicalPlan) + logicalPlan.transform[Map[String, Direction]] { + case (PatternScan(_, pattern), directionMaps) => + mergeDirectionMaps( + directionMaps :+ getEdgeLoadDirectionMapFromPattern( + pattern, + alias2TypesMap, + logicalPlan.graph)) + case (ExpandInto(_, _, pattern), directionMaps) => + mergeDirectionMaps( + directionMaps :+ getEdgeLoadDirectionMapFromPattern( + pattern, + alias2TypesMap, + logicalPlan.graph)) + case (_, directionMaps) => mergeDirectionMaps(directionMaps) + } + } + + private def getAllowIsolateVertexFromEdgeLoadDirectionMap( + edgeLoadDirectionMap: Map[String, Direction]): Set[String] = { + val isolateVertexMap = new mutable.HashMap[String, mutable.HashMap[SPO, Direction]]() + + edgeLoadDirectionMap.toList + .flatMap(entry => + if (Direction.IN.equals(entry._2)) { + val spo = new SPO(entry._1) + List((spo.getS, spo, entry._2)) + } else if (Direction.OUT.equals(entry._2)) { + val spo = new SPO(entry._1) + List((spo.getO, spo, entry._2)) + } else { + List.empty + }) + .foreach(item => { + val spo2DirectionMap = + isolateVertexMap.getOrElseUpdate(item._1, new mutable.HashMap[SPO, Direction]()) + val direction = spo2DirectionMap.getOrElse(item._2, null) + if (Direction.BOTH.equals(direction) || Direction.BOTH.equals(item._3)) { + spo2DirectionMap.put(item._2, Direction.BOTH) + } else if (null == direction) { + spo2DirectionMap.put(item._2, item._3) + } else if (Direction.IN.equals(direction)) { + if (Direction.OUT.equals(item._3)) { + spo2DirectionMap.put(item._2, Direction.BOTH) + } + } else if (Direction.OUT.equals(direction)) { + if (Direction.IN.equals(item._3)) { + spo2DirectionMap.put(item._2, Direction.BOTH) + } + } + }) + + isolateVertexMap.toList + .filter(p => !p._2.values.forall(d => Direction.BOTH.equals(d))) + .map(item => item._1) + .toSet + } + + private def getLoaderConfig(logicalPlan: LogicalOperator, catalog: Catalog) = { + val solvedModel = logicalPlan.transform[SolvedModel] { + case (Start(_, _, _, solvedModel), list) => + if (list == null || list.isEmpty) { + solvedModel + } else { + merge(solvedModel, list.head) + } + case (Driving(_, _, solvedModel), list) => + if (list == null || list.isEmpty) { + solvedModel + } else { + merge(solvedModel, list.head) + } + case (_, list) => + if (list.isEmpty) { + SolvedModel(Map.empty, Map.empty, Map.empty) + } else { + list.foldLeft(list.head)((l, r) => merge(l, r)) + } + } + val edgeLoadDirectionMap = getEdgeLoadDirectionMap(logicalPlan) + val typesAllowIsolateVertex = findAllTypesAllowIsolateVertex( + logicalPlan) ++ getAllowIsolateVertexFromEdgeLoadDirectionMap(edgeLoadDirectionMap) + val loaderConf = new GraphLoaderConfig() + val vertexLoaderConfigMap = new mutable.HashMap[String, VertexLoaderConfig]() + val edgeLoaderConfigMap = new mutable.HashMap[String, EdgeLoaderConfig]() + for (field <- solvedModel.fields.values.map(flatten(_)).flatten) { + field match { + case nodeVar: NodeVar => + for (typeName <- solvedModel.getTypes(field.name)) { + val node = logicalPlan.graph.getNode(typeName.split("/")(0)) + if (node.resolved) { + val vertexLoaderConfig = new VertexLoaderConfig() + if (typesAllowIsolateVertex.contains(node.typeName)) { + vertexLoaderConfig.setAllowIsolateVertex(true) + } + vertexLoaderConfig.setVertexType(node.typeName) + vertexLoaderConfig.setNeedProperties( + nodeVar.fields.filter(_.resolved).map(_.name).asJava) + vertexLoaderConfig.setConnection(catalog.getConnection(node.typeName)) + if (vertexLoaderConfigMap.contains(node.typeName)) { + val oldVertexLoaderConfig = vertexLoaderConfigMap(node.typeName) + vertexLoaderConfig.merge(oldVertexLoaderConfig) + } + vertexLoaderConfigMap.put(node.typeName, vertexLoaderConfig) + } + } + case edgeVar: EdgeVar => + for (typeName <- solvedModel.getTypes(field.name)) { + val edge = logicalPlan.graph.getEdge(typeName) + if (edge.resolved) { + val edgeTypeName = edge.startNode + "_" + edge.typeName + "_" + edge.endNode + val edgeLoaderConfig = new EdgeLoaderConfig() + edgeLoaderConfig.setEdgeType(edgeTypeName) + edgeLoaderConfig.setNeedProperties( + edgeVar.fields.filter(_.resolved).map(_.name).asJava) + edgeLoaderConfig.setConnection(catalog.getConnection(edgeTypeName)) + if (edgeLoaderConfigMap.contains(edgeTypeName)) { + val oldEdgeLoaderConfig = edgeLoaderConfigMap(edgeTypeName) + edgeLoaderConfig.merge(oldEdgeLoaderConfig) + } + edgeLoaderConfig.setLoadDirection( + edgeLoadDirectionMap.getOrElse(edgeTypeName, Direction.BOTH)) + edgeLoaderConfigMap.put(edgeTypeName, edgeLoaderConfig) + } + } + case _ => + } + } + loaderConf.setEdgeLoaderConfigs(edgeLoaderConfigMap.values.toSet.asJava) + loaderConf.setVertexLoaderConfigs(vertexLoaderConfigMap.values.toSet.asJava) + loaderConf + } + + private def getFilterRule(logicalPlan: LogicalOperator): Map[String, Rule] = { + val rulesMap: Map[String, mutable.MutableList[Rule]] = + logicalPlan.transform[Map[String, mutable.MutableList[Rule]]] { + case (scan: PatternScan, list) => + val curRuleMap = getRuleMap(scan.pattern) + mergeRuleMap(list :+ curRuleMap) + case (expand: ExpandInto, list) => + val curRuleMap = getRuleMap(expand.pattern) + mergeRuleMap(list :+ curRuleMap) + case (_, list) => + if (list != null && !list.isEmpty) { + mergeRuleMap(list) + } else { + Map.empty + } + } + rulesMap.toList + .map(kv => (kv._1, kv._2.toList)) + .filterNot(kv => kv._2.size > 1 || null == kv._2.head) + .map(kv => (kv._1, kv._2.head)) + .toMap + } + + private def mergeRuleMap(ruleMapList: List[Map[String, mutable.MutableList[Rule]]]) + : Map[String, mutable.MutableList[Rule]] = { + val rstRuleMap = new mutable.HashMap[String, mutable.MutableList[Rule]]() + ruleMapList.foreach(map => + map.toList.foreach(kv => + rstRuleMap.getOrElseUpdate(kv._1, new mutable.MutableList[Rule]()) ++= kv._2)) + rstRuleMap.toMap + } + + private def getRuleMap(pattern: Pattern): Map[String, mutable.MutableList[Rule]] = { + val aliasToTypes = pattern.patternTypes + val ruleMap = new mutable.HashMap[String, mutable.MutableList[Rule]]() + pattern.root.typeNames.foreach(t => { + ruleMap.getOrElseUpdate(t, new mutable.MutableList[Rule]()) += pattern.root.rule + }) + + for (conn <- pattern.topology.values.flatten) { + aliasToTypes(conn.alias).foreach(t => { + ruleMap.getOrElseUpdate(t, new mutable.MutableList[Rule]()) += conn.rule + }) + } + ruleMap.toMap + } + + private def flatten(v: Var): List[Var] = { + v match { + case v: RepeatPathVar => flatten(v.pathVar) + case v: PathVar => v.elements + case _ => List.apply(v) + } + } + +} diff --git a/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/PathConnection.scala b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/PathConnection.scala new file mode 100644 index 00000000..4a048213 --- /dev/null +++ b/reasoner/runner/runner-common/src/main/scala/com/antgroup/openspg/reasoner/util/PathConnection.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.util + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction +import com.antgroup.openspg.reasoner.lube.common.pattern.{ + Connection, + PatternConnection, + PatternElement +} +import com.antgroup.openspg.reasoner.lube.common.rule.Rule + +case class PathConnection( + alias: String, + source: String, + relTypes: Set[String], + target: String, + direction: Direction, + rule: Rule, + vertexSchemaList: List[PatternElement], + edgeSchemaList: List[PatternConnection]) + extends Connection { + + override def update(source: String, target: String): Connection = + copy(source = source, target = target) + + override def update(rule: Rule): Connection = copy(rule = rule) + + override def update(direction: Direction): Connection = copy(direction = direction) + + override def limit: Integer = 0 + +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/MemStartIdRecoderTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/MemStartIdRecoderTest.java new file mode 100644 index 00000000..f03b76e6 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/MemStartIdRecoderTest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.loader.MemStartIdRecoder; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class MemStartIdRecoderTest { + + @Before + public void init() {} + + @Test + public void testMemStartId() { + MemStartIdRecoder recoder = new MemStartIdRecoder(); + recoder.addStartId(IVertexId.from(1L, "t")); + recoder.addStartId(IVertexId.from(2L, "t")); + recoder.addStartId(IVertexId.from(2L, "t")); + recoder.flush(); + + Assert.assertEquals(2L, recoder.getStartIdCount()); + long count = 0; + while (recoder.hasNext()) { + IVertexId id = recoder.next(); + count++; + } + Assert.assertEquals(2L, count); + Assert.assertEquals(0L, recoder.getStartIdCount()); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/RunnerUtilTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/RunnerUtilTest.java new file mode 100644 index 00000000..75c78557 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/RunnerUtilTest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner; + +import com.antgroup.openspg.reasoner.batching.DynamicBatchSize; +import com.antgroup.openspg.reasoner.common.Utils; +import com.antgroup.openspg.reasoner.common.constants.Constants; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import com.google.common.collect.Lists; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +public class RunnerUtilTest { + + @Test + public void testBetween() { + Assert.assertEquals(RunnerUtil.between(null, null, 100L), true); + Assert.assertEquals(RunnerUtil.between(10L, null, 10L), true); + Assert.assertEquals(RunnerUtil.between(10L, null, 1L), false); + Assert.assertEquals(RunnerUtil.between(null, 10L, 100L), false); + Assert.assertEquals(RunnerUtil.between(null, 100L, 100L), true); + Assert.assertEquals(RunnerUtil.between(1L, 10L, 100L), false); + Assert.assertEquals(RunnerUtil.between(1L, 10L, 10L), true); + } + + @Test + public void testEdgeContext() { + IEdge edge = + new Edge<>( + IVertexId.from(0, "s"), + IVertexId.from(1, "o"), + new EdgeProperty(), + 0L, + Direction.OUT, + "s_p_o"); + Map edgeContext = RunnerUtil.edgeContext(edge, null, "E"); + Assert.assertEquals( + ((Map) edgeContext.get("E")).get(Constants.CONTEXT_LABEL), "p"); + } + + @Test + public void testDynamicBatch() { + final long allSize = 51L; + final long minSize = 5; + final long maxSize = 20L; + DynamicBatchSize batchSize = new DynamicBatchSize(allSize, minSize, maxSize, 4); + Assert.assertEquals(batchSize.remainSize(), allSize); + long size1 = batchSize.getNextBatchSize(); + System.out.println("size1=" + size1); + Utils.sleep(35); + long size2 = batchSize.getNextBatchSize(); + System.out.println("size2=" + size2); + Utils.sleep(100); + long size3 = batchSize.getNextBatchSize(); + System.out.println("size3=" + size3); + long size4 = batchSize.getNextBatchSize(); + System.out.println("size4=" + size4); + } + + @Test + public void testGetDslParams() { + Map dslParams = RunnerUtil.getOfflineDslParams(new HashMap<>(), false); + Assert.assertNotNull(dslParams); + } + + @Test + public void testStarPathLimit1() { + Map>> adjEdges = new HashMap<>(); + adjEdges.put( + "E1", + Lists.newArrayList( + new Edge<>(IVertexId.from(11L, "A"), IVertexId.from(21L, "B")), + new Edge<>(IVertexId.from(12L, "A"), IVertexId.from(22L, "B")), + new Edge<>(IVertexId.from(13L, "A"), IVertexId.from(23L, "B")), + new Edge<>(IVertexId.from(14L, "A"), IVertexId.from(24L, "B")), + new Edge<>(IVertexId.from(15L, "A"), IVertexId.from(25L, "B")), + new Edge<>(IVertexId.from(16L, "A"), IVertexId.from(26L, "B")), + new Edge<>(IVertexId.from(17L, "A"), IVertexId.from(27L, "B")), + new Edge<>(IVertexId.from(18L, "A"), IVertexId.from(28L, "B")), + new Edge<>(IVertexId.from(19L, "A"), IVertexId.from(29L, "B")))); + adjEdges.put( + "E2", + Lists.newArrayList( + new Edge<>(IVertexId.from(11L, "A"), IVertexId.from(21L, "B")), + new Edge<>(IVertexId.from(12L, "A"), IVertexId.from(22L, "B")), + new Edge<>(IVertexId.from(13L, "A"), IVertexId.from(23L, "B")), + new Edge<>(IVertexId.from(14L, "A"), IVertexId.from(24L, "B")))); + // minEdgeNum too big, will not limit edge size + RunnerUtil.doStarPathLimit(adjEdges, 1, 100); + Assert.assertEquals(adjEdges.get("E1").size(), 9); + Assert.assertEquals(adjEdges.get("E2").size(), 4); + + // edge not exceed limit + RunnerUtil.doStarPathLimit(adjEdges, 36, 3); + Assert.assertEquals(adjEdges.get("E1").size(), 9); + Assert.assertEquals(adjEdges.get("E2").size(), 4); + + // less than shrinkFactorThreshold, will not limit edge size + RunnerUtil.doStarPathLimit(adjEdges, 34, 3); + Assert.assertEquals(adjEdges.get("E1").size(), 9); + Assert.assertEquals(adjEdges.get("E2").size(), 4); + + // will do edge limit action + RunnerUtil.doStarPathLimit(adjEdges, 20, 3); + Assert.assertEquals(adjEdges.get("E1").size(), 7); + Assert.assertEquals(adjEdges.get("E2").size(), 3); + + // do nothing + RunnerUtil.doStarPathLimit(adjEdges, 20, 2); + Assert.assertEquals(adjEdges.get("E1").size(), 7); + Assert.assertEquals(adjEdges.get("E2").size(), 3); + } + + @Test + public void testCompare() { + Assert.assertEquals(0, RunnerUtil.compareTwoObject(null, null)); + Assert.assertEquals(-1, RunnerUtil.compareTwoObject(null, 1)); + Assert.assertEquals(1, RunnerUtil.compareTwoObject(1, null)); + Object obj1 = 0L; + Object obj2 = 0L; + Assert.assertEquals(0, RunnerUtil.compareTwoObject(obj1, obj2)); + obj1 = 1L; + Assert.assertEquals(1, RunnerUtil.compareTwoObject(obj1, obj2)); + obj1 = 0L; + obj2 = 1L; + Assert.assertEquals(-1, RunnerUtil.compareTwoObject(obj1, obj2)); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/ConfigCheckInitializer.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/ConfigCheckInitializer.java new file mode 100644 index 00000000..6e98aaf1 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/ConfigCheckInitializer.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.context; + +import org.junit.Assert; + +public class ConfigCheckInitializer extends BaseContextInitializer { + + @Override + public String initOnDriver() { + Assert.assertEquals(this.taskRecord.getParams().get("config1"), "1"); + Assert.assertEquals(this.taskRecord.getParams().get("config2"), "2"); + return ""; + } + + @Override + public void dispatchToWorker(String obj) { + Assert.assertEquals(this.taskRecord.getParams().get("config1"), "1"); + Assert.assertEquals(this.taskRecord.getParams().get("config2"), "2"); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/ContextManagementTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/ContextManagementTest.java new file mode 100644 index 00000000..64066c17 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/ContextManagementTest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.context; + +import com.antgroup.openspg.reasoner.task.TaskRecord; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +public class ContextManagementTest { + + @Test + public void testOnlyCallInitOnce() { + // only init once + Assert.assertEquals("0", ContextManagement.getInstance().getContext(StringInitializer.class)); + Assert.assertEquals("0", ContextManagement.getInstance().getContext(StringInitializer.class)); + Assert.assertEquals("0", ContextManagement.getInstance().getContext(StringInitializer.class)); + Assert.assertEquals("0", ContextManagement.getInstance().getContext(StringInitializer.class)); + } + + @Test + public void testJobConfigMap() { + Map paramsMap = new HashMap<>(); + paramsMap.put("config1", "1"); + paramsMap.put("config2", "2"); + + List initializerClassList = new ArrayList<>(); + initializerClassList.add("com.antgroup.openspg.reasoner.context.ConfigCheckInitializer"); + initializerClassList.add("com.antgroup.openspg.reasoner.context.StringInitializer"); + TaskRecord taskRecord = + TaskRecord.builder().params(paramsMap).initializerClassList(initializerClassList).build(); + + // init + ContextManagement.getInstance().initContextOnDriver(taskRecord); + // init again, no error + ContextManagement.getInstance().initContextOnDriver(taskRecord); + + Assert.assertEquals("0", ContextManagement.getInstance().getContext(StringInitializer.class)); + ContextManagement.getInstance().getContext(ConfigCheckInitializer.class); + + DispatchContextInfo dispatchContextInfo = + ContextManagement.getInstance().getDispatchContextInfo(); + + // create new management + ContextManagement.clear(); + + // dispatch + ContextManagement.getInstance().dispatchContextToWorker(dispatchContextInfo); + + // dispatch two times, no error + ContextManagement.getInstance().dispatchContextToWorker(dispatchContextInfo); + } +} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/FailureReasonerResult.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/StringInitializer.java similarity index 59% rename from server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/FailureReasonerResult.java rename to reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/StringInitializer.java index 58466107..7118c26b 100644 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/FailureReasonerResult.java +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/context/StringInitializer.java @@ -11,17 +11,21 @@ * or implied. */ -package com.antgroup.openspg.server.core.reasoner.model.service; +package com.antgroup.openspg.reasoner.context; -public class FailureReasonerResult extends BaseReasonerResult { +import org.junit.Assert; - private final String errorMsg; +public class StringInitializer extends BaseContextInitializer { - public FailureReasonerResult(String errorMsg) { - this.errorMsg = errorMsg; + private int count = 0; + + @Override + public String initOnDriver() { + return String.valueOf(count++); } - public String getErrorMsg() { - return errorMsg; + @Override + public void dispatchToWorker(String obj) { + Assert.assertEquals(obj, "0"); } } diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/GraphStateTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/GraphStateTest.java new file mode 100644 index 00000000..86e6a4c9 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/GraphStateTest.java @@ -0,0 +1,319 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum; +import com.antgroup.openspg.reasoner.utils.RunnerUtil; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class GraphStateTest { + + private GraphState graphState = new MemGraphState(); + + /* + * datasource is below + * vs1 + * / \ + * vt1 vt2 + */ + private LoadSourceData sourceData = new MockSourceData(); + + @Before + public void init() { + sourceData.getVertexList().forEach(vertex -> graphState.addVertex(vertex)); + sourceData + .getEdgeAggregated() + .forEach( + (s, listListTuple2) -> + graphState.addEdges(s, listListTuple2._1(), listListTuple2._2())); + } + + @Test + public void testMergeVertexProperty() { + Vertex vertex1 = new Vertex<>(); + vertex1.setId(IVertexId.from("vs2", "Student")); + IProperty property1 = new VertexVersionProperty(); + property1.put("type", "Student"); + vertex1.setValue(property1); + graphState.addVertex(vertex1); + + IVertexId vertexId = IVertexId.from("vs2", "Student"); + Map property = new HashMap<>(); + // empty + graphState.mergeVertexProperty( + IVertexId.from("empty", "Student"), property, MergeTypeEnum.REPLACE, 0L); + // replace + property.put("gmtCreate", "2023.02.07"); + graphState.mergeVertexProperty(vertexId, property, MergeTypeEnum.REPLACE, 0L); + Assert.assertTrue(graphState.getVertex(vertexId, 0L).getValue().isKeyExist("gmtCreate")); + // append + property.put("gmtCreate", "2023.02.08"); + graphState.mergeVertexProperty(vertexId, property, MergeTypeEnum.APPEND, 0L); + String val = (String) graphState.getVertex(vertexId, null).getValue().get("gmtCreate"); + Assert.assertTrue(val.contains("2023.02.07") && val.contains("2023.02.08")); + // version is null + property.put("gmtCreate", "2023.02.09"); + graphState.mergeVertexProperty(vertexId, property, MergeTypeEnum.REPLACE, null); + val = (String) graphState.getVertex(vertexId, null).getValue().get("gmtCreate"); + Assert.assertTrue( + val.contains("2023.02.09") && !val.contains("2023.02.07") && !val.contains("2023.02.08")); + + // merge new version property + property.put("gmtCreate", "2023.02.10"); + graphState.mergeVertexProperty(vertexId, property, MergeTypeEnum.REPLACE, 1L); + val = (String) graphState.getVertex(vertexId, 0L).getValue().get("gmtCreate"); + Assert.assertTrue(val.equals("2023.02.09")); + val = + (String) + ((VertexVersionProperty) graphState.getVertex(vertexId, 1L).getValue()) + .get("gmtCreate", 1L); + Assert.assertTrue(val.equals("2023.02.10")); + } + + @Test + public void testGetVertex() { + IVertexId vertexId = graphState.getVertexIterator((Set) null).next().getId(); + // version is null + IVertex v = graphState.getVertex(vertexId, null); + Assert.assertTrue(null != v); + // version is 0 + v = graphState.getVertex(vertexId, 0L); + Assert.assertTrue(null != v); + // version not exist + v = graphState.getVertex(vertexId, 1L); + // use mock data every vertex has one property "id" + Assert.assertTrue(null != v && v.getValue().getSize() == 1); + IVertexId vertexIdNotExist = IVertexId.from("not_exist_id", "Student"); + v = graphState.getVertex(vertexIdNotExist, null); + } + + private IEdge getOriginEdge( + Iterator> it, IEdge edge) { + while (it.hasNext()) { + IEdge tmp = it.next(); + if (tmp.getSourceId().equals(edge.getSourceId()) + && tmp.getTargetId().equals(edge.getTargetId()) + && tmp.getType().equals(edge.getType()) + && tmp.getVersion().equals(edge.getVersion())) { + return tmp; + } + } + return null; + } + + @Test + public void testUpdateEdgeProperty() { + IEdge edge = graphState.getEdgeIterator((Set) null).next(); + IProperty property = edge.getValue(); + Assert.assertTrue(!property.isKeyExist("xGmtCreateX")); + + // update exist version edge + property.put("xGmtCreateX", "2023.02.07"); + graphState.updateEdgeProperty( + edge.getSourceId(), edge.getType(), edge.getTargetId(), 0L, property); + Iterator> it = graphState.getEdgeIterator((Set) null); + IEdge edge1 = getOriginEdge(it, edge); + Assert.assertTrue(edge1.getValue().isKeyExist("xGmtCreateX")); + + // update not exist version edge + try { + graphState.updateEdgeProperty( + edge.getSourceId(), edge.getType(), edge.getTargetId(), 1L, property); + Assert.assertTrue(false); + } catch (Exception e) { + Assert.assertTrue(true); + } + } + + @Test + public void testMergeEdgeProperty() { + Map property = new HashMap<>(); + property.put("xGmtCreateX", "2023.02.13"); + IEdge edge = graphState.getEdgeIterator((Set) null).next(); + Assert.assertTrue(!edge.getValue().isKeyExist("xGmtCreateX")); + // merge new property + graphState.mergeEdgeProperty( + edge.getSourceId(), + edge.getType(), + edge.getTargetId(), + edge.getVersion(), + null, + property, + MergeTypeEnum.REPLACE); + Iterator> it = graphState.getEdgeIterator((Set) null); + IEdge edge1 = getOriginEdge(it, edge); + Assert.assertTrue(edge1.getValue().isKeyExist("xGmtCreateX")); + + // merge with replace + property.put("xGmtCreateX", "2023.02.13"); + property.put("xNewPropX", "newProp"); + graphState.mergeEdgeProperty( + edge.getSourceId(), + edge.getType(), + edge.getTargetId(), + edge.getVersion(), + null, + property, + MergeTypeEnum.REPLACE); + it = graphState.getEdgeIterator((Set) null); + IEdge edge2 = getOriginEdge(it, edge); + Assert.assertTrue(edge2.getValue().get("xGmtCreateX").equals("2023.02.13")); + Assert.assertTrue(edge2.getValue().isKeyExist("xNewPropX")); + + // merge with append + property.put("xGmtCreateX", "2023.02.14"); + property.put("xNewPropX2", "xNewPropX2"); + graphState.mergeEdgeProperty( + edge.getSourceId(), + edge.getType(), + edge.getTargetId(), + edge.getVersion(), + null, + property, + MergeTypeEnum.APPEND); + it = graphState.getEdgeIterator((Set) null); + IEdge edge3 = getOriginEdge(it, edge); + Assert.assertTrue(((String) edge3.getValue().get("xGmtCreateX")).contains("2023.02.13")); + Assert.assertTrue(((String) edge3.getValue().get("xGmtCreateX")).contains("2023.02.14")); + Assert.assertTrue(edge3.getValue().isKeyExist("xNewPropX2")); + } + + @Test + public void testGetEdge() { + IVertexId vertexId = graphState.getVertexIterator((Set) null).next().getId(); + IEdge edge = + graphState.getEdges(vertexId, null, null, null, Direction.BOTH).get(0); + // same condition + List> edges1 = + graphState.getEdges( + edge.getSourceId(), + edge.getVersion(), + edge.getVersion(), + new HashSet() { + { + add(edge.getType()); + } + }, + edge.getDirection()); + Assert.assertTrue(null != getOriginEdge(edges1, edge)); + + // version is null + List> edges2 = + graphState.getEdges( + edge.getSourceId(), + null, + edge.getVersion(), + new HashSet() { + { + add(edge.getType()); + } + }, + edge.getDirection()); + Assert.assertTrue(null != getOriginEdge(edges2, edge)); + + // type is empty + List> edges3 = + graphState.getEdges( + edge.getSourceId(), edge.getVersion(), null, new HashSet<>(), edge.getDirection()); + Assert.assertTrue(null == getOriginEdge(edges3, edge)); + + // vertexId not exist + IVertexId vertexIdNotExist = IVertexId.from("not_exist_id", "Student"); + List> edges4 = + graphState.getEdges( + vertexIdNotExist, edge.getVersion(), null, new HashSet<>(), edge.getDirection()); + Assert.assertTrue(null == getOriginEdge(edges4, edge)); + } + + private IEdge getOriginEdge( + List> edgeList, IEdge edge) { + for (IEdge tmp : edgeList) { + if (tmp.getSourceId().equals(edge.getSourceId()) + && tmp.getTargetId().equals(edge.getTargetId()) + && tmp.getType().equals(edge.getType()) + && tmp.getVersion().equals(edge.getVersion())) { + return tmp; + } + } + return null; + } + + @Test + public void testGetVertexIterator() { + Iterator> it = graphState.getVertexIterator((Set) null); + Assert.assertTrue(null != it); + Assert.assertTrue(it.hasNext()); + IVertex vertex = it.next(); + it = + graphState.getVertexIterator( + new HashSet() { + { + add((RunnerUtil.getVertexType(vertex))); + } + }); + Assert.assertTrue(it.hasNext()); + Assert.assertTrue(getOriginVertex(it, vertex)); + + it = graphState.getVertexIterator(vertex1 -> vertex1.getId().equals(vertex.getId())); + Assert.assertTrue(getOriginVertex(it, vertex)); + } + + private boolean getOriginVertex( + Iterator> it, IVertex vertex) { + while (it.hasNext()) { + IVertex vertex1 = it.next(); + if (vertex1.getId().equals(vertex.getId())) { + return true; + } + } + return false; + } + + @Test + public void testGetEdgeIterator() { + IVertexId vertexId = graphState.getVertexIterator((Set) null).next().getId(); + List> edgeList = + graphState.getEdges(vertexId, null, null, null, Direction.BOTH); + Assert.assertTrue(!edgeList.isEmpty()); + IEdge edge = edgeList.get(0); + Iterator> it = + graphState.getEdgeIterator( + new HashSet() { + { + add(edge.getType()); + } + }); + Assert.assertTrue(null != it); + Assert.assertTrue(getOriginEdge(it, edge) != null); + + it = graphState.getEdgeIterator(edge1 -> edge1.getType().equals(edge.getType())); + Assert.assertTrue(getOriginEdge(it, edge) != null); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/GraphStoreGraphStateTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/GraphStoreGraphStateTest.java new file mode 100644 index 00000000..867f48b7 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/GraphStoreGraphStateTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.impl.GraphStoreGraphState; +import com.antgroup.openspg.reasoner.warehouse.common.AbstractGraphLoader; +import java.util.HashSet; +import org.junit.Assert; +import org.junit.Test; + +public class GraphStoreGraphStateTest { + @Test + public void testReadVertex() { + GraphStoreGraphState graphStoreGraphState = new GraphStoreGraphState(); + AbstractGraphLoader abstractGraphLoader = new MockGraphLoader(null); + graphStoreGraphState.setGraphStoreQuery(abstractGraphLoader); + IVertex v = + graphStoreGraphState.getVertex(IVertexId.from("abc", "Test"), null); + Assert.assertTrue(v != null); + + IVertex v1 = + graphStoreGraphState.getVertex(IVertexId.from("abc2", "Test"), null); + Assert.assertTrue(v1 == null); + } + + @Test + public void testException() { + GraphStoreGraphState graphStoreGraphState = new GraphStoreGraphState(); + + try { + graphStoreGraphState.getVertexIterator(new HashSet<>()); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(true); + } + + try { + graphStoreGraphState.getVertexIterator(vertex1 -> vertex1.getId().equals("abc")); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(true); + } + + try { + graphStoreGraphState.getEdgeIterator(new HashSet<>()); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(true); + } + + try { + graphStoreGraphState.getEdgeIterator(edge1 -> edge1.getSourceId().equals("abc")); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(true); + } + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/LoadSourceData.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/LoadSourceData.java new file mode 100644 index 00000000..d0e41e6f --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/LoadSourceData.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.generator.AbstractGraphGenerator; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import java.util.Map; +import java.util.Set; + +public abstract class LoadSourceData extends AbstractGraphGenerator { + + /** + * Load alias to vertex map from source data + * + * @return + */ + public abstract Map>> loadAlias2Vertex(); + + /** + * Load alias to edge map from source data + * + * @return + */ + public abstract Map>> loadAlias2Edge(); + + /** + * Load source data schema + * + * @return + */ + public abstract Pattern loadPattern(); +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockGraphLoader.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockGraphLoader.java new file mode 100644 index 00000000..9890dbe4 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockGraphLoader.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.warehouse.common.AbstractGraphLoader; +import com.antgroup.openspg.reasoner.warehouse.common.VertexSubGraph; +import com.antgroup.openspg.reasoner.warehouse.common.config.GraphLoaderConfig; +import java.util.Iterator; + +public class MockGraphLoader extends AbstractGraphLoader { + /** + * loader + * + * @param graphLoaderConfig + */ + public MockGraphLoader(GraphLoaderConfig graphLoaderConfig) { + super(graphLoaderConfig); + } + + @Override + public void close() throws Exception {} + + @Override + public Iterator iterator() { + return null; + } + + @Override + public VertexSubGraph queryOneHotGraphState(IVertexId vertexId) { + IVertexId mockVertex = IVertexId.from("abc", "Test"); + if (mockVertex.equals(vertexId)) { + return new VertexSubGraph(new Vertex<>(mockVertex, new VertexVersionProperty())); + } + return null; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData.java new file mode 100644 index 00000000..fe818cbf --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData.java @@ -0,0 +1,78 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MockSourceData extends LoadSourceData { + /* + * vs1 + * / \ + * vt1 vt2 + */ + + @Override + public List> genVertexList() { + + return Lists.newArrayList( + constructionVersionVertex("vs1", "Student", 0), + constructionVertex("vt1", "Teacher"), + constructionVertex("vt2", "Teacher")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList( + constructionVersionEdge("vs1", "STEdge", "vt1", 0), + constructionEdge("vs1", "STEdge", "vt2")); + } + + /** + * Load alias to vertex map from source data + * + * @return + */ + @Override + public Map>> loadAlias2Vertex() { + return null; + } + + /** + * Load alias to edge map from source data + * + * @return + */ + @Override + public Map>> loadAlias2Edge() { + return null; + } + + /** + * Load source data schema + * + * @return + */ + @Override + public Pattern loadPattern() { + return null; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData2.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData2.java new file mode 100644 index 00000000..a478e413 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData2.java @@ -0,0 +1,223 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import scala.collection.JavaConversions; + +public class MockSourceData2 extends LoadSourceData { + /* + a1 a2 a3 a4 + \ / \ / + b1 b2 + \ / + \ / + c1 + */ + + /** + * Load alias to vertex map from source data + * + * @return + */ + @Override + public Map>> loadAlias2Vertex() { + Map>> alias2VertexMap = new HashMap<>(); + Vertex c1 = new Vertex<>(IVertexId.from("c1", "NULL")); + Vertex b1 = new Vertex<>(IVertexId.from("b1", "NULL")); + Vertex b2 = new Vertex<>(IVertexId.from("b2", "NULL")); + Vertex a1 = new Vertex<>(IVertexId.from("a1", "NULL")); + Vertex a2 = new Vertex<>(IVertexId.from("a2", "NULL")); + Vertex a3 = new Vertex<>(IVertexId.from("a3", "NULL")); + Vertex a4 = new Vertex<>(IVertexId.from("a4", "NULL")); + alias2VertexMap.put( + "C", + new HashSet() { + { + add(c1); + } + }); + alias2VertexMap.put( + "B", + new HashSet() { + { + add(b1); + add(b2); + } + }); + alias2VertexMap.put( + "A", + new HashSet() { + { + add(a1); + add(a2); + add(a3); + add(a4); + } + }); + return alias2VertexMap; + } + + /** + * Load alias to edge map from source data + * + * @return + */ + @Override + public Map>> loadAlias2Edge() { + Map>> alias2EdgeMap = new HashMap<>(); + Edge bc1 = + new Edge( + IVertexId.from("b1", "NULL"), + IVertexId.from("c1", "NULL"), + null, + 0L, + Direction.OUT, + "E"); + Edge bc2 = + new Edge( + IVertexId.from("b2", "NULL"), + IVertexId.from("c1", "NULL"), + null, + 0L, + Direction.OUT, + "E"); + Edge ab1 = + new Edge( + IVertexId.from("a1", "NULL"), + IVertexId.from("b1", "NULL"), + null, + 0L, + Direction.OUT, + "E"); + Edge ab2 = + new Edge( + IVertexId.from("a2", "NULL"), + IVertexId.from("b1", "NULL"), + null, + 0L, + Direction.OUT, + "E"); + Edge ab3 = + new Edge( + IVertexId.from("a3", "NULL"), + IVertexId.from("b2", "NULL"), + null, + 0L, + Direction.OUT, + "E"); + Edge ab4 = + new Edge( + IVertexId.from("a4", "NULL"), + IVertexId.from("b2", "NULL"), + null, + 0L, + Direction.OUT, + "E"); + + alias2EdgeMap.put( + "B_C", + new HashSet() { + { + add(bc1); + add(bc2); + } + }); + alias2EdgeMap.put( + "A_B", + new HashSet() { + { + add(ab1); + add(ab2); + add(ab3); + add(ab4); + } + }); + return alias2EdgeMap; + } + + /** + * Load source data schema + * + * @return + */ + @Override + public Pattern loadPattern() { + PatternElement C = new PatternElement("C", null, null); + PatternElement B = new PatternElement("B", null, null); + PatternElement A = new PatternElement("A", null, null); + + PatternConnection B_C = + new PatternConnection( + "B_C", B.alias(), null, C.alias(), Direction.OUT, null, -1, true, false); + PatternConnection A_B = + new PatternConnection( + "A_B", A.alias(), null, B.alias(), Direction.OUT, null, -1, true, false); + Set C_Connect = + new HashSet() { + { + add(B_C); + } + }; + Set B_Connect = + new HashSet() { + { + add(B_C); + add(A_B); + } + }; + Set A_Connect = + new HashSet() { + { + add(A_B); + } + }; + Map> topology = new HashMap<>(); + topology.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + topology.put(B.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + topology.put(A.alias(), Convert2ScalaUtil.toScalaImmutableSet(A_Connect)); + + Pattern schema = + new PartialGraphPattern( + A.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + return schema; + } + + @Override + public List> genVertexList() { + return null; + } + + @Override + public List> genEdgeList() { + return null; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData3.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData3.java new file mode 100644 index 00000000..536e6d2f --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/graphstate/MockSourceData3.java @@ -0,0 +1,247 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.graphstate; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import scala.collection.JavaConversions; + +public class MockSourceData3 extends LoadSourceData { + + private static final String DEFAULT_VERTEX_TYPE = "t"; + + @Override + public List> genVertexList() { + return null; + } + + @Override + public List> genEdgeList() { + return null; + } + + /** + * Load alias to vertex map from source data + * + * @return + */ + @Override + public Map>> loadAlias2Vertex() { + { + Map>> alias2VertexMap = new HashMap<>(); + IVersionProperty property1 = new VertexVersionProperty(); + Vertex c1 = new Vertex<>(IVertexId.from("c1", DEFAULT_VERTEX_TYPE), property1); + IVersionProperty property2 = new VertexVersionProperty(); + Vertex c2 = new Vertex<>(IVertexId.from("c2", DEFAULT_VERTEX_TYPE), property2); + IVersionProperty property3 = new VertexVersionProperty(); + Vertex b1 = new Vertex<>(IVertexId.from("b1", DEFAULT_VERTEX_TYPE), property3); + IVersionProperty property4 = new VertexVersionProperty(); + Vertex a1 = new Vertex<>(IVertexId.from("a1", DEFAULT_VERTEX_TYPE), property4); + IVersionProperty property5 = new VertexVersionProperty(); + Vertex a2 = new Vertex<>(IVertexId.from("a2", DEFAULT_VERTEX_TYPE), property5); + + alias2VertexMap.put( + "C", + new HashSet() { + { + add(c1); + add(c2); + } + }); + alias2VertexMap.put( + "B", + new HashSet() { + { + add(b1); + } + }); + alias2VertexMap.put( + "A", + new HashSet() { + { + add(a1); + add(a2); + } + }); + return alias2VertexMap; + } + } + + /** + * Load alias to edge map from source data + * + * @return + */ + @Override + public Map>> loadAlias2Edge() { + Map>> alias2EdgeMap = new HashMap<>(); + IProperty property1 = new EdgeProperty(); + Edge ab1 = + new Edge( + IVertexId.from("a1", DEFAULT_VERTEX_TYPE), + IVertexId.from("b1", DEFAULT_VERTEX_TYPE), + property1, + 0L, + Direction.OUT, + "directFilm"); + IProperty property2 = new EdgeProperty(); + Edge ab2 = + new Edge( + IVertexId.from("a2", DEFAULT_VERTEX_TYPE), + IVertexId.from("b1", DEFAULT_VERTEX_TYPE), + property2, + 0L, + Direction.OUT, + "directFilm"); + IProperty property3 = new EdgeProperty(); + Edge bc1 = + new Edge( + IVertexId.from("b1", DEFAULT_VERTEX_TYPE), + IVertexId.from("c1", DEFAULT_VERTEX_TYPE), + property3, + 0L, + Direction.OUT, + "workmates"); + IProperty property4 = new EdgeProperty(); + Edge bc2 = + new Edge( + IVertexId.from("b1", DEFAULT_VERTEX_TYPE), + IVertexId.from("c2", DEFAULT_VERTEX_TYPE), + property4, + 0L, + Direction.OUT, + "workmates"); + IProperty property5 = new EdgeProperty(); + Edge ac1 = + new Edge( + IVertexId.from("a1", DEFAULT_VERTEX_TYPE), + IVertexId.from("c2", DEFAULT_VERTEX_TYPE), + property5, + 0L, + Direction.OUT, + "writerOfFilm"); + IProperty property6 = new EdgeProperty(); + Edge ac2 = + new Edge( + IVertexId.from("a2", DEFAULT_VERTEX_TYPE), + IVertexId.from("c1", DEFAULT_VERTEX_TYPE), + property6, + 0L, + Direction.OUT, + "writerOfFilm"); + + alias2EdgeMap.put( + "E1", + new HashSet() { + { + add(ab1); + add(ab2); + } + }); + alias2EdgeMap.put( + "E2", + new HashSet() { + { + add(ac1); + add(ac2); + } + }); + alias2EdgeMap.put( + "E3", + new HashSet() { + { + add(bc1); + add(bc2); + } + }); + + return alias2EdgeMap; + } + + /** + * Load source data schema + * + * @return + */ + @Override + public Pattern loadPattern() { + PatternElement C = + new PatternElement("C", JavaConversions.asScalaSet(Sets.newHashSet("C")).toSet(), null); + PatternElement B = + new PatternElement("B", JavaConversions.asScalaSet(Sets.newHashSet("B")).toSet(), null); + PatternElement A = + new PatternElement("A", JavaConversions.asScalaSet(Sets.newHashSet("A")).toSet(), null); + + PatternConnection B_C = + new PatternConnection( + "E3", B.alias(), null, C.alias(), Direction.OUT, null, null, true, false); + PatternConnection A_B = + new PatternConnection( + "E1", A.alias(), null, B.alias(), Direction.OUT, null, null, true, false); + PatternConnection A_C = + new PatternConnection( + "E2", A.alias(), null, C.alias(), Direction.OUT, null, null, true, false); + + Set C_Connect = + new HashSet() { + { + add(B_C); + add(A_C); + } + }; + Set B_Connect = + new HashSet() { + { + add(B_C); + add(A_B); + } + }; + Set A_Connect = + new HashSet() { + { + add(A_B); + add(A_C); + } + }; + Map> topology = new HashMap<>(); + topology.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + topology.put(B.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + topology.put(A.alias(), Convert2ScalaUtil.toScalaImmutableSet(A_Connect)); + + Pattern schema = + new PartialGraphPattern( + A.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + return schema; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/io/Variance.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/io/Variance.java new file mode 100644 index 00000000..296cf2a5 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/io/Variance.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.io; + +public class Variance { + public static double Sum(double[] data) { + double sum = 0; + for (double datum : data) { + sum = sum + datum; + } + return sum; + } + + public static double Mean(double[] data) { + double mean = 0; + mean = Sum(data) / data.length; + return mean; + } + + public static double sampleVariance(double[] data) { + double variance = 0; + double mean = Mean(data); + for (double datum : data) { + variance = variance + (Math.pow((datum - mean), 2)); + } + variance = variance / (data.length - 1); + return variance; + } + + public static double sampleStdDev(double[] data) { + double std_dev; + std_dev = Math.sqrt(sampleVariance(data)); + return std_dev; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/EdgeCombinationIteratorTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/EdgeCombinationIteratorTest.java new file mode 100644 index 00000000..dda157d3 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/EdgeCombinationIteratorTest.java @@ -0,0 +1,712 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.kggraph.impl.EdgeCombinationIterator; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import scala.collection.JavaConversions; + +public class EdgeCombinationIteratorTest { + + private final String DEFAULT_VERTEX_TYPE = "t"; + + @Before + public void init() {} + + @Test + public void test1() { + Pattern schema = schema1(null); + Map>> alias2VertexMap = alias2VertexMap1(); + Map>> alias2EdgeMap = alias2EdgeMap1(); + + int count = 0; + + Set bIdSet = new HashSet<>(); + KgGraphSplitStaticParameters staticParameters = new KgGraphSplitStaticParameters(null, schema); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + count++; + Assert.assertEquals(1, kgGraph.getEdge("E1").size()); + Assert.assertEquals(1, kgGraph.getEdge("E2").size()); + Assert.assertEquals(1, kgGraph.getVertex("A").size()); + Assert.assertEquals(1, kgGraph.getVertex("B").size()); + Assert.assertEquals(1, kgGraph.getVertex("C").size()); + + Assert.assertEquals( + kgGraph.getEdge("E1").iterator().next().getSourceId(), + kgGraph.getVertex("B").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E1").iterator().next().getTargetId(), + kgGraph.getVertex("A").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E2").iterator().next().getSourceId(), + kgGraph.getVertex("B").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E2").iterator().next().getTargetId(), + kgGraph.getVertex("C").iterator().next().getId()); + + bIdSet.add(kgGraph.getVertex("B").iterator().next().getId()); + } + Assert.assertEquals(2, count); + Assert.assertEquals(2, bIdSet.size()); + Assert.assertTrue(bIdSet.contains(IVertexId.from(21, DEFAULT_VERTEX_TYPE))); + Assert.assertTrue(bIdSet.contains(IVertexId.from(22, DEFAULT_VERTEX_TYPE))); + } + + @Test + public void test2() { + Map> addTopology = new HashMap<>(); + addTopology.put( + "C", + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "E3", "C", null, "A", Direction.OUT, null, -1, true, false))) + .toSet()); + Pattern schema = schema1(addTopology); + + Map>> alias2VertexMap = alias2VertexMap1(); + Map>> alias2EdgeMap = alias2EdgeMap1(); + alias2EdgeMap.put( + "E3", + Sets.newHashSet( + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"), + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(12, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"), + new Edge( + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"), + new Edge( + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + IVertexId.from(12, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"))); + + int count = 0; + Set bIdSet = new HashSet<>(); + KgGraphSplitStaticParameters staticParameters = new KgGraphSplitStaticParameters(null, schema); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + count++; + Assert.assertEquals(1, kgGraph.getEdge("E1").size()); + Assert.assertEquals(1, kgGraph.getEdge("E2").size()); + Assert.assertEquals(1, kgGraph.getEdge("E3").size()); + Assert.assertEquals(1, kgGraph.getVertex("A").size()); + Assert.assertEquals(1, kgGraph.getVertex("B").size()); + Assert.assertEquals(1, kgGraph.getVertex("C").size()); + + Assert.assertEquals( + kgGraph.getEdge("E1").iterator().next().getSourceId(), + kgGraph.getVertex("B").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E1").iterator().next().getTargetId(), + kgGraph.getVertex("A").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E2").iterator().next().getSourceId(), + kgGraph.getVertex("B").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E2").iterator().next().getTargetId(), + kgGraph.getVertex("C").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E3").iterator().next().getSourceId(), + kgGraph.getVertex("C").iterator().next().getId()); + Assert.assertEquals( + kgGraph.getEdge("E3").iterator().next().getTargetId(), + kgGraph.getVertex("A").iterator().next().getId()); + + bIdSet.add(kgGraph.getVertex("B").iterator().next().getId()); + } + Assert.assertEquals(2, count); + Assert.assertEquals(2, bIdSet.size()); + Assert.assertTrue(bIdSet.contains(IVertexId.from(21, DEFAULT_VERTEX_TYPE))); + Assert.assertTrue(bIdSet.contains(IVertexId.from(22, DEFAULT_VERTEX_TYPE))); + } + + @Test + public void test3() { + Pattern schema = schema2(); + Map>> alias2VertexMap = alias2VertexMap2(); + Map>> alias2EdgeMap = alias2EdgeMap2(); + + int count = 0; + + Set aIdSet = new HashSet<>(); + KgGraphSplitStaticParameters staticParameters = new KgGraphSplitStaticParameters(null, schema); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + count++; + Assert.assertEquals(1, kgGraph.getEdge("E1").size()); + Assert.assertEquals(1, kgGraph.getEdge("E2").size()); + Assert.assertEquals(1, kgGraph.getEdge("E3").size()); + Assert.assertEquals(1, kgGraph.getVertex("A").size()); + Assert.assertEquals(1, kgGraph.getVertex("B").size()); + Assert.assertEquals(1, kgGraph.getVertex("C").size()); + Assert.assertEquals(1, kgGraph.getVertex("D").size()); + + aIdSet.add(kgGraph.getVertex("A").iterator().next().getId()); + } + + Assert.assertEquals(8, count); + Assert.assertEquals(1, aIdSet.size()); + Assert.assertTrue(aIdSet.contains(IVertexId.from(11, DEFAULT_VERTEX_TYPE))); + } + + @Test + public void test4() { + Pattern schema = schema2(); + Map>> alias2VertexMap = alias2VertexMap2(); + Map>> alias2EdgeMap = alias2EdgeMap2(); + + alias2EdgeMap + .get("E3") + .add( + new Edge( + IVertexId.from(15, DEFAULT_VERTEX_TYPE), + IVertexId.from(45, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T")); + + int count = 0; + + Set aIdSet = new HashSet<>(); + KgGraphSplitStaticParameters staticParameters = new KgGraphSplitStaticParameters(null, schema); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + count++; + Assert.assertEquals(1, kgGraph.getEdge("E1").size()); + Assert.assertEquals(1, kgGraph.getEdge("E2").size()); + Assert.assertEquals(1, kgGraph.getEdge("E3").size()); + Assert.assertEquals(1, kgGraph.getVertex("A").size()); + Assert.assertEquals(1, kgGraph.getVertex("B").size()); + Assert.assertEquals(1, kgGraph.getVertex("C").size()); + Assert.assertEquals(1, kgGraph.getVertex("D").size()); + + aIdSet.add(kgGraph.getVertex("A").iterator().next().getId()); + } + + Assert.assertEquals(9, count); + Assert.assertEquals(2, aIdSet.size()); + Assert.assertTrue(aIdSet.contains(IVertexId.from(11, DEFAULT_VERTEX_TYPE))); + Assert.assertTrue(aIdSet.contains(IVertexId.from(15, DEFAULT_VERTEX_TYPE))); + } + + @Test + public void test5() { + Pattern schema = schema1(null); + Map>> alias2VertexMap = alias2VertexMap1(); + Map>> alias2EdgeMap = alias2EdgeMap1(); + + final int num = 10; + IVertexId b = IVertexId.from(2000, DEFAULT_VERTEX_TYPE); + alias2VertexMap.get("B").add(new Vertex<>(b, null)); + for (int i = 0; i < num; ++i) { + IVertexId a = IVertexId.from(1000 + i, DEFAULT_VERTEX_TYPE); + IVertexId c = IVertexId.from(3000 + i, DEFAULT_VERTEX_TYPE); + alias2VertexMap.get("A").add(new Vertex<>(a, null)); + alias2VertexMap.get("C").add(new Vertex<>(c, null)); + alias2EdgeMap.get("E1").add(new Edge<>(b, a, null, 0L, Direction.IN, "E1T")); + alias2EdgeMap.get("E2").add(new Edge<>(b, c, null, 0L, Direction.OUT, "E2T")); + } + + long count = 0; + KgGraphSplitStaticParameters staticParameters = new KgGraphSplitStaticParameters(null, schema); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + it.setLogCount(num); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + count++; + } + Assert.assertEquals(num * num + 2, count); + } + + // duplicate data + @Test + public void test6() { + PatternElement did = + new PatternElement( + "did", JavaConversions.asScalaSet(Sets.newHashSet("ABM.Apdid")).toSet(), null); + PatternElement pkg = + new PatternElement( + "pkg", JavaConversions.asScalaSet(Sets.newHashSet("ABM.Pkg")).toSet(), null); + PatternElement pkg2 = + new PatternElement( + "pkg2", JavaConversions.asScalaSet(Sets.newHashSet("ABM.Pkg")).toSet(), null); + PatternElement b = + new PatternElement( + "b", JavaConversions.asScalaSet(Sets.newHashSet("ABM.BundleApp")).toSet(), null); + PatternElement c = + new PatternElement( + "c", JavaConversions.asScalaSet(Sets.newHashSet("ABM.BundleApp")).toSet(), null); + PatternElement family = + new PatternElement( + "family", + JavaConversions.asScalaSet(Sets.newHashSet("ABM.BundleAppFamily")).toSet(), + null); + Map> topology = new HashMap<>(); + topology.put( + did.alias(), + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "r2", "did", null, "pkg", Direction.OUT, null, -1, true, false), + new PatternConnection( + "r1", "did", null, "pkg2", Direction.OUT, null, -1, true, false))) + .toSet()); + topology.put( + pkg.alias(), + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "e1", "pkg", null, "b", Direction.OUT, null, -1, true, false))) + .toSet()); + topology.put( + b.alias(), + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "e2", "b", null, "family", Direction.OUT, null, -1, true, false))) + .toSet()); + topology.put( + pkg2.alias(), + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "e4", "pkg2", null, "c", Direction.OUT, null, -1, true, false))) + .toSet()); + topology.put( + family.alias(), + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "e3", "family", null, "c", Direction.IN, null, null, true, false))) + .toSet()); + Map map = new HashMap<>(); + map.put("did", did); + map.put("pkg", pkg); + map.put("pkg2", pkg2); + map.put("b", b); + map.put("c", c); + map.put("family", family); + Pattern schema = + new PartialGraphPattern( + family.alias(), + JavaConversions.mapAsScalaMap(map).toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + + Map>> alias2VertexMap = new HashMap<>(); + alias2VertexMap.put( + "did", Sets.newHashSet(new Vertex<>(IVertexId.from(11, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "pkg", Sets.newHashSet(new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "pkg2", + Sets.newHashSet( + new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "b", Sets.newHashSet(new Vertex<>(IVertexId.from(31, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "c", Sets.newHashSet(new Vertex<>(IVertexId.from(32, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "family", Sets.newHashSet(new Vertex<>(IVertexId.from(41, DEFAULT_VERTEX_TYPE), null))); + + Map>> alias2EdgeMap = new HashMap<>(); + alias2EdgeMap.put( + "r1", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "r"))); + alias2EdgeMap.put( + "r2", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "r"), + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "r"))); + alias2EdgeMap.put( + "e4", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "e"))); + alias2EdgeMap.put( + "e1", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "e"), + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "e"))); + alias2EdgeMap.put( + "e3", + Sets.newHashSet( + new Edge( + IVertexId.from(41, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.IN, + "ef"))); + alias2EdgeMap.put( + "e2", + Sets.newHashSet( + new Edge( + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + IVertexId.from(41, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "ef"))); + + int count = 0; + KgGraphSplitStaticParameters staticParameters = new KgGraphSplitStaticParameters(null, schema); + EdgeCombinationIterator it = + new EdgeCombinationIterator( + staticParameters.getEdgeIterateInfoList(), + staticParameters.getEdgeIterateOrderMap(), + alias2VertexMap, + alias2EdgeMap); + while (it.hasNext()) { + KgGraph kgGraph = it.next(); + if (null == kgGraph) { + continue; + } + count++; + } + Assert.assertEquals(1, count); + } + + private Pattern schema1(Map> addTopology) { + PatternElement A = + new PatternElement("A", JavaConversions.asScalaSet(Sets.newHashSet("A")).toSet(), null); + PatternElement B = + new PatternElement("B", JavaConversions.asScalaSet(Sets.newHashSet("B")).toSet(), null); + PatternElement C = + new PatternElement("C", JavaConversions.asScalaSet(Sets.newHashSet("C")).toSet(), null); + Map> topology = new HashMap<>(); + if (null != addTopology) { + topology.putAll(addTopology); + } + topology.put( + "B", + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "E2", "B", null, "C", Direction.OUT, null, -1, true, false), + new PatternConnection( + "E1", "B", null, "A", Direction.IN, null, null, true, false))) + .toSet()); + return new PartialGraphPattern( + B.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + } + + private Map>> alias2VertexMap1() { + Map>> alias2VertexMap = new HashMap<>(); + alias2VertexMap.put( + "A", + Sets.newHashSet( + new Vertex<>(IVertexId.from(11, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(12, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "B", + Sets.newHashSet( + new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "C", + Sets.newHashSet( + new Vertex<>(IVertexId.from(31, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(32, DEFAULT_VERTEX_TYPE), null))); + return alias2VertexMap; + } + + private Map>> alias2EdgeMap1() { + Map>> alias2EdgeMap = new HashMap<>(); + alias2EdgeMap.put( + "E1", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.IN, + "E1T"), + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(12, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.IN, + "E1T"))); + + alias2EdgeMap.put( + "E2", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"), + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"))); + return alias2EdgeMap; + } + + private Pattern schema2() { + PatternElement A = + new PatternElement("A", JavaConversions.asScalaSet(Sets.newHashSet("A")).toSet(), null); + PatternElement B = + new PatternElement("B", JavaConversions.asScalaSet(Sets.newHashSet("B")).toSet(), null); + PatternElement C = + new PatternElement("C", JavaConversions.asScalaSet(Sets.newHashSet("C")).toSet(), null); + PatternElement D = + new PatternElement("D", JavaConversions.asScalaSet(Sets.newHashSet("D")).toSet(), null); + Map> topology = new HashMap<>(); + topology.put( + "A", + JavaConversions.asScalaSet( + Sets.newHashSet( + new PatternConnection( + "E1", "A", null, "B", Direction.OUT, null, -1, true, false), + new PatternConnection( + "E2", "A", null, "C", Direction.OUT, null, -1, true, false), + new PatternConnection( + "E3", "A", null, "D", Direction.OUT, null, -1, true, false))) + .toSet()); + return new PartialGraphPattern( + B.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C, "D", D)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + } + + private Map>> alias2VertexMap2() { + Map>> alias2VertexMap = new HashMap<>(); + alias2VertexMap.put( + "A", + Sets.newHashSet( + new Vertex<>(IVertexId.from(11, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(15, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "B", + Sets.newHashSet( + new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(25, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "C", + Sets.newHashSet( + new Vertex<>(IVertexId.from(31, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(32, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(35, DEFAULT_VERTEX_TYPE), null))); + alias2VertexMap.put( + "D", + Sets.newHashSet( + new Vertex<>(IVertexId.from(41, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(42, DEFAULT_VERTEX_TYPE), null))); + return alias2VertexMap; + } + + private Map>> alias2EdgeMap2() { + Map>> alias2EdgeMap = new HashMap<>(); + alias2EdgeMap.put( + "E1", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"), + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"), + new Edge( + IVertexId.from(15, DEFAULT_VERTEX_TYPE), + IVertexId.from(25, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"))); + + alias2EdgeMap.put( + "E2", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"), + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(32, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"), + new Edge( + IVertexId.from(15, DEFAULT_VERTEX_TYPE), + IVertexId.from(35, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"))); + + alias2EdgeMap.put( + "E3", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(41, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"), + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(42, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"))); + return alias2EdgeMap; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/KgGraphSimpleSplitTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/KgGraphSimpleSplitTest.java new file mode 100644 index 00000000..55890489 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/KgGraphSimpleSplitTest.java @@ -0,0 +1,772 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import scala.collection.JavaConversions; + +public class KgGraphSimpleSplitTest { + + public static final String DEFAULT_VERTEX_TYPE = "t"; + + @Before + public void init() {} + + @Test + public void testNotNeedSplit() { + Pattern schema = pattern1(); + KgGraph kgGraph = kgGraph1(); + + Set splitVertexAliasSet = Sets.newHashSet("B", "C", "D"); + List> rst1 = + kgGraph.split( + splitVertexAliasSet, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet, schema), + null, + null); + Assert.assertEquals(1, rst1.size()); + Assert.assertEquals(rst1.get(0), kgGraph); + + Set splitVertexAliasSet2 = Sets.newHashSet("D"); + List> rst2 = + kgGraph.split( + splitVertexAliasSet2, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet2, schema), + null, + null); + Assert.assertEquals(1, rst2.size()); + Assert.assertEquals(rst2.get(0), kgGraph); + } + + @Test + public void testSampleSplit1() { + Pattern schema = pattern1(); + KgGraph kgGraph = kgGraph1(); + + Set splitVertexAliasSet = Sets.newHashSet("A"); + List> rst1 = + kgGraph.split( + splitVertexAliasSet, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet, schema), + null, + null); + Assert.assertEquals(2, rst1.size()); + rst1.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + + Set splitVertexAliasSet2 = Sets.newHashSet("A", "C"); + List> rst2 = + kgGraph.split( + splitVertexAliasSet2, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet2, schema), + null, + null); + Assert.assertEquals(2, rst2.size()); + rst2.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + } + + @Test + public void testSampleSplit2() { + Pattern schema = pattern1(); + KgGraphImpl kgGraph = kgGraph1(); + kgGraph + .getAlias2VertexMap() + .get("A") + .addAll( + Lists.newArrayList( + new Vertex<>(IVertexId.from(13, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(14, DEFAULT_VERTEX_TYPE), null))); + + kgGraph + .getAlias2VertexMap() + .get("B") + .add(new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null)); + + kgGraph + .getAlias2EdgeMap() + .get("E1") + .add( + new Edge( + IVertexId.from(13, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T")); + kgGraph + .getAlias2EdgeMap() + .get("E1") + .add( + new Edge( + IVertexId.from(14, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T")); + + kgGraph + .getAlias2EdgeMap() + .get("E2") + .add( + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T")); + + Set splitVertexAliasSet1 = Sets.newHashSet("A", "B"); + List> rst1 = + kgGraph.split( + splitVertexAliasSet1, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet1, schema), + null, + null); + Assert.assertEquals(4, rst1.size()); + rst1.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + + Set splitVertexAliasSet2 = Sets.newHashSet("A"); + List> rst2 = + kgGraph.split( + splitVertexAliasSet2, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet2, schema), + null, + null); + Assert.assertEquals(4, rst2.size()); + rst2.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + + Set splitVertexAliasSet3 = Sets.newHashSet("B"); + List> rst3 = + kgGraph.split( + splitVertexAliasSet3, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet3, schema), + null, + null); + Assert.assertEquals(2, rst3.size()); + rst3.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(2, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + + kgGraph + .getAlias2VertexMap() + .get("D") + .add(new Vertex<>(IVertexId.from(42, DEFAULT_VERTEX_TYPE), null)); + + kgGraph + .getAlias2EdgeMap() + .get("E3") + .add( + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(42, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T")); + + Set splitVertexAliasSet4 = Sets.newHashSet("D"); + List> rst4 = + kgGraph.split( + splitVertexAliasSet4, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet4, schema), + null, + null); + Assert.assertEquals(2, rst4.size()); + rst4.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(4, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + } + + @Test + public void testSampleSplit3() { + Pattern schema = pattern2(); + KgGraphImpl kgGraph = kgGraph2(); + + Set splitVertexAliasSet1 = Sets.newHashSet("C", "D"); + List> rst1 = + kgGraph.split( + splitVertexAliasSet1, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet1, schema), + null, + null); + Assert.assertEquals(1, rst1.size()); + Assert.assertEquals(rst1.get(0), kgGraph); + + kgGraph + .getAlias2VertexMap() + .get("B") + .add(new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null)); + + kgGraph + .getAlias2EdgeMap() + .get("E1") + .add( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T")); + kgGraph + .getAlias2EdgeMap() + .get("E2") + .add( + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T")); + + kgGraph + .getAlias2VertexMap() + .get("D") + .add(new Vertex<>(IVertexId.from(42, DEFAULT_VERTEX_TYPE), null)); + + kgGraph + .getAlias2EdgeMap() + .get("E3") + .add( + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(42, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T")); + kgGraph + .getAlias2EdgeMap() + .get("E4") + .add( + new Edge( + IVertexId.from(42, DEFAULT_VERTEX_TYPE), + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E4T")); + + Set splitVertexAliasSet2 = Sets.newHashSet("B", "D"); + List> rst2 = + kgGraph.split( + splitVertexAliasSet2, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet2, schema), + null, + null); + Assert.assertEquals(4, rst2.size()); + rst2.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + }); + + Set splitVertexAliasSet3 = Sets.newHashSet("B"); + List> rst3 = + kgGraph.split( + splitVertexAliasSet3, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet3, schema), + null, + null); + Assert.assertEquals(2, rst3.size()); + rst3.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getVertex("D").size()); + }); + } + + @Test + public void testSampleSplit4() { + Pattern schema = pattern3(); + KgGraphImpl kgGraph = kgGraph3(); + + Set splitVertexAliasSet1 = Sets.newHashSet("B"); + List> rst1 = + kgGraph.split( + splitVertexAliasSet1, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet1, schema), + null, + null); + Assert.assertEquals(1, rst1.size()); + rst1.stream() + .forEach( + iVertexIdKgGraph -> { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("A").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("C").size()); + Assert.assertEquals( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + iVertexIdKgGraph.getVertex("B").get(0).getId()); + }); + } + + public static Pattern pattern1() { + PatternElement A = + new PatternElement("A", JavaConversions.asScalaSet(Sets.newHashSet("A")).toSet(), null); + PatternElement B = + new PatternElement("B", JavaConversions.asScalaSet(Sets.newHashSet("B")).toSet(), null); + PatternElement C = + new PatternElement("C", JavaConversions.asScalaSet(Sets.newHashSet("C")).toSet(), null); + PatternElement D = + new PatternElement("D", JavaConversions.asScalaSet(Sets.newHashSet("D")).toSet(), null); + + Connection A_B = + new PatternConnection( + "E1", A.alias(), null, B.alias(), Direction.OUT, null, -1, true, false); + Connection B_C = + new PatternConnection( + "E2", B.alias(), null, C.alias(), Direction.OUT, null, -1, true, false); + Connection C_D = + new PatternConnection( + "E3", C.alias(), null, D.alias(), Direction.OUT, null, -1, true, false); + + Set C_Connect = + new HashSet() { + { + add(C_D); + add(B_C); + } + }; + Set B_Connect = + new HashSet() { + { + add(B_C); + add(A_B); + } + }; + Set A_Connect = + new HashSet() { + { + add(A_B); + } + }; + Set D_Connect = + new HashSet() { + { + add(C_D); + } + }; + + Map> topology = new HashMap<>(); + topology.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + topology.put(B.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + topology.put(A.alias(), Convert2ScalaUtil.toScalaImmutableSet(A_Connect)); + topology.put(D.alias(), Convert2ScalaUtil.toScalaImmutableSet(D_Connect)); + + Pattern schema = + new PartialGraphPattern( + C.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C, "D", D)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + return schema; + } + + public static Pattern pattern2() { + PatternElement A = + new PatternElement("A", JavaConversions.asScalaSet(Sets.newHashSet("A")).toSet(), null); + PatternElement B = + new PatternElement("B", JavaConversions.asScalaSet(Sets.newHashSet("B")).toSet(), null); + PatternElement C = + new PatternElement("C", JavaConversions.asScalaSet(Sets.newHashSet("C")).toSet(), null); + PatternElement D = + new PatternElement("D", JavaConversions.asScalaSet(Sets.newHashSet("D")).toSet(), null); + + PatternConnection A_B = + new PatternConnection( + "E1", A.alias(), null, B.alias(), Direction.OUT, null, -1, true, false); + PatternConnection B_C = + new PatternConnection( + "E2", B.alias(), null, C.alias(), Direction.OUT, null, -1, true, false); + PatternConnection C_D = + new PatternConnection( + "E3", C.alias(), null, D.alias(), Direction.OUT, null, -1, true, false); + PatternConnection D_A = + new PatternConnection( + "E4", D.alias(), null, A.alias(), Direction.OUT, null, -1, true, false); + + Set C_Connect = + new HashSet() { + { + add(C_D); + add(B_C); + } + }; + Set B_Connect = + new HashSet() { + { + add(B_C); + add(A_B); + } + }; + Set A_Connect = + new HashSet() { + { + add(A_B); + add(D_A); + } + }; + Set D_Connect = + new HashSet() { + { + add(C_D); + add(D_A); + } + }; + + Map> topology = new HashMap<>(); + topology.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + topology.put(B.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + topology.put(A.alias(), Convert2ScalaUtil.toScalaImmutableSet(A_Connect)); + topology.put(D.alias(), Convert2ScalaUtil.toScalaImmutableSet(D_Connect)); + + Pattern schema = + new PartialGraphPattern( + C.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C, "D", D)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + return schema; + } + + public static Pattern pattern3() { + PatternElement A = + new PatternElement("A", JavaConversions.asScalaSet(Sets.newHashSet("A")).toSet(), null); + PatternElement B = + new PatternElement("B", JavaConversions.asScalaSet(Sets.newHashSet("B")).toSet(), null); + PatternElement C = + new PatternElement("C", JavaConversions.asScalaSet(Sets.newHashSet("C")).toSet(), null); + + PatternConnection A_B = + new PatternConnection( + "E1", A.alias(), null, B.alias(), Direction.OUT, null, -1, true, false); + PatternConnection A_C = + new PatternConnection( + "E2", A.alias(), null, C.alias(), Direction.OUT, null, -1, true, false); + PatternConnection B_C = + new PatternConnection( + "E3", B.alias(), null, C.alias(), Direction.OUT, null, -1, true, false); + + Set C_Connect = + new HashSet() { + { + add(B_C); + add(A_C); + } + }; + Set B_Connect = + new HashSet() { + { + add(B_C); + add(A_B); + } + }; + Set A_Connect = + new HashSet() { + { + add(A_B); + add(A_C); + } + }; + Map> topology = new HashMap<>(); + topology.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + topology.put(B.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + topology.put(A.alias(), Convert2ScalaUtil.toScalaImmutableSet(A_Connect)); + + Pattern schema = + new PartialGraphPattern( + A.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A, "B", B, "C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + return schema; + } + + public static KgGraphImpl kgGraph1() { + KgGraphImpl kgGraph = new KgGraphImpl(); + kgGraph + .getAlias2VertexMap() + .put( + "A", + Sets.newHashSet( + new Vertex<>(IVertexId.from(11, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(12, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("B", Sets.newHashSet(new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("C", Sets.newHashSet(new Vertex<>(IVertexId.from(31, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("D", Sets.newHashSet(new Vertex<>(IVertexId.from(41, DEFAULT_VERTEX_TYPE), null))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E1", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"), + new Edge( + IVertexId.from(12, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E2", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E3", + Sets.newHashSet( + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(41, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"))); + return kgGraph; + } + + public static KgGraphImpl kgGraph2() { + KgGraphImpl kgGraph = new KgGraphImpl(); + kgGraph + .getAlias2VertexMap() + .put("A", Sets.newHashSet(new Vertex<>(IVertexId.from(11, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("B", Sets.newHashSet(new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("C", Sets.newHashSet(new Vertex<>(IVertexId.from(31, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("D", Sets.newHashSet(new Vertex<>(IVertexId.from(41, DEFAULT_VERTEX_TYPE), null))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E1", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E2", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E3", + Sets.newHashSet( + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(41, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"))); + kgGraph + .getAlias2EdgeMap() + .put( + "E4", + Sets.newHashSet( + new Edge( + IVertexId.from(41, DEFAULT_VERTEX_TYPE), + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E4T"))); + + return kgGraph; + } + + public static KgGraphImpl kgGraph3() { + KgGraphImpl kgGraph = new KgGraphImpl(); + kgGraph + .getAlias2VertexMap() + .put("A", Sets.newHashSet(new Vertex<>(IVertexId.from(11, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put( + "B", + Sets.newHashSet( + new Vertex<>(IVertexId.from(21, DEFAULT_VERTEX_TYPE), null), + new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null))); + kgGraph + .getAlias2VertexMap() + .put("C", Sets.newHashSet(new Vertex<>(IVertexId.from(31, DEFAULT_VERTEX_TYPE), null))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E1", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"), + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T"))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E2", + Sets.newHashSet( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T"))); + + kgGraph + .getAlias2EdgeMap() + .put( + "E3", + Sets.newHashSet( + new Edge( + IVertexId.from(21, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T"))); + return kgGraph; + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/KgGraphTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/KgGraphTest.java new file mode 100644 index 00000000..c4ea3207 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/KgGraphTest.java @@ -0,0 +1,488 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.graphstate.LoadSourceData; +import com.antgroup.openspg.reasoner.graphstate.MockSourceData3; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.*; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import scala.collection.JavaConversions; + +public class KgGraphTest { + + private KgGraphImpl kgGraph = new KgGraphImpl(); + + private LoadSourceData sourceData = new MockSourceData3(); + + private Pattern schema; + + private static final String DEFAULT_VERTEX_TYPE = "t"; + + @Before + public void init() { + kgGraph.setAlias2VertexMap(sourceData.loadAlias2Vertex()); + kgGraph.setAlias2EdgeMap(sourceData.loadAlias2Edge()); + schema = sourceData.loadPattern(); + } + + @Test + public void testSplit() { + // split on A + Set splitVertexAliasSet1 = Sets.newHashSet("A"); + List> splitRst = + kgGraph.split( + splitVertexAliasSet1, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet1, schema), + null, + null); + Assert.assertTrue(splitRst.size() == kgGraph.getAlias2VertexMap().get("A").size()); + splitRst.stream() + .map(kgGraph -> ((KgGraphImpl) kgGraph)) + .forEach(kgGraph -> Assert.assertTrue(kgGraph.getAlias2VertexMap().get("A").size() == 1)); + splitRst.stream().forEach(kgGraph -> kgGraph.show()); + + // split on B + Pattern schemaB = + new PartialGraphPattern( + "B", + JavaConversions.mapAsScalaMap( + ImmutableMap.of( + "B", + schema.getNode("B"), + "A", + schema.getNode("A"), + "C", + schema.getNode("C"))) + .toMap(scala.Predef$.MODULE$.conforms()), + schema.topology()); + Set splitVertexAliasSet2 = Sets.newHashSet("B"); + splitRst = + kgGraph.split( + splitVertexAliasSet2, + schemaB, + new KgGraphSplitStaticParameters(splitVertexAliasSet2, schemaB), + null, + null); + Assert.assertTrue(splitRst.size() == kgGraph.getAlias2VertexMap().get("B").size()); + splitRst.stream() + .map(kgGraph -> ((KgGraphImpl) kgGraph)) + .forEach(kgGraph -> Assert.assertTrue(kgGraph.getAlias2VertexMap().get("B").size() == 1)); + splitRst.stream().forEach(kgGraph -> kgGraph.show()); + + // split on C + Pattern schemaC = + new PartialGraphPattern( + "C", + JavaConversions.mapAsScalaMap( + ImmutableMap.of( + "A", + schema.getNode("A"), + "B", + schema.getNode("B"), + "C", + schema.getNode("C"))) + .toMap(scala.Predef$.MODULE$.conforms()), + schema.topology()); + Set splitVertexAliasSet3 = Sets.newHashSet("C"); + splitRst = + kgGraph.split( + splitVertexAliasSet3, + schemaC, + new KgGraphSplitStaticParameters(splitVertexAliasSet3, schemaC), + null, + null); + Assert.assertTrue(splitRst.size() == kgGraph.getAlias2VertexMap().get("C").size()); + splitRst.stream() + .map(kgGraph -> ((KgGraphImpl) kgGraph)) + .forEach(kgGraph -> Assert.assertTrue(kgGraph.getAlias2VertexMap().get("C").size() == 1)); + splitRst.stream().forEach(kgGraph -> kgGraph.show()); + } + + @Test + public void testGetPath() { + Iterator> it = + kgGraph.getPath(new KgGraphSplitStaticParameters(null, schema), null); + int count = 0; + while (it.hasNext()) { + KgGraphImpl graph = ((KgGraphImpl) it.next()); + if (null != graph) { + graph.show(); + count++; + // one vertex alias has only one vertex + for (String pe : JavaConversions.setAsJavaSet(schema.topology().keySet())) { + Assert.assertTrue(graph.getAlias2VertexMap().get(pe).size() == 1); + } + + for (String edgeAlias : graph.getAlias2EdgeMap().keySet()) { + Assert.assertTrue(graph.getAlias2EdgeMap().get(edgeAlias).size() == 1); + } + } + } + Assert.assertTrue(count == 2); + + // schema is null + try { + it = kgGraph.getPath(null, null); + } catch (NullPointerException e) { + Assert.assertTrue(true); + } + } + + @Test + public void testGetPathSingleVertex() { + KgGraphImpl kgGraph1 = new KgGraphImpl(); + Map>> alias2VertexMap = new HashMap<>(); + IVertex v1 = new Vertex(IVertexId.from(1L, DEFAULT_VERTEX_TYPE)); + IVertex v2 = new Vertex(IVertexId.from(2L, DEFAULT_VERTEX_TYPE)); + IVertex v3 = new Vertex(IVertexId.from(3L, DEFAULT_VERTEX_TYPE)); + alias2VertexMap.put("A", Sets.newHashSet(v1, v2, v3)); + kgGraph1.setAlias2VertexMap(alias2VertexMap); + + PatternElement A = new PatternElement("A", null, null); + Map> topology = new HashMap<>(); + Pattern schema1 = + new PartialGraphPattern( + A.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + Iterator> it = + kgGraph1.getPath(new KgGraphSplitStaticParameters(null, schema1), null); + int count = 0; + while (it.hasNext()) { + KgGraph tmp = it.next(); + Assert.assertTrue(tmp.getVertex("A").size() == 1); + count++; + } + Assert.assertTrue(count == 3); + } + + @Test + public void testPipeline() { + // Schema of total data is B -> C <- A + + PatternElement C = new PatternElement("C", null, null); + PatternElement B = new PatternElement("B", null, null); + PatternElement A = new PatternElement("A", null, null); + PatternConnection B_C = + new PatternConnection( + "BC", B.alias(), null, C.alias(), Direction.OUT, null, -1, true, false); + PatternConnection A_C = + new PatternConnection( + "CA", C.alias(), null, A.alias(), Direction.IN, null, -1, true, false); + Set C_Connect = + new HashSet() { + { + add(A_C); + } + }; + Set B_Connect = + new HashSet() { + { + add(B_C); + } + }; + Map> topology = new HashMap<>(); + topology.put(B.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + topology.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(B_Connect)); + // schema1 is B -> C + Pattern schema1 = + new PartialGraphPattern( + B.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("C", C, "B", B, "A", A)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + + // patternScan on B + IVertex b1 = + new Vertex<>(IVertexId.from("b1", DEFAULT_VERTEX_TYPE), new VertexVersionProperty()); + Edge bc1 = + new Edge( + IVertexId.from("b1", DEFAULT_VERTEX_TYPE), + IVertexId.from("c1", DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "BC"); + Edge bc2 = + new Edge( + IVertexId.from("b1", DEFAULT_VERTEX_TYPE), + IVertexId.from("c2", DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "BC"); + Map>> alias2EdgeMap = new HashMap<>(); + alias2EdgeMap.put( + "BC", + new ArrayList() { + { + add(bc1); + add(bc2); + } + }); + + KgGraphImpl kgGraph1 = new KgGraphImpl(); + kgGraph1.init(b1, alias2EdgeMap, schema1); + Assert.assertTrue(kgGraph1.getAlias2VertexMap().get("B").size() == 1); + Set> virtualCSet = kgGraph1.getAlias2VertexMap().get("C"); + for (IVertex cVertex : virtualCSet) { + // vertex C is virtual + Assert.assertTrue(cVertex.getValue() == null); + } + Assert.assertTrue(kgGraph1.getAlias2EdgeMap().get("BC").size() == 2); + + // split on C + schema1 = + new PartialGraphPattern( + C.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("B", B, "C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology)); + Set splitVertexAliasSet1 = Sets.newHashSet("C"); + List> splitRst = + kgGraph1.split( + splitVertexAliasSet1, + schema1, + new KgGraphSplitStaticParameters(splitVertexAliasSet1, schema1), + null, + null); + // splitRst.stream().forEach(g -> g.show()); + Assert.assertTrue(splitRst.size() == 2); + KgGraphImpl c1Msg = ((KgGraphImpl) splitRst.get(0)); + KgGraphImpl c2Msg = ((KgGraphImpl) splitRst.get(1)); + + // Omit shuffle to C and merge on C + + // schema2 is C <- A + Map> topology2 = new HashMap<>(); + topology2.put(C.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + topology2.put(A.alias(), Convert2ScalaUtil.toScalaImmutableSet(C_Connect)); + Pattern schema2 = + new PartialGraphPattern( + C.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("C", C, "A", A)) + .toMap(scala.Predef$.MODULE$.conforms()), + Convert2ScalaUtil.toScalaImmutableMap(topology2)); + + // pattern scan C + // worker1 + IVertex c1 = + new Vertex<>(IVertexId.from("c1", DEFAULT_VERTEX_TYPE), new VertexVersionProperty()); + Edge ca1 = + new Edge( + IVertexId.from("c1", DEFAULT_VERTEX_TYPE), + IVertexId.from("a1", DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.IN, + "CA"); + Map>> alias2EdgeMap2 = new HashMap<>(); + alias2EdgeMap2.put( + "CA", + new ArrayList() { + { + add(ca1); + } + }); + KgGraph kgGraph2 = new KgGraphImpl(); + kgGraph2.init(c1, alias2EdgeMap2, schema2); + System.out.println("c1 <- a1:"); + kgGraph2.show(); + + // pattern scan C + // worker2 + IVertex c2 = + new Vertex<>(IVertexId.from("c2", DEFAULT_VERTEX_TYPE), new VertexVersionProperty()); + Edge ca2 = + new Edge( + IVertexId.from("c2", DEFAULT_VERTEX_TYPE), + IVertexId.from("a1", DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.IN, + "CA"); + Map>> alias2EdgeMap3 = new HashMap<>(); + alias2EdgeMap3.put( + "CA", + new ArrayList() { + { + add(ca2); + } + }); + KgGraph kgGraph3 = new KgGraphImpl(); + kgGraph3.init(c2, alias2EdgeMap3, schema2); + System.out.println("c2 <- a1:"); + kgGraph3.show(); + + // expand on C + KgGraph c1MsgExpandKgGraph = kgGraph2; + KgGraph c2MsgExpandKgGraph = kgGraph3; + if (kgGraph2.getVertex("C").get(0).getId().getInternalId() + != c1Msg.getVertex("C").get(0).getId().getInternalId()) { + c1MsgExpandKgGraph = kgGraph3; + c2MsgExpandKgGraph = kgGraph2; + } + c1Msg.expand( + c1MsgExpandKgGraph, + new PartialGraphPattern( + "C", + JavaConversions.mapAsScalaMap(ImmutableMap.of("C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + null)); + c1Msg.show(); + Assert.assertTrue(c1Msg.getAlias2EdgeMap().containsKey("CA")); + Assert.assertTrue(c1Msg.getVertex("C").size() == 1); + // c1 is already a real vertex, not a virtual vertex + Assert.assertTrue(c1Msg.getVertex("C").get(0).getValue() != null); + // a1 is a virtual vertex + Assert.assertTrue(c1Msg.getVertex("A").get(0).getValue() == null); + + c2Msg.expand( + c2MsgExpandKgGraph, + new PartialGraphPattern( + "C", + JavaConversions.mapAsScalaMap(ImmutableMap.of("C", C)) + .toMap(scala.Predef$.MODULE$.conforms()), + null)); + c2Msg.show(); + + // merge on A, there are two messages send to A + KgGraph kgGraph4 = new KgGraphImpl(); + List> msgs = new ArrayList<>(); + msgs.add(c1Msg); + msgs.add(c2Msg); + kgGraph4.merge(msgs, new PartialGraphPattern(null, null, null)); + kgGraph4.show(); + Assert.assertTrue(kgGraph4.getVertex("C").size() == 2); + Assert.assertTrue(kgGraph4.getVertex("B").size() == 1); + Assert.assertTrue(kgGraph4.getEdge("CA").size() == 2); + Assert.assertTrue(kgGraph4.getEdge("BC").size() == 2); + // A is an virtual vertex + Assert.assertTrue(kgGraph4.getVertex("A").get(0).getValue() == null); + + // schema3 has only one vertex and no edges + Pattern schema3 = + new PartialGraphPattern( + A.alias(), + JavaConversions.mapAsScalaMap(ImmutableMap.of("A", A)) + .toMap(scala.Predef$.MODULE$.conforms()), + new scala.collection.immutable.HashMap<>()); + IVertex a1 = + new Vertex<>(IVertexId.from("a1", DEFAULT_VERTEX_TYPE), new VertexVersionProperty()); + // pattern on A + KgGraph kgGraph5 = new KgGraphImpl(); + kgGraph5.init(a1, new HashMap<>(), schema3); + + // expand on A + kgGraph4.expand(kgGraph5, schema3); + kgGraph4.show(); + // A is a real vertex + Assert.assertTrue(kgGraph5.getVertex("A").get(0).getValue() != null); + } + + @Test + public void testSetProperty() { + Map propertyMap = new HashMap<>(); + propertyMap.put("newProp", 1); + // set vertex property, specific version + kgGraph.setVertexProperty("A", propertyMap, 1L); + List> aVertexSet = kgGraph.getVertex("A"); + for (IVertex vertex : aVertexSet) { + Object val = ((IVersionProperty) vertex.getValue()).get("newProp", 1L); + Assert.assertTrue(val.equals(1)); + } + + // set vertex property, not specific version + kgGraph.setVertexProperty("A", propertyMap, null); + aVertexSet = kgGraph.getVertex("A"); + for (IVertex vertex : aVertexSet) { + Object val = ((IVersionProperty) vertex.getValue()).get("newProp"); + Assert.assertTrue(val.equals(1)); + } + + // set edge property + kgGraph.setEdgeProperty("E1", propertyMap); + List> e1EdgeSet = kgGraph.getEdge("E1"); + for (IEdge edge : e1EdgeSet) { + Assert.assertTrue(edge.getValue().get("newProp").equals(1)); + } + + try { + // vertex alias not in kgGraph + kgGraph.setVertexProperty("NOT_EXIST", propertyMap, 1L); + + // edge alias not in kgGraph + kgGraph.setEdgeProperty("NOT_EXIST", propertyMap); + Assert.assertTrue(false); + } catch (Exception e) { + Assert.assertTrue(true); + } + + // delete vertex property + propertyMap.put("newProp", null); + kgGraph.setVertexProperty("A", propertyMap, null); + aVertexSet = kgGraph.getVertex("A"); + for (IVertex vertex : aVertexSet) { + Object val = ((IVersionProperty) vertex.getValue()).get("newProp"); + Assert.assertTrue(val == null); + } + + // delete vertex + kgGraph.setVertexProperty("A", null, null); + aVertexSet = kgGraph.getVertex("A"); + for (IVertex vertex : aVertexSet) { + Assert.assertTrue(vertex.getValue().getKeySet().isEmpty()); + } + + // delete edge property + propertyMap.put("newProp", null); + kgGraph.setEdgeProperty("E1", propertyMap); + e1EdgeSet = kgGraph.getEdge("E1"); + for (IEdge edge : e1EdgeSet) { + Assert.assertTrue(edge.getValue().get("newProp") == null); + } + + // delete edge + kgGraph.setEdgeProperty("E1", null); + e1EdgeSet = kgGraph.getEdge("E1"); + for (IEdge edge : e1EdgeSet) { + Assert.assertTrue(edge.getValue().getKeySet().isEmpty()); + } + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/SplitPathLimitTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/SplitPathLimitTest.java new file mode 100644 index 00000000..29b8775a --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/SplitPathLimitTest.java @@ -0,0 +1,117 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.google.common.collect.Sets; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class SplitPathLimitTest { + + @Before + public void init() {} + + @Test + public void testSplitToWorkerIndexLimit() { + Pattern schema = KgGraphSimpleSplitTest.pattern1(); + KgGraphImpl kgGraph = KgGraphSimpleSplitTest.kgGraph1(); + + // without filter, do simple copy split + // max path limit not effective + Map> splitMap = + kgGraph.splitToWorkerIndex( + "A", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (11L == vertexId.getInternalId()) { + return 0; + } + return 1; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("A"), schema), + null, + 1L); + Assert.assertEquals(2, splitMap.size()); + + // with filter, expand all paths + // max path limit effective, will return 1 message + splitMap = + kgGraph.splitToWorkerIndex( + "A", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (11L == vertexId.getInternalId()) { + return 0; + } + return 1; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("A"), schema), + new Predicate>() { + @Override + public boolean test(KgGraph iVertexIdKgGraph) { + return true; + } + }, + 1L); + Assert.assertEquals(1, splitMap.size()); + } + + @Test + public void testSplitLimit() { + Pattern schema = KgGraphSimpleSplitTest.pattern1(); + KgGraphImpl kgGraph = KgGraphSimpleSplitTest.kgGraph1(); + + // without filter, do simple copy split + // max path limit also effective, will return 1 message + Set splitVertexAliasSet = Sets.newHashSet("A"); + List> splitResult = + kgGraph.split( + splitVertexAliasSet, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet, schema), + null, + 1L); + Assert.assertEquals(1, splitResult.size()); + + // with filter, max path limit effective, will return 1 message + splitResult = + kgGraph.split( + splitVertexAliasSet, + schema, + new KgGraphSplitStaticParameters(splitVertexAliasSet, schema), + new Predicate>() { + @Override + public boolean test(KgGraph iVertexIdKgGraph) { + return true; + } + }, + 1L); + Assert.assertEquals(1, splitResult.size()); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/SplitToWorkerIndexTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/SplitToWorkerIndexTest.java new file mode 100644 index 00000000..9a295a1a --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/kggraph/SplitToWorkerIndexTest.java @@ -0,0 +1,448 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.kggraph; + +import static com.antgroup.openspg.reasoner.kggraph.KgGraphSimpleSplitTest.DEFAULT_VERTEX_TYPE; + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphImpl; +import com.antgroup.openspg.reasoner.kggraph.impl.KgGraphSplitStaticParameters; +import com.antgroup.openspg.reasoner.lube.common.pattern.Pattern; +import com.google.common.collect.Sets; +import java.util.Iterator; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Predicate; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class SplitToWorkerIndexTest { + + @Before + public void init() {} + + @Test + public void testSimpleSplit() { + Pattern schema = KgGraphSimpleSplitTest.pattern1(); + KgGraph kgGraph = KgGraphSimpleSplitTest.kgGraph1(); + + Map> splitMap = + kgGraph.splitToWorkerIndex( + "A", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (11L == vertexId.getInternalId()) { + return 0; + } else { + return 1; + } + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("A"), schema), + null, + null); + + Assert.assertEquals(2, splitMap.size()); + Assert.assertEquals(11L, splitMap.get(0).getVertex("A").get(0).getId().getInternalId()); + Assert.assertEquals(12L, splitMap.get(1).getVertex("A").get(0).getId().getInternalId()); + } + + @Test + public void testSimpleSplit2() { + Pattern schema = KgGraphSimpleSplitTest.pattern1(); + KgGraph kgGraph = KgGraphSimpleSplitTest.kgGraph1(); + + Map> splitMap = + kgGraph.splitToWorkerIndex( + "A", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + return 0; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("A"), schema), + null, + null); + + Assert.assertEquals(1, splitMap.size()); + Assert.assertEquals(2, splitMap.get(0).getVertex("A").size()); + Assert.assertEquals(2, splitMap.get(0).getEdge("E1").size()); + + int countPath = 0; + Iterator> pathIt = + splitMap.get(0).getPath(new KgGraphSplitStaticParameters(null, schema), null); + while (pathIt.hasNext()) { + KgGraph path = pathIt.next(); + if (null == path) { + continue; + } + countPath++; + } + Assert.assertEquals(2, countPath); + } + + @Test + public void testSplit3() { + Pattern schema = KgGraphSimpleSplitTest.pattern2(); + KgGraph kgGraph = KgGraphSimpleSplitTest.kgGraph2(); + + Map> splitMap = + kgGraph.splitToWorkerIndex( + "C", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + return 0; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("C"), schema), + null, + null); + + Assert.assertEquals(1, splitMap.size()); + Assert.assertEquals(1, splitMap.get(0).getVertex("A").size()); + Assert.assertEquals(1, splitMap.get(0).getEdge("E1").size()); + + int countPath = 0; + Iterator> pathIt = + splitMap.get(0).getPath(new KgGraphSplitStaticParameters(null, schema), null); + while (pathIt.hasNext()) { + KgGraph path = pathIt.next(); + if (null == path) { + continue; + } + countPath++; + } + Assert.assertEquals(1, countPath); + } + + @Test + public void testSimpleSplit4() { + Pattern schema = KgGraphSimpleSplitTest.pattern2(); + KgGraphImpl kgGraph = KgGraphSimpleSplitTest.kgGraph2(); + + kgGraph + .getAlias2VertexMap() + .get("B") + .add(new Vertex<>(IVertexId.from(22, DEFAULT_VERTEX_TYPE), null)); + + kgGraph + .getAlias2EdgeMap() + .get("E1") + .add( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T")); + kgGraph + .getAlias2EdgeMap() + .get("E2") + .add( + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E2T")); + + kgGraph + .getAlias2VertexMap() + .get("D") + .add(new Vertex<>(IVertexId.from(42, DEFAULT_VERTEX_TYPE), null)); + + kgGraph + .getAlias2EdgeMap() + .get("E3") + .add( + new Edge( + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + IVertexId.from(42, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T")); + kgGraph + .getAlias2EdgeMap() + .get("E4") + .add( + new Edge( + IVertexId.from(42, DEFAULT_VERTEX_TYPE), + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E4T")); + + Map> splitMap = + kgGraph.splitToWorkerIndex( + "B", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (21L == vertexId.getInternalId()) { + return 0; + } else { + return 1; + } + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("C"), schema), + null, + null); + + Assert.assertEquals(2, splitMap.size()); + splitMap.forEach( + new BiConsumer>() { + @Override + public void accept(Integer integer, KgGraph iVertexIdKgGraph) { + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getVertex("D").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getEdge("E3").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getEdge("E4").size()); + + int countPath = 0; + Iterator> pathIt = + iVertexIdKgGraph.getPath(new KgGraphSplitStaticParameters(null, schema), null); + while (pathIt.hasNext()) { + KgGraph path = pathIt.next(); + if (null == path) { + continue; + } + countPath++; + } + Assert.assertEquals(2, countPath); + } + }); + + Map> splitMap2 = + kgGraph.splitToWorkerIndex( + "D", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (41L == vertexId.getInternalId()) { + return 0; + } else { + return 1; + } + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("D"), schema), + null, + null); + + Assert.assertEquals(2, splitMap2.size()); + splitMap2.forEach( + new BiConsumer>() { + @Override + public void accept(Integer integer, KgGraph iVertexIdKgGraph) { + + Assert.assertEquals(1, iVertexIdKgGraph.getVertex("D").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getVertex("B").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getEdge("E1").size()); + Assert.assertEquals(2, iVertexIdKgGraph.getEdge("E2").size()); + + int countPath = 0; + Iterator> pathIt = + iVertexIdKgGraph.getPath(new KgGraphSplitStaticParameters(null, schema), null); + while (pathIt.hasNext()) { + KgGraph path = pathIt.next(); + if (null == path) { + continue; + } + countPath++; + } + Assert.assertEquals(2, countPath); + } + }); + } + + @Test + public void testSplit5() { + Pattern schema = KgGraphSimpleSplitTest.pattern3(); + KgGraphImpl kgGraph = KgGraphSimpleSplitTest.kgGraph3(); + + Map> splitMap = + kgGraph.splitToWorkerIndex( + "A", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + return 0; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("A"), schema), + new Predicate>() { + @Override + public boolean test(KgGraph iVertexIdKgGraph) { + return true; + } + }, + null); + Assert.assertEquals(1, splitMap.size()); + } + + @Test + public void testSplit6() { + Pattern schema = KgGraphSimpleSplitTest.pattern3(); + KgGraphImpl kgGraph = KgGraphSimpleSplitTest.kgGraph3(); + + Map> splitMap = + kgGraph.splitToWorkerIndex( + "B", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (21L == vertexId.getInternalId()) { + return 0; + } + return 1; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("B"), schema), + new Predicate>() { + @Override + public boolean test(KgGraph iVertexIdKgGraph) { + return true; + } + }, + null); + Assert.assertEquals(1, splitMap.size()); + + kgGraph + .getAlias2EdgeMap() + .get("E3") + .add( + new Edge( + IVertexId.from(22, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T")); + + Map> splitMap2 = + kgGraph.splitToWorkerIndex( + "B", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (21L == vertexId.getInternalId()) { + return 0; + } + return 1; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("B"), schema), + new Predicate>() { + @Override + public boolean test(KgGraph iVertexIdKgGraph) { + return true; + } + }, + null); + Assert.assertEquals(2, splitMap2.size()); + + kgGraph + .getAlias2VertexMap() + .get("B") + .add(new Vertex<>(IVertexId.from(23, DEFAULT_VERTEX_TYPE), null)); + kgGraph + .getAlias2EdgeMap() + .get("E3") + .add( + new Edge( + IVertexId.from(23, DEFAULT_VERTEX_TYPE), + IVertexId.from(31, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E3T")); + + kgGraph + .getAlias2EdgeMap() + .get("E1") + .add( + new Edge( + IVertexId.from(11, DEFAULT_VERTEX_TYPE), + IVertexId.from(23, DEFAULT_VERTEX_TYPE), + null, + 0L, + Direction.OUT, + "E1T")); + + Map> splitMap3 = + kgGraph.splitToWorkerIndex( + "B", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (21L == vertexId.getInternalId()) { + return 0; + } + return 1; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("B"), schema), + new Predicate>() { + @Override + public boolean test(KgGraph iVertexIdKgGraph) { + return true; + } + }, + null); + Assert.assertEquals(2, splitMap3.size()); + Assert.assertEquals(1, splitMap3.get(0).getVertex("B").size()); + Assert.assertEquals(2, splitMap3.get(1).getVertex("B").size()); + + Map> splitMap4 = + kgGraph.splitToWorkerIndex( + "B", + new IVertexId2WorkerIndex() { + @Override + public int workerIndex(IVertexId vertexId) { + if (21L == vertexId.getInternalId()) { + return 0; + } + return 1; + } + }, + schema, + new KgGraphSplitStaticParameters(Sets.newHashSet("B"), schema), + null, + null); + Assert.assertEquals(2, splitMap4.size()); + Assert.assertEquals(1, splitMap4.get(0).getVertex("B").size()); + Assert.assertEquals(2, splitMap4.get(1).getVertex("B").size()); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/pattern/PatternMatcherTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/pattern/PatternMatcherTest.java new file mode 100644 index 00000000..79432063 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/pattern/PatternMatcherTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.pattern; + +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.kggraph.KgGraph; +import org.junit.Assert; +import org.junit.Test; + +public class PatternMatcherTest { + @Test + public void testTimeOut() throws InterruptedException { + PatternMatcher patternMatcher = new PatternMatcher("test", new MemGraphState()); + Thread.sleep(100); + KgGraph res = + patternMatcher.patternMatch( + IVertexId.from("test", "test"), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + 10); + Assert.assertTrue(res == null); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/rdg/GraphProcessTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/rdg/GraphProcessTest.java new file mode 100644 index 00000000..beaee307 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/rdg/GraphProcessTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ +package com.antgroup.openspg.reasoner.rdg; + +import com.antgroup.openspg.reasoner.lube.common.expr.AggOpExpr; +import com.antgroup.openspg.reasoner.lube.common.expr.AggUdf$; +import com.antgroup.openspg.reasoner.lube.common.expr.Aggregator; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.common.expr.Ref$; +import com.antgroup.openspg.reasoner.lube.common.expr.VString$; +import com.antgroup.openspg.reasoner.rdg.common.groupProcess.AggOpProcessBaseGroupProcess; +import com.google.common.collect.Sets; +import java.util.Set; +import org.junit.Assert; +import org.junit.Test; +import scala.collection.JavaConversions; + +public class GraphProcessTest { + @Test + public void testGetUnDefineUdaf() { + Set args = Sets.newHashSet(VString$.MODULE$.apply("test")); + Aggregator aggregator = + new AggOpExpr( + AggUdf$.MODULE$.apply("unDefineUdf", JavaConversions.asScalaSet(args).toList()), + Ref$.MODULE$.apply("test")); + try { + AggOpProcessBaseGroupProcess groupProcess = + new AggOpProcessBaseGroupProcess("", null, aggregator); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("unsupported aggregator function, type=unDefineUdf")); + } + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/rule/RuleRunnerTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/rule/RuleRunnerTest.java new file mode 100644 index 00000000..3b81a2d4 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/rule/RuleRunnerTest.java @@ -0,0 +1,427 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.rule; + +import com.alibaba.fastjson.JSON; +import com.antgroup.openspg.reasoner.lube.common.expr.Expr; +import com.antgroup.openspg.reasoner.lube.utils.transformer.impl.Expr2QlexpressTransformer; +import com.antgroup.openspg.reasoner.parser.expr.RuleExprParser; +import com.antgroup.openspg.reasoner.udf.utils.DateUtils; +import com.google.common.collect.Lists; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import scala.collection.JavaConversions; + +public class RuleRunnerTest { + + @Before + public void init() { + DateUtils.timeZone = TimeZone.getTimeZone("Asia/Shanghai"); + } + + /** test overloading */ + @Test + public void testOverloading() { + Map context = new HashMap<>(); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = concat('1', '2') == '12'", + "R2 = concat('x', 'y', 'z', 1, 2, 3) == 'xyz123'", + "R3 = current_time_millis() / 1000 - unix_timestamp() < 10", + "R1 && R2 && R3"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testArrayResult() { + Map context = new HashMap<>(); + Object rst = + RuleRunner.getInstance() + .executeExpression(context, Lists.newArrayList("['abc', 'cdf']"), ""); + System.out.println(JSON.toJSONString(rst)); + Assert.assertTrue(rst instanceof String[]); + rst = RuleRunner.getInstance().executeExpression(context, Lists.newArrayList("'abcdef'"), ""); + System.out.println(JSON.toJSONString(rst)); + } + + @Test + public void testRLike() { + Map context = new HashMap<>(); + context.put( + "a", + new HashMap() { + { + put("phone", "1345-23456"); + } + }); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "a.phone rlike '(63-)|(60-)|(66-)|(81-)|(84-)|(852-)|(855-)|(91-)|(95-)|(62-)|(853-)|(856-)|(886-)|(1345-)'"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testStrUDF() { + Map context = new HashMap<>(); + context.put( + "InsurancePeriod", + new HashMap() { + { + put("value", "70岁"); + } + }); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "str_contains(InsurancePeriod.value,'岁') && cast_type(sub_str(InsurancePeriod.value,1,in_str(InsurancePeriod.value,'岁') - 1),'int') >= 70 || str_contains(InsurancePeriod.value,'年') && cast_type(sub_str(InsurancePeriod.value,1,in_str(InsurancePeriod.value,'年') - 1),'int') > 200"), + ""); + Assert.assertTrue(rst); + } + + /** test context */ + @Test + public void testContext() { + Map context = new HashMap<>(); + context.put("A", 1000); + context.put("B", 100); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = A + B == 1100", + "R2 = A > B", + "R3 = A >= B", + "R4 = B < A", + "R5 = B <= A", + "R6 = A != B", + "R1 && R2 && R3 && R4 && R5 && R6"), + ""); + Assert.assertTrue(rst); + } + + /** test context loss */ + @Test + public void testContextLoss() { + Map context = new HashMap<>(); + context.put("A", 1000); + boolean rst = RuleRunner.getInstance().check(context, Lists.newArrayList("A > B"), ""); + Assert.assertFalse(rst); + } + + @Test + public void testConsts() { + RuleExprParser parser = new RuleExprParser(); + + Expr expr = parser.parse("\"\""); + Expr2QlexpressTransformer transformer = new Expr2QlexpressTransformer(); + List rules = + Lists.newArrayList(JavaConversions.asJavaCollection(transformer.transform(expr))); + Map context = new HashMap<>(); + Object rst = RuleRunner.getInstance().executeExpression(context, rules, ""); + Assert.assertEquals(rst, ""); + } + + /** test context loss */ + @Test + public void testContextLoss2() { + Map context = new HashMap<>(); + Map property = new HashMap<>(); + context.put("A", property); + property.put("id", "A1"); + property.put("name", "A1Name"); + property.put("age", 18); + property.put("amt", 22.3); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = A.id == 'A1'", + "R2 = A.age == 18", + "R3 = A.notExist == null", + "R1 && R2 && R3"), + ""); + Assert.assertTrue(rst); + + rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = A.id == 'A1'", + "R2 = A.age == 18", + "R3 = A.notExist == '123'", + "R1 && R2 && R3"), + ""); + Assert.assertFalse(rst); + } + + /** test In op */ + @Test + public void testIn() { + Map context = new HashMap<>(); + Map propertyA = new HashMap<>(); + context.put("A", propertyA); + propertyA.put("id", "A1"); + Map propertyB = new HashMap<>(); + context.put("B", propertyB); + propertyB.put("idList", Lists.newArrayList("A1", "C1")); + + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = 'str' in ['str']", + "R2 = 'str' in 'str22'", + "R3 = A.id in 'A1_suffix'", + "R4 = A.id in ['A1', 'xx']", + "R5 = A.id in B.idList", + "R6 = 12 in 12", + "R7 = 12.0 in [12.0, 13.0]", + "R1 && R2 && R3 && R4 && R5 && R6 && R7"), + ""); + Assert.assertTrue(rst); + } + + /** test concat have a lot of parameters */ + @Test + public void testUdfQuery() { + boolean rst = + RuleRunner.getInstance() + .check( + new HashMap<>(), + Lists.newArrayList("'0123456789' == concat(0,1,2,3,4,5,6,7,8,9)"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testMax() { + Map context = new HashMap<>(); + context.put("list", Lists.newArrayList(1, 2, 3, 4, 5)); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList("R1 = max(list) == 5", "R2 = min(list) == 1", "R1 && R2"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testHourOfDay() { + Map context = new HashMap<>(); + Map t = new HashMap<>(); + t.put("payDate", 1691950562L); + context.put("t", t); + boolean rst = + RuleRunner.getInstance() + .check(context, Lists.newArrayList("hourOfDay(t.payDate * 1000) < 10"), ""); + Assert.assertTrue(rst); + } + + @Test + public void testProject() { + boolean rst = + RuleRunner.getInstance() + .check( + new HashMap<>(), Lists.newArrayList("R1 = 1 == 1", "R2 = 10 > 9", "R1 && R2"), ""); + Assert.assertTrue(rst); + } + + @Test + public void testRuleValue() { + boolean rst = + RuleRunner.getInstance() + .check( + new HashMap<>(), + Lists.newArrayList( + "R1 = rule_value(true, 1, 2) == 1", + "R2 = rule_value(false, 1, 2) == 2", + "R1 && R2 && rule_value(false, 1, rule_value(true, 'x', 'y')) == 'x'"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testRuleCombine2() { + Map context = new HashMap<>(); + context.put("sex", "男"); + context.put("MonthAmount", 3000); + context.put("DayliyAmount", 301); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = sex == '男'", + "R3 = DayliyAmount > 300", + "R4 = MonthAmount < 500", + "R3 && R1 && !(R4 && R1)"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testRuleArrayParameter() { + Map context = new HashMap<>(); + context.put("dateArray", new Date[] {new Date(0), new Date(1000)}); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = \"男女\" == concat([\"男\", \"女\"])", + "R2 = \"123\" == concat([1, 2, 3])", + "R3 = \"123\" == concat([1L, 2L, 3L])", + "R4 = \"\" == concat(dateArray)", + "R1 && R2 && R3 && !R4"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testRuleListParameter() { + Map context = new HashMap<>(); + context.put("dateList", Lists.newArrayList(new Date(), new Date(1000))); + context.put("charList", Lists.newArrayList('a', 'b')); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = \"\" == concat([])", + "R2 = \"abc\" == concat(['a', 'b', 'c'])", + "R3 = \"1.02.0\" == concat([1.0, 2.0])", + "R4 = \"truefalse\" == concat([true, false])", + "R5 = \"\" == concat(dateList)", + "R6 = \"ab\" == concat(charList)", + "R1 && R2 && R3 && R4 && !R5 && R6"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testStrRules() { + Map context = new HashMap<>(); + context.put("strList", Lists.newArrayList("a", "b", "c")); + boolean rst = + RuleRunner.getInstance() + .check( + context, + Lists.newArrayList( + "R1 = concat_ws(',', 1, 2) == '1,2'", + "R2 = hash(1) == 1", + "R3 = is_blank('')", + "R4 = contains_tag('1,2,3,4', '4,5,6,7', ',')", + "R5 = 7 == str_length('1,2,3,4')", + "R6 = concat_ws('|', strList) == 'a|b|c'", + "R7 = str_length(null) == 0", + "R1 && R2 && R3 && R4 && R5 && R6 && R7"), + ""); + Assert.assertTrue(rst); + } + + @Test + public void testContainsAny2() { + Map context = new HashMap<>(); + context.put("l", Lists.newArrayList("a", "b", "c")); + context.put( + "s", + new HashMap() { + { + put("entity", "影像学检查"); + } + }); + List rules = + Lists.newArrayList( + "rule_value(contains_any(s.entity,[\"包膜\"]) && contains_any(s.inspection,[\"MRI\",\"CT\",\"PETCT\",\"PETMRI\"]) && contains_any(s.index,[\"累及\",\"侵犯\",\"膨隆\",\"突破包膜\"]) || contains_any(s.entity,\"神经血管束\",\"血管神经束\",\"DVC\") && contains_any(s.inspection,[\"MRI\",\"PETMRI\"]) && contains_any(s.index,[\"累及\",\"侵犯\"]),\"TRUE\",rule_value(contains_any(s.entity,\"包膜\") && contains_any(s.inspection,[\"MRI\",\"CT\",\"PETCT\",\"PETMRI\"]) && contains_any(s.index,[\"完整\"]) || contains_any(s.entity,\"神经血管束\",\"血管神经束\",\"DVC\") && contains_any(s.inspection,[\"MRI\",\"PETMRI\"]) && contains_any(s.index,[\"完整\"]),\"FALSE\",\"\"))"); + boolean rst = RuleRunner.getInstance().check(context, rules, ""); + Assert.assertFalse(rst); + } + + @Test + public void testContainsAny() { + RuleExprParser ruleExprParser = new RuleExprParser(); + Expr e = + ruleExprParser.parse( + "contains_any(s.entity, [\"包膜\"]) " + + "and contains_any(s.inspection, [\"MRI\", \"CT\", \"PETCT\", \"PETMRI\"]) " + + "and contains_any(s.status, [\"完整\"]) " + + "and " + + "not contains_any(s.status, [\"无\", \"不\", \"未见\"])"); + + Expr2QlexpressTransformer transformer = new Expr2QlexpressTransformer(); + + List rules = + Lists.newArrayList(JavaConversions.asJavaCollection(transformer.transform(e))); + Map context = new HashMap<>(); + context.put("l", Lists.newArrayList("a", "b", "c")); + context.put( + "s", + new HashMap() { + { + put("entity", "包膜"); + put("inspection", "MRI"); + put("status", "完整"); + } + }); + + boolean rst = RuleRunner.getInstance().check(context, rules, ""); + Assert.assertTrue(rst); + } + + @Test + public void testContainsAny3() { + RuleExprParser ruleExprParser = new RuleExprParser(); + Expr e = ruleExprParser.parse("contains_any(s.entity, '包膜') and contains_any(s.entity, a)"); + + Expr2QlexpressTransformer transformer = new Expr2QlexpressTransformer(); + + List rules = + Lists.newArrayList(JavaConversions.asJavaCollection(transformer.transform(e))); + Map context = new HashMap<>(); + context.put("l", Lists.newArrayList("a", "b", "c")); + context.put( + "s", + new HashMap() { + { + put("entity", "包膜"); + put("inspection", "MRI"); + put("status", "完整"); + } + }); + + boolean rst = RuleRunner.getInstance().check(context, rules, ""); + Assert.assertFalse(rst); + } +} diff --git a/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/utils/ObjSerdeTest.java b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/utils/ObjSerdeTest.java new file mode 100644 index 00000000..32ef5174 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/java/com/antgroup/openspg/reasoner/utils/ObjSerdeTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.openspg.reasoner.utils; + +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.graphstate.generator.AbstractGraphGenerator; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.catalog.PropertyGraphSchema; +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog; +import com.antgroup.openspg.reasoner.util.Convert2ScalaUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import scala.Tuple2; +import scala.collection.immutable.Set; + +public class ObjSerdeTest { + + private String encode; + + @Before + public void init() throws Exception { + Catalog catalogSer = createCatalog(); + encode = SimpleObjSerde.ser(catalogSer); + } + + @Test + public void decodeSchema() { + // add mock data to job + AbstractGraphGenerator graphGenerator = + new AbstractGraphGenerator() { + @Override + public List> genVertexList() { + return Lists.newArrayList( + constructionVertex("张三", "CustFundKG.Account"), + constructionVertex("李四", "CustFundKG.Account")); + } + + @Override + public List> genEdgeList() { + return Lists.newArrayList(constructionEdge("张三", "accountFundContact", "李四")); + } + }; + Tuple2>, List>> graphData2 = + graphGenerator.getGraphData(); + String encodeStr = SimpleObjSerde.ser(graphData2); + + Tuple2>, List>> graphData = + (Tuple2>, List>>) + SimpleObjSerde.de(encodeStr); + Assert.assertEquals(graphData._1.size(), 2); + } + + private Catalog createCatalog() { + Map> schema = new HashMap<>(); + schema.put("User", Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet("id", "zhixin"))); + schema.put( + "creditCardPaymentBindEvent", + Convert2ScalaUtil.toScalaImmutableSet( + Sets.newHashSet("id", "cardBank", "accountQuery", "bindSelf", "cardNum"))); + schema.put( + "creditCardPaymentBindEvent_relateCreditCardPaymentBindEvent_User", + Convert2ScalaUtil.toScalaImmutableSet(Sets.newHashSet())); + Catalog catalog = new PropertyGraphCatalog(Convert2ScalaUtil.toScalaImmutableMap(schema)); + catalog.init(); + return catalog; + } + + @Test + public void serdeTest() throws Exception { + Catalog catalogSer = createCatalog(); + Catalog catalogDe = (Catalog) SimpleObjSerde.de(encode); + + PropertyGraphSchema propertyGraphSchemaSer = + catalogSer.getGraph(Catalog.defaultGraphName()).graphSchema(); + PropertyGraphSchema propertyGraphSchemaDe = + catalogDe.getGraph(Catalog.defaultGraphName()).graphSchema(); + + Assert.assertEquals( + propertyGraphSchemaSer.nodes().get("User"), propertyGraphSchemaDe.nodes().get("User")); + } +} diff --git a/dev/test/docker-compose-no-es.sh b/reasoner/runner/runner-common/src/test/resources/log4j.properties similarity index 60% rename from dev/test/docker-compose-no-es.sh rename to reasoner/runner/runner-common/src/test/resources/log4j.properties index 4f8b9d94..0c3263d5 100644 --- a/dev/test/docker-compose-no-es.sh +++ b/reasoner/runner/runner-common/src/test/resources/log4j.properties @@ -1,4 +1,3 @@ -# # Copyright 2023 Ant Group CO., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except @@ -9,6 +8,11 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. -# -docker-compose -f docker-compose-no-es.yml up -d +# logger config for test +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n \ No newline at end of file diff --git a/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptyRDG.scala b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptyRDG.scala new file mode 100644 index 00000000..abbe1675 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptyRDG.scala @@ -0,0 +1,167 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.reasoner.session + +import com.antgroup.openspg.reasoner.lube.block.{DDLOp, SortItem} +import com.antgroup.openspg.reasoner.lube.common.expr.{Aggregator, Expr} +import com.antgroup.openspg.reasoner.lube.common.pattern.{EdgePattern, LinkedPatternConnection, Pattern, PatternElement} +import com.antgroup.openspg.reasoner.lube.common.rule.Rule +import com.antgroup.openspg.reasoner.lube.logical.{RichVar, Var} +import com.antgroup.openspg.reasoner.lube.physical.planning.JoinType +import com.antgroup.openspg.reasoner.lube.physical.rdg.RDG + +class EmptyRDG extends RDG[EmptyRDG] { + override type Records = EmptyRow + + /** + * Match the giving pattern on Graph + * + * @param pattern specific pattern, see more [[Pattern]] + * @return + */ + override def patternScan(pattern: Pattern): EmptyRDG = this + + /** + * Grow a new pattern from the matched patterns, which means that the new pattern is connected + * to the matched patterns. + * + * @param target the target of growth + * @param pattern specific pattern, see more [[Pattern]] + * @return + */ + override def expandInto(target: PatternElement, pattern: Pattern): EmptyRDG = this + + /** + * Returns a tabular data containing only the given columns. + * The column order within the table is aligned with the argument. + * + * @param cols columns to select. + * @return + */ + override def select(cols: List[Var], as: List[String]): EmptyRow = new EmptyRow(cols, this) + + /** + * Returns a [[RDG]] containing only data where the given expression evaluates to + * + * @param expr filter expression + * @return + */ + override def filter(expr: Rule): EmptyRDG = this + + /** + * returns a [[RDG]] that is ordered by the given columns. + * + * @param groupKey query variables to group, return global sort if groupKey is empty + * @param sortItems a sequence of column names and their order + * (i.e.[[com.antgroup.openspg.reasoner.lube.block.Asc]] + * [[com.antgroup.openspg.reasoner.lube.block.Desc]] + * @param limit return limit number of sorted data + * @return + */ + override def orderBy(groupKey: List[Var], sortItems: List[SortItem], limit: Int): EmptyRDG = + this + + /** + * Groups the intermediate data by the given query variables. + * Additionally a set of aggregations can be apply on the grouped [[RDG]]. + * + * @param by query variables to group by. + * @param aggregations map of aggregations functions, the key of map is [[Var]] of output + * @return + */ + override def groupBy(by: List[Var], aggregations: Map[Var, Aggregator]): EmptyRDG = this + + /** + * Add fields in [[RDG]] + * + * @param fields the key of map is [[Var]] of output + * @return + */ + override def addFields(fields: Map[Var, Expr]): EmptyRDG = this + + /** + * Remove fields in [[RDG]] according to fields. + * + * @param fields + * @return + */ + override def dropFields(fields: Set[Var]): EmptyRDG = this + + /** + * Returns a [[RDG]] containing the first n rows of the current RDG. + * + * @param n number of rows to return. + * @return + */ + override def limit(n: Long): EmptyRDG = this + + /** + * Print the result, usually used for debug. + * + * @param rows number of rows to print + */ + override def show(rows: Int): Unit = {} + + /** + * compute the linked edge based on pattern + * + * @param pattern + * @return + */ + override def linkedExpand(pattern: EdgePattern[LinkedPatternConnection]): EmptyRDG = this + + /** + * cache current RDG + */ + override def cache(): EmptyRDG = this + + /** + * Do DDL operations on the basis of GraphState, such as adding vertices, + * adding attributes to vertex, adding edges, etc. The main goal is to reuse. + * + * @param ddlOps + * @return + */ + override def ddl(ddlOps: List[DDLOp]): EmptyRDG = this + + /** + * fold left var to the right var. + * eg. + * a RDG with schema "A -[E]-> B -[E1] -> B1", then fold ((E1 -> E, B1 -> B)) + * the schema with output rdg is "A -[E]-> B" + * the data of E1 merge in E, B1 merge in B + * + * @param windMapping + * @return + */ + def fold(foldMapping: List[(List[Var], RichVar)]): EmptyRDG = this + + /** + * Join current rdg with an other rdg + * + * @param other other rdg + * @param joinType see more [[JoinType]] + * @param onAlias Joining two RDGs based on onAlias + * @param lhsSchemaMapping left rdg schema mapping after join. + * @param rhsSchemaMapping right rdg schema mapping after join. + */ + override def join( + other: EmptyRDG, + joinType: JoinType, + onAlias: List[(String, String)], + lhsSchemaMapping: Map[Var, Var], + rhsSchemaMapping: Map[Var, Var]): EmptyRDG = this + + override def unfold(mapping: List[(RichVar, List[Var])]): EmptyRDG = this +} diff --git a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClient.java b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptyRow.scala similarity index 55% rename from cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClient.java rename to reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptyRow.scala index 6f62b882..71d9a77d 100644 --- a/cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClient.java +++ b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptyRow.scala @@ -11,13 +11,18 @@ * or implied. */ -package com.antgroup.openspg.cloudext.interfaces.objectstore; +package com.antgroup.reasoner.session -import com.antgroup.openspg.cloudext.interfaces.objectstore.cmd.ObjectStoreSaveCmd; -import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath; -import com.antgroup.openspg.common.util.cloudext.CloudExtClient; +import com.antgroup.openspg.reasoner.lube.logical.Var +import com.antgroup.openspg.reasoner.lube.physical.rdg.Row -public interface ObjectStoreClient extends CloudExtClient { +class EmptyRow(orderedFields: List[Var], rdg: EmptyRDG) + extends Row[EmptyRDG](orderedFields, rdg) { - ObjectStorePath save(ObjectStoreSaveCmd cmd); + /** + * Print the result, usually used for debug. + * + * @param rows number of rows to print + */ + override def show(rows: Int): Unit = {} } diff --git a/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptySession.scala b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptySession.scala new file mode 100644 index 00000000..f15d7637 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/EmptySession.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.reasoner.session + +import com.antgroup.openspg.reasoner.lube.catalog.Catalog +import com.antgroup.openspg.reasoner.lube.logical.RepeatPathVar +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface +import com.antgroup.openspg.reasoner.lube.physical.PropertyGraph +import com.antgroup.openspg.reasoner.warehouse.common.config.GraphLoaderConfig +import com.antgroup.openspg.reasoner.session.KGReasonerSession + +class EmptyPropertyGraph extends PropertyGraph[EmptyRDG] { + + /** + * Start with ids according to the types of start nodes. + * + * @param types + * @return + */ + override def createRDG(alias: String, types: Set[String]): EmptyRDG = new EmptyRDG() + + /** + * Start with specific rdg with specific alias. + * + * @param rdg + * @param alias + * @return + */ + override def createRDG(alias: String, rdg: EmptyRDG): EmptyRDG = rdg + + /** + * Start with specific rdg with specific alias which in [[RepeatPathVar]] + * + * @param repeatVar + * @param alias + * @param rdg + * @return + */ + override def createRDGFromPath( + repeatVar: RepeatPathVar, + alias: String, + rdg: EmptyRDG): EmptyRDG = rdg + +} + +class EmptySession(parser: ParserInterface, catalog: Catalog) + extends KGReasonerSession[EmptyRDG](parser, catalog) { + + /** + * Load a graph from Knowledge Graph to [[KGReasonerSession]] + * + * @param graphLoaderConfig + * @return + */ + override def loadGraph(graphLoaderConfig: GraphLoaderConfig): PropertyGraph[EmptyRDG] = + new EmptyPropertyGraph() + +} diff --git a/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/KgGraphSchemaTest.scala b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/KgGraphSchemaTest.scala new file mode 100644 index 00000000..0024f64e --- /dev/null +++ b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/KgGraphSchemaTest.scala @@ -0,0 +1,301 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.reasoner.session + +import com.antgroup.openspg.reasoner.common.graph.edge.Direction +import com.antgroup.openspg.reasoner.lube.common.pattern.{ + PartialGraphPattern, + PatternConnection, + PatternElement +} +import com.antgroup.openspg.reasoner.lube.logical.{EdgeVar, NodeVar} +import com.antgroup.openspg.reasoner.rdg.common.FoldRepeatEdgeInfo +import com.antgroup.openspg.reasoner.util.{KgGraphSchema, PathConnection} +import org.scalatest.flatspec.AnyFlatSpec + +class KgGraphSchemaTest extends AnyFlatSpec { + + "convert2KgGraphSchema()" should "alias can not change" in { + val schema1 = KgGraphSchema.convert2KgGraphSchema( + PartialGraphPattern( + "A", + Map.apply(("A" -> PatternElement("A", Set.apply("Film"), null))), + Map.apply())) + + assert(schema1.root.alias == "A") + } + + "convert2KgGraphSchema()" should "connection's source alias equals root's alias" in { + val root = new PatternElement("A", Set.apply("Film"), null) + val b = PatternElement("B", Set.apply("FilmDirector"), null) + val schema1 = KgGraphSchema.convert2KgGraphSchema( + PartialGraphPattern( + "A", + Map.apply(("A" -> root), ("B" -> b)), + Map.apply( + root.alias -> Set.apply( + new PatternConnection( + "E1", + "B", + Set.apply("directFilm"), + root.alias, + Direction.IN, + null))))) + + assert(schema1.root.alias == "A") + assert(schema1.topology.get(schema1.root.alias).get.head.source == "A") + assert(schema1.topology.get(schema1.root.alias).get.head.direction == Direction.OUT) + } + + "expandSchema()" should "add new pattern" in { + val aliasA = new PatternElement("A", Set.apply("Film"), null) + val aliasB = new PatternElement("B", Set.apply("FilmDirector"), null) + val p1 = PartialGraphPattern( + aliasA.alias, + Map.apply(("A" -> aliasA), ("B" -> aliasB)), + Map.apply( + aliasA.alias -> Set.apply( + new PatternConnection( + "E1", + aliasB.alias, + Set.apply("directFilm"), + aliasA.alias, + Direction.IN, + null)))) + + val aliasC = new PatternElement("C", Set.apply("FilmWriter"), null) + val p2 = PartialGraphPattern( + aliasB.alias, + Map.apply(("B" -> aliasB), ("C" -> aliasC)), + Map.apply( + aliasB.alias -> Set.apply( + new PatternConnection( + "E3", + aliasB.alias, + Set.apply("workmates"), + aliasC.alias, + Direction.OUT, + null)))) + + val schema1 = KgGraphSchema.convert2KgGraphSchema(p1) + val schema2 = KgGraphSchema.schemaChangeRoot(schema1, "B") + val schema3 = KgGraphSchema.expandSchema(schema2, p2) + assert(schema3.root.alias == "B") + } + + "foldPathEdgeSchema()" should "create new path connection" in { + val aliasA = PatternElement("A", Set.apply("Account"), null) + val aliasB = PatternElement("B1", Set.apply("Account"), null) + val schema = PartialGraphPattern( + aliasA.alias, + Map.apply("A" -> aliasA, "B1" -> aliasB), + Map.apply( + aliasA.alias -> Set.apply( + new PatternConnection( + "E1", + aliasA.alias, + Set.apply("trade"), + aliasB.alias, + Direction.OUT, + null)))) + + val foldSchema1 = + KgGraphSchema.foldPathEdgeSchema(schema, new FoldRepeatEdgeInfo("E1", "E", "B1", "B")) + assert(foldSchema1.root == aliasA) + assert(foldSchema1.rootAlias == "A") + assert(foldSchema1.nodes("A") == aliasA) + assert(foldSchema1.nodes("B").alias == "B") + val pc: PathConnection = foldSchema1.topology("A").head.asInstanceOf[PathConnection] + assert(pc.alias == "E") + assert(pc.source == "A") + assert(pc.target == "B") + assert(pc.vertexSchemaList.isEmpty) + assert(pc.edgeSchemaList.length == 1) + + val aliasB2 = PatternElement("B2", Set.apply("Account"), null) + val schema2 = PartialGraphPattern( + "B", + Map.apply("B" -> foldSchema1.nodes("B"), "B2" -> aliasB2), + Map.apply( + aliasA.alias -> Set.apply( + new PatternConnection( + "E2", + "B", + Set.apply("trade"), + aliasB2.alias, + Direction.OUT, + null)))) + val expendSchema = KgGraphSchema.expandSchema(foldSchema1, schema2) + + val foldSchema2 = + KgGraphSchema.foldPathEdgeSchema(expendSchema, new FoldRepeatEdgeInfo("E2", "E", "B2", "B")) + assert(foldSchema2.root == null) + assert(foldSchema2.rootAlias == "E.B1") + assert(foldSchema2.nodes("A") == aliasA) + assert(foldSchema2.nodes("B").alias == "B") + val pc2: PathConnection = foldSchema2.topology("A").head.asInstanceOf[PathConnection] + assert(pc2.alias == "E") + assert(pc2.source == "A") + assert(pc2.target == "B") + assert(pc2.vertexSchemaList.length == 1) + assert(pc2.edgeSchemaList.length == 2) + + val aliasB3 = PatternElement("B3", Set.apply("Account"), null) + val schema3 = PartialGraphPattern( + "B", + Map.apply("B" -> foldSchema2.nodes("B"), "B3" -> aliasB3), + Map.apply( + aliasA.alias -> Set.apply( + new PatternConnection( + "E3", + "B", + Set.apply("trade"), + aliasB3.alias, + Direction.OUT, + null)))) + val expendSchema2 = KgGraphSchema.expandSchema(foldSchema2, schema3) + val foldSchema3 = KgGraphSchema.foldPathEdgeSchema( + expendSchema2, + new FoldRepeatEdgeInfo("E3", "E", "B3", "B")) + assert(foldSchema3.root == null) + assert(foldSchema3.rootAlias == "E.B2") + assert(foldSchema3.nodes("A") == aliasA) + assert(foldSchema3.nodes("B").alias == "B") + val pc3: PathConnection = foldSchema3.topology("A").head.asInstanceOf[PathConnection] + assert(pc3.alias == "E") + assert(pc3.source == "A") + assert(pc3.target == "B") + assert(pc3.vertexSchemaList.length == 2) + assert(pc3.edgeSchemaList.length == 3) + + } + + "foldPathEdgeSchema()" should "handling complex paths" in { + val aliasA = PatternElement("A", Set.apply("User"), null) + val aliasB = PatternElement("B", Set.apply("Account"), null) + val aliasC = PatternElement("C1", Set.apply("Account"), null) + val schema = PartialGraphPattern( + aliasC.alias, + Map.apply("A" -> aliasA, "B" -> aliasB, "C1" -> aliasC), + Map.apply( + aliasA.alias -> Set.apply( + new PatternConnection( + "E1", + aliasA.alias, + Set.apply("has"), + aliasB.alias, + Direction.OUT, + null)), + aliasB.alias -> Set.apply( + new PatternConnection( + "E1", + aliasA.alias, + Set.apply("has"), + aliasB.alias, + Direction.OUT, + null), + new PatternConnection( + "E2", + aliasB.alias, + Set.apply("trade"), + aliasC.alias, + Direction.OUT, + null)), + aliasC.alias -> Set.apply( + new PatternConnection( + "E2", + aliasB.alias, + Set.apply("trade"), + aliasC.alias, + Direction.OUT, + null)))) + + val foldSchema1 = + KgGraphSchema.foldPathEdgeSchema(schema, new FoldRepeatEdgeInfo("E2", "E", "C1", "C")) + assert(foldSchema1.root.typeNames == aliasC.typeNames) + assert(foldSchema1.rootAlias == "C") + assert(foldSchema1.nodes("A") == aliasA) + assert(foldSchema1.nodes("B").alias == "B") + val pc: PathConnection = foldSchema1.topology("C").head.asInstanceOf[PathConnection] + assert(pc.alias == "E") + assert(pc.source == "B") + assert(pc.target == "C") + assert(pc.vertexSchemaList.isEmpty) + assert(pc.edgeSchemaList.length == 1) + + val aliasC2 = PatternElement("C2", Set.apply("Account"), null) + val schema2 = PartialGraphPattern( + "C", + Map.apply("C" -> foldSchema1.nodes("C"), "C2" -> aliasC2), + Map.apply( + "C" -> Set.apply( + new PatternConnection( + "E2", + "C", + Set.apply("trade"), + aliasC2.alias, + Direction.OUT, + null)))) + val expendSchema = KgGraphSchema.expandSchema(foldSchema1, schema2) + + val foldSchema2 = + KgGraphSchema.foldPathEdgeSchema(expendSchema, new FoldRepeatEdgeInfo("E2", "E", "C2", "C")) + assert(foldSchema2.root == null) + assert(foldSchema2.rootAlias == "E.C1") + assert(foldSchema2.nodes("A") == aliasA) + assert(foldSchema2.nodes("B") == aliasB) + val pc2: PathConnection = foldSchema2.topology("C").head.asInstanceOf[PathConnection] + assert(pc2.alias == "E") + assert(pc2.source == "B") + assert(pc2.target == "C") + assert(pc2.vertexSchemaList.length == 1) + assert(pc2.edgeSchemaList.length == 2) + + } + + "schemaAliasMapping()" should "works" in { + val aliasA = PatternElement("A", Set.apply("User"), null) + val aliasB = PatternElement("B", Set.apply("Account"), null) + val aliasC = PatternElement("C", Set.apply("Account"), null) + val schema = PartialGraphPattern( + aliasB.alias, + Map.apply("A" -> aliasA, "B" -> aliasB, "C" -> aliasC), + Map.apply( + aliasB.alias -> Set.apply( + new PatternConnection( + "E1", + aliasB.alias, + Set.apply("has"), + aliasA.alias, + Direction.IN, + null), + new PatternConnection( + "E2", + aliasB.alias, + Set.apply("trade"), + aliasC.alias, + Direction.OUT, + null)))) + + val schema2 = KgGraphSchema.convert2KgGraphSchema(schema) + + val schema3 = KgGraphSchema.schemaAliasMapping( + schema2, + Map.apply( + NodeVar("A", Set.empty) -> NodeVar("A_rename", Set.empty), + NodeVar("B", Set.empty) -> NodeVar("B_rename", Set.empty), + EdgeVar("E1", Set.empty) -> EdgeVar("E1_rename", Set.empty))) + assert(schema3.rootAlias == "B_rename") + } +} diff --git a/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/PhysicalOpUtilTests.scala b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/PhysicalOpUtilTests.scala new file mode 100644 index 00000000..291a4939 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/PhysicalOpUtilTests.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.reasoner.session + +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog +import com.antgroup.openspg.reasoner.lube.physical.util.PhysicalOperatorUtil +import com.antgroup.openspg.reasoner.parser.KgDslParser +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, equal} + +class PhysicalOpUtilTests extends AnyFunSpec { + it("test physical operator utils") { + val dsl = + """ + |Define (s:User)-[p:tradeInfo]->(u:User) { + | GraphStructure { + | (s)-[t:trade]->(u:User) + | } Rule { + | R1("交易时间在90天内"): t.trade_time < -90 + | trade_num = group(s,u).count(t) + | p.trade_num = trade_num + | } + |} + | + |GraphStructure { + | (s:User)-[t:tradeInfo]->(u:User) + |} + |Rule { + | R1: t.trade_num > 100 + |} + |Action { + | get(s.id) + |} + | + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "User" -> Set.apply("id"), + "User_trade_User" -> Set.apply("trade_num", "trade_time")) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan(dsl, Map.empty) + PhysicalOperatorUtil.getStartTypes(rst.head) should equal (Set.apply("User")) + } +} diff --git a/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/ReasonerSessionTests.scala b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/ReasonerSessionTests.scala new file mode 100644 index 00000000..07dd4763 --- /dev/null +++ b/reasoner/runner/runner-common/src/test/scala/com/antgroup/reasoner/session/ReasonerSessionTests.scala @@ -0,0 +1,727 @@ +/* + * Copyright 2023 Ant Group CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. + */ + +package com.antgroup.reasoner.session + +import scala.collection.mutable.ListBuffer + +import com.antgroup.openspg.reasoner.common.constants.Constants +import com.antgroup.openspg.reasoner.common.exception.SchemaException +import com.antgroup.openspg.reasoner.lube.catalog.impl.PropertyGraphCatalog +import com.antgroup.openspg.reasoner.lube.common.expr.{AggIfOpExpr, BAnd, BinaryOpExpr} +import com.antgroup.openspg.reasoner.lube.common.graph.IRGraph +import com.antgroup.openspg.reasoner.lube.logical.optimizer.LogicalOptimizer +import com.antgroup.openspg.reasoner.lube.logical.planning.{LogicalPlanner, LogicalPlannerContext} +import com.antgroup.openspg.reasoner.lube.physical.operators._ +import com.antgroup.openspg.reasoner.lube.physical.rdg.RDG +import com.antgroup.openspg.reasoner.parser.KgDslParser +import com.antgroup.openspg.reasoner.rule.RuleRunner +import com.antgroup.openspg.reasoner.util.LoaderUtil +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers.{contain, convertToAnyShouldWrapper, equal} + +class ReasonerSessionTests extends AnyFunSpec { + + // scalastyle:off + it("test rule reference") { + val dsl = + """ + |Define (user:TuringCore.AlipayUser)-[teCount:teCount]->(o:Long) { + | GraphStructure { + | (user) -[pwl:workLoc]-> (aa1:CKG.AdministrativeArea) + | (te:TuringCore.TravelEvent) -[ptler:traveler]-> (user) + | (te) -[ptm:travelMode]-> (tm:TuringCore.TravelMode) + | (te) -[pte:travelEndpoint]-> (aa1:CKG.AdministrativeArea) + | } + | Rule { + | R1('LivedInHangzhou'): aa1.id == '中国-浙江省-杭州市' + | R2('CommutersAtWeekdays'): dayOfWeek(te.eventTime) in [1, 2, 3, 4, 5] + | and hourOfDay(te.eventTime) in [6, 7, 8, 9, 10, 17, 18, 19, 20, 21] + | R3('BusAndSubway'): tm.id in ['bus', 'subway'] + | tmCount('NumberOfTrips') = group(user).count(te.id) + | R4('TripsOver3'): tmCount >= 3 + | R5('idNotNull'): user.id != '' + | o = tmCount + | } + |} + | + |GraphStructure { + | (user:TuringCore.AlipayUser) + |} + |Rule{ + |} + |Action{ + | get(user.teCount) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "TuringCore.AlipayUser" -> Set.apply("id"), + "CKG.AdministrativeArea" -> Set.apply("id"), + "TuringCore.TravelEvent" -> Set.apply("eventTime", "id"), + "TuringCore.TravelMode" -> Set.apply("id"), + "TuringCore.AlipayUser_workLoc_CKG.AdministrativeArea" -> Set.empty, + "TuringCore.TravelEvent_traveler_TuringCore.AlipayUser" -> Set.empty, + "TuringCore.TravelEvent_travelMode_TuringCore.TravelMode" -> Set.empty, + "TuringCore.TravelEvent_travelEndpoint_CKG.AdministrativeArea" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan(dsl, Map.empty) + rst.foreach(session.getResult(_)) + val node = catalog.getGraph(IRGraph.defaultGraphName).getNode("TuringCore.AlipayUser") + node.properties.map(_.name) should contain("teCount") + } + + it("dependency_analysis") { + val dsl = + """ + |GraphStructure { + | (s:Park)-[e:nearby(s.boundary, o.center, 10.1)]->(o:Subway) + | } + | Rule{} + | Action {get(s.boundary)} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.apply("boundary"), + "Subway" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val physicalOperatorList = session.plan(dsl, Map.empty) + val definePhysicalOp = physicalOperatorList.head + val expandIntoCount = definePhysicalOp.transform[Int] { + case (LinkedExpand(in, pattern, meta), count) => + in.isInstanceOf[PatternScan[EmptyRDG]] should equal(true) + count.head + case (ExpandInto(_, _, _, _), count) => + count.head + 1 + case (_, count) => + if (count.isEmpty) { + 0 + } else { + count.head + } + } + expandIntoCount should equal(2) + } + + it("dependency_analysis_subquery") { + val dsl = + """ + |GraphStructure { + | (s:Park)-[e:nearby(s.boundary, o.center, 10.1)]->(o:Subway) + | } + | Rule{} + | Action {get(s.boundary, o.center)} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.apply("boundary"), + "Subway" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty, + "Park_boundaryS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val physicalOperatorList = session.plan( + dsl, + Map + .apply( + Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE -> true, + Constants.START_LABEL -> "Park", + "1" -> "1") + .asInstanceOf[Map[String, Object]]) + physicalOperatorList.size should equal(1) + var subqueryPhysicalOp = physicalOperatorList.head + val forwardDirectionPhysicalOpOrder = getPhysicalPlanOrder(subqueryPhysicalOp) + + val reversePhysicalOperatorList = session.plan( + dsl, + Map + .apply( + Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE -> true, + Constants.START_LABEL -> "Subway", + Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE -> true, + "1" -> "1") + .asInstanceOf[Map[String, Object]]) + subqueryPhysicalOp = reversePhysicalOperatorList.head + val reverseDirectionPhysicalOpOrder = getPhysicalPlanOrder(subqueryPhysicalOp) + forwardDirectionPhysicalOpOrder.equals(reverseDirectionPhysicalOpOrder) should equal(true) + println(forwardDirectionPhysicalOpOrder) + forwardDirectionPhysicalOpOrder should equal( + "Start,Cache,DrivingRDG,PatternScan,LinkedExpand,ExpandInto,ExpandInto,Drop,Filter,Drop,DDL,Join,PatternScan,ExpandInto,Drop,Select") + } + + it("multiple_st_edge_test") { + val dsl: String = + """ + |GraphStructure{ + | (poi1: PE.AntPOI)-[e1:within(poi1.location, aoi1.coords)]->(aoi1:PE.AntAOI)<-[e2:within(poi2.location, aoi1.coords)]-(poi2:PE.AntPOI) + |} + |Rule { + | R1: poi1.id != poi2.id + |} + |Action { + | get(poi1.id, aoi1.id, poi2.id) + |}""".stripMargin + + val schema: Map[String, Set[String]] = Map.apply( + "PE.AntPOI" -> Set.apply("id", "location"), + "PE.AntAOI" -> Set.apply("id", "coords"), + "STD.S2CellId" -> Set.empty, + "PE.AntAOI_coordsS2CellId_STD.S2CellId" -> Set.empty, + "PE.AntPOI_locationS2CellId_STD.S2CellId" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val physicalOperatorList = session.plan( + dsl, + Map + .apply( + Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE -> true, + Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE -> true, + Constants.START_LABEL -> "PE.AntPOI", + "1" -> "1") + .asInstanceOf[Map[String, Object]]) + physicalOperatorList.size should equal(1) + var subqueryPhysicalOp = physicalOperatorList.head + val physicalOpOrder = getPhysicalPlanOrder(subqueryPhysicalOp) + println(physicalOpOrder) + physicalOpOrder should equal( + "Start,Cache,DrivingRDG,PatternScan,LinkedExpand,ExpandInto,ExpandInto,Drop,Filter,Drop,DDL,Join,PatternScan,Cache,DrivingRDG,PatternScan,LinkedExpand,ExpandInto,ExpandInto,Drop,Filter,Drop,DDL,Join,ExpandInto,ExpandInto,Drop,Filter,Drop,Select") + } + + private def getPhysicalPlanOrder[T <: RDG[T]](physicalOperator: PhysicalOperator[T]) = { + val opNameList: ListBuffer[String] = ListBuffer.empty + physicalOperator.transform[Unit] { + case (addInto: AddInto[T], _) => + opNameList += "AddInto" + case (agg: Aggregate[T], _) => + opNameList += "Aggregate" + case (cache: Cache[T], _) => + opNameList += "Cache" + case (ddl: DDL[T], _) => + opNameList += "DDL" + case (drivingRDG: DrivingRDG[T], _) => + opNameList += "DrivingRDG" + case (drop: Drop[T], _) => + opNameList += "Drop" + case (expandInto: ExpandInto[T], _) => + opNameList += "ExpandInto" + case (filter: Filter[T], _) => + opNameList += "Filter" + case (join: Join[T], _) => + opNameList += "Join" + case (linkedExpand: LinkedExpand[T], _) => + opNameList += "LinkedExpand" + case (orderBy: OrderBy[T], _) => + opNameList += "OrderBy" + case (scan: PatternScan[T], _) => + opNameList += "PatternScan" + case (select: Select[T], _) => + opNameList += "Select" + case (start: Start[T], _) => + opNameList += "Start" + case (_, _) => + } + opNameList.mkString(",") + } + + it("testBehindLinkedEdge") { + val dsl = + """ + |GraphStructure { + | (p:Park)-[e1:SameCity]->(b:BusinessArea)-[e2:nearby(b.center, s.center, 5)]->(s:Subway) + |} + |Rule{R1(""): e1.city = 'Beijing'} + |Action{get(s.center)} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "Park" -> Set.empty, + "BusinessArea" -> Set.apply("center"), + "Subway" -> Set.apply("center"), + "STD.S2CellId" -> Set.empty, + "Subway_centerS2CellId_STD.S2CellId" -> Set.empty, + "Park_SameCity_BusinessArea" -> Set.apply("city")) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val physicalOperatorList = session.plan(dsl, Map.empty) + physicalOperatorList.size should equal(2) + val definePhysicalPlan = physicalOperatorList.head + val expandIntoCount = definePhysicalPlan.transform[Int] { + case (LinkedExpand(in, pattern, meta), count) => + in.isInstanceOf[PatternScan[EmptyRDG]] should equal(true) + count.head + case (ExpandInto(_, _, _, _), count) => + count.head + 1 + case (_, count) => + if (count.isEmpty) { + 0 + } else { + count.head + } + } + expandIntoCount should equal(2) + + val originBlock = physicalOperatorList(1) + originBlock.isInstanceOf[Select[EmptyRDG]] should equal(true) + originBlock.transform[Unit] { + case (PatternScan(_, pattern, _), _) => + pattern.topology("b").size should equal(2) + case (LinkedExpand(_, _, _), _) => + true should equal(false) + case _ => + } + } + + // scalastyle:on + + it("test add predicate") { + val dsl = + """ + |Define (s:User)-[p:tradeInfo]->(u:User) { + | GraphStructure { + | (s)-[t:trade]->(u:User) + | } Rule { + | R1("交易时间在90天内"): t.trade_time < -90 + | trade_num = group(s,u).count(t) + | p.trade_num = trade_num + | } + |} + | + |GraphStructure { + | (s:User)-[t:tradeInfo]->(u:User) + |} + |Rule { + | R1: t.trade_num > 100 + |} + |Action { + | get(s.id) + |} + | + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "User" -> Set.apply("id"), + "User_trade_User" -> Set.apply("trade_num", "trade_time")) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan(dsl, Map.empty) + rst.foreach(session.getResult(_)) + val edge = catalog.getGraph(IRGraph.defaultGraphName).getEdge("User_tradeInfo_User") + edge.properties.map(_.name) should contain("trade_num") + } + + it("test filter push down to loader config") { + val dsl = + """ + |GraphStructure { + | (s: User)-[t:trade]->(u:User) + |} + |Rule { + | R1("交易时间在90天内"): t.trade_time < -90 + | trade_num = group(s,u).count(t) + | p.trade_num = trade_num + |} + |Action { + | get(s.id) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "User" -> Set.apply("id"), + "User_trade_User" -> Set.apply("trade_num", "trade_time")) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val parser = new KgDslParser() + val block = parser.parse(dsl) + implicit val context: LogicalPlannerContext = + LogicalPlannerContext(catalog, parser, Map.empty) + val logicalOp = LogicalPlanner.plan(block).head + val op = LogicalOptimizer.optimize(logicalOp) + val loaderConfig = LoaderUtil.getLoaderConfig(List.apply(op), catalog) + loaderConfig + .getEdgeLoaderConfigs() + .iterator() + .next() + .getPropertiesFilterRules + .size() should equal(1) + } + + it("test sub query depends with belongTo parse 2") { + val dsl = + """ + |Define (s:AttributePOC.BrinsonAttribute)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfBrinsonAttribute`/`交易`) { + | GraphStructure { + | (s)-[p1:factorValue]->(u1:AttributePOC.BrinsonAttribute) + | } + | + | Rule { + | } + |} + + |GraphStructure { + | (s:`AttributePOC.TaxonomyOfBrinsonAttribute`/`交易`) + |} + |Rule { + | } + |Action { + | get(s.id) + |}""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "AttributePOC.TracebackDay" -> Set.apply("id"), + "AttributePOC.BrinsonAttribute" -> Set.apply("id", "factorValue", "factorType"), + "AttributePOC.TaxonomyOfBrinsonAttribute" -> Set.apply("id"), + "AttributePOC.TracebackDay_day_AttributePOC.BrinsonAttribute" -> Set.empty, + "AttributePOC.BrinsonAttribute_factorValue_AttributePOC.BrinsonAttribute" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan( + dsl, + Map + .apply((Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + val cnt = rst.head.transform[Int] { + case (join: Join[EmptyRDG], cnt) => cnt.sum + 1 + case (_, cnt) => + if (cnt.isEmpty) { + 0 + } else { + cnt.sum + } + } + cnt should equal(1) + } + + it("test sub query depends with belongTo parse exception") { + val dsl = + """ + |Define (u2:AttributePOC.BrinsonAttribute)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfBrinsonAttribute`/`交易`) { + | GraphStructure { + | (s:AttributePOC.BrinsonAttribute)-[p1:factorValue]->(u1:AttributePOC.BrinsonAttribute) + | (s)-[p2:factorValue]->(u2) + | (s)-[p3:factorValue]->(u3:AttributePOC.BrinsonAttribute) + | } + | + | Rule { + | R1: u1.factorType == "cluster" + | R2: u2.factorType == "trade" + | R3: u3.factorType == "stock" + | R6: s.factorType == "total" + | v = (u1.factorValue/ s.factorValue + u2.factorValue / s.factorValue) + | R4("必须大于50%"): v > 0.5 + | R5("交易收益大于选股"): u2.factorValue > u3.factorValue + | } + |} + + |GraphStructure { + | (s:AttributePOC.BrinsonAttribute)-[p:belongTo]->(o:`AttributePOC.TaxonomyOfBrinsonAttribute`/`交易`), + | (o1:AttributePOC.BrinsonAttribute)-[p2:factorValue]->(o) + |} + |Rule { + | } + |Action { + | get(s.id) + |}""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "AttributePOC.TracebackDay" -> Set.apply("id"), + "AttributePOC.BrinsonAttribute" -> Set.apply("id", "factorValue", "factorType"), + "AttributePOC.TaxonomyOfBrinsonAttribute" -> Set.apply("id", "tmp"), + "AttributePOC.TracebackDay_day_AttributePOC.BrinsonAttribute" -> Set.empty, + "AttributePOC.BrinsonAttribute_factorValue_AttributePOC.BrinsonAttribute" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + try { + val rst = session.plan( + dsl, + Map + .apply((Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + } catch { + case ex: SchemaException => + ex.getMessage.contains("BelongTo find conflict") should equal(true) + } + + } + + it("test transitive") { + val dsl = + """ + |GraphStructure { + | A [FilmPerson] + | C,D [FilmDirector] + | A->C [test] as e1 + | C->D [t1] repeat(2,5) as e2 + |} + |Rule { + |} + |Action { + | get(A.name,C.name) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "FilmPerson" -> Set.apply("name"), + "FilmDirector" -> Set.apply("name"), + "FilmPerson_test_FilmDirector" -> Set.empty, + "FilmDirector_t1_FilmDirector" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan( + dsl, + Map + .apply( + (Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true), + (Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + val cnt = rst.head.transform[Int] { + case (join: Join[EmptyRDG], cnt) => cnt.sum + 1 + case (_, cnt) => + if (cnt.isEmpty) { + 0 + } else { + cnt.sum + } + } + cnt should equal(5) + } + + it("test agg count") { + val dsl = + """ + |GraphStructure { + | (s:AMLz50.Userinfo)-[:expand_linked_alipay_id(s.id)]->(B:CustFundKG.Yeb)-[t:transfer]->(C:CustFundKG.Yeb) + |} + |Rule { + | R1:date_diff(from_unix_time(now(), 'yyyyMMdd'),from_unix_time(t.payDate, 'yyyyMMdd')) <= 90 + | R2:t.bizComment rlike "(赌博)|(赌)|(上分)|(下分)|(输)|(赢)|(福利)|(好运)|(反水)" + | transnum = group(s).countIf(R1 and R2, t) + | o = rule_value(transnum > 0, true, false) + | } + |Action { + | get(s.id, o, transnum) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = Map.apply( + "AMLz50.Userinfo" -> Set.empty, + "CustFundKG.Yeb" -> Set.empty, + "CustFundKG.Yeb_transfer_CustFundKG.Yeb" -> Set.apply("payDate", "bizComment")) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan( + dsl, + Map + .apply( + (Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true), + (Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + val cnt = rst.head.transform[Int] { + case (agg: Aggregate[EmptyRDG], cnt) => + val sum = agg.aggregations + .map(a => { + a._2 match { + case AggIfOpExpr( + _, + BinaryOpExpr(BAnd, BinaryOpExpr(_, _, _), BinaryOpExpr(_, _, _))) => + 1 + case _ => 0 + } + }) + .sum + cnt.sum + sum + case (_, cnt) => + if (cnt.isEmpty) { + 0 + } else { + cnt.sum + } + } + cnt should equal(1) + } + + it("test id push down") { + val dsl = + """ + | + |GraphStructure { + | A [User, __start__='true'] + | B,C [User] + | A->B [trans] as p1 + | B->C [trans] as p2 + | C->A [trans] as p3 + | } + |Rule { + |R1: A.id in $idSet1 + |R2: B.id in $idSet2 + |R3: C.id in $idSet2 + |} + |Action { + | get(A.id, B.id, C.id) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = + Map.apply("User" -> Set.apply("id"), "User_trans_User" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan( + dsl, + Map + .apply((Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + val cnt = rst.head.transform[Int] { + case (expandInto: ExpandInto[EmptyRDG], cnt) => + var ele = expandInto.pattern.getNode("B") + if (ele == null) { + ele = expandInto.pattern.getNode("C") + } + ele.rule should equal(null) + cnt.sum + 1 + case (_, cnt) => + if (cnt.isEmpty) { + 0 + } else { + cnt.sum + } + } + cnt should equal(2) + } + + it("test id push down 4") { + val dsl = + """ + | + |GraphStructure { + | A [User, __start__='true'] + | B,C [User] + | A->B [trans] as p1 + | B->C [trans] as p2 + | C->A [trans] as p3 + | } + |Rule { + |R1: A.id in $idSet1 + |} + |Action { + | get(A.id, C.id) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = + Map.apply("User" -> Set.apply("id"), "User_trans_User" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + val rst = session.plan( + dsl, + Map + .apply( + (Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true), + (Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + val cnt = rst.head.transform[Int] { + case (expandInto: ExpandInto[EmptyRDG], cnt) => + var ele = expandInto.pattern.getNode("C") + (ele != null) should equal(true) + ele.rule should equal(null) + cnt.sum + 1 + case (_, cnt) => + if (cnt.isEmpty) { + 0 + } else { + cnt.sum + } + } + cnt should equal(1) + } + + it("test schema absent test") { + val dsl = + """ + | + |GraphStructure { + | A [User, __start__='true'] + | B,C [User] + | A->B [trans] as p1 + | B->C [trans] as p2 + | C->A [test] as p3 + | } + |Rule { + |R1: A.id in $idSet1 + |} + |Action { + | get(A.id, C.id) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = + Map.apply("User" -> Set.apply("id"), "User_trans_User" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + try { + session.plan( + dsl, + Map + .apply( + (Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true), + (Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + } catch { + case ex: SchemaException => + println(ex.getMessage) + ex.getMessage.contains("Cannot find p3 types in") should equal(true) + } + } + + it("test schema absent test2") { + val dsl = + """ + | + |GraphStructure { + | A [User, __start__='true'] + | B,C [User] + | D [TEST] + | A->B [trans] as p1 + | B->C [trans] as p2 + | C->A [trans] as p3 + | } + |Rule { + |R1: A.id in $idSet1 + |} + |Action { + | get(A.id, C.id) + |} + |""".stripMargin + val schema: Map[String, Set[String]] = + Map.apply("User" -> Set.apply("id"), "User_trans_User" -> Set.empty) + val catalog = new PropertyGraphCatalog(schema) + catalog.init() + val session = new EmptySession(new KgDslParser(), catalog) + try { + session.plan( + dsl, + Map + .apply( + (Constants.SPG_REASONER_LUBE_SUBQUERY_ENABLE, true), + (Constants.SPG_REASONER_PLAN_PRETTY_PRINT_LOGGER_ENABLE, true)) + .asInstanceOf[Map[String, Object]]) + } catch { + case ex: SchemaException => + println(ex.getMessage) + ex.getMessage.contains("Cannot find IRNode(D,Set()).name") should equal(true) + } + } +} diff --git a/reasoner/warehouse/cloudext-warehouse/pom.xml b/reasoner/warehouse/cloudext-warehouse/pom.xml new file mode 100644 index 00000000..fd1754ea --- /dev/null +++ b/reasoner/warehouse/cloudext-warehouse/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + com.antgroup.openspg.reasoner + reasoner-parent + 0.0.1-SNAPSHOT + ../../pom.xml + + + reasoner-cloudext-warehouse + + + + com.antgroup.openspg.reasoner + reasoner-runner-common + + + com.antgroup.openspg.reasoner + reasoner-common + + + com.antgroup.openspg.cloudext + cloudext-interface-graph-store + + + com.antgroup.openspg.cloudext + cloudext-impl-graph-store-tugraph + + + diff --git a/reasoner/warehouse/cloudext-warehouse/src/main/java/com/antgroup/openspg/reasoner/warehouse/cloudext/CloudExtGraphState.java b/reasoner/warehouse/cloudext-warehouse/src/main/java/com/antgroup/openspg/reasoner/warehouse/cloudext/CloudExtGraphState.java new file mode 100644 index 00000000..826aaa90 --- /dev/null +++ b/reasoner/warehouse/cloudext-warehouse/src/main/java/com/antgroup/openspg/reasoner/warehouse/cloudext/CloudExtGraphState.java @@ -0,0 +1,171 @@ +package com.antgroup.openspg.reasoner.warehouse.cloudext; + +import com.antgroup.openspg.cloudext.interfaces.graphstore.BaseLPGGraphStoreClient; +import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClientDriverManager; +import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.OneHopLPGRecordQuery; +import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.ScanLPGRecordQuery; +import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.VertexLPGRecordQuery; +import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.EdgeRecord; +import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord; +import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.struct.GraphLPGRecordStruct; +import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName; +import com.antgroup.openspg.reasoner.common.graph.edge.Direction; +import com.antgroup.openspg.reasoner.common.graph.edge.IEdge; +import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge; +import com.antgroup.openspg.reasoner.common.graph.property.IProperty; +import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty; +import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexVersionProperty; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex; +import com.antgroup.openspg.reasoner.common.graph.vertex.impl.VertexId; +import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState; +import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum; +import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; +import java.util.*; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +@Slf4j +public class CloudExtGraphState extends MemGraphState { + + private BaseLPGGraphStoreClient lpgGraphStoreClient; + + @Override + public void init(Map param) { + super.init(param); + + GraphStoreConnectionInfo connInfo = new GraphStoreConnectionInfo(); + connInfo.setScheme(param.getOrDefault("cloudext.graphstore.schema", "tugraph")); + connInfo.setParams(Collections.unmodifiableMap(param)); + lpgGraphStoreClient = + (BaseLPGGraphStoreClient) GraphStoreClientDriverManager.getClient(connInfo); + } + + @Override + public IVertex getVertex(IVertexId id, Long version) { + IVersionProperty iProperty = (IVersionProperty) vertexMap.get(id); + if (iProperty != null) { + return super.getVertex(id, version); + } + + GraphLPGRecordStruct recordStruct = + (GraphLPGRecordStruct) + lpgGraphStoreClient.queryRecord(new VertexLPGRecordQuery(id.getBizId(), id.getType())); + + if (recordStruct.isEmpty()) { + return null; + } + + for (VertexRecord vertexRecord : recordStruct.getVertices()) { + VertexVersionProperty property = + new VertexVersionProperty(vertexRecord.toPropertyMapWithIdAndVersion()); + return new Vertex<>( + new VertexId(vertexRecord.getId(), vertexRecord.getVertexType()), property); + } + return null; + } + + @Override + public void mergeVertexProperty( + IVertexId id, Map property, MergeTypeEnum mergeType, Long version) { + IProperty iProperty = vertexMap.get(id); + if (iProperty == null) { + IVertex vertex = getVertex(id, version); + addVertex(vertex, version); + } + super.mergeVertexProperty(id, property, mergeType, version); + } + + @Override + public void mergeEdgeProperty( + IVertexId s, + String p, + IVertexId o, + Long version, + Direction direction, + Map property, + MergeTypeEnum mergeType) { + throw new UnsupportedOperationException(); + } + + @Override + public Iterator> getVertexIterator(Set vertexTypes) { + if (CollectionUtils.isEmpty(vertexTypes)) { + return Collections.emptyIterator(); + } + + List vertexRecords = new ArrayList<>(); + for (String vertexType : vertexTypes) { + GraphLPGRecordStruct recordStruct = + (GraphLPGRecordStruct) + lpgGraphStoreClient.queryRecord(new ScanLPGRecordQuery(vertexType, null)); + vertexRecords.addAll(recordStruct.getVertices()); + } + + if (CollectionUtils.isEmpty(vertexRecords)) { + return Collections.emptyIterator(); + } + + return vertexRecords.stream() + .map( + vertexRecord -> { + VertexVersionProperty property = + new VertexVersionProperty(vertexRecord.toPropertyMapWithIdAndVersion()); + return (IVertex) + new Vertex( + new VertexId(vertexRecord.getId(), vertexRecord.getVertexType()), property); + }) + .iterator(); + } + + @Override + public List> getEdges( + IVertexId vertexId, + Long startVersion, + Long endVersion, + Set types, + Direction direction) { + List> edges = + super.getEdges(vertexId, startVersion, endVersion, types, direction); + if (CollectionUtils.isNotEmpty(edges)) { + return edges; + } + + GraphLPGRecordStruct recordStruct = + (GraphLPGRecordStruct) + lpgGraphStoreClient.queryRecord( + new OneHopLPGRecordQuery( + vertexId.getBizId(), + vertexId.getType(), + types.stream().map(EdgeTypeName::parse).collect(Collectors.toSet()), + com.antgroup.openspg.cloudext.interfaces.graphstore.model.Direction.valueOf( + direction.name()))); + + List edgeRecords = recordStruct.getEdges(); + List> results = new ArrayList<>(edgeRecords.size()); + for (EdgeRecord edgeRecord : edgeRecords) { + VertexId srcVertexId = + new VertexId(edgeRecord.getSrcId(), edgeRecord.getEdgeType().getStartVertexType()); + VertexId dstVertexId = + new VertexId(edgeRecord.getDstId(), edgeRecord.getEdgeType().getEndVertexType()); + if (Direction.IN.equals(direction)) { + // 当是IN边时,这里交换下src和dst,由于图存的方向和kgdsl需要的方向不一致 + VertexId tmp = srcVertexId; + srcVertexId = dstVertexId; + dstVertexId = tmp; + } + results.add( + new Edge<>( + srcVertexId, + dstVertexId, + new EdgeProperty(edgeRecord.toPropertyMapWithId()), + 0L, + direction, + edgeRecord.getEdgeType().toString())); + } + return results; + } +} diff --git a/server/api/facade/pom.xml b/server/api/facade/pom.xml index d3a627cd..62ac92f7 100644 --- a/server/api/facade/pom.xml +++ b/server/api/facade/pom.xml @@ -28,10 +28,6 @@ com.antgroup.openspg.server core-schema-model - - com.antgroup.openspg.server - core-reasoner-model - com.google.code.gson gson diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/JSON.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java similarity index 74% rename from server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/JSON.java rename to server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java index 89d8e2a7..ae3fe6ec 100644 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/JSON.java +++ b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java @@ -41,25 +41,13 @@ import com.antgroup.openspg.core.schema.model.type.EntityType; import com.antgroup.openspg.core.schema.model.type.EventType; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; import com.antgroup.openspg.core.schema.model.type.StandardType; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.JobReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.model.service.KgdslReasonerContent; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerContentTypeEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerReceiptTypeEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.SuccessReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.model.service.VertexReasonerContent; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.ToNumberPolicy; import com.google.gson.typeadapters.RuntimeTypeAdapterFactory; import java.lang.reflect.Type; -public class JSON { +public class SchemaJsonUtils { public static final String DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static final String DEFAULT_TYPE_FIELD_NAME = "@type"; @@ -103,32 +91,13 @@ public class JSON { .registerSubtype( OperatorIdentifier.class, SPGIdentifierTypeEnum.OPERATOR.name()) .recognizeSubtypes()) - .registerTypeAdapterFactory( - RuntimeTypeAdapterFactory.of(BaseReasonerResult.class, DEFAULT_TYPE_FIELD_NAME) - .registerSubtype(FailureReasonerResult.class, JobInstStatusEnum.FAILURE.name()) - .registerSubtype(SuccessReasonerResult.class, JobInstStatusEnum.SUCCESS.name()) - .recognizeSubtypes()) // BaseConceptSemantic .registerTypeAdapterFactory( RuntimeTypeAdapterFactory.of(BaseConceptSemantic.class, DEFAULT_TYPE_FIELD_NAME) .registerSubtype(DynamicTaxonomySemantic.class) .registerSubtype(LogicalCausationSemantic.class) .recognizeSubtypes()) - // BaseReasonerContent - .registerTypeAdapterFactory( - RuntimeTypeAdapterFactory.of(BaseReasonerContent.class, DEFAULT_TYPE_FIELD_NAME) - .registerSubtype( - KgdslReasonerContent.class, ReasonerContentTypeEnum.KGDSL.name()) - .registerSubtype( - VertexReasonerContent.class, ReasonerContentTypeEnum.VERTEX.name()) - .recognizeSubtypes()) // BaseReasonerReceipt - .registerTypeAdapterFactory( - RuntimeTypeAdapterFactory.of(BaseReasonerReceipt.class, DEFAULT_TYPE_FIELD_NAME) - .registerSubtype( - TableReasonerReceipt.class, ReasonerReceiptTypeEnum.TABLE.name()) - .registerSubtype(JobReasonerReceipt.class, ReasonerReceiptTypeEnum.JOB.name()) - .recognizeSubtypes()) .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) .setDateFormat(DATA_FORMAT) .create(); diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerDslRunRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerDslRunRequest.java deleted file mode 100644 index f2febdee..00000000 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerDslRunRequest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.facade.dto.reasoner.request; - -import com.antgroup.openspg.server.common.model.base.BaseRequest; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent; -import java.util.Map; - -public class ReasonerDslRunRequest extends BaseRequest { - - private Long projectId; - - private BaseReasonerContent content; - - private Map params; - - public BaseReasonerContent getContent() { - return content; - } - - public void setContent(BaseReasonerContent content) { - this.content = content; - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public Map getParams() { - return params; - } - - public void setParams(Map params) { - this.params = params; - } -} diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInfoQuery.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInfoQuery.java deleted file mode 100644 index bd3e7735..00000000 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInfoQuery.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.facade.dto.reasoner.request; - -import com.antgroup.openspg.server.common.model.base.BaseQuery; - -public class ReasonerJobInfoQuery extends BaseQuery { - - private Long reasonerJobInfoId; - - private String externalJobInfoId; - - public Long getReasonerJobInfoId() { - return reasonerJobInfoId; - } - - public ReasonerJobInfoQuery setReasonerJobInfoId(Long reasonerJobInfoId) { - this.reasonerJobInfoId = reasonerJobInfoId; - return this; - } - - public String getExternalJobInfoId() { - return externalJobInfoId; - } - - public ReasonerJobInfoQuery setExternalJobInfoId(String externalJobInfoId) { - this.externalJobInfoId = externalJobInfoId; - return this; - } -} diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInstQuery.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInstQuery.java deleted file mode 100644 index 29ad61c9..00000000 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInstQuery.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.facade.dto.reasoner.request; - -import com.antgroup.openspg.server.common.model.base.BaseQuery; - -public class ReasonerJobInstQuery extends BaseQuery { - - private Long reasonerJobInstId; - - private String externalJobInstId; - - public Long getReasonerJobInstId() { - return reasonerJobInstId; - } - - public ReasonerJobInstQuery setReasonerJobInstId(Long reasonerJobInstId) { - this.reasonerJobInstId = reasonerJobInstId; - return this; - } - - public String getExternalJobInstId() { - return externalJobInstId; - } - - public ReasonerJobInstQuery setExternalJobInstId(String externalJobInstId) { - this.externalJobInstId = externalJobInstId; - return this; - } -} diff --git a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobSubmitRequest.java b/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobSubmitRequest.java deleted file mode 100644 index 4a9e5845..00000000 --- a/server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobSubmitRequest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.facade.dto.reasoner.request; - -import com.antgroup.openspg.server.common.model.base.BaseRequest; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent; -import java.util.Map; - -public class ReasonerJobSubmitRequest extends BaseRequest { - - private String jobName; - - private Long projectId; - - private BaseReasonerContent content; - - private String cron; - - private String idempotentId; - - private Map params; - - public BaseReasonerContent getContent() { - return content; - } - - public void setContent(BaseReasonerContent content) { - this.content = content; - } - - public String getJobName() { - return jobName; - } - - public void setJobName(String jobName) { - this.jobName = jobName; - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public String getCron() { - return cron; - } - - public void setCron(String cron) { - this.cron = cron; - } - - public String getIdempotentId() { - return idempotentId; - } - - public void setIdempotentId(String idempotentId) { - this.idempotentId = idempotentId; - } - - public Map getParams() { - return params; - } - - public void setParams(Map params) { - this.params = params; - } -} diff --git a/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java b/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java index 6d3caa5f..3b2eb5c0 100644 --- a/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java +++ b/server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.server.api.http.client.forest; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.dtflys.forest.converter.ConvertOptions; import com.dtflys.forest.converter.json.ForestJsonConverter; import com.dtflys.forest.http.ForestRequest; @@ -40,7 +40,7 @@ public class GsonConvertor implements ForestJsonConverter { @Override public String encodeToString(Object obj) { - return JSON.serialize(obj); + return SchemaJsonUtils.serialize(obj); } @Override @@ -76,7 +76,7 @@ public class GsonConvertor implements ForestJsonConverter { if (obj instanceof CharSequence) { return convertToJavaObject(obj.toString(), LinkedHashMap.class); } - final Gson gson = JSON.gson; + final Gson gson = SchemaJsonUtils.gson; final JsonElement jsonElement = gson.toJsonTree(obj); return toMap(jsonElement.getAsJsonObject(), true); } @@ -88,7 +88,7 @@ public class GsonConvertor implements ForestJsonConverter { @Override public String getDateFormat() { - return JSON.DATA_FORMAT; + return SchemaJsonUtils.DATA_FORMAT; } @Override @@ -96,7 +96,7 @@ public class GsonConvertor implements ForestJsonConverter { if (StringUtils.isBlank(source)) { return null; } - return JSON.deserialize(source, targetType); + return SchemaJsonUtils.deserialize(source, targetType); } @Override diff --git a/server/api/http-server/pom.xml b/server/api/http-server/pom.xml index 503c7c09..335d9319 100644 --- a/server/api/http-server/pom.xml +++ b/server/api/http-server/pom.xml @@ -36,10 +36,6 @@ com.antgroup.openspg.server biz-common - - com.antgroup.openspg.server - biz-reasoner - org.springdoc springdoc-openapi-ui diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java index 410e6314..f6804a77 100644 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java +++ b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java @@ -13,7 +13,7 @@ package com.antgroup.openspg.server.api.http.server; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import java.util.Collections; import java.util.List; import org.springframework.context.annotation.Configuration; @@ -28,7 +28,7 @@ public class WebMvcConfig implements WebMvcConfigurer { @Override public void extendMessageConverters(List> converters) { GsonHttpMessageConverter gsonHttpMessageConverter = new GsonHttpMessageConverter(); - gsonHttpMessageConverter.setGson(JSON.gson); + gsonHttpMessageConverter.setGson(SchemaJsonUtils.gson); gsonHttpMessageConverter.setSupportedMediaTypes( Collections.singletonList(MediaType.APPLICATION_JSON)); converters.add(0, gsonHttpMessageConverter); diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ObjectStoreController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ObjectStoreController.java deleted file mode 100644 index b4af3aaa..00000000 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ObjectStoreController.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.http.server.openapi; - -import com.antgroup.openspg.server.api.facade.dto.common.request.ObjectStoreRequest; -import com.antgroup.openspg.server.api.facade.dto.common.response.ObjectStoreResponse; -import com.antgroup.openspg.server.api.http.server.HttpBizCallback; -import com.antgroup.openspg.server.api.http.server.HttpBizTemplate; -import com.antgroup.openspg.server.biz.common.ObjectStoreManager; -import java.io.IOException; -import java.io.InputStream; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.multipart.MultipartFile; - -@Controller -@RequestMapping("/public/v1/objectStore") -public class ObjectStoreController { - - @Autowired private ObjectStoreManager objectStoreManager; - - @RequestMapping(method = RequestMethod.POST, consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity upload(ObjectStoreRequest request, MultipartFile file) { - return HttpBizTemplate.execute( - new HttpBizCallback() { - @Override - public void check() {} - - @Override - public ObjectStoreResponse action() { - InputStream inputStream = null; - try { - inputStream = file.getInputStream(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return objectStoreManager.objectStore(request, inputStream); - } - }); - } -} diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ReasonerController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ReasonerController.java deleted file mode 100644 index 86990113..00000000 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ReasonerController.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.http.server.openapi; - -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerDslRunRequest; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobSubmitRequest; -import com.antgroup.openspg.server.api.http.server.BaseController; -import com.antgroup.openspg.server.api.http.server.HttpBizCallback; -import com.antgroup.openspg.server.api.http.server.HttpBizTemplate; -import com.antgroup.openspg.server.biz.reasoner.ReasonerManager; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; - -@Controller -@RequestMapping("/public/v1/reasoner") -public class ReasonerController extends BaseController { - - @Autowired private ReasonerManager reasonerManager; - - @RequestMapping(value = "/runDsl", method = RequestMethod.POST) - public ResponseEntity runDsl(@RequestBody ReasonerDslRunRequest request) { - return HttpBizTemplate.execute( - new HttpBizCallback() { - @Override - public void check() {} - - @Override - public BaseReasonerReceipt action() { - return reasonerManager.runDsl(request); - } - }); - } - - @RequestMapping(value = "/submitJobInfo", method = RequestMethod.POST) - public ResponseEntity submitJobInfo(@RequestBody ReasonerJobSubmitRequest request) { - return HttpBizTemplate.execute( - new HttpBizCallback() { - @Override - public void check() {} - - @Override - public BaseReasonerReceipt action() { - return reasonerManager.submitJob(request); - } - }); - } - - @RequestMapping(value = "/queryJobInst", method = RequestMethod.GET) - public ResponseEntity queryJobInst(@RequestParam(required = false) Long jobInstId) { - return HttpBizTemplate.execute( - new HttpBizCallback>() { - @Override - public void check() {} - - @Override - public List action() { - ReasonerJobInstQuery query = new ReasonerJobInstQuery(); - query.setReasonerJobInstId(jobInstId); - return reasonerManager.queryJobInst(query); - } - }); - } -} diff --git a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/TableStoreController.java b/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/TableStoreController.java deleted file mode 100644 index 44e4ff57..00000000 --- a/server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/TableStoreController.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.api.http.server.openapi; - -import com.antgroup.openspg.server.api.http.server.BaseController; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import org.springframework.core.io.InputStreamResource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; - -@Controller -@RequestMapping("/public/v1/tableStore") -public class TableStoreController extends BaseController { - - @RequestMapping(path = "/download", method = RequestMethod.GET) - public ResponseEntity download(String fileName) { - File file = new File(fileName); - FileInputStream fileInputStream = null; - try { - fileInputStream = new FileInputStream(file); - } catch (FileNotFoundException e) { - return ResponseEntity.notFound().build(); - } - InputStreamResource resource = new InputStreamResource(fileInputStream); - - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fileName) - .contentLength(file.length()) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resource); - } -} diff --git a/server/arks/sofaboot/pom.xml b/server/arks/sofaboot/pom.xml index 8e6ff8e6..c4c545eb 100644 --- a/server/arks/sofaboot/pom.xml +++ b/server/arks/sofaboot/pom.xml @@ -27,10 +27,6 @@ com.antgroup.openspg.server api-http-server - - com.antgroup.openspg.server - biz-reasoner - com.antgroup.openspg.server biz-schema @@ -71,7 +67,6 @@ 2.5.7 true - ../../dev/release/openspg/target executable diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ObjectStoreManager.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ObjectStoreManager.java deleted file mode 100644 index b94f4b9f..00000000 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ObjectStoreManager.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.biz.common; - -import com.antgroup.openspg.server.api.facade.dto.common.request.ObjectStoreRequest; -import com.antgroup.openspg.server.api.facade.dto.common.response.ObjectStoreResponse; -import java.io.InputStream; - -public interface ObjectStoreManager { - - ObjectStoreResponse objectStore(ObjectStoreRequest request, InputStream file); -} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ObjectStoreManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ObjectStoreManagerImpl.java deleted file mode 100644 index 48c1af7d..00000000 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ObjectStoreManagerImpl.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.biz.common.impl; - -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient; -import com.antgroup.openspg.cloudext.interfaces.objectstore.cmd.ObjectStoreSaveCmd; -import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath; -import com.antgroup.openspg.server.api.facade.dto.common.request.ObjectStoreRequest; -import com.antgroup.openspg.server.api.facade.dto.common.response.ObjectStoreResponse; -import com.antgroup.openspg.server.biz.common.ObjectStoreManager; -import com.antgroup.openspg.server.common.service.datasource.DataSourceService; -import java.io.InputStream; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class ObjectStoreManagerImpl implements ObjectStoreManager { - - @Autowired private DataSourceService dataSourceService; - - @Override - public ObjectStoreResponse objectStore(ObjectStoreRequest request, InputStream file) { - ObjectStoreClient objectStoreClient = dataSourceService.buildSharedFileStoreClient(); - - ObjectStorePath filePath = - objectStoreClient.save( - new ObjectStoreSaveCmd(new ObjectStorePath(request.getName()), file)); - return new ObjectStoreResponse() - .setRelativePath(filePath.getRelativePath()) - .setAbsolutePath(filePath.getAbsolutePath()); - } -} diff --git a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/SearchEngineManagerImpl.java b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/SearchEngineManagerImpl.java index ddf4505e..17b3510b 100644 --- a/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/SearchEngineManagerImpl.java +++ b/server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/SearchEngineManagerImpl.java @@ -14,7 +14,7 @@ package com.antgroup.openspg.server.biz.common.impl; import com.antgroup.openspg.cloudext.interfaces.searchengine.SearchEngineClient; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.api.facade.dto.common.request.SearchEngineIndexRequest; import com.antgroup.openspg.server.api.facade.dto.common.response.SearchEngineIndexResponse; import com.antgroup.openspg.server.biz.common.SearchEngineManager; @@ -38,6 +38,6 @@ public class SearchEngineManagerImpl implements SearchEngineManager { (SearchEngineConnectionInfo) searchEngineClient.getConnInfo(); return new SearchEngineIndexResponse() .setIndexName(convertedIndexName) - .setConnInfo(JSON.serialize(connInfo)); + .setConnInfo(SchemaJsonUtils.serialize(connInfo)); } } diff --git a/server/biz/reasoner/pom.xml b/server/biz/reasoner/pom.xml deleted file mode 100644 index a06d34b9..00000000 --- a/server/biz/reasoner/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../pom.xml - - - biz-reasoner - - - com.antgroup.openspg.server - api-http-client - - - com.antgroup.openspg.server - common-service - - - com.antgroup.openspg.server - core-reasoner-service - - - org.springframework - spring-context - - - diff --git a/server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/ReasonerManager.java b/server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/ReasonerManager.java deleted file mode 100644 index 8a2bbbfc..00000000 --- a/server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/ReasonerManager.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.biz.reasoner; - -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerDslRunRequest; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobSubmitRequest; -import com.antgroup.openspg.server.core.reasoner.model.service.JobReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; -import java.util.List; - -public interface ReasonerManager { - - TableReasonerReceipt runDsl(ReasonerDslRunRequest request); - - JobReasonerReceipt submitJob(ReasonerJobSubmitRequest request); - - List queryJobInst(ReasonerJobInstQuery query); -} diff --git a/server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/impl/ReasonerManagerImpl.java b/server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/impl/ReasonerManagerImpl.java deleted file mode 100644 index b950556e..00000000 --- a/server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/impl/ReasonerManagerImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.biz.reasoner.impl; - -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd; -import com.antgroup.openspg.common.util.StringUtils; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerDslRunRequest; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobSubmitRequest; -import com.antgroup.openspg.server.biz.reasoner.ReasonerManager; -import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.common.service.config.AppEnvConfig; -import com.antgroup.openspg.server.common.service.datasource.DataSourceService; -import com.antgroup.openspg.server.core.reasoner.model.service.JobReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInfoService; -import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInstService; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class ReasonerManagerImpl implements ReasonerManager { - - @Autowired private AppEnvConfig appEnvConfig; - - @Autowired private ReasonerJobInfoService reasonerJobInfoService; - - @Autowired private ReasonerJobInstService reasonerJobInstService; - - @Autowired private DataSourceService dataSourceService; - - @Override - public TableReasonerReceipt runDsl(ReasonerDslRunRequest request) { - ComputingClient computingClient = dataSourceService.buildSharedComputingClient(); - return computingClient.run( - new ReasonerJobRunCmd( - request.getProjectId(), - appEnvConfig.getSchemaUri(), - (GraphStoreConnectionInfo) dataSourceService.buildSharedKgStoreClient().getConnInfo(), - request.getContent())); - } - - @Override - public JobReasonerReceipt submitJob(ReasonerJobSubmitRequest request) { - ReasonerJobInfo reasonerJobInfo = - new ReasonerJobInfo( - request.getJobName(), - request.getProjectId(), - request.getContent(), - request.getCron(), - JobInfoStateEnum.ENABLE, - request.getParams()); - - // create a reasoner job - Long reasonerJobInfoId = reasonerJobInfoService.create(reasonerJobInfo); - - // if the cron expression is empty, create a reasoner job instance - Long reasonerJobInstId = null; - if (StringUtils.isBlank(request.getCron())) { - ReasonerJobInst reasonerJobInst = - new ReasonerJobInst( - reasonerJobInfoId, - reasonerJobInfo.getProjectId(), - JobInstStatusEnum.INIT, - null, - null, - null, - null, - null); - reasonerJobInstId = reasonerJobInstService.create(reasonerJobInfo, reasonerJobInst); - } - return new JobReasonerReceipt(reasonerJobInfoId, reasonerJobInstId); - } - - @Override - public List queryJobInst(ReasonerJobInstQuery query) { - return reasonerJobInstService.query(query); - } -} diff --git a/server/cloudext/impl/computing/local/pom.xml b/server/cloudext/impl/computing/local/pom.xml deleted file mode 100644 index da8cc399..00000000 --- a/server/cloudext/impl/computing/local/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../../../pom.xml - - - cloudext-impl-computing-local - - - - com.antgroup.openspg.server - cloudext-interface-computing - - - com.antgroup.openspg.cloudext - cloudext-interface-table-store - - - com.antgroup.kg.reasoner - local - ${kgreasoner-version} - system - ${project.basedir}/../../../../lib/spgreasoner-local-0.0.1.jar - - - com.antgroup.openspg.server - core-reasoner-service - - - diff --git a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClient.java b/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClient.java deleted file mode 100644 index 165f4cab..00000000 --- a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClient.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.computing.local; - -import com.antgroup.openspg.cloudext.impl.computing.local.impl.LocalReasonerExecutorImpl; -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd; -import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class LocalComputingClient implements ComputingClient { - - private static final String BUILDER_NUMBER_OF_THREAD = "builder.nThreads"; - - @Getter private final ComputingConnectionInfo connInfo; - private final LocalReasonerExecutor localReasonerExecutor; - - public LocalComputingClient(ComputingConnectionInfo connInfo) { - this.connInfo = connInfo; - localReasonerExecutor = - new LocalReasonerExecutorImpl( - (String) connInfo.getParamOrDefault(BUILDER_NUMBER_OF_THREAD, "*2")); - } - - @Override - public ReasonerStatusWithProgress query(ReasonerJobProcessQuery query) { - return localReasonerExecutor.query(query); - } - - @Override - public boolean canSubmit(ReasonerJobCanSubmitQuery query) { - return localReasonerExecutor.canSubmit(query); - } - - @Override - public String submit(ReasonerJobSubmitCmd cmd) { - return localReasonerExecutor.submit(cmd); - } - - @Override - public TableReasonerReceipt run(ReasonerJobRunCmd cmd) { - return localReasonerExecutor.run(cmd); - } -} diff --git a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClientDriver.java b/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClientDriver.java deleted file mode 100644 index 809d8cb4..00000000 --- a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClientDriver.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.computing.local; - -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient; -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriver; -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriverManager; -import com.antgroup.openspg.common.util.cloudext.CachedCloudExtClientDriver; -import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo; - -public class LocalComputingClientDriver - extends CachedCloudExtClientDriver - implements ComputingClientDriver { - - static { - ComputingClientDriverManager.registerDriver(new LocalComputingClientDriver()); - } - - @Override - public String driverScheme() { - return "local"; - } - - @Override - protected ComputingClient innerConnect(ComputingConnectionInfo connInfo) { - return new LocalComputingClient(connInfo); - } -} diff --git a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalReasonerExecutor.java b/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalReasonerExecutor.java deleted file mode 100644 index 687a208a..00000000 --- a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalReasonerExecutor.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.computing.local; - -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; - -public interface LocalReasonerExecutor { - - ReasonerStatusWithProgress query(ReasonerJobProcessQuery query); - - boolean canSubmit(ReasonerJobCanSubmitQuery query); - - String submit(ReasonerJobSubmitCmd cmd); - - TableReasonerReceipt run(ReasonerJobRunCmd cmd); -} diff --git a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/impl/LocalReasonerExecutorImpl.java b/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/impl/LocalReasonerExecutorImpl.java deleted file mode 100644 index a718b72f..00000000 --- a/server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/impl/LocalReasonerExecutorImpl.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.computing.local.impl; - -import com.antgroup.kg.reasoner.catalog.impl.KgSchemaConnectionInfo; -import com.antgroup.kg.reasoner.local.KGReasonerLocalRunner; -import com.antgroup.kg.reasoner.local.model.LocalReasonerResult; -import com.antgroup.kg.reasoner.local.model.LocalReasonerTask; -import com.antgroup.openspg.cloudext.impl.computing.local.LocalReasonerExecutor; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableFileHandler; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriverManager; -import com.antgroup.openspg.cloudext.interfaces.tablestore.cmd.TableFileCreateCmd; -import com.antgroup.openspg.cloudext.interfaces.tablestore.model.ColumnMeta; -import com.antgroup.openspg.cloudext.interfaces.tablestore.model.TableRecord; -import com.antgroup.openspg.common.util.StringUtils; -import com.antgroup.openspg.common.util.thread.ThreadUtils; -import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.core.reasoner.model.ReasonerException; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.KgdslReasonerContent; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerProgress; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import com.antgroup.openspg.server.core.reasoner.model.service.SuccessReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; -import com.antgroup.openspg.server.core.reasoner.service.util.LocalRunnerUtils; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; - -@Slf4j -public class LocalReasonerExecutorImpl implements LocalReasonerExecutor { - - // for reasoner job - private final ThreadPoolExecutor jobDriver; - private final ThreadPoolExecutor jobWorker; - private final Map> runningReasonerJobInst; - - // for olap - private final ThreadPoolExecutor olapDriver; - private final ThreadPoolExecutor olapWorker; - - public LocalReasonerExecutorImpl(String nThreads) { - jobDriver = ThreadUtils.newDaemonFixedThreadPool(1, "localReasonerDriver"); - jobWorker = - ThreadUtils.newDaemonFixedThreadPool(ThreadUtils.nThreads(nThreads), "localReasonerWorker"); - - olapDriver = - ThreadUtils.newDaemonFixedThreadPool(ThreadUtils.nThreads("*1"), "olapReasonerDriver"); - olapWorker = - ThreadUtils.newDaemonFixedThreadPool(ThreadUtils.nThreads(nThreads), "olapReasonerWorker"); - runningReasonerJobInst = new ConcurrentHashMap<>(); - } - - @Override - public ReasonerStatusWithProgress query(ReasonerJobProcessQuery query) { - Future future = - runningReasonerJobInst.get(query.getComputingJobInstId()); - if (future == null) { - return null; - } - if (future.isDone()) { - // if finished, remove it from runningBuilderJobInst - runningReasonerJobInst.remove(query.getComputingJobInstId()); - ReasonerStatusWithProgress progress = null; - try { - progress = future.get(); - } catch (Throwable e) { - throw ReasonerException.reasonerError(e); - } - return progress; - } else { - return new ReasonerStatusWithProgress(JobInstStatusEnum.RUNNING); - } - } - - @Override - public boolean canSubmit(ReasonerJobCanSubmitQuery query) { - return runningReasonerJobInst.isEmpty(); - } - - @Override - public String submit(ReasonerJobSubmitCmd cmd) { - ReasonerJobInst jobInst = cmd.getJobInst(); - if (runningReasonerJobInst.isEmpty()) { - synchronized (runningReasonerJobInst) { - if (runningReasonerJobInst.isEmpty()) { - Future future = doSubmit(cmd); - String computingJobInstId = String.valueOf(jobInst.getJobInstId()); - runningReasonerJobInst.put(computingJobInstId, future); - return computingJobInstId; - } - } - } - return null; - } - - @Override - public TableReasonerReceipt run(ReasonerJobRunCmd cmd) { - Future future = - olapDriver.submit( - () -> { - LocalReasonerResult localReasonerResult = - doRun( - cmd.getProjectId(), - cmd.getSchemaUrl(), - cmd.getConnInfo(), - cmd.getContent(), - olapWorker); - return buildReceipt(localReasonerResult); - }); - try { - return future.get(3, TimeUnit.MINUTES); - } catch (InterruptedException | ExecutionException e) { - throw ReasonerException.reasonerError(e); - } catch (TimeoutException e) { - future.cancel(true); - throw ReasonerException.timeout(3); - } - } - - private Future doSubmit(ReasonerJobSubmitCmd cmd) { - return jobDriver.submit( - () -> { - try { - ReasonerJobInfo jobInfo = cmd.getJobInfo(); - LocalReasonerResult localReasonerResult = - doRun( - jobInfo.getProjectId(), - cmd.getSchemaUrl(), - cmd.getGraphStoreConnInfo(), - jobInfo.getContent(), - jobWorker); - JobInstStatusEnum status = null; - BaseReasonerResult result = null; - ReasonerProgress progress = new ReasonerProgress(); - if (StringUtils.isNotBlank(localReasonerResult.getErrMsg())) { - status = JobInstStatusEnum.FAILURE; - result = new FailureReasonerResult(localReasonerResult.getErrMsg()); - } else { - status = JobInstStatusEnum.SUCCESS; - result = writeResult2TableFile(cmd, localReasonerResult); - } - return new ReasonerStatusWithProgress(status, result, progress); - } catch (Throwable e) { - throw ReasonerException.reasonerError(e); - } - }); - } - - private LocalReasonerResult doRun( - Long projectId, - String schemaUrl, - GraphStoreConnectionInfo connInfo, - BaseReasonerContent content, - ThreadPoolExecutor executor) { - KGReasonerLocalRunner localRunner = new KGReasonerLocalRunner(); - LocalReasonerTask reasonerTask = new LocalReasonerTask(); - reasonerTask.setThreadPoolExecutor(executor); - reasonerTask.setCatalog( - LocalRunnerUtils.buildCatalog(projectId, new KgSchemaConnectionInfo(schemaUrl, ""))); - reasonerTask.setGraphState(LocalRunnerUtils.buildGraphState(connInfo)); - reasonerTask.setDsl(((KgdslReasonerContent) content).getKgdsl()); - return localRunner.run(reasonerTask); - } - - private SuccessReasonerResult writeResult2TableFile( - ReasonerJobSubmitCmd cmd, LocalReasonerResult result) throws Exception { - TableStoreClient tableStoreClient = - TableStoreClientDriverManager.getClient(cmd.getTableStoreConnInfo()); - - if (CollectionUtils.isEmpty(result.getColumns())) { - return new SuccessReasonerResult(null); - } - String tableName = cmd.tableName(); - TableFileHandler fileHandler = - tableStoreClient.create( - new TableFileCreateCmd( - tableName, - result.getColumns().stream().map(ColumnMeta::new).toArray(ColumnMeta[]::new))); - - fileHandler.batchWrite( - result.getRows().stream().map(TableRecord::new).collect(Collectors.toList())); - fileHandler.close(); - return new SuccessReasonerResult(fileHandler.getTableName()); - } - - private TableReasonerReceipt buildReceipt(LocalReasonerResult reasonerResult) { - if (StringUtils.isNotBlank(reasonerResult.getErrMsg())) { - throw ReasonerException.reasonerError(reasonerResult.getErrMsg()); - } - return new TableReasonerReceipt( - reasonerResult.getColumns(), - reasonerResult.getRows().stream().map(Arrays::asList).collect(Collectors.toList())); - } -} diff --git a/server/cloudext/impl/job-scheduler/local/pom.xml b/server/cloudext/impl/job-scheduler/local/pom.xml deleted file mode 100644 index 567ac994..00000000 --- a/server/cloudext/impl/job-scheduler/local/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../../../pom.xml - - - cloudext-impl-job-scheduler-local - - - - com.antgroup.openspg.server - cloudext-interface-job-scheduler - - - com.antgroup.openspg.server - common-service - - - diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClient.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClient.java deleted file mode 100644 index 1bae870e..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClient.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.scheduler.JobExecuteScheduler; -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInfoService; -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInstService; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.SchedulerCallback; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo; -import com.antgroup.openspg.server.common.service.spring.SpringContextHolder; -import java.util.List; -import lombok.Getter; -import lombok.Setter; - -public class LocalJobSchedulerClient implements JobSchedulerClient { - - @Getter private final JobSchedulerConnectionInfo connInfo; - @Getter private final List schedulerCallbacks; - - @Setter private SchedulerJobInfoService jobInfoService; - @Setter private SchedulerJobInstService jobInstService; - - public LocalJobSchedulerClient(JobSchedulerConnectionInfo connInfo) { - this.connInfo = connInfo; - jobInfoService = SpringContextHolder.getBean(SchedulerJobInfoService.class); - jobInstService = SpringContextHolder.getBean(SchedulerJobInstService.class); - schedulerCallbacks = SpringContextHolder.getBeans(SchedulerCallback.class); - new JobExecuteScheduler(jobInstService, schedulerCallbacks).init(); - } - - @Override - public String createJobInfo(SchedulerJobInfo jobInfo) { - return jobInfoService.create(jobInfo); - } - - @Override - public String createJobInst(SchedulerJobInst jobInst) { - return jobInstService.create(jobInst); - } - - @Override - public void close() throws Exception {} -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClientDriver.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClientDriver.java deleted file mode 100644 index 11590b51..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClientDriver.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriver; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriverManager; -import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo; - -public class LocalJobSchedulerClientDriver implements JobSchedulerClientDriver { - - private static final JobSchedulerClient INSTANCE = - new LocalJobSchedulerClient(new JobSchedulerConnectionInfo().setScheme("local")); - - static { - JobSchedulerClientDriverManager.registerDriver(new LocalJobSchedulerClientDriver()); - } - - @Override - public String driverScheme() { - return "local"; - } - - @Override - public JobSchedulerClient connect(JobSchedulerConnectionInfo connInfo) { - return INSTANCE; - } -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInfoRepository.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInfoRepository.java deleted file mode 100644 index cf06b17e..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInfoRepository.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; - -public interface SchedulerJobInfoRepository { - - String save(SchedulerJobInfo jobInfo); -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInstRepository.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInstRepository.java deleted file mode 100644 index bef5f1d9..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInstRepository.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd.SchedulerJobInstQuery; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import java.util.List; - -public interface SchedulerJobInstRepository { - - String save(SchedulerJobInst jobInst); - - List query(SchedulerJobInstQuery query); - - int updateStatus(String jobInstId, JobInstStatusEnum status); -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/JobExecuteScheduler.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/JobExecuteScheduler.java deleted file mode 100644 index 430e7be6..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/JobExecuteScheduler.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.scheduler; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInstService; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.SchedulerCallback; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.CallbackResult; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.common.util.thread.SPGThread; -import com.antgroup.openspg.common.util.thread.ThreadUtils; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; - -@Slf4j -public class JobExecuteScheduler extends BaseScheduler { - - private final SchedulerJobInstService schedulerJobInstService; - private final Map schedulerCallbacks; - - public JobExecuteScheduler( - SchedulerJobInstService schedulerJobInstService, List schedulerCallbacks) { - this.schedulerJobInstService = schedulerJobInstService; - - this.schedulerCallbacks = new HashMap<>(JobTypeEnum.values().length); - for (SchedulerCallback schedulerCallback : schedulerCallbacks) { - for (JobTypeEnum jobType : schedulerCallback.accept()) { - this.schedulerCallbacks.put(jobType.name(), schedulerCallback); - } - } - } - - public void init() { - log.info("init JobExecuteScheduler..."); - new SPGThread( - "JobExecuteScheduler", - () -> { - while (true) { - try { - doExecute(); - ThreadUtils.sleep(3000); - } catch (Throwable e) { - log.warn("JobExecuteScheduler execute fail", e); - } - } - }) - .start(); - } - - private void doExecute() { - List runningJobInsts = schedulerJobInstService.queryRunningJobInsts(); - if (CollectionUtils.isNotEmpty(runningJobInsts)) { - processJobInsts(runningJobInsts); - return; - } - - List queuedJobInsts = schedulerJobInstService.queryToRunJobInsts(); - if (CollectionUtils.isNotEmpty(queuedJobInsts)) { - processJobInsts(queuedJobInsts); - } - } - - private void processJobInsts(List jobInsts) { - for (SchedulerJobInst jobInst : jobInsts) { - log.info( - "polling jobType={} schedulerJobInstId={}", jobInst.getJobType(), jobInst.getJobInstId()); - - SchedulerCallback callback = schedulerCallbacks.get(jobInst.getJobType()); - if (callback == null) { - updateStatus(jobInst, JobInstStatusEnum.FAILURE); - continue; - } - CallbackResult result = callback.polling(jobInst); - JobInstStatusEnum newStatus = result.getStatus(); - updateStatus(jobInst, newStatus); - } - } - - private void updateStatus(SchedulerJobInst jobInst, JobInstStatusEnum newStatus) { - if (!newStatus.equals(jobInst.getStatus())) { - log.info( - "update schedulerJobInstId={} from status={} to status={}", - jobInst.getJobInstId(), - jobInst.getStatus(), - newStatus); - schedulerJobInstService.updateStatus(jobInst.getJobInstId(), newStatus); - } - } -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInfoService.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInfoService.java deleted file mode 100644 index 7d23eb38..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInfoService.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; - -public interface SchedulerJobInfoService { - - String create(SchedulerJobInfo jobInfo); -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInstService.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInstService.java deleted file mode 100644 index 04c3f3d5..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInstService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import java.util.List; - -public interface SchedulerJobInstService { - - String create(SchedulerJobInst jobInst); - - List queryRunningJobInsts(); - - List queryToRunJobInsts(); - - void updateStatus(String jobInstId, JobInstStatusEnum status); -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInfoServiceImpl.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInfoServiceImpl.java deleted file mode 100644 index e2115b9d..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInfoServiceImpl.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.impl; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo.SchedulerJobInfoRepository; -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInfoService; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class SchedulerJobInfoServiceImpl implements SchedulerJobInfoService { - - @Autowired private SchedulerJobInfoRepository jobInfoRepository; - - @Override - public String create(SchedulerJobInfo jobInfo) { - return jobInfoRepository.save(jobInfo); - } -} diff --git a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInstServiceImpl.java b/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInstServiceImpl.java deleted file mode 100644 index cc54947c..00000000 --- a/server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInstServiceImpl.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.impl; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd.SchedulerJobInstQuery; -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo.SchedulerJobInstRepository; -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInstService; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.google.common.collect.Sets; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class SchedulerJobInstServiceImpl implements SchedulerJobInstService { - - @Autowired private SchedulerJobInstRepository schedulerJobInstRepository; - - @Override - public String create(SchedulerJobInst jobInst) { - return schedulerJobInstRepository.save(jobInst); - } - - @Override - public List queryRunningJobInsts() { - SchedulerJobInstQuery jobInstQuery = - new SchedulerJobInstQuery() - .setStatus(Sets.newHashSet(JobInstStatusEnum.RUNNING_STATUS)) - .setOrderBy("id asc"); - return schedulerJobInstRepository.query(jobInstQuery); - } - - @Override - public List queryToRunJobInsts() { - SchedulerJobInstQuery jobInstQuery = - new SchedulerJobInstQuery() - .setStatus(Sets.newHashSet(JobInstStatusEnum.QUEUE, JobInstStatusEnum.INIT)) - .setOrderBy("id asc"); - return schedulerJobInstRepository.query(jobInstQuery); - } - - @Override - public void updateStatus(String jobInstId, JobInstStatusEnum status) { - schedulerJobInstRepository.updateStatus(jobInstId, status); - } -} diff --git a/server/cloudext/interface/computing/pom.xml b/server/cloudext/interface/computing/pom.xml deleted file mode 100644 index 799b5f23..00000000 --- a/server/cloudext/interface/computing/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../../pom.xml - - - cloudext-interface-computing - - - - com.antgroup.openspg - common-util - - - com.antgroup.openspg.server - core-reasoner-model - - - diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClient.java b/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClient.java deleted file mode 100644 index 0c7720a6..00000000 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.computing; - -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd; -import com.antgroup.openspg.common.util.cloudext.CloudExtClient; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt; - -public interface ComputingClient extends CloudExtClient { - - /* ----------------------- * - | Reasoner Job | - * ----------------------- */ - - ReasonerStatusWithProgress query(ReasonerJobProcessQuery query); - - boolean canSubmit(ReasonerJobCanSubmitQuery query); - - String submit(ReasonerJobSubmitCmd cmd); - - TableReasonerReceipt run(ReasonerJobRunCmd cmd); -} diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriver.java b/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriver.java deleted file mode 100644 index e591affe..00000000 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriver.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.computing; - -import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver; -import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo; - -public interface ComputingClientDriver - extends CloudExtClientDriver {} diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriverManager.java b/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriverManager.java deleted file mode 100644 index dfa56ce3..00000000 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriverManager.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.computing; - -import com.antgroup.openspg.common.util.DriverManagerUtils; -import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo; -import com.antgroup.openspg.server.common.model.exception.CloudExtException; -import java.util.concurrent.CopyOnWriteArrayList; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ComputingClientDriverManager { - - private static final CopyOnWriteArrayList registeredDrivers = - new CopyOnWriteArrayList<>(); - - private ComputingClientDriverManager() {} - - static { - DriverManagerUtils.loadDrivers("cloudext.computing.drivers", ComputingClientDriver.class); - log.info("computing DriverManager initialized"); - } - - public static synchronized void registerDriver(ComputingClientDriver driver) { - if (driver != null) { - registeredDrivers.addIfAbsent(driver); - } else { - throw new NullPointerException(); - } - log.info("registerDriver: {}", driver); - } - - public static ComputingClient getClient(ComputingConnectionInfo config) { - for (ComputingClientDriver driver : registeredDrivers) { - if (driver.acceptsConfig(config)) { - return driver.connect(config); - } - } - throw CloudExtException.driverNotExist(config); - } -} diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobProcessQuery.java b/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobProcessQuery.java deleted file mode 100644 index 7ba73739..00000000 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobProcessQuery.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.computing.cmd; - -import com.antgroup.openspg.server.common.model.base.BaseQuery; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public class ReasonerJobProcessQuery extends BaseQuery { - - private final String computingJobInstId; -} diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobRunCmd.java b/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobRunCmd.java deleted file mode 100644 index 113e3a53..00000000 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobRunCmd.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.computing.cmd; - -import com.antgroup.openspg.server.common.model.base.BaseCmd; -import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public class ReasonerJobRunCmd extends BaseCmd { - - private final Long projectId; - - private final String schemaUrl; - - private final GraphStoreConnectionInfo connInfo; - - private final BaseReasonerContent content; -} diff --git a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobSubmitCmd.java b/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobSubmitCmd.java deleted file mode 100644 index eb89cdea..00000000 --- a/server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobSubmitCmd.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.computing.cmd; - -import com.antgroup.openspg.server.common.model.base.BaseCmd; -import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import java.util.Map; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public class ReasonerJobSubmitCmd extends BaseCmd { - - private final ReasonerJobInst jobInst; - - private final ReasonerJobInfo jobInfo; - - private final GraphStoreConnectionInfo graphStoreConnInfo; - - private final TableStoreConnectionInfo tableStoreConnInfo; - - private final String schemaUrl; - - private final Map params; - - public String tableName() { - return String.format("spgreasoner_%s_%s_result", jobInfo.getJobName(), jobInst.getJobInstId()); - } -} diff --git a/server/cloudext/interface/computing/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriver b/server/cloudext/interface/computing/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriver deleted file mode 100644 index 98a48d4a..00000000 --- a/server/cloudext/interface/computing/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriver +++ /dev/null @@ -1 +0,0 @@ -com.antgroup.openspg.cloudext.impl.computing.local.LocalComputingClientDriver diff --git a/server/cloudext/interface/job-scheduler/pom.xml b/server/cloudext/interface/job-scheduler/pom.xml deleted file mode 100644 index b08bd31a..00000000 --- a/server/cloudext/interface/job-scheduler/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../../pom.xml - - - cloudext-interface-job-scheduler - - - - com.antgroup.openspg - common-util - - - diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClient.java b/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClient.java deleted file mode 100644 index e6ef0b07..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClient.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.jobscheduler; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.common.util.cloudext.CloudExtClient; - -public interface JobSchedulerClient extends CloudExtClient { - - String createJobInfo(SchedulerJobInfo jobInfo); - - String createJobInst(SchedulerJobInst jobInst); - - void close() throws Exception; -} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriver.java b/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriver.java deleted file mode 100644 index 00e48c8c..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriver.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.jobscheduler; - -import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver; -import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo; - -public interface JobSchedulerClientDriver - extends CloudExtClientDriver {} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriverManager.java b/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriverManager.java deleted file mode 100644 index b6c13f9e..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriverManager.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.jobscheduler; - -import com.antgroup.openspg.common.util.DriverManagerUtils; -import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo; -import com.antgroup.openspg.server.common.model.exception.CloudExtException; -import java.util.concurrent.CopyOnWriteArrayList; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class JobSchedulerClientDriverManager { - - private static final CopyOnWriteArrayList registeredDrivers = - new CopyOnWriteArrayList<>(); - - private JobSchedulerClientDriverManager() {} - - static { - DriverManagerUtils.loadDrivers("cloudext.jobscheduler.drivers", JobSchedulerClientDriver.class); - log.info("job-scheduler DriverManager initialized"); - } - - public static synchronized void registerDriver(JobSchedulerClientDriver driver) { - if (driver != null) { - registeredDrivers.addIfAbsent(driver); - } else { - throw new NullPointerException(); - } - log.info("registerDriver: {}", driver); - } - - public static JobSchedulerClient getClient(JobSchedulerConnectionInfo config) { - for (JobSchedulerClientDriver driver : registeredDrivers) { - if (driver.acceptsConfig(config)) { - return driver.connect(config); - } - } - throw CloudExtException.driverNotExist(config); - } -} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/SchedulerCallback.java b/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/SchedulerCallback.java deleted file mode 100644 index 5431535c..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/SchedulerCallback.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.jobscheduler; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.CallbackResult; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import java.util.Set; - -public interface SchedulerCallback { - - Set accept(); - - CallbackResult polling(SchedulerJobInst jobInst); -} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInfo.java b/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInfo.java deleted file mode 100644 index 30e7b5e8..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInfo.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model; - -import com.antgroup.openspg.server.common.model.base.BaseValObj; -import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; - -@Getter -@AllArgsConstructor -public class SchedulerJobInfo extends BaseValObj { - - @Setter private String jobId; - - private final String jobName; - - private final String jobType; - - private final String cron; - - private final JobInfoStateEnum status; - - private final String idempotentId; -} diff --git a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInst.java b/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInst.java deleted file mode 100644 index a74c207b..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInst.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model; - -import com.antgroup.openspg.server.common.model.base.BaseValObj; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public class SchedulerJobInst extends BaseValObj { - - private final String jobInstId; - - private final String jobId; - - private final String jobType; - - private final JobInstStatusEnum status; - - private final String host; - - private final String idempotentId; -} diff --git a/server/cloudext/interface/job-scheduler/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriver b/server/cloudext/interface/job-scheduler/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriver deleted file mode 100644 index c2bc6592..00000000 --- a/server/cloudext/interface/job-scheduler/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriver +++ /dev/null @@ -1 +0,0 @@ -com.antgroup.openspg.cloudext.impl.jobscheduler.local.LocalJobSchedulerClientDriver diff --git a/server/common/service/pom.xml b/server/common/service/pom.xml index 79f86148..2c7642f1 100644 --- a/server/common/service/pom.xml +++ b/server/common/service/pom.xml @@ -36,26 +36,10 @@ com.antgroup.openspg.cloudext cloudext-interface-graph-store - - com.antgroup.openspg.cloudext - cloudext-interface-object-store - com.antgroup.openspg.cloudext cloudext-interface-search-engine - - com.antgroup.openspg.server - cloudext-interface-job-scheduler - - - com.antgroup.openspg.server - cloudext-interface-computing - - - com.antgroup.openspg.cloudext - cloudext-interface-table-store - org.springframework spring-beans diff --git a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/datasource/DataSourceService.java b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/datasource/DataSourceService.java index 8dacf93f..af724002 100644 --- a/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/datasource/DataSourceService.java +++ b/server/common/service/src/main/java/com/antgroup/openspg/server/common/service/datasource/DataSourceService.java @@ -13,18 +13,10 @@ package com.antgroup.openspg.server.common.service.datasource; -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient; -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriverManager; import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClient; import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClientDriverManager; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriverManager; -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient; -import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriverManager; import com.antgroup.openspg.cloudext.interfaces.searchengine.SearchEngineClient; import com.antgroup.openspg.cloudext.interfaces.searchengine.SearchEngineClientDriverManager; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient; -import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriverManager; import com.antgroup.openspg.server.common.model.datasource.DataSource; import com.antgroup.openspg.server.common.model.datasource.DataSourceUsageTypeEnum; import com.antgroup.openspg.server.common.model.datasource.connection.*; @@ -44,43 +36,9 @@ public interface DataSourceService { (GraphStoreConnectionInfo) graphStore.getConnectionInfo()); } - default ObjectStoreClient buildSharedOperatorStoreClient() { - DataSource objectStore = - getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.OPERATOR_STORE); - return ObjectStoreClientDriverManager.getClient( - (ObjectStoreConnectionInfo) objectStore.getConnectionInfo()); - } - - default ObjectStoreClient buildSharedFileStoreClient() { - DataSource objectStore = - getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.FILE_STORE); - return ObjectStoreClientDriverManager.getClient( - (ObjectStoreConnectionInfo) objectStore.getConnectionInfo()); - } - default SearchEngineClient buildSharedSearchEngineClient() { DataSource searchEngine = getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.SEARCH); return SearchEngineClientDriverManager.getClient( (SearchEngineConnectionInfo) searchEngine.getConnectionInfo()); } - - default JobSchedulerClient buildSharedJobSchedulerClient() { - DataSource scheduler = - getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.JOB_SCHEDULER); - return JobSchedulerClientDriverManager.getClient( - (JobSchedulerConnectionInfo) scheduler.getConnectionInfo()); - } - - default ComputingClient buildSharedComputingClient() { - DataSource computing = getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.COMPUTING); - return ComputingClientDriverManager.getClient( - (ComputingConnectionInfo) computing.getConnectionInfo()); - } - - default TableStoreClient buildSharedTableStoreClient() { - DataSource computing = - getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.TABLE_STORE); - return TableStoreClientDriverManager.getClient( - (TableStoreConnectionInfo) computing.getConnectionInfo()); - } } diff --git a/server/core/reasoner/model/pom.xml b/server/core/reasoner/model/pom.xml deleted file mode 100644 index 5d74ea75..00000000 --- a/server/core/reasoner/model/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../../pom.xml - - - core-reasoner-model - - - com.antgroup.openspg.server - common-model - - - diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/ReasonerException.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/ReasonerException.java deleted file mode 100644 index c47e7e25..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/ReasonerException.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model; - -import com.antgroup.openspg.server.common.model.exception.OpenSPGException; - -public class ReasonerException extends OpenSPGException { - - private ReasonerException(Throwable cause, String messagePattern, Object... args) { - super(cause, true, true, messagePattern, args); - } - - private ReasonerException(String messagePattern, Object... args) { - this(null, messagePattern, args); - } - - public static ReasonerException reasonerError(Throwable e) { - return new ReasonerException(e, e.getMessage()); - } - - public static ReasonerException reasonerError(String errorMsg) { - return new ReasonerException(errorMsg); - } - - public static ReasonerException timeout(int minutes) { - return new ReasonerException( - "reasoner cannot finish in {} minutes, please submit reasoner job", minutes); - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerContent.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerContent.java deleted file mode 100644 index a4913117..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerContent.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.common.model.base.BaseToString; - -/** - * The base class of reasoning content, subclasses can inherit this base class to enrich the - * information under different reasoning content types - */ -public abstract class BaseReasonerContent extends BaseToString { - - /** reasoning content types */ - private final ReasonerContentTypeEnum paramType; - - public BaseReasonerContent(ReasonerContentTypeEnum paramType) { - this.paramType = paramType; - } - - public ReasonerContentTypeEnum getParamType() { - return paramType; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerReceipt.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerReceipt.java deleted file mode 100644 index 0a0cbb07..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/BaseReasonerReceipt.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.common.model.base.BaseToString; - -/** - * Base class for reasoning receipts. - * - *

Contains the reasoning mode and the reasoning receipt type for the current reasoning instance - */ -public abstract class BaseReasonerReceipt extends BaseToString { - - /** the reasoning receipt type */ - private final ReasonerReceiptTypeEnum receiptType; - - public BaseReasonerReceipt(ReasonerReceiptTypeEnum receiptType) { - this.receiptType = receiptType; - } - - public ReasonerReceiptTypeEnum getReceiptType() { - return receiptType; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/JobReasonerReceipt.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/JobReasonerReceipt.java deleted file mode 100644 index b4dc1a96..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/JobReasonerReceipt.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -public class JobReasonerReceipt extends BaseReasonerReceipt { - - private final Long reasonerJobInfoId; - - private final Long reasonerJobInstId; - - public JobReasonerReceipt(Long reasonerJobInfoId, Long reasonerJobInstId) { - super(ReasonerReceiptTypeEnum.JOB); - this.reasonerJobInfoId = reasonerJobInfoId; - this.reasonerJobInstId = reasonerJobInstId; - } - - public Long getReasonerJobInfoId() { - return reasonerJobInfoId; - } - - public Long getReasonerJobInstId() { - return reasonerJobInstId; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/KgdslReasonerContent.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/KgdslReasonerContent.java deleted file mode 100644 index 4697f77d..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/KgdslReasonerContent.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.core.reasoner.model.struct.StartingVertex; -import java.util.Collections; -import java.util.List; - -/** - * Reasoning based on KGDSL. This class contains a KGDSL script and possible starting vertices. When - * there is a starting vertex, the task starts from that starting point for reasoning. If there is - * no starting vertex, the task will trigger reasoning from a batch of vertices of the same type. - */ -public class KgdslReasonerContent extends BaseReasonerContent { - - /** A short reasoning script */ - private final String kgdsl; - - /** - * starting vertices for reasoning, may be empty. - * - *

If it is empty, it usually consumes more resources, it is recommended to run in cluster - * mode. - */ - private final List startingVertices; - - public KgdslReasonerContent(String kgdsl, List startingVertices) { - super(ReasonerContentTypeEnum.KGDSL); - this.kgdsl = kgdsl; - this.startingVertices = startingVertices; - } - - public String getKgdsl() { - return kgdsl; - } - - public List getStartingVertices() { - if (startingVertices == null) { - return Collections.emptyList(); - } - return startingVertices; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerContentTypeEnum.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerContentTypeEnum.java deleted file mode 100644 index 4cf50ada..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerContentTypeEnum.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -/** - * This class defines the input content types of knowledge reasoning tasks. - * - *

In actual scenarios, knowledge reasoning can be based on vertices or edges (when there are - * attributes expressed by logical rules). In addition, the most direct reasoning method is to - * reason based on a piece of Kgdsl - */ -public enum ReasonerContentTypeEnum { - /** - * Vertex-based query or reasoning. The user passes in the id of a vertex. If some attributes of - * the vertex are defined by logical rules, the logical rules will be calculated first and then - * returned; If all the attributes of the vertex are defined by non-logical rules, the vertex’s - * attributes can be returned directly. - */ - VERTEX, - - /** - * KGDSL-based reasoning. The user passes in a piece of KGDSL, and the knowledge reasoning engine - * will run this piece of KGDSL based on the factual data to get the reasoning result. If this - * KGDSL refers to other KGDSL definitions, it will execute the dependent KGDSL first, and then - * run the current KGDSL like a Java method call; - */ - KGDSL, - ; -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerJobInfo.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerJobInfo.java deleted file mode 100644 index fc795678..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerJobInfo.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.common.model.base.BaseModel; -import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum; -import java.util.HashMap; -import java.util.Map; - -public class ReasonerJobInfo extends BaseModel { - - private Long jobId; - - private final String jobName; - - private final Long projectId; - - private final BaseReasonerContent content; - - private final String cron; - - private final JobInfoStateEnum status; - - private String externalJobInfoId; - - private final Map params; - - public ReasonerJobInfo( - String jobName, - Long projectId, - BaseReasonerContent content, - String cron, - JobInfoStateEnum status, - Map params) { - this.jobName = jobName; - this.projectId = projectId; - this.content = content; - this.cron = cron; - this.status = status; - this.params = params == null ? new HashMap<>(5) : params; - } - - public Long getJobId() { - return jobId; - } - - public ReasonerJobInfo setJobId(Long jobId) { - this.jobId = jobId; - return this; - } - - public String getJobName() { - return jobName; - } - - public Long getProjectId() { - return projectId; - } - - public BaseReasonerContent getContent() { - return content; - } - - public String getCron() { - return cron; - } - - public JobInfoStateEnum getStatus() { - return status; - } - - public String getExternalJobInfoId() { - return externalJobInfoId; - } - - public ReasonerJobInfo setExternalJobInfoId(String externalJobInfoId) { - this.externalJobInfoId = externalJobInfoId; - return this; - } - - public Map getParams() { - return params; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerJobInst.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerJobInst.java deleted file mode 100644 index ea546607..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerJobInst.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.common.model.base.BaseModel; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import java.util.Date; - -public class ReasonerJobInst extends BaseModel { - - private final Long jobId; - private final Long projectId; - private final Date startTime; - private final Date endTime; - private final String logInfo; - private ReasonerProgress progress; - private BaseReasonerResult result; - private Long jobInstId; - private JobInstStatusEnum status; - private String externalJobInstId; - - public ReasonerJobInst( - Long jobId, - Long projectId, - JobInstStatusEnum status, - BaseReasonerResult result, - Date startTime, - Date endTime, - ReasonerProgress progress, - String logInfo) { - this.jobId = jobId; - this.projectId = projectId; - this.status = status; - this.startTime = startTime; - this.endTime = endTime; - this.result = result; - this.progress = progress; - this.logInfo = logInfo; - } - - public boolean isFinished() { - return status.isFinished(); - } - - public boolean isRunning() { - return status.isRunning(); - } - - public Long getJobInstId() { - return jobInstId; - } - - public ReasonerJobInst setJobInstId(Long jobInstId) { - this.jobInstId = jobInstId; - return this; - } - - public Long getJobId() { - return jobId; - } - - public Long getProjectId() { - return projectId; - } - - public JobInstStatusEnum getStatus() { - return status; - } - - public void setProgress(ReasonerStatusWithProgress progress) { - this.status = progress.getStatus(); - this.result = progress.getResult(); - this.progress = progress.getProgress(); - } - - public Date getStartTime() { - return startTime; - } - - public Date getEndTime() { - return endTime; - } - - public BaseReasonerResult getResult() { - return result; - } - - public ReasonerProgress getProgress() { - return progress; - } - - public String getLogInfo() { - return logInfo; - } - - public String getExternalJobInstId() { - return externalJobInstId; - } - - public ReasonerJobInst setExternalJobInstId(String externalJobInstId) { - this.externalJobInstId = externalJobInstId; - return this; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerReceiptTypeEnum.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerReceiptTypeEnum.java deleted file mode 100644 index c39384cb..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerReceiptTypeEnum.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -/** - * The type of receipt returned by the knowledge reasoning engine. - * - *

Different reasoning modes produce different reasoning receipts. When reasoning locally, the - * reasoning receipt is the reasoning result, which is usually expressed in the form of a table; in - * remote cluster reasoning, the reasoning receipt may be a UUID, used for front-end continuous - * polling to obtain reasoning results - */ -public enum ReasonerReceiptTypeEnum { - /** - * When the reasoning engine finally executes a segment of KGDSL, the result is carried in a table - * data structure; - */ - TABLE, - - /** - * However, when a reasoning task is sent to a remote cluster, the reasoning result returned by - * the inference service to the front-end may temporarily be a UUID, and the front-end will then - * obtain the real reasoning result from the reasoning service with the UUID, that is, a table - */ - JOB, - ; -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerStatusWithProgress.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerStatusWithProgress.java deleted file mode 100644 index 4673d4ad..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/ReasonerStatusWithProgress.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.common.model.base.BaseValObj; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; - -public class ReasonerStatusWithProgress extends BaseValObj { - - private final JobInstStatusEnum status; - - private final BaseReasonerResult result; - - private final ReasonerProgress progress; - - public ReasonerStatusWithProgress(JobInstStatusEnum status) { - this(status, null, null); - } - - public ReasonerStatusWithProgress( - JobInstStatusEnum status, BaseReasonerResult result, ReasonerProgress progress) { - this.status = status; - this.result = result; - this.progress = progress; - } - - public JobInstStatusEnum getStatus() { - return status; - } - - public BaseReasonerResult getResult() { - return result; - } - - public ReasonerProgress getProgress() { - return progress; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/TableReasonerReceipt.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/TableReasonerReceipt.java deleted file mode 100644 index 97ec378d..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/TableReasonerReceipt.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import java.util.List; - -public class TableReasonerReceipt extends BaseReasonerReceipt { - - // todo TableLpgRecordStruct - private final List columns; - - private final List> cells; - - public TableReasonerReceipt(List columns, List> cells) { - super(ReasonerReceiptTypeEnum.TABLE); - this.columns = columns; - this.cells = cells; - } - - public List getColumns() { - return columns; - } - - public List> getCells() { - return cells; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/VertexReasonerContent.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/VertexReasonerContent.java deleted file mode 100644 index 789ef2cf..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/service/VertexReasonerContent.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.service; - -import com.antgroup.openspg.server.core.reasoner.model.struct.StartingVertex; -import java.util.List; - -/** - * Query or reason about the vertices passed in. - * - *

When the attribute of a vertex is defined by non-logical rules, we call this attribute fact - * data, when the attribute of a vertex is defined by logical rules, we call this attribute data - * defined by derived rules. - * - *

When a vertex is passed in, we will perform reasoning calculations on the attributes defined - * by its logical rules, and query the attributes defined by its non-logical rules, and finally - * return them together - */ -public class VertexReasonerContent extends BaseReasonerContent { - - /** the vertices to reason about */ - private final List startingVertices; - - public VertexReasonerContent(List startingVertices) { - super(ReasonerContentTypeEnum.VERTEX); - this.startingVertices = startingVertices; - } - - public List getStartingVertices() { - return startingVertices; - } -} diff --git a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/struct/StartingVertex.java b/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/struct/StartingVertex.java deleted file mode 100644 index e8b59d0b..00000000 --- a/server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/struct/StartingVertex.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.model.struct; - -import com.antgroup.openspg.server.common.model.base.BaseToString; -import java.util.Set; - -public class StartingVertex extends BaseToString { - - private String typeName; - - private Set bizIds; - - public String getTypeName() { - return typeName; - } - - public void setTypeName(String typeName) { - this.typeName = typeName; - } - - public Set getBizIds() { - return bizIds; - } - - public void setBizIds(Set bizIds) { - this.bizIds = bizIds; - } -} diff --git a/server/core/reasoner/service/pom.xml b/server/core/reasoner/service/pom.xml deleted file mode 100644 index 3b86f0bb..00000000 --- a/server/core/reasoner/service/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - 4.0.0 - - com.antgroup.openspg.server - server-parent - 0.0.1-SNAPSHOT - ../../../pom.xml - - - core-reasoner-service - - - com.antgroup.openspg.server - common-service - - - com.antgroup.openspg.server - api-http-client - - - com.antgroup.openspg.server - core-reasoner-model - - - com.antgroup.openspg.server - cloudext-interface-job-scheduler - - - com.antgroup.kg.reasoner - local - ${kgreasoner-version} - system - ${project.basedir}/../../../lib/spgreasoner-local-0.0.1.jar - - - org.scala-lang - scala-compiler - - - org.scala-lang - scala-reflect - - - org.scala-lang - scala-library - - - org.springframework - spring-context - - - diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/ReasonerJobInfoService.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/ReasonerJobInfoService.java deleted file mode 100644 index 734af51a..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/ReasonerJobInfoService.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service; - -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInfoQuery; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import java.util.List; - -public interface ReasonerJobInfoService { - - Long create(ReasonerJobInfo reasonerJobInfo); - - ReasonerJobInfo queryById(Long jobId); - - List query(ReasonerJobInfoQuery query); -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/ReasonerJobInstService.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/ReasonerJobInstService.java deleted file mode 100644 index 686a7981..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/ReasonerJobInstService.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import java.util.List; - -public interface ReasonerJobInstService { - - Long create(ReasonerJobInfo reasonerJobInfo, ReasonerJobInst reasonerJobInst); - - List query(ReasonerJobInstQuery query); - - /* ----------------------- * - | For Scheduler | - * ----------------------- */ - - /** - * Triggered by the scheduling system to poll the status of the reasoner job, the main process - * includes: - * - *

1. If the job is in the final state, return directly; 2. If the job is running, query the - * computing pool to determine whether the task is completed; 3. If the job is in waiting state, - * try to submit the task; - * - * @param jobInst scheduling job instance - * @return reasoner job instance - */ - ReasonerJobInst pollingReasonerJob(SchedulerJobInst jobInst); - - ReasonerJobInst queryByExternalJobInstId(String externalJobInstId); - - int updateToFailure(Long jobInstId, FailureReasonerResult result); -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerJobInfoServiceImpl.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerJobInfoServiceImpl.java deleted file mode 100644 index cdd237ee..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerJobInfoServiceImpl.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service.impl; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInfoQuery; -import com.antgroup.openspg.server.common.service.datasource.DataSourceService; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInfoService; -import com.antgroup.openspg.server.core.reasoner.service.repo.ReasonerJobInfoRepository; -import java.util.List; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class ReasonerJobInfoServiceImpl implements ReasonerJobInfoService { - - @Autowired private DataSourceService dataSourceService; - - @Autowired private ReasonerJobInfoRepository reasonerJobInfoRepository; - - @Override - public Long create(ReasonerJobInfo reasonerJobInfo) { - JobSchedulerClient jobSchedulerClient = dataSourceService.buildSharedJobSchedulerClient(); - Long reasonerJobInfoId = reasonerJobInfoRepository.save(reasonerJobInfo); - - SchedulerJobInfo schedulerJobInfo = - new SchedulerJobInfo( - null, - reasonerJobInfo.getJobName(), - JobTypeEnum.REASONING.name(), - reasonerJobInfo.getCron(), - reasonerJobInfo.getStatus(), - String.valueOf(reasonerJobInfoId)); - String schedulerJobInfoId = jobSchedulerClient.createJobInfo(schedulerJobInfo); - - reasonerJobInfo.setExternalJobInfoId(schedulerJobInfoId); - reasonerJobInfoRepository.updateExternalJobId(reasonerJobInfoId, schedulerJobInfoId); - return reasonerJobInfoId; - } - - @Override - public ReasonerJobInfo queryById(Long jobId) { - List reasonerJobInfos = - query(new ReasonerJobInfoQuery().setReasonerJobInfoId(jobId)); - if (CollectionUtils.isNotEmpty(reasonerJobInfos)) { - return reasonerJobInfos.get(0); - } - return null; - } - - @Override - public List query(ReasonerJobInfoQuery query) { - return reasonerJobInfoRepository.query(query); - } -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerJobInstServiceImpl.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerJobInstServiceImpl.java deleted file mode 100644 index 9cc8b4a2..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerJobInstServiceImpl.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service.impl; - -import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery; -import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.common.service.config.AppEnvConfig; -import com.antgroup.openspg.server.common.service.datasource.DataSourceService; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInfoService; -import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInstService; -import com.antgroup.openspg.server.core.reasoner.service.repo.ReasonerJobInstRepository; -import java.util.HashMap; -import java.util.List; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class ReasonerJobInstServiceImpl implements ReasonerJobInstService { - - @Autowired private AppEnvConfig appEnvConfig; - - @Autowired private ReasonerJobInstRepository reasonerJobInstRepository; - - @Autowired private DataSourceService dataSourceService; - - @Autowired private ReasonerJobInfoService reasonerJobInfoService; - - @Override - public Long create(ReasonerJobInfo reasonerJobInfo, ReasonerJobInst reasonerJobInst) { - JobSchedulerClient jobSchedulerClient = dataSourceService.buildSharedJobSchedulerClient(); - Long buildingJobInstId = reasonerJobInstRepository.save(reasonerJobInst); - - SchedulerJobInst schedulerJobInst = - new SchedulerJobInst( - null, - reasonerJobInfo.getExternalJobInfoId(), - JobTypeEnum.REASONING.name(), - reasonerJobInst.getStatus(), - null, - String.valueOf(buildingJobInstId)); - String schedulerJobInstId = jobSchedulerClient.createJobInst(schedulerJobInst); - - reasonerJobInst.setExternalJobInstId(schedulerJobInstId); - reasonerJobInstRepository.updateExternalJobId(buildingJobInstId, schedulerJobInstId); - return buildingJobInstId; - } - - @Override - public List query(ReasonerJobInstQuery query) { - return reasonerJobInstRepository.query(query); - } - - @Override - public ReasonerJobInst pollingReasonerJob(SchedulerJobInst jobInst) { - ComputingClient computingClient = dataSourceService.buildSharedComputingClient(); - - ReasonerJobInst reasonerJobInst = queryByExternalJobInstId(jobInst.getJobInstId()); - if (reasonerJobInst.isFinished()) { - return reasonerJobInst; - } else if (reasonerJobInst.isRunning()) { - ReasonerStatusWithProgress progress = - computingClient.query( - new ReasonerJobProcessQuery(String.valueOf(reasonerJobInst.getJobInstId()))); - if (progress == null) { - // if status is null, rerun it - progress = new ReasonerStatusWithProgress(JobInstStatusEnum.QUEUE); - reasonerJobInst.setProgress(progress); - reasonerJobInstRepository.updateStatus(reasonerJobInst.getJobInstId(), progress); - return reasonerJobInst; - } else if (progress.getStatus().isRunning()) { - reasonerJobInstRepository.updateStatus(reasonerJobInst.getJobInstId(), progress); - return reasonerJobInst; - } else if (progress.getStatus().isFinished()) { - reasonerJobInst.setProgress(progress); - reasonerJobInstRepository.updateStatus(reasonerJobInst.getJobInstId(), progress); - return reasonerJobInst; - } - } - - // The task is not in finished or running status, try to submit the task - if (!computingClient.canSubmit(new ReasonerJobCanSubmitQuery())) { - return reasonerJobInst; - } - - ReasonerJobInfo reasonerJobInfo = reasonerJobInfoService.queryById(reasonerJobInst.getJobId()); - String submit = - computingClient.submit( - new ReasonerJobSubmitCmd( - reasonerJobInst, - reasonerJobInfo, - (GraphStoreConnectionInfo) - dataSourceService.buildSharedKgStoreClient().getConnInfo(), - (TableStoreConnectionInfo) - dataSourceService.buildSharedTableStoreClient().getConnInfo(), - appEnvConfig.getSchemaUri(), - new HashMap<>(5))); - if (submit != null) { - // The task is submitted successfully and the task is set to running status. - ReasonerStatusWithProgress progress = - new ReasonerStatusWithProgress(JobInstStatusEnum.RUNNING, null, null); - reasonerJobInstRepository.updateStatus(reasonerJobInst.getJobInstId(), progress); - return reasonerJobInst; - } - return reasonerJobInst; - } - - @Override - public ReasonerJobInst queryByExternalJobInstId(String externalJobInstId) { - ReasonerJobInstQuery query = new ReasonerJobInstQuery().setExternalJobInstId(externalJobInstId); - List jobInsts = reasonerJobInstRepository.query(query); - return CollectionUtils.isNotEmpty(jobInsts) ? jobInsts.get(0) : null; - } - - @Override - public int updateToFailure(Long jobInstId, FailureReasonerResult result) { - return reasonerJobInstRepository.updateToFailure(jobInstId, result); - } -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerSchedulerCallback.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerSchedulerCallback.java deleted file mode 100644 index 385b4d38..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/impl/ReasonerSchedulerCallback.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service.impl; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.SchedulerCallback; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.CallbackResult; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInstService; -import com.google.common.collect.Sets; -import java.util.Set; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class ReasonerSchedulerCallback implements SchedulerCallback { - - @Autowired private ReasonerJobInstService reasonerJobInstService; - - @Override - public Set accept() { - return Sets.newHashSet(JobTypeEnum.REASONING); - } - - @Override - public CallbackResult polling(SchedulerJobInst jobInst) { - ReasonerJobInst reasonerJobInst = null; - try { - reasonerJobInst = reasonerJobInstService.pollingReasonerJob(jobInst); - } catch (Throwable e) { - log.warn("polling schedulerJobInstId={} for reasoner error", jobInst.getJobInstId(), e); - reasonerJobInst = reasonerJobInstService.queryByExternalJobInstId(jobInst.getJobInstId()); - FailureReasonerResult result = new FailureReasonerResult(e.getMessage()); - if (reasonerJobInst != null) { - reasonerJobInstService.updateToFailure(reasonerJobInst.getJobInstId(), result); - } - return new CallbackResult(JobInstStatusEnum.FAILURE, result); - } - return new CallbackResult(reasonerJobInst.getStatus(), reasonerJobInst.getResult()); - } -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/repo/ReasonerJobInfoRepository.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/repo/ReasonerJobInfoRepository.java deleted file mode 100644 index 83bdb527..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/repo/ReasonerJobInfoRepository.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service.repo; - -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInfoQuery; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import java.util.List; - -public interface ReasonerJobInfoRepository { - - Long save(ReasonerJobInfo jobInfo); - - int updateExternalJobId(Long reasonerJobInfoId, String externalJobInfoId); - - List query(ReasonerJobInfoQuery query); -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/repo/ReasonerJobInstRepository.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/repo/ReasonerJobInstRepository.java deleted file mode 100644 index 058c0ef2..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/repo/ReasonerJobInstRepository.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service.repo; - -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import java.util.List; - -public interface ReasonerJobInstRepository { - - Long save(ReasonerJobInst jobInst); - - int updateExternalJobId(Long reasonerJobInstId, String externalJobInstId); - - List query(ReasonerJobInstQuery query); - - int updateStatus(Long jobInstId, ReasonerStatusWithProgress process); - - int updateToFailure(Long jobInstId, FailureReasonerResult result); -} diff --git a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/util/LocalRunnerUtils.java b/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/util/LocalRunnerUtils.java deleted file mode 100644 index 6ce6b47b..00000000 --- a/server/core/reasoner/service/src/main/java/com/antgroup/openspg/server/core/reasoner/service/util/LocalRunnerUtils.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.core.reasoner.service.util; - -import com.antgroup.kg.reasoner.catalog.impl.KgSchemaConnectionInfo; -import com.antgroup.kg.reasoner.catalog.impl.OpenKgCatalog; -import com.antgroup.kg.reasoner.common.graph.vertex.IVertexId; -import com.antgroup.kg.reasoner.graphstate.GraphState; -import com.antgroup.kg.reasoner.graphstate.impl.CloudExtGraphState; -import com.antgroup.kg.reasoner.lube.catalog.Catalog; -import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -@SuppressWarnings({"unchecked", "rawtypes"}) -public class LocalRunnerUtils { - - public static Catalog buildCatalog(Long projectId, KgSchemaConnectionInfo connInfo) { - Catalog catalog = new OpenKgCatalog(projectId, connInfo, null); - catalog.init(); - return catalog; - } - - public static GraphState buildGraphState(GraphStoreConnectionInfo connInfo) { - CloudExtGraphState cloudExtGraphState = new CloudExtGraphState(); - - Map params = new HashMap<>(); - params.put("cloudext.graphstore.schema", connInfo.getScheme()); - params.putAll(connInfo.getParams()); - cloudExtGraphState.init((Map) Collections.unmodifiableMap(params)); - return cloudExtGraphState; - } -} diff --git a/server/core/schema/service/pom.xml b/server/core/schema/service/pom.xml index ab64766a..69948358 100644 --- a/server/core/schema/service/pom.xml +++ b/server/core/schema/service/pom.xml @@ -24,6 +24,10 @@ core-schema-service + + com.antgroup.openspg.reasoner + reasoner-local-runner + com.antgroup.openspg.server core-schema-model @@ -36,33 +40,10 @@ com.antgroup.openspg.cloudext cloudext-impl-graph-store-tugraph - - com.antgroup.openspg.cloudext - cloudext-impl-object-store-local - - - com.antgroup.openspg.cloudext - cloudext-impl-table-store-local - com.antgroup.openspg.cloudext cloudext-impl-search-engine-elasticsearch - - com.antgroup.openspg.server - cloudext-impl-job-scheduler-local - - - com.antgroup.openspg.server - cloudext-impl-computing-local - - - com.antgroup.kg.reasoner - local - ${kgreasoner-version} - system - ${project.basedir}/../../../lib/spgreasoner-local-0.0.1.jar - org.scala-lang scala-library diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/PropertyChecker.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/PropertyChecker.java index fcd6ed90..613aa790 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/PropertyChecker.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/PropertyChecker.java @@ -13,13 +13,6 @@ package com.antgroup.openspg.server.core.schema.service.alter.check; -import com.antgroup.kg.reasoner.catalog.impl.OpenKgCatalog; -import com.antgroup.kg.reasoner.lube.block.Block; -import com.antgroup.kg.reasoner.lube.catalog.Catalog; -import com.antgroup.kg.reasoner.lube.logical.planning.LogicalPlannerContext; -import com.antgroup.kg.reasoner.lube.logical.validate.Validator; -import com.antgroup.kg.reasoner.lube.parser.ParserInterface; -import com.antgroup.kg.reasoner.parser.KgDslParser; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.DslSyntaxError; import com.antgroup.openspg.core.schema.model.SchemaConstants; @@ -28,6 +21,13 @@ import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.predicate.Property; import com.antgroup.openspg.core.schema.model.type.*; import com.antgroup.openspg.core.schema.model.type.BasicType.TextBasicType; +import com.antgroup.openspg.reasoner.catalog.impl.OpenSPGCatalog; +import com.antgroup.openspg.reasoner.lube.block.Block; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; +import com.antgroup.openspg.reasoner.lube.logical.planning.LogicalPlannerContext; +import com.antgroup.openspg.reasoner.lube.logical.validate.Validator; +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface; +import com.antgroup.openspg.reasoner.parser.KgDslParser; import com.antgroup.openspg.server.core.schema.service.type.model.BuiltInPropertyEnum; import java.util.*; import java.util.regex.Pattern; @@ -165,7 +165,7 @@ public class PropertyChecker { protected Catalog buildCatalog(SchemaCheckContext context) { List spgTypes = context.getMergedSchema(); ProjectSchema projectSchema = new ProjectSchema(spgTypes); - Catalog catalog = new OpenKgCatalog(context.getProjectId(), null, projectSchema); + Catalog catalog = new OpenSPGCatalog(context.getProjectId(), null, projectSchema); catalog.init(); return catalog; } diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/RelationChecker.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/RelationChecker.java index bebdda57..9b397620 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/RelationChecker.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/RelationChecker.java @@ -13,11 +13,11 @@ package com.antgroup.openspg.server.core.schema.service.alter.check; -import com.antgroup.kg.reasoner.lube.catalog.Catalog; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.predicate.Relation; import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType; +import com.antgroup.openspg.reasoner.lube.catalog.Catalog; import java.util.ArrayList; import java.util.HashSet; import java.util.List; diff --git a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/semantic/impl/LogicalRuleServiceImpl.java b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/semantic/impl/LogicalRuleServiceImpl.java index 487e5c1e..e1bc886a 100644 --- a/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/semantic/impl/LogicalRuleServiceImpl.java +++ b/server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/semantic/impl/LogicalRuleServiceImpl.java @@ -13,12 +13,12 @@ package com.antgroup.openspg.server.core.schema.service.semantic.impl; -import com.antgroup.kg.reasoner.lube.parser.ParserInterface; -import com.antgroup.kg.reasoner.parser.KgDslParser; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.DslSyntaxError; import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; import com.antgroup.openspg.core.schema.model.semantic.RuleCode; +import com.antgroup.openspg.reasoner.lube.parser.ParserInterface; +import com.antgroup.openspg.reasoner.parser.KgDslParser; import com.antgroup.openspg.server.core.schema.service.semantic.LogicalRuleService; import com.antgroup.openspg.server.core.schema.service.semantic.model.DslCheckResult; import com.antgroup.openspg.server.core.schema.service.semantic.repository.LogicalRuleRepository; diff --git a/server/infra/dao/pom.xml b/server/infra/dao/pom.xml index 0645d8b5..466791a8 100644 --- a/server/infra/dao/pom.xml +++ b/server/infra/dao/pom.xml @@ -36,14 +36,6 @@ com.antgroup.openspg.server common-service - - com.antgroup.openspg.server - core-reasoner-service - - - com.antgroup.openspg.server - cloudext-impl-job-scheduler-local - org.springframework spring-context diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInfoDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInfoDO.java deleted file mode 100644 index a8e672e2..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInfoDO.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.Date; - -public class JobInfoDO { - private Long id; - - private Date gmtCreate; - - private Date gmtModified; - - private String name; - - private String type; - - private String cron; - - private String status; - - private String idempotentId; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type == null ? null : type.trim(); - } - - public String getCron() { - return cron; - } - - public void setCron(String cron) { - this.cron = cron == null ? null : cron.trim(); - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status == null ? null : status.trim(); - } - - public String getIdempotentId() { - return idempotentId; - } - - public void setIdempotentId(String idempotentId) { - this.idempotentId = idempotentId == null ? null : idempotentId.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInfoDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInfoDOExample.java deleted file mode 100644 index d0f0ca59..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInfoDOExample.java +++ /dev/null @@ -1,744 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class JobInfoDOExample { - protected String orderByClause; - - protected boolean distinct; - - protected List oredCriteria; - - public JobInfoDOExample() { - oredCriteria = new ArrayList(); - } - - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - public String getOrderByClause() { - return orderByClause; - } - - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - public boolean isDistinct() { - return distinct; - } - - public List getOredCriteria() { - return oredCriteria; - } - - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNull() { - addCriterion("gmt_create is null"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNotNull() { - addCriterion("gmt_create is not null"); - return (Criteria) this; - } - - public Criteria andGmtCreateEqualTo(Date value) { - addCriterion("gmt_create =", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotEqualTo(Date value) { - addCriterion("gmt_create <>", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThan(Date value) { - addCriterion("gmt_create >", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_create >=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThan(Date value) { - addCriterion("gmt_create <", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThanOrEqualTo(Date value) { - addCriterion("gmt_create <=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateIn(List values) { - addCriterion("gmt_create in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotIn(List values) { - addCriterion("gmt_create not in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateBetween(Date value1, Date value2) { - addCriterion("gmt_create between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotBetween(Date value1, Date value2) { - addCriterion("gmt_create not between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNull() { - addCriterion("gmt_modified is null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNotNull() { - addCriterion("gmt_modified is not null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedEqualTo(Date value) { - addCriterion("gmt_modified =", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotEqualTo(Date value) { - addCriterion("gmt_modified <>", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThan(Date value) { - addCriterion("gmt_modified >", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_modified >=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThan(Date value) { - addCriterion("gmt_modified <", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThanOrEqualTo(Date value) { - addCriterion("gmt_modified <=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIn(List values) { - addCriterion("gmt_modified in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotIn(List values) { - addCriterion("gmt_modified not in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedBetween(Date value1, Date value2) { - addCriterion("gmt_modified between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotBetween(Date value1, Date value2) { - addCriterion("gmt_modified not between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andCronIsNull() { - addCriterion("cron is null"); - return (Criteria) this; - } - - public Criteria andCronIsNotNull() { - addCriterion("cron is not null"); - return (Criteria) this; - } - - public Criteria andCronEqualTo(String value) { - addCriterion("cron =", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotEqualTo(String value) { - addCriterion("cron <>", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronGreaterThan(String value) { - addCriterion("cron >", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronGreaterThanOrEqualTo(String value) { - addCriterion("cron >=", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronLessThan(String value) { - addCriterion("cron <", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronLessThanOrEqualTo(String value) { - addCriterion("cron <=", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronLike(String value) { - addCriterion("cron like", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotLike(String value) { - addCriterion("cron not like", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronIn(List values) { - addCriterion("cron in", values, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotIn(List values) { - addCriterion("cron not in", values, "cron"); - return (Criteria) this; - } - - public Criteria andCronBetween(String value1, String value2) { - addCriterion("cron between", value1, value2, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotBetween(String value1, String value2) { - addCriterion("cron not between", value1, value2, "cron"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(String value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(String value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(String value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(String value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(String value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(String value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLike(String value) { - addCriterion("status like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotLike(String value) { - addCriterion("status not like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(String value1, String value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(String value1, String value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andIdempotentIdIsNull() { - addCriterion("idempotent_id is null"); - return (Criteria) this; - } - - public Criteria andIdempotentIdIsNotNull() { - addCriterion("idempotent_id is not null"); - return (Criteria) this; - } - - public Criteria andIdempotentIdEqualTo(String value) { - addCriterion("idempotent_id =", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotEqualTo(String value) { - addCriterion("idempotent_id <>", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdGreaterThan(String value) { - addCriterion("idempotent_id >", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdGreaterThanOrEqualTo(String value) { - addCriterion("idempotent_id >=", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdLessThan(String value) { - addCriterion("idempotent_id <", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdLessThanOrEqualTo(String value) { - addCriterion("idempotent_id <=", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdLike(String value) { - addCriterion("idempotent_id like", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotLike(String value) { - addCriterion("idempotent_id not like", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdIn(List values) { - addCriterion("idempotent_id in", values, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotIn(List values) { - addCriterion("idempotent_id not in", values, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdBetween(String value1, String value2) { - addCriterion("idempotent_id between", value1, value2, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotBetween(String value1, String value2) { - addCriterion("idempotent_id not between", value1, value2, "idempotentId"); - return (Criteria) this; - } - } - - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInstDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInstDO.java deleted file mode 100644 index 5ec81b15..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInstDO.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.Date; - -public class JobInstDO { - private Long id; - - private Date gmtCreate; - - private Date gmtModified; - - private Long jobId; - - private String type; - - private String status; - - private String host; - - private String traceId; - - private String idempotentId; - - private String result; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public Long getJobId() { - return jobId; - } - - public void setJobId(Long jobId) { - this.jobId = jobId; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type == null ? null : type.trim(); - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status == null ? null : status.trim(); - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host == null ? null : host.trim(); - } - - public String getTraceId() { - return traceId; - } - - public void setTraceId(String traceId) { - this.traceId = traceId == null ? null : traceId.trim(); - } - - public String getIdempotentId() { - return idempotentId; - } - - public void setIdempotentId(String idempotentId) { - this.idempotentId = idempotentId == null ? null : idempotentId.trim(); - } - - public String getResult() { - return result; - } - - public void setResult(String result) { - this.result = result == null ? null : result.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInstDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInstDOExample.java deleted file mode 100644 index f6806aab..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/JobInstDOExample.java +++ /dev/null @@ -1,804 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class JobInstDOExample { - protected String orderByClause; - - protected boolean distinct; - - protected List oredCriteria; - - public JobInstDOExample() { - oredCriteria = new ArrayList(); - } - - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - public String getOrderByClause() { - return orderByClause; - } - - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - public boolean isDistinct() { - return distinct; - } - - public List getOredCriteria() { - return oredCriteria; - } - - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNull() { - addCriterion("gmt_create is null"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNotNull() { - addCriterion("gmt_create is not null"); - return (Criteria) this; - } - - public Criteria andGmtCreateEqualTo(Date value) { - addCriterion("gmt_create =", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotEqualTo(Date value) { - addCriterion("gmt_create <>", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThan(Date value) { - addCriterion("gmt_create >", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_create >=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThan(Date value) { - addCriterion("gmt_create <", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThanOrEqualTo(Date value) { - addCriterion("gmt_create <=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateIn(List values) { - addCriterion("gmt_create in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotIn(List values) { - addCriterion("gmt_create not in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateBetween(Date value1, Date value2) { - addCriterion("gmt_create between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotBetween(Date value1, Date value2) { - addCriterion("gmt_create not between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNull() { - addCriterion("gmt_modified is null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNotNull() { - addCriterion("gmt_modified is not null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedEqualTo(Date value) { - addCriterion("gmt_modified =", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotEqualTo(Date value) { - addCriterion("gmt_modified <>", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThan(Date value) { - addCriterion("gmt_modified >", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_modified >=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThan(Date value) { - addCriterion("gmt_modified <", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThanOrEqualTo(Date value) { - addCriterion("gmt_modified <=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIn(List values) { - addCriterion("gmt_modified in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotIn(List values) { - addCriterion("gmt_modified not in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedBetween(Date value1, Date value2) { - addCriterion("gmt_modified between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotBetween(Date value1, Date value2) { - addCriterion("gmt_modified not between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andJobIdIsNull() { - addCriterion("job_id is null"); - return (Criteria) this; - } - - public Criteria andJobIdIsNotNull() { - addCriterion("job_id is not null"); - return (Criteria) this; - } - - public Criteria andJobIdEqualTo(Long value) { - addCriterion("job_id =", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdNotEqualTo(Long value) { - addCriterion("job_id <>", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdGreaterThan(Long value) { - addCriterion("job_id >", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdGreaterThanOrEqualTo(Long value) { - addCriterion("job_id >=", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdLessThan(Long value) { - addCriterion("job_id <", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdLessThanOrEqualTo(Long value) { - addCriterion("job_id <=", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdIn(List values) { - addCriterion("job_id in", values, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdNotIn(List values) { - addCriterion("job_id not in", values, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdBetween(Long value1, Long value2) { - addCriterion("job_id between", value1, value2, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdNotBetween(Long value1, Long value2) { - addCriterion("job_id not between", value1, value2, "jobId"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(String value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(String value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(String value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(String value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(String value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(String value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLike(String value) { - addCriterion("status like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotLike(String value) { - addCriterion("status not like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(String value1, String value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(String value1, String value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andHostIsNull() { - addCriterion("host is null"); - return (Criteria) this; - } - - public Criteria andHostIsNotNull() { - addCriterion("host is not null"); - return (Criteria) this; - } - - public Criteria andHostEqualTo(String value) { - addCriterion("host =", value, "host"); - return (Criteria) this; - } - - public Criteria andHostNotEqualTo(String value) { - addCriterion("host <>", value, "host"); - return (Criteria) this; - } - - public Criteria andHostGreaterThan(String value) { - addCriterion("host >", value, "host"); - return (Criteria) this; - } - - public Criteria andHostGreaterThanOrEqualTo(String value) { - addCriterion("host >=", value, "host"); - return (Criteria) this; - } - - public Criteria andHostLessThan(String value) { - addCriterion("host <", value, "host"); - return (Criteria) this; - } - - public Criteria andHostLessThanOrEqualTo(String value) { - addCriterion("host <=", value, "host"); - return (Criteria) this; - } - - public Criteria andHostLike(String value) { - addCriterion("host like", value, "host"); - return (Criteria) this; - } - - public Criteria andHostNotLike(String value) { - addCriterion("host not like", value, "host"); - return (Criteria) this; - } - - public Criteria andHostIn(List values) { - addCriterion("host in", values, "host"); - return (Criteria) this; - } - - public Criteria andHostNotIn(List values) { - addCriterion("host not in", values, "host"); - return (Criteria) this; - } - - public Criteria andHostBetween(String value1, String value2) { - addCriterion("host between", value1, value2, "host"); - return (Criteria) this; - } - - public Criteria andHostNotBetween(String value1, String value2) { - addCriterion("host not between", value1, value2, "host"); - return (Criteria) this; - } - - public Criteria andTraceIdIsNull() { - addCriterion("trace_id is null"); - return (Criteria) this; - } - - public Criteria andTraceIdIsNotNull() { - addCriterion("trace_id is not null"); - return (Criteria) this; - } - - public Criteria andTraceIdEqualTo(String value) { - addCriterion("trace_id =", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdNotEqualTo(String value) { - addCriterion("trace_id <>", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdGreaterThan(String value) { - addCriterion("trace_id >", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdGreaterThanOrEqualTo(String value) { - addCriterion("trace_id >=", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdLessThan(String value) { - addCriterion("trace_id <", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdLessThanOrEqualTo(String value) { - addCriterion("trace_id <=", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdLike(String value) { - addCriterion("trace_id like", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdNotLike(String value) { - addCriterion("trace_id not like", value, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdIn(List values) { - addCriterion("trace_id in", values, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdNotIn(List values) { - addCriterion("trace_id not in", values, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdBetween(String value1, String value2) { - addCriterion("trace_id between", value1, value2, "traceId"); - return (Criteria) this; - } - - public Criteria andTraceIdNotBetween(String value1, String value2) { - addCriterion("trace_id not between", value1, value2, "traceId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdIsNull() { - addCriterion("idempotent_id is null"); - return (Criteria) this; - } - - public Criteria andIdempotentIdIsNotNull() { - addCriterion("idempotent_id is not null"); - return (Criteria) this; - } - - public Criteria andIdempotentIdEqualTo(String value) { - addCriterion("idempotent_id =", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotEqualTo(String value) { - addCriterion("idempotent_id <>", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdGreaterThan(String value) { - addCriterion("idempotent_id >", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdGreaterThanOrEqualTo(String value) { - addCriterion("idempotent_id >=", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdLessThan(String value) { - addCriterion("idempotent_id <", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdLessThanOrEqualTo(String value) { - addCriterion("idempotent_id <=", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdLike(String value) { - addCriterion("idempotent_id like", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotLike(String value) { - addCriterion("idempotent_id not like", value, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdIn(List values) { - addCriterion("idempotent_id in", values, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotIn(List values) { - addCriterion("idempotent_id not in", values, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdBetween(String value1, String value2) { - addCriterion("idempotent_id between", value1, value2, "idempotentId"); - return (Criteria) this; - } - - public Criteria andIdempotentIdNotBetween(String value1, String value2) { - addCriterion("idempotent_id not between", value1, value2, "idempotentId"); - return (Criteria) this; - } - } - - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorOverviewDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorOverviewDO.java deleted file mode 100644 index 806c73d7..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorOverviewDO.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.Date; - -public class OperatorOverviewDO { - private Long id; - - private String name; - - private String type; - - private Date gmtCreate; - - private Date gmtModified; - - private String description; - - private String lang; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type == null ? null : type.trim(); - } - - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description == null ? null : description.trim(); - } - - public String getLang() { - return lang; - } - - public void setLang(String lang) { - this.lang = lang == null ? null : lang.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorOverviewDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorOverviewDOExample.java deleted file mode 100644 index df359e4e..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorOverviewDOExample.java +++ /dev/null @@ -1,674 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class OperatorOverviewDOExample { - protected String orderByClause; - - protected boolean distinct; - - protected List oredCriteria; - - public OperatorOverviewDOExample() { - oredCriteria = new ArrayList(); - } - - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - public String getOrderByClause() { - return orderByClause; - } - - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - public boolean isDistinct() { - return distinct; - } - - public List getOredCriteria() { - return oredCriteria; - } - - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNull() { - addCriterion("gmt_create is null"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNotNull() { - addCriterion("gmt_create is not null"); - return (Criteria) this; - } - - public Criteria andGmtCreateEqualTo(Date value) { - addCriterion("gmt_create =", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotEqualTo(Date value) { - addCriterion("gmt_create <>", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThan(Date value) { - addCriterion("gmt_create >", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_create >=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThan(Date value) { - addCriterion("gmt_create <", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThanOrEqualTo(Date value) { - addCriterion("gmt_create <=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateIn(List values) { - addCriterion("gmt_create in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotIn(List values) { - addCriterion("gmt_create not in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateBetween(Date value1, Date value2) { - addCriterion("gmt_create between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotBetween(Date value1, Date value2) { - addCriterion("gmt_create not between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNull() { - addCriterion("gmt_modified is null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNotNull() { - addCriterion("gmt_modified is not null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedEqualTo(Date value) { - addCriterion("gmt_modified =", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotEqualTo(Date value) { - addCriterion("gmt_modified <>", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThan(Date value) { - addCriterion("gmt_modified >", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_modified >=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThan(Date value) { - addCriterion("gmt_modified <", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThanOrEqualTo(Date value) { - addCriterion("gmt_modified <=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIn(List values) { - addCriterion("gmt_modified in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotIn(List values) { - addCriterion("gmt_modified not in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedBetween(Date value1, Date value2) { - addCriterion("gmt_modified between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotBetween(Date value1, Date value2) { - addCriterion("gmt_modified not between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNull() { - addCriterion("description is null"); - return (Criteria) this; - } - - public Criteria andDescriptionIsNotNull() { - addCriterion("description is not null"); - return (Criteria) this; - } - - public Criteria andDescriptionEqualTo(String value) { - addCriterion("description =", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotEqualTo(String value) { - addCriterion("description <>", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThan(String value) { - addCriterion("description >", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionGreaterThanOrEqualTo(String value) { - addCriterion("description >=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThan(String value) { - addCriterion("description <", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLessThanOrEqualTo(String value) { - addCriterion("description <=", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionLike(String value) { - addCriterion("description like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotLike(String value) { - addCriterion("description not like", value, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionIn(List values) { - addCriterion("description in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotIn(List values) { - addCriterion("description not in", values, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionBetween(String value1, String value2) { - addCriterion("description between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andDescriptionNotBetween(String value1, String value2) { - addCriterion("description not between", value1, value2, "description"); - return (Criteria) this; - } - - public Criteria andLangIsNull() { - addCriterion("lang is null"); - return (Criteria) this; - } - - public Criteria andLangIsNotNull() { - addCriterion("lang is not null"); - return (Criteria) this; - } - - public Criteria andLangEqualTo(String value) { - addCriterion("lang =", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangNotEqualTo(String value) { - addCriterion("lang <>", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangGreaterThan(String value) { - addCriterion("lang >", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangGreaterThanOrEqualTo(String value) { - addCriterion("lang >=", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangLessThan(String value) { - addCriterion("lang <", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangLessThanOrEqualTo(String value) { - addCriterion("lang <=", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangLike(String value) { - addCriterion("lang like", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangNotLike(String value) { - addCriterion("lang not like", value, "lang"); - return (Criteria) this; - } - - public Criteria andLangIn(List values) { - addCriterion("lang in", values, "lang"); - return (Criteria) this; - } - - public Criteria andLangNotIn(List values) { - addCriterion("lang not in", values, "lang"); - return (Criteria) this; - } - - public Criteria andLangBetween(String value1, String value2) { - addCriterion("lang between", value1, value2, "lang"); - return (Criteria) this; - } - - public Criteria andLangNotBetween(String value1, String value2) { - addCriterion("lang not between", value1, value2, "lang"); - return (Criteria) this; - } - } - - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDO.java deleted file mode 100644 index 1fbd6db1..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDO.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.Date; - -public class OperatorVersionDO { - private Long id; - - private Long overviewId; - - private String mainClass; - - private String jarAddress; - - private Date gmtCreate; - - private Date gmtModified; - - private Integer version; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getOverviewId() { - return overviewId; - } - - public void setOverviewId(Long overviewId) { - this.overviewId = overviewId; - } - - public String getMainClass() { - return mainClass; - } - - public void setMainClass(String mainClass) { - this.mainClass = mainClass == null ? null : mainClass.trim(); - } - - public String getJarAddress() { - return jarAddress; - } - - public void setJarAddress(String jarAddress) { - this.jarAddress = jarAddress == null ? null : jarAddress.trim(); - } - - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public Integer getVersion() { - return version; - } - - public void setVersion(Integer version) { - this.version = version; - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDOExample.java deleted file mode 100644 index 88147ea6..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDOExample.java +++ /dev/null @@ -1,654 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class OperatorVersionDOExample { - protected String orderByClause; - - protected boolean distinct; - - protected List oredCriteria; - - public OperatorVersionDOExample() { - oredCriteria = new ArrayList(); - } - - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - public String getOrderByClause() { - return orderByClause; - } - - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - public boolean isDistinct() { - return distinct; - } - - public List getOredCriteria() { - return oredCriteria; - } - - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andOverviewIdIsNull() { - addCriterion("overview_id is null"); - return (Criteria) this; - } - - public Criteria andOverviewIdIsNotNull() { - addCriterion("overview_id is not null"); - return (Criteria) this; - } - - public Criteria andOverviewIdEqualTo(Long value) { - addCriterion("overview_id =", value, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdNotEqualTo(Long value) { - addCriterion("overview_id <>", value, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdGreaterThan(Long value) { - addCriterion("overview_id >", value, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdGreaterThanOrEqualTo(Long value) { - addCriterion("overview_id >=", value, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdLessThan(Long value) { - addCriterion("overview_id <", value, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdLessThanOrEqualTo(Long value) { - addCriterion("overview_id <=", value, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdIn(List values) { - addCriterion("overview_id in", values, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdNotIn(List values) { - addCriterion("overview_id not in", values, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdBetween(Long value1, Long value2) { - addCriterion("overview_id between", value1, value2, "overviewId"); - return (Criteria) this; - } - - public Criteria andOverviewIdNotBetween(Long value1, Long value2) { - addCriterion("overview_id not between", value1, value2, "overviewId"); - return (Criteria) this; - } - - public Criteria andMainClassIsNull() { - addCriterion("main_class is null"); - return (Criteria) this; - } - - public Criteria andMainClassIsNotNull() { - addCriterion("main_class is not null"); - return (Criteria) this; - } - - public Criteria andMainClassEqualTo(String value) { - addCriterion("main_class =", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassNotEqualTo(String value) { - addCriterion("main_class <>", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassGreaterThan(String value) { - addCriterion("main_class >", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassGreaterThanOrEqualTo(String value) { - addCriterion("main_class >=", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassLessThan(String value) { - addCriterion("main_class <", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassLessThanOrEqualTo(String value) { - addCriterion("main_class <=", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassLike(String value) { - addCriterion("main_class like", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassNotLike(String value) { - addCriterion("main_class not like", value, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassIn(List values) { - addCriterion("main_class in", values, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassNotIn(List values) { - addCriterion("main_class not in", values, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassBetween(String value1, String value2) { - addCriterion("main_class between", value1, value2, "mainClass"); - return (Criteria) this; - } - - public Criteria andMainClassNotBetween(String value1, String value2) { - addCriterion("main_class not between", value1, value2, "mainClass"); - return (Criteria) this; - } - - public Criteria andJarAddressIsNull() { - addCriterion("jar_address is null"); - return (Criteria) this; - } - - public Criteria andJarAddressIsNotNull() { - addCriterion("jar_address is not null"); - return (Criteria) this; - } - - public Criteria andJarAddressEqualTo(String value) { - addCriterion("jar_address =", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressNotEqualTo(String value) { - addCriterion("jar_address <>", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressGreaterThan(String value) { - addCriterion("jar_address >", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressGreaterThanOrEqualTo(String value) { - addCriterion("jar_address >=", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressLessThan(String value) { - addCriterion("jar_address <", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressLessThanOrEqualTo(String value) { - addCriterion("jar_address <=", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressLike(String value) { - addCriterion("jar_address like", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressNotLike(String value) { - addCriterion("jar_address not like", value, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressIn(List values) { - addCriterion("jar_address in", values, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressNotIn(List values) { - addCriterion("jar_address not in", values, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressBetween(String value1, String value2) { - addCriterion("jar_address between", value1, value2, "jarAddress"); - return (Criteria) this; - } - - public Criteria andJarAddressNotBetween(String value1, String value2) { - addCriterion("jar_address not between", value1, value2, "jarAddress"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNull() { - addCriterion("gmt_create is null"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNotNull() { - addCriterion("gmt_create is not null"); - return (Criteria) this; - } - - public Criteria andGmtCreateEqualTo(Date value) { - addCriterion("gmt_create =", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotEqualTo(Date value) { - addCriterion("gmt_create <>", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThan(Date value) { - addCriterion("gmt_create >", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_create >=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThan(Date value) { - addCriterion("gmt_create <", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThanOrEqualTo(Date value) { - addCriterion("gmt_create <=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateIn(List values) { - addCriterion("gmt_create in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotIn(List values) { - addCriterion("gmt_create not in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateBetween(Date value1, Date value2) { - addCriterion("gmt_create between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotBetween(Date value1, Date value2) { - addCriterion("gmt_create not between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNull() { - addCriterion("gmt_modified is null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNotNull() { - addCriterion("gmt_modified is not null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedEqualTo(Date value) { - addCriterion("gmt_modified =", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotEqualTo(Date value) { - addCriterion("gmt_modified <>", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThan(Date value) { - addCriterion("gmt_modified >", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_modified >=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThan(Date value) { - addCriterion("gmt_modified <", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThanOrEqualTo(Date value) { - addCriterion("gmt_modified <=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIn(List values) { - addCriterion("gmt_modified in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotIn(List values) { - addCriterion("gmt_modified not in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedBetween(Date value1, Date value2) { - addCriterion("gmt_modified between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotBetween(Date value1, Date value2) { - addCriterion("gmt_modified not between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andVersionIsNull() { - addCriterion("version is null"); - return (Criteria) this; - } - - public Criteria andVersionIsNotNull() { - addCriterion("version is not null"); - return (Criteria) this; - } - - public Criteria andVersionEqualTo(Integer value) { - addCriterion("version =", value, "version"); - return (Criteria) this; - } - - public Criteria andVersionNotEqualTo(Integer value) { - addCriterion("version <>", value, "version"); - return (Criteria) this; - } - - public Criteria andVersionGreaterThan(Integer value) { - addCriterion("version >", value, "version"); - return (Criteria) this; - } - - public Criteria andVersionGreaterThanOrEqualTo(Integer value) { - addCriterion("version >=", value, "version"); - return (Criteria) this; - } - - public Criteria andVersionLessThan(Integer value) { - addCriterion("version <", value, "version"); - return (Criteria) this; - } - - public Criteria andVersionLessThanOrEqualTo(Integer value) { - addCriterion("version <=", value, "version"); - return (Criteria) this; - } - - public Criteria andVersionIn(List values) { - addCriterion("version in", values, "version"); - return (Criteria) this; - } - - public Criteria andVersionNotIn(List values) { - addCriterion("version not in", values, "version"); - return (Criteria) this; - } - - public Criteria andVersionBetween(Integer value1, Integer value2) { - addCriterion("version between", value1, value2, "version"); - return (Criteria) this; - } - - public Criteria andVersionNotBetween(Integer value1, Integer value2) { - addCriterion("version not between", value1, value2, "version"); - return (Criteria) this; - } - } - - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDOWithBLOBs.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDOWithBLOBs.java deleted file mode 100644 index c3d282ff..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/OperatorVersionDOWithBLOBs.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -public class OperatorVersionDOWithBLOBs extends OperatorVersionDO { - private String operatorConfig; - - private String config; - - public String getOperatorConfig() { - return operatorConfig; - } - - public void setOperatorConfig(String operatorConfig) { - this.operatorConfig = operatorConfig == null ? null : operatorConfig.trim(); - } - - public String getConfig() { - return config; - } - - public void setConfig(String config) { - this.config = config == null ? null : config.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInfoDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInfoDO.java deleted file mode 100644 index 00af3ea5..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInfoDO.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.Date; - -public class SPGJobInfoDO { - private Long id; - - private Date gmtCreate; - - private Date gmtModified; - - private String name; - - private String type; - - private Long projectId; - - private String cron; - - private String status; - - private String extInfo; - - private String externalJobInfoId; - - private String content; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name == null ? null : name.trim(); - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type == null ? null : type.trim(); - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public String getCron() { - return cron; - } - - public void setCron(String cron) { - this.cron = cron == null ? null : cron.trim(); - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status == null ? null : status.trim(); - } - - public String getExtInfo() { - return extInfo; - } - - public void setExtInfo(String extInfo) { - this.extInfo = extInfo == null ? null : extInfo.trim(); - } - - public String getExternalJobInfoId() { - return externalJobInfoId; - } - - public void setExternalJobInfoId(String externalJobInfoId) { - this.externalJobInfoId = externalJobInfoId == null ? null : externalJobInfoId.trim(); - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content == null ? null : content.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInfoDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInfoDOExample.java deleted file mode 100644 index 990623cd..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInfoDOExample.java +++ /dev/null @@ -1,874 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class SPGJobInfoDOExample { - protected String orderByClause; - - protected boolean distinct; - - protected List oredCriteria; - - public SPGJobInfoDOExample() { - oredCriteria = new ArrayList(); - } - - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - public String getOrderByClause() { - return orderByClause; - } - - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - public boolean isDistinct() { - return distinct; - } - - public List getOredCriteria() { - return oredCriteria; - } - - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNull() { - addCriterion("gmt_create is null"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNotNull() { - addCriterion("gmt_create is not null"); - return (Criteria) this; - } - - public Criteria andGmtCreateEqualTo(Date value) { - addCriterion("gmt_create =", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotEqualTo(Date value) { - addCriterion("gmt_create <>", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThan(Date value) { - addCriterion("gmt_create >", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_create >=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThan(Date value) { - addCriterion("gmt_create <", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThanOrEqualTo(Date value) { - addCriterion("gmt_create <=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateIn(List values) { - addCriterion("gmt_create in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotIn(List values) { - addCriterion("gmt_create not in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateBetween(Date value1, Date value2) { - addCriterion("gmt_create between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotBetween(Date value1, Date value2) { - addCriterion("gmt_create not between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNull() { - addCriterion("gmt_modified is null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNotNull() { - addCriterion("gmt_modified is not null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedEqualTo(Date value) { - addCriterion("gmt_modified =", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotEqualTo(Date value) { - addCriterion("gmt_modified <>", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThan(Date value) { - addCriterion("gmt_modified >", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_modified >=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThan(Date value) { - addCriterion("gmt_modified <", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThanOrEqualTo(Date value) { - addCriterion("gmt_modified <=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIn(List values) { - addCriterion("gmt_modified in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotIn(List values) { - addCriterion("gmt_modified not in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedBetween(Date value1, Date value2) { - addCriterion("gmt_modified between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotBetween(Date value1, Date value2) { - addCriterion("gmt_modified not between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andNameIsNull() { - addCriterion("name is null"); - return (Criteria) this; - } - - public Criteria andNameIsNotNull() { - addCriterion("name is not null"); - return (Criteria) this; - } - - public Criteria andNameEqualTo(String value) { - addCriterion("name =", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotEqualTo(String value) { - addCriterion("name <>", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThan(String value) { - addCriterion("name >", value, "name"); - return (Criteria) this; - } - - public Criteria andNameGreaterThanOrEqualTo(String value) { - addCriterion("name >=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThan(String value) { - addCriterion("name <", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLessThanOrEqualTo(String value) { - addCriterion("name <=", value, "name"); - return (Criteria) this; - } - - public Criteria andNameLike(String value) { - addCriterion("name like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameNotLike(String value) { - addCriterion("name not like", value, "name"); - return (Criteria) this; - } - - public Criteria andNameIn(List values) { - addCriterion("name in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameNotIn(List values) { - addCriterion("name not in", values, "name"); - return (Criteria) this; - } - - public Criteria andNameBetween(String value1, String value2) { - addCriterion("name between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andNameNotBetween(String value1, String value2) { - addCriterion("name not between", value1, value2, "name"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andProjectIdIsNull() { - addCriterion("project_id is null"); - return (Criteria) this; - } - - public Criteria andProjectIdIsNotNull() { - addCriterion("project_id is not null"); - return (Criteria) this; - } - - public Criteria andProjectIdEqualTo(Long value) { - addCriterion("project_id =", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdNotEqualTo(Long value) { - addCriterion("project_id <>", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdGreaterThan(Long value) { - addCriterion("project_id >", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdGreaterThanOrEqualTo(Long value) { - addCriterion("project_id >=", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdLessThan(Long value) { - addCriterion("project_id <", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdLessThanOrEqualTo(Long value) { - addCriterion("project_id <=", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdIn(List values) { - addCriterion("project_id in", values, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdNotIn(List values) { - addCriterion("project_id not in", values, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdBetween(Long value1, Long value2) { - addCriterion("project_id between", value1, value2, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdNotBetween(Long value1, Long value2) { - addCriterion("project_id not between", value1, value2, "projectId"); - return (Criteria) this; - } - - public Criteria andCronIsNull() { - addCriterion("cron is null"); - return (Criteria) this; - } - - public Criteria andCronIsNotNull() { - addCriterion("cron is not null"); - return (Criteria) this; - } - - public Criteria andCronEqualTo(String value) { - addCriterion("cron =", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotEqualTo(String value) { - addCriterion("cron <>", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronGreaterThan(String value) { - addCriterion("cron >", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronGreaterThanOrEqualTo(String value) { - addCriterion("cron >=", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronLessThan(String value) { - addCriterion("cron <", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronLessThanOrEqualTo(String value) { - addCriterion("cron <=", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronLike(String value) { - addCriterion("cron like", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotLike(String value) { - addCriterion("cron not like", value, "cron"); - return (Criteria) this; - } - - public Criteria andCronIn(List values) { - addCriterion("cron in", values, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotIn(List values) { - addCriterion("cron not in", values, "cron"); - return (Criteria) this; - } - - public Criteria andCronBetween(String value1, String value2) { - addCriterion("cron between", value1, value2, "cron"); - return (Criteria) this; - } - - public Criteria andCronNotBetween(String value1, String value2) { - addCriterion("cron not between", value1, value2, "cron"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(String value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(String value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(String value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(String value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(String value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(String value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLike(String value) { - addCriterion("status like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotLike(String value) { - addCriterion("status not like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(String value1, String value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(String value1, String value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andExtInfoIsNull() { - addCriterion("ext_info is null"); - return (Criteria) this; - } - - public Criteria andExtInfoIsNotNull() { - addCriterion("ext_info is not null"); - return (Criteria) this; - } - - public Criteria andExtInfoEqualTo(String value) { - addCriterion("ext_info =", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoNotEqualTo(String value) { - addCriterion("ext_info <>", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoGreaterThan(String value) { - addCriterion("ext_info >", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoGreaterThanOrEqualTo(String value) { - addCriterion("ext_info >=", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoLessThan(String value) { - addCriterion("ext_info <", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoLessThanOrEqualTo(String value) { - addCriterion("ext_info <=", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoLike(String value) { - addCriterion("ext_info like", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoNotLike(String value) { - addCriterion("ext_info not like", value, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoIn(List values) { - addCriterion("ext_info in", values, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoNotIn(List values) { - addCriterion("ext_info not in", values, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoBetween(String value1, String value2) { - addCriterion("ext_info between", value1, value2, "extInfo"); - return (Criteria) this; - } - - public Criteria andExtInfoNotBetween(String value1, String value2) { - addCriterion("ext_info not between", value1, value2, "extInfo"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdIsNull() { - addCriterion("external_job_info_id is null"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdIsNotNull() { - addCriterion("external_job_info_id is not null"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdEqualTo(String value) { - addCriterion("external_job_info_id =", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdNotEqualTo(String value) { - addCriterion("external_job_info_id <>", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdGreaterThan(String value) { - addCriterion("external_job_info_id >", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdGreaterThanOrEqualTo(String value) { - addCriterion("external_job_info_id >=", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdLessThan(String value) { - addCriterion("external_job_info_id <", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdLessThanOrEqualTo(String value) { - addCriterion("external_job_info_id <=", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdLike(String value) { - addCriterion("external_job_info_id like", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdNotLike(String value) { - addCriterion("external_job_info_id not like", value, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdIn(List values) { - addCriterion("external_job_info_id in", values, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdNotIn(List values) { - addCriterion("external_job_info_id not in", values, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdBetween(String value1, String value2) { - addCriterion("external_job_info_id between", value1, value2, "externalJobInfoId"); - return (Criteria) this; - } - - public Criteria andExternalJobInfoIdNotBetween(String value1, String value2) { - addCriterion("external_job_info_id not between", value1, value2, "externalJobInfoId"); - return (Criteria) this; - } - } - - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDO.java deleted file mode 100644 index 99df73b4..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDO.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.Date; - -public class SPGJobInstDO { - private Long id; - - private Date gmtCreate; - - private Date gmtModified; - - private Long jobId; - - private String type; - - private Long projectId; - - private String status; - - private Date startTime; - - private Date endTime; - - private String externalJobInstId; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Date getGmtCreate() { - return gmtCreate; - } - - public void setGmtCreate(Date gmtCreate) { - this.gmtCreate = gmtCreate; - } - - public Date getGmtModified() { - return gmtModified; - } - - public void setGmtModified(Date gmtModified) { - this.gmtModified = gmtModified; - } - - public Long getJobId() { - return jobId; - } - - public void setJobId(Long jobId) { - this.jobId = jobId; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type == null ? null : type.trim(); - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status == null ? null : status.trim(); - } - - public Date getStartTime() { - return startTime; - } - - public void setStartTime(Date startTime) { - this.startTime = startTime; - } - - public Date getEndTime() { - return endTime; - } - - public void setEndTime(Date endTime) { - this.endTime = endTime; - } - - public String getExternalJobInstId() { - return externalJobInstId; - } - - public void setExternalJobInstId(String externalJobInstId) { - this.externalJobInstId = externalJobInstId == null ? null : externalJobInstId.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDOExample.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDOExample.java deleted file mode 100644 index d5dfcf86..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDOExample.java +++ /dev/null @@ -1,844 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -public class SPGJobInstDOExample { - protected String orderByClause; - - protected boolean distinct; - - protected List oredCriteria; - - public SPGJobInstDOExample() { - oredCriteria = new ArrayList(); - } - - public void setOrderByClause(String orderByClause) { - this.orderByClause = orderByClause; - } - - public String getOrderByClause() { - return orderByClause; - } - - public void setDistinct(boolean distinct) { - this.distinct = distinct; - } - - public boolean isDistinct() { - return distinct; - } - - public List getOredCriteria() { - return oredCriteria; - } - - public void or(Criteria criteria) { - oredCriteria.add(criteria); - } - - public Criteria or() { - Criteria criteria = createCriteriaInternal(); - oredCriteria.add(criteria); - return criteria; - } - - public Criteria createCriteria() { - Criteria criteria = createCriteriaInternal(); - if (oredCriteria.size() == 0) { - oredCriteria.add(criteria); - } - return criteria; - } - - protected Criteria createCriteriaInternal() { - Criteria criteria = new Criteria(); - return criteria; - } - - public void clear() { - oredCriteria.clear(); - orderByClause = null; - distinct = false; - } - - protected abstract static class GeneratedCriteria { - protected List criteria; - - protected GeneratedCriteria() { - super(); - criteria = new ArrayList(); - } - - public boolean isValid() { - return criteria.size() > 0; - } - - public List getAllCriteria() { - return criteria; - } - - public List getCriteria() { - return criteria; - } - - protected void addCriterion(String condition) { - if (condition == null) { - throw new RuntimeException("Value for condition cannot be null"); - } - criteria.add(new Criterion(condition)); - } - - protected void addCriterion(String condition, Object value, String property) { - if (value == null) { - throw new RuntimeException("Value for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value)); - } - - protected void addCriterion(String condition, Object value1, Object value2, String property) { - if (value1 == null || value2 == null) { - throw new RuntimeException("Between values for " + property + " cannot be null"); - } - criteria.add(new Criterion(condition, value1, value2)); - } - - public Criteria andIdIsNull() { - addCriterion("id is null"); - return (Criteria) this; - } - - public Criteria andIdIsNotNull() { - addCriterion("id is not null"); - return (Criteria) this; - } - - public Criteria andIdEqualTo(Long value) { - addCriterion("id =", value, "id"); - return (Criteria) this; - } - - public Criteria andIdNotEqualTo(Long value) { - addCriterion("id <>", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThan(Long value) { - addCriterion("id >", value, "id"); - return (Criteria) this; - } - - public Criteria andIdGreaterThanOrEqualTo(Long value) { - addCriterion("id >=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThan(Long value) { - addCriterion("id <", value, "id"); - return (Criteria) this; - } - - public Criteria andIdLessThanOrEqualTo(Long value) { - addCriterion("id <=", value, "id"); - return (Criteria) this; - } - - public Criteria andIdIn(List values) { - addCriterion("id in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdNotIn(List values) { - addCriterion("id not in", values, "id"); - return (Criteria) this; - } - - public Criteria andIdBetween(Long value1, Long value2) { - addCriterion("id between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andIdNotBetween(Long value1, Long value2) { - addCriterion("id not between", value1, value2, "id"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNull() { - addCriterion("gmt_create is null"); - return (Criteria) this; - } - - public Criteria andGmtCreateIsNotNull() { - addCriterion("gmt_create is not null"); - return (Criteria) this; - } - - public Criteria andGmtCreateEqualTo(Date value) { - addCriterion("gmt_create =", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotEqualTo(Date value) { - addCriterion("gmt_create <>", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThan(Date value) { - addCriterion("gmt_create >", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_create >=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThan(Date value) { - addCriterion("gmt_create <", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateLessThanOrEqualTo(Date value) { - addCriterion("gmt_create <=", value, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateIn(List values) { - addCriterion("gmt_create in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotIn(List values) { - addCriterion("gmt_create not in", values, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateBetween(Date value1, Date value2) { - addCriterion("gmt_create between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtCreateNotBetween(Date value1, Date value2) { - addCriterion("gmt_create not between", value1, value2, "gmtCreate"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNull() { - addCriterion("gmt_modified is null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIsNotNull() { - addCriterion("gmt_modified is not null"); - return (Criteria) this; - } - - public Criteria andGmtModifiedEqualTo(Date value) { - addCriterion("gmt_modified =", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotEqualTo(Date value) { - addCriterion("gmt_modified <>", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThan(Date value) { - addCriterion("gmt_modified >", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedGreaterThanOrEqualTo(Date value) { - addCriterion("gmt_modified >=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThan(Date value) { - addCriterion("gmt_modified <", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedLessThanOrEqualTo(Date value) { - addCriterion("gmt_modified <=", value, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedIn(List values) { - addCriterion("gmt_modified in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotIn(List values) { - addCriterion("gmt_modified not in", values, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedBetween(Date value1, Date value2) { - addCriterion("gmt_modified between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andGmtModifiedNotBetween(Date value1, Date value2) { - addCriterion("gmt_modified not between", value1, value2, "gmtModified"); - return (Criteria) this; - } - - public Criteria andJobIdIsNull() { - addCriterion("job_id is null"); - return (Criteria) this; - } - - public Criteria andJobIdIsNotNull() { - addCriterion("job_id is not null"); - return (Criteria) this; - } - - public Criteria andJobIdEqualTo(Long value) { - addCriterion("job_id =", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdNotEqualTo(Long value) { - addCriterion("job_id <>", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdGreaterThan(Long value) { - addCriterion("job_id >", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdGreaterThanOrEqualTo(Long value) { - addCriterion("job_id >=", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdLessThan(Long value) { - addCriterion("job_id <", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdLessThanOrEqualTo(Long value) { - addCriterion("job_id <=", value, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdIn(List values) { - addCriterion("job_id in", values, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdNotIn(List values) { - addCriterion("job_id not in", values, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdBetween(Long value1, Long value2) { - addCriterion("job_id between", value1, value2, "jobId"); - return (Criteria) this; - } - - public Criteria andJobIdNotBetween(Long value1, Long value2) { - addCriterion("job_id not between", value1, value2, "jobId"); - return (Criteria) this; - } - - public Criteria andTypeIsNull() { - addCriterion("type is null"); - return (Criteria) this; - } - - public Criteria andTypeIsNotNull() { - addCriterion("type is not null"); - return (Criteria) this; - } - - public Criteria andTypeEqualTo(String value) { - addCriterion("type =", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotEqualTo(String value) { - addCriterion("type <>", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThan(String value) { - addCriterion("type >", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeGreaterThanOrEqualTo(String value) { - addCriterion("type >=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThan(String value) { - addCriterion("type <", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLessThanOrEqualTo(String value) { - addCriterion("type <=", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeLike(String value) { - addCriterion("type like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotLike(String value) { - addCriterion("type not like", value, "type"); - return (Criteria) this; - } - - public Criteria andTypeIn(List values) { - addCriterion("type in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotIn(List values) { - addCriterion("type not in", values, "type"); - return (Criteria) this; - } - - public Criteria andTypeBetween(String value1, String value2) { - addCriterion("type between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andTypeNotBetween(String value1, String value2) { - addCriterion("type not between", value1, value2, "type"); - return (Criteria) this; - } - - public Criteria andProjectIdIsNull() { - addCriterion("project_id is null"); - return (Criteria) this; - } - - public Criteria andProjectIdIsNotNull() { - addCriterion("project_id is not null"); - return (Criteria) this; - } - - public Criteria andProjectIdEqualTo(Long value) { - addCriterion("project_id =", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdNotEqualTo(Long value) { - addCriterion("project_id <>", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdGreaterThan(Long value) { - addCriterion("project_id >", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdGreaterThanOrEqualTo(Long value) { - addCriterion("project_id >=", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdLessThan(Long value) { - addCriterion("project_id <", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdLessThanOrEqualTo(Long value) { - addCriterion("project_id <=", value, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdIn(List values) { - addCriterion("project_id in", values, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdNotIn(List values) { - addCriterion("project_id not in", values, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdBetween(Long value1, Long value2) { - addCriterion("project_id between", value1, value2, "projectId"); - return (Criteria) this; - } - - public Criteria andProjectIdNotBetween(Long value1, Long value2) { - addCriterion("project_id not between", value1, value2, "projectId"); - return (Criteria) this; - } - - public Criteria andStatusIsNull() { - addCriterion("status is null"); - return (Criteria) this; - } - - public Criteria andStatusIsNotNull() { - addCriterion("status is not null"); - return (Criteria) this; - } - - public Criteria andStatusEqualTo(String value) { - addCriterion("status =", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotEqualTo(String value) { - addCriterion("status <>", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThan(String value) { - addCriterion("status >", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusGreaterThanOrEqualTo(String value) { - addCriterion("status >=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThan(String value) { - addCriterion("status <", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLessThanOrEqualTo(String value) { - addCriterion("status <=", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusLike(String value) { - addCriterion("status like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotLike(String value) { - addCriterion("status not like", value, "status"); - return (Criteria) this; - } - - public Criteria andStatusIn(List values) { - addCriterion("status in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotIn(List values) { - addCriterion("status not in", values, "status"); - return (Criteria) this; - } - - public Criteria andStatusBetween(String value1, String value2) { - addCriterion("status between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStatusNotBetween(String value1, String value2) { - addCriterion("status not between", value1, value2, "status"); - return (Criteria) this; - } - - public Criteria andStartTimeIsNull() { - addCriterion("start_time is null"); - return (Criteria) this; - } - - public Criteria andStartTimeIsNotNull() { - addCriterion("start_time is not null"); - return (Criteria) this; - } - - public Criteria andStartTimeEqualTo(Date value) { - addCriterion("start_time =", value, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeNotEqualTo(Date value) { - addCriterion("start_time <>", value, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeGreaterThan(Date value) { - addCriterion("start_time >", value, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeGreaterThanOrEqualTo(Date value) { - addCriterion("start_time >=", value, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeLessThan(Date value) { - addCriterion("start_time <", value, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeLessThanOrEqualTo(Date value) { - addCriterion("start_time <=", value, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeIn(List values) { - addCriterion("start_time in", values, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeNotIn(List values) { - addCriterion("start_time not in", values, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeBetween(Date value1, Date value2) { - addCriterion("start_time between", value1, value2, "startTime"); - return (Criteria) this; - } - - public Criteria andStartTimeNotBetween(Date value1, Date value2) { - addCriterion("start_time not between", value1, value2, "startTime"); - return (Criteria) this; - } - - public Criteria andEndTimeIsNull() { - addCriterion("end_time is null"); - return (Criteria) this; - } - - public Criteria andEndTimeIsNotNull() { - addCriterion("end_time is not null"); - return (Criteria) this; - } - - public Criteria andEndTimeEqualTo(Date value) { - addCriterion("end_time =", value, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeNotEqualTo(Date value) { - addCriterion("end_time <>", value, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeGreaterThan(Date value) { - addCriterion("end_time >", value, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeGreaterThanOrEqualTo(Date value) { - addCriterion("end_time >=", value, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeLessThan(Date value) { - addCriterion("end_time <", value, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeLessThanOrEqualTo(Date value) { - addCriterion("end_time <=", value, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeIn(List values) { - addCriterion("end_time in", values, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeNotIn(List values) { - addCriterion("end_time not in", values, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeBetween(Date value1, Date value2) { - addCriterion("end_time between", value1, value2, "endTime"); - return (Criteria) this; - } - - public Criteria andEndTimeNotBetween(Date value1, Date value2) { - addCriterion("end_time not between", value1, value2, "endTime"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdIsNull() { - addCriterion("external_job_inst_id is null"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdIsNotNull() { - addCriterion("external_job_inst_id is not null"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdEqualTo(String value) { - addCriterion("external_job_inst_id =", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdNotEqualTo(String value) { - addCriterion("external_job_inst_id <>", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdGreaterThan(String value) { - addCriterion("external_job_inst_id >", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdGreaterThanOrEqualTo(String value) { - addCriterion("external_job_inst_id >=", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdLessThan(String value) { - addCriterion("external_job_inst_id <", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdLessThanOrEqualTo(String value) { - addCriterion("external_job_inst_id <=", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdLike(String value) { - addCriterion("external_job_inst_id like", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdNotLike(String value) { - addCriterion("external_job_inst_id not like", value, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdIn(List values) { - addCriterion("external_job_inst_id in", values, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdNotIn(List values) { - addCriterion("external_job_inst_id not in", values, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdBetween(String value1, String value2) { - addCriterion("external_job_inst_id between", value1, value2, "externalJobInstId"); - return (Criteria) this; - } - - public Criteria andExternalJobInstIdNotBetween(String value1, String value2) { - addCriterion("external_job_inst_id not between", value1, value2, "externalJobInstId"); - return (Criteria) this; - } - } - - public static class Criteria extends GeneratedCriteria { - - protected Criteria() { - super(); - } - } - - public static class Criterion { - private String condition; - - private Object value; - - private Object secondValue; - - private boolean noValue; - - private boolean singleValue; - - private boolean betweenValue; - - private boolean listValue; - - private String typeHandler; - - public String getCondition() { - return condition; - } - - public Object getValue() { - return value; - } - - public Object getSecondValue() { - return secondValue; - } - - public boolean isNoValue() { - return noValue; - } - - public boolean isSingleValue() { - return singleValue; - } - - public boolean isBetweenValue() { - return betweenValue; - } - - public boolean isListValue() { - return listValue; - } - - public String getTypeHandler() { - return typeHandler; - } - - protected Criterion(String condition) { - super(); - this.condition = condition; - this.typeHandler = null; - this.noValue = true; - } - - protected Criterion(String condition, Object value, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.typeHandler = typeHandler; - if (value instanceof List) { - this.listValue = true; - } else { - this.singleValue = true; - } - } - - protected Criterion(String condition, Object value) { - this(condition, value, null); - } - - protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { - super(); - this.condition = condition; - this.value = value; - this.secondValue = secondValue; - this.typeHandler = typeHandler; - this.betweenValue = true; - } - - protected Criterion(String condition, Object value, Object secondValue) { - this(condition, value, secondValue, null); - } - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDOWithBLOBs.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDOWithBLOBs.java deleted file mode 100644 index 3a311fde..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/dataobject/SPGJobInstDOWithBLOBs.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.dataobject; - -public class SPGJobInstDOWithBLOBs extends SPGJobInstDO { - private String result; - - private String progress; - - private String logInfo; - - public String getResult() { - return result; - } - - public void setResult(String result) { - this.result = result == null ? null : result.trim(); - } - - public String getProgress() { - return progress; - } - - public void setProgress(String progress) { - this.progress = progress == null ? null : progress.trim(); - } - - public String getLogInfo() { - return logInfo; - } - - public void setLogInfo(String logInfo) { - this.logInfo = logInfo == null ? null : logInfo.trim(); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/JobInfoDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/JobInfoDOMapper.java deleted file mode 100644 index 0a6d7956..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/JobInfoDOMapper.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.mapper; - -import com.antgroup.openspg.server.infra.dao.dataobject.JobInfoDO; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInfoDOExample; -import java.util.List; -import org.apache.ibatis.annotations.Param; - -public interface JobInfoDOMapper { - long countByExample(JobInfoDOExample example); - - int deleteByExample(JobInfoDOExample example); - - int deleteByPrimaryKey(Long id); - - int insert(JobInfoDO record); - - int insertSelective(JobInfoDO record); - - List selectByExample(JobInfoDOExample example); - - JobInfoDO selectByPrimaryKey(Long id); - - int updateByExampleSelective( - @Param("record") JobInfoDO record, @Param("example") JobInfoDOExample example); - - int updateByExample( - @Param("record") JobInfoDO record, @Param("example") JobInfoDOExample example); - - int updateByPrimaryKeySelective(JobInfoDO record); - - int updateByPrimaryKey(JobInfoDO record); -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/JobInstDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/JobInstDOMapper.java deleted file mode 100644 index 972e3581..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/JobInstDOMapper.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.mapper; - -import com.antgroup.openspg.server.infra.dao.dataobject.JobInstDO; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInstDOExample; -import java.util.List; -import org.apache.ibatis.annotations.Param; - -public interface JobInstDOMapper { - long countByExample(JobInstDOExample example); - - int deleteByExample(JobInstDOExample example); - - int deleteByPrimaryKey(Long id); - - int insert(JobInstDO record); - - int insertSelective(JobInstDO record); - - List selectByExampleWithBLOBs(JobInstDOExample example); - - List selectByExample(JobInstDOExample example); - - JobInstDO selectByPrimaryKey(Long id); - - int updateByExampleSelective( - @Param("record") JobInstDO record, @Param("example") JobInstDOExample example); - - int updateByExampleWithBLOBs( - @Param("record") JobInstDO record, @Param("example") JobInstDOExample example); - - int updateByExample( - @Param("record") JobInstDO record, @Param("example") JobInstDOExample example); - - int updateByPrimaryKeySelective(JobInstDO record); - - int updateByPrimaryKeyWithBLOBs(JobInstDO record); - - int updateByPrimaryKey(JobInstDO record); -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OperatorOverviewDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OperatorOverviewDOMapper.java deleted file mode 100644 index 644127d5..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OperatorOverviewDOMapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.mapper; - -import com.antgroup.openspg.server.infra.dao.dataobject.OperatorOverviewDO; -import com.antgroup.openspg.server.infra.dao.dataobject.OperatorOverviewDOExample; -import java.util.List; -import org.apache.ibatis.annotations.Param; - -public interface OperatorOverviewDOMapper { - long countByExample(OperatorOverviewDOExample example); - - int deleteByExample(OperatorOverviewDOExample example); - - int deleteByPrimaryKey(Long id); - - int insert(OperatorOverviewDO record); - - int insertSelective(OperatorOverviewDO record); - - List selectByExample(OperatorOverviewDOExample example); - - OperatorOverviewDO selectByPrimaryKey(Long id); - - int updateByExampleSelective( - @Param("record") OperatorOverviewDO record, - @Param("example") OperatorOverviewDOExample example); - - int updateByExample( - @Param("record") OperatorOverviewDO record, - @Param("example") OperatorOverviewDOExample example); - - int updateByPrimaryKeySelective(OperatorOverviewDO record); - - int updateByPrimaryKey(OperatorOverviewDO record); -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OperatorVersionDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OperatorVersionDOMapper.java deleted file mode 100644 index a23ead0c..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/OperatorVersionDOMapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.mapper; - -import com.antgroup.openspg.server.infra.dao.dataobject.OperatorVersionDO; -import com.antgroup.openspg.server.infra.dao.dataobject.OperatorVersionDOExample; -import java.util.List; -import org.apache.ibatis.annotations.Param; - -public interface OperatorVersionDOMapper { - long countByExample(OperatorVersionDOExample example); - - int deleteByExample(OperatorVersionDOExample example); - - int deleteByPrimaryKey(Long id); - - int insert(OperatorVersionDO record); - - int insertSelective(OperatorVersionDO record); - - List selectByExample(OperatorVersionDOExample example); - - OperatorVersionDO selectByPrimaryKey(Long id); - - int updateByExampleSelective( - @Param("record") OperatorVersionDO record, - @Param("example") OperatorVersionDOExample example); - - int updateByExample( - @Param("record") OperatorVersionDO record, - @Param("example") OperatorVersionDOExample example); - - int updateByPrimaryKeySelective(OperatorVersionDO record); - - int updateByPrimaryKey(OperatorVersionDO record); -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/SPGJobInfoDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/SPGJobInfoDOMapper.java deleted file mode 100644 index 61213e99..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/SPGJobInfoDOMapper.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.mapper; - -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInfoDO; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInfoDOExample; -import java.util.List; -import org.apache.ibatis.annotations.Param; - -public interface SPGJobInfoDOMapper { - long countByExample(SPGJobInfoDOExample example); - - int deleteByExample(SPGJobInfoDOExample example); - - int deleteByPrimaryKey(Long id); - - int insert(SPGJobInfoDO record); - - int insertSelective(SPGJobInfoDO record); - - List selectByExampleWithBLOBs(SPGJobInfoDOExample example); - - List selectByExample(SPGJobInfoDOExample example); - - SPGJobInfoDO selectByPrimaryKey(Long id); - - int updateByExampleSelective( - @Param("record") SPGJobInfoDO record, @Param("example") SPGJobInfoDOExample example); - - int updateByExampleWithBLOBs( - @Param("record") SPGJobInfoDO record, @Param("example") SPGJobInfoDOExample example); - - int updateByExample( - @Param("record") SPGJobInfoDO record, @Param("example") SPGJobInfoDOExample example); - - int updateByPrimaryKeySelective(SPGJobInfoDO record); - - int updateByPrimaryKeyWithBLOBs(SPGJobInfoDO record); - - int updateByPrimaryKey(SPGJobInfoDO record); -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/SPGJobInstDOMapper.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/SPGJobInstDOMapper.java deleted file mode 100644 index b9351593..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/mapper/SPGJobInstDOMapper.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.mapper; - -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInstDO; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInstDOExample; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInstDOWithBLOBs; -import java.util.List; -import org.apache.ibatis.annotations.Param; - -public interface SPGJobInstDOMapper { - long countByExample(SPGJobInstDOExample example); - - int deleteByExample(SPGJobInstDOExample example); - - int deleteByPrimaryKey(Long id); - - int insert(SPGJobInstDOWithBLOBs record); - - int insertSelective(SPGJobInstDOWithBLOBs record); - - List selectByExampleWithBLOBs(SPGJobInstDOExample example); - - List selectByExample(SPGJobInstDOExample example); - - SPGJobInstDOWithBLOBs selectByPrimaryKey(Long id); - - int updateByExampleSelective( - @Param("record") SPGJobInstDOWithBLOBs record, @Param("example") SPGJobInstDOExample example); - - int updateByExampleWithBLOBs( - @Param("record") SPGJobInstDOWithBLOBs record, @Param("example") SPGJobInstDOExample example); - - int updateByExample( - @Param("record") SPGJobInstDO record, @Param("example") SPGJobInstDOExample example); - - int updateByPrimaryKeySelective(SPGJobInstDOWithBLOBs record); - - int updateByPrimaryKeyWithBLOBs(SPGJobInstDOWithBLOBs record); - - int updateByPrimaryKey(SPGJobInstDO record); -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/SchedulerJobInfoRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/SchedulerJobInfoRepositoryImpl.java deleted file mode 100644 index b6d03a16..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/SchedulerJobInfoRepositoryImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.scheduler; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo.SchedulerJobInfoRepository; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInfoDO; -import com.antgroup.openspg.server.infra.dao.mapper.JobInfoDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.scheduler.convertor.SchedulerJobInfoConvertor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Repository; - -@Repository -public class SchedulerJobInfoRepositoryImpl implements SchedulerJobInfoRepository { - - @Autowired private JobInfoDOMapper jobInfoDOMapper; - - @Override - public String save(SchedulerJobInfo jobInfo) { - JobInfoDO jobInfoDO = SchedulerJobInfoConvertor.toDO(jobInfo); - jobInfoDOMapper.insert(jobInfoDO); - return String.valueOf(jobInfoDO.getId()); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/SchedulerJobInstRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/SchedulerJobInstRepositoryImpl.java deleted file mode 100644 index cb85972d..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/SchedulerJobInstRepositoryImpl.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.scheduler; - -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd.SchedulerJobInstQuery; -import com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo.SchedulerJobInstRepository; -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.common.util.CollectionsUtils; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInstDO; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInstDOExample; -import com.antgroup.openspg.server.infra.dao.mapper.JobInstDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.scheduler.convertor.SchedulerJobInstConvertor; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Repository; - -@Repository -public class SchedulerJobInstRepositoryImpl implements SchedulerJobInstRepository { - - @Autowired private JobInstDOMapper jobInstInfoDOMapper; - - @Override - public String save(SchedulerJobInst jobInst) { - JobInstDO jobInstDO = SchedulerJobInstConvertor.toDO(jobInst); - jobInstInfoDOMapper.insert(jobInstDO); - return String.valueOf(jobInstDO.getId()); - } - - @Override - public List query(SchedulerJobInstQuery query) { - JobInstDOExample example = new JobInstDOExample(); - JobInstDOExample.Criteria criteria = example.createCriteria(); - - if (query.getStatus() != null) { - criteria.andStatusIn(CollectionsUtils.listMap(query.getStatus(), Enum::name)); - } - if (query.getOrderBy() != null) { - example.setOrderByClause(query.getOrderBy()); - } - - List jobInstDOS = jobInstInfoDOMapper.selectByExample(example); - return CollectionsUtils.listMap(jobInstDOS, SchedulerJobInstConvertor::toModel); - } - - @Override - public int updateStatus(String jobInstId, JobInstStatusEnum status) { - JobInstDOExample example = new JobInstDOExample(); - example.createCriteria().andIdEqualTo(Long.valueOf(jobInstId)); - - JobInstDO jobInstDO = new JobInstDO(); - jobInstDO.setStatus(status.name()); - return jobInstInfoDOMapper.updateByExampleSelective(jobInstDO, example); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/convertor/SchedulerJobInfoConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/convertor/SchedulerJobInfoConvertor.java deleted file mode 100644 index 21e21cb0..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/convertor/SchedulerJobInfoConvertor.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.scheduler.convertor; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo; -import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInfoDO; - -public class SchedulerJobInfoConvertor { - - public static JobInfoDO toDO(SchedulerJobInfo jobInfo) { - JobInfoDO jobInfoDO = new JobInfoDO(); - - if (jobInfo.getJobId() != null) { - jobInfoDO.setId(Long.valueOf(jobInfo.getJobId())); - } - jobInfoDO.setName(jobInfo.getJobName()); - jobInfoDO.setType(jobInfo.getJobType()); - jobInfoDO.setCron(jobInfo.getCron()); - jobInfoDO.setStatus(jobInfo.getStatus().name()); - jobInfoDO.setIdempotentId(jobInfo.getIdempotentId()); - return jobInfoDO; - } - - public static SchedulerJobInfo toModel(JobInfoDO jobInfoDO) { - if (jobInfoDO == null) { - return null; - } - return new SchedulerJobInfo( - String.valueOf(jobInfoDO.getId()), - jobInfoDO.getName(), - jobInfoDO.getType(), - jobInfoDO.getCron(), - JobInfoStateEnum.valueOf(jobInfoDO.getStatus()), - jobInfoDO.getIdempotentId()); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/convertor/SchedulerJobInstConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/convertor/SchedulerJobInstConvertor.java deleted file mode 100644 index e129a18b..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/scheduler/convertor/SchedulerJobInstConvertor.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.scheduler.convertor; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst; -import com.antgroup.openspg.common.util.StringUtils; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.infra.dao.dataobject.JobInstDO; - -public class SchedulerJobInstConvertor { - - public static JobInstDO toDO(SchedulerJobInst jobInst) { - JobInstDO jobInstDO = new JobInstDO(); - - if (StringUtils.isNotBlank(jobInst.getJobInstId())) { - jobInstDO.setId(Long.valueOf(jobInst.getJobInstId())); - } - jobInstDO.setJobId(Long.valueOf(jobInst.getJobId())); - jobInstDO.setType(jobInst.getJobType()); - jobInstDO.setStatus(jobInst.getStatus().name()); - jobInstDO.setIdempotentId(jobInst.getIdempotentId()); - return jobInstDO; - } - - public static SchedulerJobInst toModel(JobInstDO jobInstDO) { - if (jobInstDO == null) { - return null; - } - return new SchedulerJobInst( - String.valueOf(jobInstDO.getId()), - String.valueOf(jobInstDO.getJobId()), - jobInstDO.getType(), - JobInstStatusEnum.valueOf(jobInstDO.getStatus()), - jobInstDO.getHost(), - jobInstDO.getIdempotentId()); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/ConstraintRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/ConstraintRepositoryImpl.java similarity index 94% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/ConstraintRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/ConstraintRepositoryImpl.java index 9b956792..5ebde3b2 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/ConstraintRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/ConstraintRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.core.schema.model.constraint.Constraint; @@ -20,7 +20,7 @@ import com.antgroup.openspg.server.core.schema.service.predicate.repository.Cons import com.antgroup.openspg.server.infra.dao.dataobject.ConstraintDO; import com.antgroup.openspg.server.infra.dao.dataobject.ConstraintDOExample; import com.antgroup.openspg.server.infra.dao.mapper.ConstraintDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.ConstraintDOConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.ConstraintDOConvertor; import java.util.Collections; import java.util.Date; import java.util.List; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/LogicalRuleRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/LogicalRuleRepositoryImpl.java similarity index 95% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/LogicalRuleRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/LogicalRuleRepositoryImpl.java index 2a387bdd..d73b7b46 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/LogicalRuleRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/LogicalRuleRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; @@ -21,7 +21,7 @@ import com.antgroup.openspg.server.core.schema.service.semantic.repository.Logic import com.antgroup.openspg.server.infra.dao.dataobject.LogicRuleDO; import com.antgroup.openspg.server.infra.dao.dataobject.LogicRuleDOExample; import com.antgroup.openspg.server.infra.dao.mapper.LogicRuleDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.LogicalRuleConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.LogicalRuleConvertor; import java.util.Date; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/OntologyParentRelRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/OntologyParentRelRepositoryImpl.java similarity index 93% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/OntologyParentRelRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/OntologyParentRelRepositoryImpl.java index 6a08518c..7f767ab5 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/OntologyParentRelRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/OntologyParentRelRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum; @@ -24,8 +24,8 @@ import com.antgroup.openspg.server.infra.dao.dataobject.OntologyParentRelDO; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyParentRelDOExample; import com.antgroup.openspg.server.infra.dao.mapper.OntologyDOMapper; import com.antgroup.openspg.server.infra.dao.mapper.OntologyParentRelDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.OntologyParentRelConvertor; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ValidStatusEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.OntologyParentRelConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/ProjectOntologyRelRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/ProjectOntologyRelRepositoryImpl.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/ProjectOntologyRelRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/ProjectOntologyRelRepositoryImpl.java index e14a299d..528123b4 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/ProjectOntologyRelRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/ProjectOntologyRelRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum; @@ -22,9 +22,9 @@ import com.antgroup.openspg.server.core.schema.service.type.repository.ProjectOn import com.antgroup.openspg.server.infra.dao.dataobject.ProjectOntologyRelDO; import com.antgroup.openspg.server.infra.dao.dataobject.ProjectOntologyRelDOExample; import com.antgroup.openspg.server.infra.dao.mapper.ProjectOntologyRelDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.ProjectOntologyRelConvertor; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ProjectEntityTypeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.YesOrNoEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.ProjectOntologyRelConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ProjectEntityTypeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.YesOrNoEnum; import com.google.common.collect.Lists; import java.util.List; import org.apache.commons.collections4.CollectionUtils; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/PropertyRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java similarity index 95% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/PropertyRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java index 3591026a..e7cd88ff 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/PropertyRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/PropertyRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum; @@ -24,9 +24,9 @@ import com.antgroup.openspg.server.core.schema.service.type.repository.SPGTypeRe import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDO; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDOExample; import com.antgroup.openspg.server.infra.dao.mapper.OntologyPropertyDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.SimplePropertyConvertor; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.MapTypeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.PropertyCategoryEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.SimplePropertyConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.MapTypeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.PropertyCategoryEnum; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Collections; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SPGTypeRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SPGTypeRepositoryImpl.java similarity index 95% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SPGTypeRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SPGTypeRepositoryImpl.java index 4f185631..d4dffb96 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SPGTypeRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SPGTypeRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum; import com.antgroup.openspg.core.schema.model.semantic.SPGOntologyEnum; @@ -25,9 +25,9 @@ import com.antgroup.openspg.server.core.schema.service.type.repository.SPGTypeRe import com.antgroup.openspg.server.infra.dao.dataobject.OntologyDOExample; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyDOWithBLOBs; import com.antgroup.openspg.server.infra.dao.mapper.OntologyDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.OntologyConvertor; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.EntityCategoryEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ValidStatusEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.OntologyConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.EntityCategoryEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; import java.util.Collections; import java.util.Date; import java.util.List; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SemanticRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SemanticRepositoryImpl.java similarity index 97% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SemanticRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SemanticRepositoryImpl.java index 78463d8b..a40d6fac 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SemanticRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SemanticRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.common.util.StringUtils; @@ -24,7 +24,7 @@ import com.antgroup.openspg.server.core.schema.service.semantic.repository.Seman import com.antgroup.openspg.server.infra.dao.dataobject.SemanticDO; import com.antgroup.openspg.server.infra.dao.dataobject.SemanticDOExample; import com.antgroup.openspg.server.infra.dao.mapper.SemanticDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.SimpleSemanticConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.SimpleSemanticConvertor; import java.util.Date; import java.util.List; import org.apache.commons.collections4.CollectionUtils; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SubPropertyRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SubPropertyRepositoryImpl.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SubPropertyRepositoryImpl.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SubPropertyRepositoryImpl.java index de30bb00..deb2b31c 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/SubPropertyRepositoryImpl.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/SubPropertyRepositoryImpl.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema; +package com.antgroup.openspg.server.infra.dao.repository.schema; import com.antgroup.openspg.common.util.CollectionsUtils; import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum; @@ -21,9 +21,9 @@ import com.antgroup.openspg.server.core.schema.service.predicate.repository.SubP import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDO; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDOExample; import com.antgroup.openspg.server.infra.dao.mapper.OntologyPropertyDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor.SimpleSubPropertyConvertor; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.MapTypeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.PropertyCategoryEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.convertor.SimpleSubPropertyConvertor; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.MapTypeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.PropertyCategoryEnum; import java.util.Collections; import java.util.List; import org.apache.commons.collections4.CollectionUtils; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/Constants.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/Constants.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/Constants.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/Constants.java index 719873bc..cbf4d765 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/Constants.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/Constants.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.config; +package com.antgroup.openspg.server.infra.dao.repository.schema.config; public class Constants { diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/ConstraintItemConfigDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/ConstraintItemConfigDO.java similarity index 90% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/ConstraintItemConfigDO.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/ConstraintItemConfigDO.java index 20ae1214..7acc5d27 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/ConstraintItemConfigDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/ConstraintItemConfigDO.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.config; +package com.antgroup.openspg.server.infra.dao.repository.schema.config; import lombok.Data; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/MultiVersionConfigDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/MultiVersionConfigDO.java similarity index 92% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/MultiVersionConfigDO.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/MultiVersionConfigDO.java index 1b05f2a4..442526b0 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/MultiVersionConfigDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/MultiVersionConfigDO.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.config; +package com.antgroup.openspg.server.infra.dao.repository.schema.config; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/OntologyEntityName.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/OntologyEntityName.java similarity index 97% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/OntologyEntityName.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/OntologyEntityName.java index 26ada493..da833b72 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/OntologyEntityName.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/OntologyEntityName.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.config; +package com.antgroup.openspg.server.infra.dao.repository.schema.config; import lombok.Getter; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/OperatorConfigDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/OperatorConfigDO.java similarity index 95% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/OperatorConfigDO.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/OperatorConfigDO.java index 2615044b..1c25ba6b 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/OperatorConfigDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/OperatorConfigDO.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.config; +package com.antgroup.openspg.server.infra.dao.repository.schema.config; import lombok.Data; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/StandardConfigDO.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/StandardConfigDO.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/StandardConfigDO.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/StandardConfigDO.java index 15f99f50..76dff3dc 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/config/StandardConfigDO.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/config/StandardConfigDO.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.config; +package com.antgroup.openspg.server.infra.dao.repository.schema.config; import com.antgroup.openspg.server.common.model.base.BaseToString; import lombok.AllArgsConstructor; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ConstraintDOConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ConstraintDOConvertor.java similarity index 96% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ConstraintDOConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ConstraintDOConvertor.java index 135c5d0d..5efa6cc1 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ConstraintDOConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ConstraintDOConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.alibaba.fastjson.JSON; import com.antgroup.openspg.core.schema.model.constraint.BaseConstraintItem; @@ -24,8 +24,8 @@ import com.antgroup.openspg.core.schema.model.constraint.RangeConstraint; import com.antgroup.openspg.core.schema.model.constraint.RegularConstraint; import com.antgroup.openspg.core.schema.model.constraint.UniqueConstraint; import com.antgroup.openspg.server.infra.dao.dataobject.ConstraintDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ConstraintRangeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.YesOrNoEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ConstraintRangeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.YesOrNoEnum; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ConstraintItemConfigDOConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ConstraintItemConfigDOConvertor.java similarity index 94% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ConstraintItemConfigDOConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ConstraintItemConfigDOConvertor.java index 9bc31d74..e406d2e1 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ConstraintItemConfigDOConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ConstraintItemConfigDOConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.core.schema.model.constraint.BaseConstraintItem; import com.antgroup.openspg.core.schema.model.constraint.EnumConstraint; @@ -20,8 +20,8 @@ import com.antgroup.openspg.core.schema.model.constraint.NotNullConstraint; import com.antgroup.openspg.core.schema.model.constraint.RangeConstraint; import com.antgroup.openspg.core.schema.model.constraint.RegularConstraint; import com.antgroup.openspg.core.schema.model.constraint.UniqueConstraint; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.config.ConstraintItemConfigDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ConstraintEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.config.ConstraintItemConfigDO; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ConstraintEnum; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Collections; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ExtConfigConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java similarity index 90% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ExtConfigConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java index a404b15c..ba19218d 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ExtConfigConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ExtConfigConvertor.java @@ -11,11 +11,11 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.core.schema.model.SchemaConstants; import com.antgroup.openspg.core.schema.model.SchemaExtInfo; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleProperty; import com.antgroup.openspg.server.core.schema.service.type.model.SimpleSPGType; import java.util.stream.Collectors; @@ -50,7 +50,7 @@ public class ExtConfigConvertor { default: break; } - return JSON.serialize(extInfo); + return SchemaJsonUtils.serialize(extInfo); } public static String getExtConfig(SimpleProperty simplePredicate) { @@ -71,7 +71,7 @@ public class ExtConfigConvertor { extInfo.put(SchemaConstants.PROPERTY_GROUP_KEY, simplePredicate.getPropertyGroup().name()); } extInfo.put(SchemaConstants.VALUE_TYPE_KEY, simplePredicate.getObjectTypeEnum().name()); - return JSON.serialize(extInfo); + return SchemaJsonUtils.serialize(extInfo); } public static T get(SchemaExtInfo schemaExtInfo, String key, Class clazz) { @@ -81,6 +81,6 @@ public class ExtConfigConvertor { if (!schemaExtInfo.containsKey(key)) { return null; } - return JSON.deserialize(schemaExtInfo.get(key).toString(), clazz); + return SchemaJsonUtils.deserialize(schemaExtInfo.get(key).toString(), clazz); } } diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/LogicalRuleConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/LogicalRuleConvertor.java similarity index 96% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/LogicalRuleConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/LogicalRuleConvertor.java index bca5da62..5b838508 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/LogicalRuleConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/LogicalRuleConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.core.schema.model.semantic.LogicalRule; import com.antgroup.openspg.core.schema.model.semantic.RuleCode; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/OntologyConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java similarity index 89% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/OntologyConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java index 1088fdaf..7948825f 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/OntologyConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.BasicInfo; @@ -27,18 +27,18 @@ import com.antgroup.openspg.core.schema.model.type.MultiVersionConfig; import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; import com.antgroup.openspg.core.schema.model.type.VisibleScopeEnum; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.type.model.OperatorConfig; import com.antgroup.openspg.server.core.schema.service.type.model.ProjectOntologyRel; import com.antgroup.openspg.server.core.schema.service.type.model.SimpleSPGType; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyDO; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyDOWithBLOBs; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.config.ConstraintItemConfigDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.config.OntologyEntityName; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.EntityCategoryEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.LayerEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.TrueOrFalseEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ValidStatusEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.config.ConstraintItemConfigDO; +import com.antgroup.openspg.server.infra.dao.repository.schema.config.OntologyEntityName; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.EntityCategoryEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.LayerEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.TrueOrFalseEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; import com.google.gson.reflect.TypeToken; import java.util.ArrayList; import java.util.Date; @@ -74,7 +74,7 @@ public class OntologyConvertor { ontologyDO.setVersion(1); ontologyDO.setVersionStatus(AlterStatusEnum.ONLINE.name()); ontologyDO.setTransformerId(0L); - ontologyDO.setOperatorConfig(JSON.serialize(advancedType.getOperatorConfigs())); + ontologyDO.setOperatorConfig(SchemaJsonUtils.serialize(advancedType.getOperatorConfigs())); ontologyDO.setConfig(ExtConfigConvertor.getExtConfig(advancedType)); return ontologyDO; } @@ -95,7 +95,7 @@ public class OntologyConvertor { : TrueOrFalseEnum.FALSE.name()); ontologyDO.setScope(advancedType.getVisibleScope().name()); ontologyDO.setVersion(1); - ontologyDO.setOperatorConfig(JSON.serialize(advancedType.getOperatorConfigs())); + ontologyDO.setOperatorConfig(SchemaJsonUtils.serialize(advancedType.getOperatorConfigs())); ontologyDO.setConfig(ExtConfigConvertor.getExtConfig(advancedType)); return ontologyDO; } @@ -139,9 +139,10 @@ public class OntologyConvertor { ontologyDO.getDescription()); SPGTypeEnum spgTypeEnum = EntityCategoryEnum.toSpgType(ontologyDO.getEntityCategory()); VisibleScopeEnum visibleScopeEnum = VisibleScopeEnum.toEnum(ontologyDO.getScope()); - SchemaExtInfo extConfig = JSON.deserialize(ontologyDO.getConfig(), SchemaExtInfo.class); + SchemaExtInfo extConfig = + SchemaJsonUtils.deserialize(ontologyDO.getConfig(), SchemaExtInfo.class); List operatorConfigs = - JSON.deserialize( + SchemaJsonUtils.deserialize( ontologyDO.getOperatorConfig(), new TypeToken>() {}.getType()); Long projectId = projectOntologyRel == null ? null : projectOntologyRel.getProjectId(); @@ -166,8 +167,8 @@ public class OntologyConvertor { Object constraint = extConfig.get(SchemaConstants.STANDARD_CONSTRAINT_KEY); if (constraint instanceof List) { List constraintItemConfigDOS = - JSON.deserialize( - JSON.serialize(constraint), + SchemaJsonUtils.deserialize( + SchemaJsonUtils.serialize(constraint), new TypeToken>() {}.getType()); constraintItems = ConstraintItemConfigDOConvertor.toConstraintItem(constraintItemConfigDOS); diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/OntologyParentRelConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyParentRelConvertor.java similarity index 88% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/OntologyParentRelConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyParentRelConvertor.java index 0956c4f5..84425f21 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/OntologyParentRelConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/OntologyParentRelConvertor.java @@ -11,17 +11,17 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyDO; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyDOWithBLOBs; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyParentRelDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.config.Constants; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.config.OntologyEntityName; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ValidStatusEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.YesOrNoEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.config.Constants; +import com.antgroup.openspg.server.infra.dao.repository.schema.config.OntologyEntityName; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.YesOrNoEnum; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ProjectOntologyRelConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ProjectOntologyRelConvertor.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ProjectOntologyRelConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ProjectOntologyRelConvertor.java index e20db87a..c9b31000 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/ProjectOntologyRelConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/ProjectOntologyRelConvertor.java @@ -11,14 +11,14 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum; import com.antgroup.openspg.core.schema.model.type.RefSourceEnum; import com.antgroup.openspg.server.core.schema.service.type.model.ProjectOntologyRel; import com.antgroup.openspg.server.infra.dao.dataobject.ProjectOntologyRelDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ProjectEntityTypeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.YesOrNoEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ProjectEntityTypeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.YesOrNoEnum; import java.util.Date; public class ProjectOntologyRelConvertor { diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimplePropertyConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimplePropertyConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java index 38e6529a..8afda2f6 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimplePropertyConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimplePropertyConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.BasicInfo; @@ -27,12 +27,12 @@ import com.antgroup.openspg.core.schema.model.semantic.RuleCode; import com.antgroup.openspg.core.schema.model.semantic.SPGOntologyEnum; import com.antgroup.openspg.core.schema.model.type.MultiVersionConfig; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; -import com.antgroup.openspg.server.api.facade.JSON; +import com.antgroup.openspg.server.api.facade.SchemaJsonUtils; import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleProperty; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.MapTypeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.PropertyCategoryEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ValidStatusEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.MapTypeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.PropertyCategoryEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; import java.util.Date; public class SimplePropertyConvertor { @@ -80,7 +80,7 @@ public class SimplePropertyConvertor { propertyRangeDO.setMultiverConfig( simpleProperty.getMultiVersionConfig() == null ? null - : JSON.serialize(simpleProperty.getMultiVersionConfig())); + : SchemaJsonUtils.serialize(simpleProperty.getMultiVersionConfig())); propertyRangeDO.setStorePropertyName(basicInfo.getName().getName()); propertyRangeDO.setPropertySource(null); propertyRangeDO.setTransformerId(0L); @@ -108,7 +108,7 @@ public class SimplePropertyConvertor { propertyRangeDO.setMultiverConfig( simpleProperty.getMultiVersionConfig() == null ? null - : JSON.serialize(simpleProperty.getMultiVersionConfig())); + : SchemaJsonUtils.serialize(simpleProperty.getMultiVersionConfig())); propertyRangeDO.setPropertyConfig(ExtConfigConvertor.getExtConfig(simpleProperty)); return propertyRangeDO; } @@ -120,13 +120,13 @@ public class SimplePropertyConvertor { propertyRangeDO.getPropertyNameZh(), propertyRangeDO.getPropertyDesc()); SchemaExtInfo schemaExtInfo = - JSON.deserialize(propertyRangeDO.getPropertyConfig(), SchemaExtInfo.class); + SchemaJsonUtils.deserialize(propertyRangeDO.getPropertyConfig(), SchemaExtInfo.class); if (schemaExtInfo == null) { schemaExtInfo = new SchemaExtInfo(); } MultiVersionConfig multiVersionConfig = - JSON.deserialize(propertyRangeDO.getMultiverConfig(), MultiVersionConfig.class); + SchemaJsonUtils.deserialize(propertyRangeDO.getMultiverConfig(), MultiVersionConfig.class); MountedConceptConfig mountedConceptConfig = ExtConfigConvertor.get( schemaExtInfo, SchemaConstants.MOUNT_CONCEPT_CONFIG_KEY, MountedConceptConfig.class); diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimpleSemanticConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSemanticConvertor.java similarity index 97% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimpleSemanticConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSemanticConvertor.java index 987f10f6..7641408f 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimpleSemanticConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSemanticConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.antgroup.openspg.common.util.StringUtils; import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimpleSubPropertyConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java similarity index 94% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimpleSubPropertyConvertor.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java index 6d538d77..0fecfab5 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/convertor/SimpleSubPropertyConvertor.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/convertor/SimpleSubPropertyConvertor.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.convertor; +package com.antgroup.openspg.server.infra.dao.repository.schema.convertor; import com.alibaba.fastjson.JSON; import com.antgroup.openspg.core.schema.model.BasicInfo; @@ -23,9 +23,9 @@ import com.antgroup.openspg.core.schema.model.predicate.EncryptTypeEnum; import com.antgroup.openspg.core.schema.model.type.MultiVersionConfig; import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleSubProperty; import com.antgroup.openspg.server.infra.dao.dataobject.OntologyPropertyDO; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.MapTypeEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.PropertyCategoryEnum; -import com.antgroup.openspg.server.infra.dao.repository.spgschema.enums.ValidStatusEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.MapTypeEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.PropertyCategoryEnum; +import com.antgroup.openspg.server.infra.dao.repository.schema.enums.ValidStatusEnum; import java.util.Date; public class SimpleSubPropertyConvertor { diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ConstraintEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ConstraintEnum.java similarity index 96% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ConstraintEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ConstraintEnum.java index d15c06ec..b416d801 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ConstraintEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ConstraintEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; public enum ConstraintEnum { diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ConstraintRangeEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ConstraintRangeEnum.java similarity index 97% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ConstraintRangeEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ConstraintRangeEnum.java index c0168076..77fb2f4d 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ConstraintRangeEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ConstraintRangeEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; import com.antgroup.openspg.core.schema.model.constraint.RangeConstraint; import org.apache.commons.lang3.StringUtils; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/EntityCategoryEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java similarity index 96% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/EntityCategoryEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java index fc9003ac..20735ee3 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/EntityCategoryEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/EntityCategoryEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/LayerEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/LayerEnum.java similarity index 87% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/LayerEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/LayerEnum.java index 01475135..a3bcb0c9 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/LayerEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/LayerEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; public enum LayerEnum { CORE, diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/MapTypeEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/MapTypeEnum.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/MapTypeEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/MapTypeEnum.java index 7a47158d..5290dc23 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/MapTypeEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/MapTypeEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; public enum MapTypeEnum { EDGE, diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ProjectEntityTypeEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ProjectEntityTypeEnum.java similarity index 94% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ProjectEntityTypeEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ProjectEntityTypeEnum.java index dd1ba6d6..e2d39aa2 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ProjectEntityTypeEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ProjectEntityTypeEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; import com.antgroup.openspg.core.schema.model.semantic.SPGOntologyEnum; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/PropertyCategoryEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/PropertyCategoryEnum.java similarity index 92% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/PropertyCategoryEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/PropertyCategoryEnum.java index be1257ce..880be198 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/PropertyCategoryEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/PropertyCategoryEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; public enum PropertyCategoryEnum { /** Basic type. */ diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/RelationDirectEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/RelationDirectEnum.java similarity index 91% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/RelationDirectEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/RelationDirectEnum.java index de15d09e..15a70548 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/RelationDirectEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/RelationDirectEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; public enum RelationDirectEnum { DOUBLE, diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/TrueOrFalseEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/TrueOrFalseEnum.java similarity index 93% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/TrueOrFalseEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/TrueOrFalseEnum.java index 39f8a7df..c1c3a5b9 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/TrueOrFalseEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/TrueOrFalseEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; /** True or false enum. */ public enum TrueOrFalseEnum { diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ValidStatusEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ValidStatusEnum.java similarity index 90% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ValidStatusEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ValidStatusEnum.java index ee7a7228..0e8ede24 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/ValidStatusEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/ValidStatusEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; public enum ValidStatusEnum { VALID("1"), diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/YesOrNoEnum.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/YesOrNoEnum.java similarity index 93% rename from server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/YesOrNoEnum.java rename to server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/YesOrNoEnum.java index 24df9581..261f4deb 100644 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgschema/enums/YesOrNoEnum.java +++ b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/schema/enums/YesOrNoEnum.java @@ -11,7 +11,7 @@ * or implied. */ -package com.antgroup.openspg.server.infra.dao.repository.spgschema.enums; +package com.antgroup.openspg.server.infra.dao.repository.schema.enums; import com.google.common.collect.Lists; import java.util.List; diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/ReasonerJobInfoRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/ReasonerJobInfoRepositoryImpl.java deleted file mode 100644 index a5feef9b..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/ReasonerJobInfoRepositoryImpl.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.spgreasoner; - -import com.antgroup.openspg.common.util.CollectionsUtils; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInfoQuery; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.core.reasoner.service.repo.ReasonerJobInfoRepository; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInfoDO; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInfoDOExample; -import com.antgroup.openspg.server.infra.dao.mapper.SPGJobInfoDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgreasoner.convertor.ReasonerJobInfoConvertor; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Repository; - -@Repository -public class ReasonerJobInfoRepositoryImpl implements ReasonerJobInfoRepository { - - @Autowired private SPGJobInfoDOMapper spgJobInfoDOMapper; - - @Override - public Long save(ReasonerJobInfo jobInfo) { - SPGJobInfoDO jobInfoDO = ReasonerJobInfoConvertor.toDO(jobInfo); - spgJobInfoDOMapper.insert(jobInfoDO); - return jobInfoDO.getId(); - } - - @Override - public int updateExternalJobId(Long reasonerJobInfoId, String externalJobInfoId) { - SPGJobInfoDOExample example = new SPGJobInfoDOExample(); - example.createCriteria().andIdEqualTo(reasonerJobInfoId); - - SPGJobInfoDO jobInfoDO = new SPGJobInfoDO(); - jobInfoDO.setExternalJobInfoId(externalJobInfoId); - return spgJobInfoDOMapper.updateByExampleSelective(jobInfoDO, example); - } - - @Override - public List query(ReasonerJobInfoQuery query) { - SPGJobInfoDOExample example = new SPGJobInfoDOExample(); - SPGJobInfoDOExample.Criteria criteria = example.createCriteria(); - if (query.getReasonerJobInfoId() != null) { - criteria.andIdEqualTo(query.getReasonerJobInfoId()); - } - if (query.getExternalJobInfoId() != null) { - criteria.andExternalJobInfoIdEqualTo(query.getExternalJobInfoId()); - } - example.setOrderByClause("id desc"); - List spgJobInfoDOS = spgJobInfoDOMapper.selectByExampleWithBLOBs(example); - return CollectionsUtils.listMap(spgJobInfoDOS, ReasonerJobInfoConvertor::toModel); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/ReasonerJobInstRepositoryImpl.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/ReasonerJobInstRepositoryImpl.java deleted file mode 100644 index 271b4bf5..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/ReasonerJobInstRepositoryImpl.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.spgreasoner; - -import com.antgroup.openspg.common.util.CollectionsUtils; -import com.antgroup.openspg.server.api.facade.JSON; -import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress; -import com.antgroup.openspg.server.core.reasoner.service.repo.ReasonerJobInstRepository; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInstDOExample; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInstDOWithBLOBs; -import com.antgroup.openspg.server.infra.dao.mapper.SPGJobInstDOMapper; -import com.antgroup.openspg.server.infra.dao.repository.spgreasoner.convertor.ReasonerJobInstConvertor; -import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Repository; - -@Repository -public class ReasonerJobInstRepositoryImpl implements ReasonerJobInstRepository { - - @Autowired private SPGJobInstDOMapper spgJobInstDOMapper; - - @Override - public Long save(ReasonerJobInst jobInst) { - SPGJobInstDOWithBLOBs jobInstDO = ReasonerJobInstConvertor.toDO(jobInst); - spgJobInstDOMapper.insert(jobInstDO); - return jobInstDO.getId(); - } - - @Override - public int updateExternalJobId(Long reasonerJobInstId, String externalJobInstId) { - SPGJobInstDOExample example = new SPGJobInstDOExample(); - example.createCriteria().andIdEqualTo(reasonerJobInstId); - - SPGJobInstDOWithBLOBs jobInstDO = new SPGJobInstDOWithBLOBs(); - jobInstDO.setExternalJobInstId(externalJobInstId); - return spgJobInstDOMapper.updateByExampleSelective(jobInstDO, example); - } - - @Override - public List query(ReasonerJobInstQuery query) { - SPGJobInstDOExample example = new SPGJobInstDOExample(); - SPGJobInstDOExample.Criteria criteria = example.createCriteria(); - if (query.getReasonerJobInstId() != null) { - criteria.andIdEqualTo(query.getReasonerJobInstId()); - } - if (query.getExternalJobInstId() != null) { - criteria.andExternalJobInstIdEqualTo(query.getExternalJobInstId()); - } - - List spgJobInstDOS = - spgJobInstDOMapper.selectByExampleWithBLOBs(example); - return CollectionsUtils.listMap(spgJobInstDOS, ReasonerJobInstConvertor::toModel); - } - - @Override - public int updateStatus(Long jobInstId, ReasonerStatusWithProgress process) { - SPGJobInstDOExample example = new SPGJobInstDOExample(); - example.createCriteria().andIdEqualTo(jobInstId); - - SPGJobInstDOWithBLOBs jobInstDO = new SPGJobInstDOWithBLOBs(); - jobInstDO.setStatus(process.getStatus().name()); - jobInstDO.setResult(JSON.serialize(process.getResult())); - jobInstDO.setProgress(JSON.serialize(process.getProgress())); - return spgJobInstDOMapper.updateByExampleSelective(jobInstDO, example); - } - - @Override - public int updateToFailure(Long jobInstId, FailureReasonerResult result) { - SPGJobInstDOExample example = new SPGJobInstDOExample(); - example.createCriteria().andIdEqualTo(jobInstId); - - SPGJobInstDOWithBLOBs jobInstDO = new SPGJobInstDOWithBLOBs(); - jobInstDO.setStatus(JobInstStatusEnum.FAILURE.name()); - jobInstDO.setResult(JSON.serialize(result)); - return spgJobInstDOMapper.updateByExampleSelective(jobInstDO, example); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/convertor/ReasonerJobInfoConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/convertor/ReasonerJobInfoConvertor.java deleted file mode 100644 index 92c558eb..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/convertor/ReasonerJobInfoConvertor.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.spgreasoner.convertor; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.server.api.facade.JSON; -import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInfoDO; -import com.google.gson.reflect.TypeToken; -import java.util.Map; - -public class ReasonerJobInfoConvertor { - - public static SPGJobInfoDO toDO(ReasonerJobInfo reasonerJobInfo) { - SPGJobInfoDO jobInfoDO = new SPGJobInfoDO(); - jobInfoDO.setId(reasonerJobInfo.getJobId()); - jobInfoDO.setName(reasonerJobInfo.getJobName()); - jobInfoDO.setType(JobTypeEnum.REASONING.name()); - jobInfoDO.setProjectId(reasonerJobInfo.getProjectId()); - jobInfoDO.setCron(reasonerJobInfo.getCron()); - jobInfoDO.setStatus(reasonerJobInfo.getStatus().name()); - jobInfoDO.setExtInfo(JSON.serialize(reasonerJobInfo.getParams())); - jobInfoDO.setContent(JSON.serialize(reasonerJobInfo.getContent())); - jobInfoDO.setExternalJobInfoId(reasonerJobInfo.getExternalJobInfoId()); - return jobInfoDO; - } - - public static ReasonerJobInfo toModel(SPGJobInfoDO jobInfoDO) { - if (jobInfoDO == null) { - return null; - } - if (!JobTypeEnum.REASONING.name().equals(jobInfoDO.getType())) { - return null; - } - - return new ReasonerJobInfo( - jobInfoDO.getName(), - jobInfoDO.getProjectId(), - JSON.deserialize(jobInfoDO.getContent(), BaseReasonerContent.class), - jobInfoDO.getCron(), - JobInfoStateEnum.valueOf(jobInfoDO.getStatus()), - JSON.deserialize( - jobInfoDO.getExtInfo(), new TypeToken>() {}.getType())) - .setJobId(jobInfoDO.getId()); - } -} diff --git a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/convertor/ReasonerJobInstConvertor.java b/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/convertor/ReasonerJobInstConvertor.java deleted file mode 100644 index 81653f94..00000000 --- a/server/infra/dao/src/main/java/com/antgroup/openspg/server/infra/dao/repository/spgreasoner/convertor/ReasonerJobInstConvertor.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2023 Ant Group CO., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. - */ - -package com.antgroup.openspg.server.infra.dao.repository.spgreasoner.convertor; - -import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum; -import com.antgroup.openspg.server.api.facade.JSON; -import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum; -import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerResult; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst; -import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerProgress; -import com.antgroup.openspg.server.infra.dao.dataobject.SPGJobInstDOWithBLOBs; - -public class ReasonerJobInstConvertor { - - public static ReasonerJobInst toModel(SPGJobInstDOWithBLOBs jobInstDO) { - if (jobInstDO == null) { - return null; - } - if (!JobTypeEnum.REASONING.name().equals(jobInstDO.getType())) { - return null; - } - return new ReasonerJobInst( - jobInstDO.getJobId(), - jobInstDO.getProjectId(), - JobInstStatusEnum.valueOf(jobInstDO.getStatus()), - JSON.deserialize(jobInstDO.getResult(), BaseReasonerResult.class), - jobInstDO.getStartTime(), - jobInstDO.getEndTime(), - JSON.deserialize(jobInstDO.getProgress(), ReasonerProgress.class), - jobInstDO.getLogInfo()) - .setJobInstId(jobInstDO.getId()) - .setExternalJobInstId(jobInstDO.getExternalJobInstId()); - } - - public static SPGJobInstDOWithBLOBs toDO(ReasonerJobInst jobInst) { - SPGJobInstDOWithBLOBs jobInstDO = new SPGJobInstDOWithBLOBs(); - - jobInstDO.setId(jobInst.getJobInstId()); - jobInstDO.setJobId(jobInst.getJobId()); - jobInstDO.setType(JobTypeEnum.REASONING.name()); - jobInstDO.setProjectId(jobInst.getProjectId()); - jobInstDO.setStatus(jobInst.getStatus().name()); - jobInstDO.setStartTime(jobInst.getStartTime()); - jobInstDO.setEndTime(jobInst.getEndTime()); - jobInstDO.setProgress(JSON.serialize(jobInst.getProgress())); - jobInstDO.setExternalJobInstId(jobInst.getExternalJobInstId()); - jobInstDO.setResult(JSON.serialize(jobInst.getResult())); - jobInstDO.setLogInfo(jobInst.getLogInfo()); - jobInstDO.setExternalJobInstId(jobInst.getExternalJobInstId()); - return jobInstDO; - } -} diff --git a/server/infra/dao/src/main/resources/mapper/JobInfoDOMapper.xml b/server/infra/dao/src/main/resources/mapper/JobInfoDOMapper.xml deleted file mode 100644 index 9c72e9e8..00000000 --- a/server/infra/dao/src/main/resources/mapper/JobInfoDOMapper.xml +++ /dev/null @@ -1,271 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, gmt_create, gmt_modified, name, type, cron, status, idempotent_id - - - - - delete from kg_schedule_job_info - where id = #{id,jdbcType=BIGINT} - - - delete from kg_schedule_job_info - - - - - - - SELECT LAST_INSERT_ID() - - insert into kg_schedule_job_info (gmt_create, gmt_modified, name, - type, cron, status, - idempotent_id) - values (#{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{name,jdbcType=VARCHAR}, - #{type,jdbcType=VARCHAR}, #{cron,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, - #{idempotentId,jdbcType=VARCHAR}) - - - - SELECT LAST_INSERT_ID() - - insert into kg_schedule_job_info - - - gmt_create, - - - gmt_modified, - - - name, - - - type, - - - cron, - - - status, - - - idempotent_id, - - - - - #{gmtCreate,jdbcType=TIMESTAMP}, - - - #{gmtModified,jdbcType=TIMESTAMP}, - - - #{name,jdbcType=VARCHAR}, - - - #{type,jdbcType=VARCHAR}, - - - #{cron,jdbcType=VARCHAR}, - - - #{status,jdbcType=VARCHAR}, - - - #{idempotentId,jdbcType=VARCHAR}, - - - - - - update kg_schedule_job_info - - - id = #{record.id,jdbcType=BIGINT}, - - - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - - - name = #{record.name,jdbcType=VARCHAR}, - - - type = #{record.type,jdbcType=VARCHAR}, - - - cron = #{record.cron,jdbcType=VARCHAR}, - - - status = #{record.status,jdbcType=VARCHAR}, - - - idempotent_id = #{record.idempotentId,jdbcType=VARCHAR}, - - - - - - - - update kg_schedule_job_info - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - name = #{record.name,jdbcType=VARCHAR}, - type = #{record.type,jdbcType=VARCHAR}, - cron = #{record.cron,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=VARCHAR}, - idempotent_id = #{record.idempotentId,jdbcType=VARCHAR} - - - - - - update kg_schedule_job_info - - - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - - - name = #{name,jdbcType=VARCHAR}, - - - type = #{type,jdbcType=VARCHAR}, - - - cron = #{cron,jdbcType=VARCHAR}, - - - status = #{status,jdbcType=VARCHAR}, - - - idempotent_id = #{idempotentId,jdbcType=VARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update kg_schedule_job_info - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - name = #{name,jdbcType=VARCHAR}, - type = #{type,jdbcType=VARCHAR}, - cron = #{cron,jdbcType=VARCHAR}, - status = #{status,jdbcType=VARCHAR}, - idempotent_id = #{idempotentId,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/JobInstDOMapper.xml b/server/infra/dao/src/main/resources/mapper/JobInstDOMapper.xml deleted file mode 100644 index 1c148d05..00000000 --- a/server/infra/dao/src/main/resources/mapper/JobInstDOMapper.xml +++ /dev/null @@ -1,353 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, gmt_create, gmt_modified, job_id, type, status, host, trace_id, idempotent_id - - - result - - - - - - delete from kg_schedule_job_inst - where id = #{id,jdbcType=BIGINT} - - - delete from kg_schedule_job_inst - - - - - - - SELECT LAST_INSERT_ID() - - insert into kg_schedule_job_inst (gmt_create, gmt_modified, job_id, - type, status, host, - trace_id, idempotent_id, result - ) - values (#{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{jobId,jdbcType=BIGINT}, - #{type,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{host,jdbcType=VARCHAR}, - #{traceId,jdbcType=VARCHAR}, #{idempotentId,jdbcType=VARCHAR}, #{result,jdbcType=LONGVARCHAR} - ) - - - - SELECT LAST_INSERT_ID() - - insert into kg_schedule_job_inst - - - gmt_create, - - - gmt_modified, - - - job_id, - - - type, - - - status, - - - host, - - - trace_id, - - - idempotent_id, - - - result, - - - - - #{gmtCreate,jdbcType=TIMESTAMP}, - - - #{gmtModified,jdbcType=TIMESTAMP}, - - - #{jobId,jdbcType=BIGINT}, - - - #{type,jdbcType=VARCHAR}, - - - #{status,jdbcType=VARCHAR}, - - - #{host,jdbcType=VARCHAR}, - - - #{traceId,jdbcType=VARCHAR}, - - - #{idempotentId,jdbcType=VARCHAR}, - - - #{result,jdbcType=LONGVARCHAR}, - - - - - - update kg_schedule_job_inst - - - id = #{record.id,jdbcType=BIGINT}, - - - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - - - job_id = #{record.jobId,jdbcType=BIGINT}, - - - type = #{record.type,jdbcType=VARCHAR}, - - - status = #{record.status,jdbcType=VARCHAR}, - - - host = #{record.host,jdbcType=VARCHAR}, - - - trace_id = #{record.traceId,jdbcType=VARCHAR}, - - - idempotent_id = #{record.idempotentId,jdbcType=VARCHAR}, - - - result = #{record.result,jdbcType=LONGVARCHAR}, - - - - - - - - update kg_schedule_job_inst - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - job_id = #{record.jobId,jdbcType=BIGINT}, - type = #{record.type,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=VARCHAR}, - host = #{record.host,jdbcType=VARCHAR}, - trace_id = #{record.traceId,jdbcType=VARCHAR}, - idempotent_id = #{record.idempotentId,jdbcType=VARCHAR}, - result = #{record.result,jdbcType=LONGVARCHAR} - - - - - - update kg_schedule_job_inst - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - job_id = #{record.jobId,jdbcType=BIGINT}, - type = #{record.type,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=VARCHAR}, - host = #{record.host,jdbcType=VARCHAR}, - trace_id = #{record.traceId,jdbcType=VARCHAR}, - idempotent_id = #{record.idempotentId,jdbcType=VARCHAR} - - - - - - update kg_schedule_job_inst - - - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - - - job_id = #{jobId,jdbcType=BIGINT}, - - - type = #{type,jdbcType=VARCHAR}, - - - status = #{status,jdbcType=VARCHAR}, - - - host = #{host,jdbcType=VARCHAR}, - - - trace_id = #{traceId,jdbcType=VARCHAR}, - - - idempotent_id = #{idempotentId,jdbcType=VARCHAR}, - - - result = #{result,jdbcType=LONGVARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update kg_schedule_job_inst - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - job_id = #{jobId,jdbcType=BIGINT}, - type = #{type,jdbcType=VARCHAR}, - status = #{status,jdbcType=VARCHAR}, - host = #{host,jdbcType=VARCHAR}, - trace_id = #{traceId,jdbcType=VARCHAR}, - idempotent_id = #{idempotentId,jdbcType=VARCHAR}, - result = #{result,jdbcType=LONGVARCHAR} - where id = #{id,jdbcType=BIGINT} - - - update kg_schedule_job_inst - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - job_id = #{jobId,jdbcType=BIGINT}, - type = #{type,jdbcType=VARCHAR}, - status = #{status,jdbcType=VARCHAR}, - host = #{host,jdbcType=VARCHAR}, - trace_id = #{traceId,jdbcType=VARCHAR}, - idempotent_id = #{idempotentId,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/OperatorOverviewDOMapper.xml b/server/infra/dao/src/main/resources/mapper/OperatorOverviewDOMapper.xml deleted file mode 100644 index 2ec31fbf..00000000 --- a/server/infra/dao/src/main/resources/mapper/OperatorOverviewDOMapper.xml +++ /dev/null @@ -1,256 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, name, type, gmt_create, gmt_modified, description, lang - - - - - delete from kg_operator_overview - where id = #{id,jdbcType=BIGINT} - - - delete from kg_operator_overview - - - - - - - SELECT LAST_INSERT_ID() - - insert into kg_operator_overview (name, type, gmt_create, - gmt_modified, description, lang - ) - values (#{name,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{gmtCreate,jdbcType=TIMESTAMP}, - #{gmtModified,jdbcType=TIMESTAMP}, #{description,jdbcType=VARCHAR}, #{lang,jdbcType=VARCHAR} - ) - - - - SELECT LAST_INSERT_ID() - - insert into kg_operator_overview - - - name, - - - type, - - - gmt_create, - - - gmt_modified, - - - description, - - - lang, - - - - - #{name,jdbcType=VARCHAR}, - - - #{type,jdbcType=VARCHAR}, - - - #{gmtCreate,jdbcType=TIMESTAMP}, - - - #{gmtModified,jdbcType=TIMESTAMP}, - - - #{description,jdbcType=VARCHAR}, - - - #{lang,jdbcType=VARCHAR}, - - - - - - update kg_operator_overview - - - id = #{record.id,jdbcType=BIGINT}, - - - name = #{record.name,jdbcType=VARCHAR}, - - - type = #{record.type,jdbcType=VARCHAR}, - - - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - - - description = #{record.description,jdbcType=VARCHAR}, - - - lang = #{record.lang,jdbcType=VARCHAR}, - - - - - - - - update kg_operator_overview - set id = #{record.id,jdbcType=BIGINT}, - name = #{record.name,jdbcType=VARCHAR}, - type = #{record.type,jdbcType=VARCHAR}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - description = #{record.description,jdbcType=VARCHAR}, - lang = #{record.lang,jdbcType=VARCHAR} - - - - - - update kg_operator_overview - - - name = #{name,jdbcType=VARCHAR}, - - - type = #{type,jdbcType=VARCHAR}, - - - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - - - description = #{description,jdbcType=VARCHAR}, - - - lang = #{lang,jdbcType=VARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update kg_operator_overview - set name = #{name,jdbcType=VARCHAR}, - type = #{type,jdbcType=VARCHAR}, - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - description = #{description,jdbcType=VARCHAR}, - lang = #{lang,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/OperatorVersionDOMapper.xml b/server/infra/dao/src/main/resources/mapper/OperatorVersionDOMapper.xml deleted file mode 100644 index eade3bb5..00000000 --- a/server/infra/dao/src/main/resources/mapper/OperatorVersionDOMapper.xml +++ /dev/null @@ -1,256 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, overview_id, main_class, jar_address, gmt_create, gmt_modified, version - - - - - delete from kg_operator_version - where id = #{id,jdbcType=BIGINT} - - - delete from kg_operator_version - - - - - - - SELECT LAST_INSERT_ID() - - insert into kg_operator_version (overview_id, main_class, jar_address, - gmt_create, gmt_modified, version - ) - values (#{overviewId,jdbcType=BIGINT}, #{mainClass,jdbcType=VARCHAR}, #{jarAddress,jdbcType=VARCHAR}, - #{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{version,jdbcType=INTEGER} - ) - - - - SELECT LAST_INSERT_ID() - - insert into kg_operator_version - - - overview_id, - - - main_class, - - - jar_address, - - - gmt_create, - - - gmt_modified, - - - version, - - - - - #{overviewId,jdbcType=BIGINT}, - - - #{mainClass,jdbcType=VARCHAR}, - - - #{jarAddress,jdbcType=VARCHAR}, - - - #{gmtCreate,jdbcType=TIMESTAMP}, - - - #{gmtModified,jdbcType=TIMESTAMP}, - - - #{version,jdbcType=INTEGER}, - - - - - - update kg_operator_version - - - id = #{record.id,jdbcType=BIGINT}, - - - overview_id = #{record.overviewId,jdbcType=BIGINT}, - - - main_class = #{record.mainClass,jdbcType=VARCHAR}, - - - jar_address = #{record.jarAddress,jdbcType=VARCHAR}, - - - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - - - version = #{record.version,jdbcType=INTEGER}, - - - - - - - - update kg_operator_version - set id = #{record.id,jdbcType=BIGINT}, - overview_id = #{record.overviewId,jdbcType=BIGINT}, - main_class = #{record.mainClass,jdbcType=VARCHAR}, - jar_address = #{record.jarAddress,jdbcType=VARCHAR}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - version = #{record.version,jdbcType=INTEGER} - - - - - - update kg_operator_version - - - overview_id = #{overviewId,jdbcType=BIGINT}, - - - main_class = #{mainClass,jdbcType=VARCHAR}, - - - jar_address = #{jarAddress,jdbcType=VARCHAR}, - - - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - - - version = #{version,jdbcType=INTEGER}, - - - where id = #{id,jdbcType=BIGINT} - - - update kg_operator_version - set overview_id = #{overviewId,jdbcType=BIGINT}, - main_class = #{mainClass,jdbcType=VARCHAR}, - jar_address = #{jarAddress,jdbcType=VARCHAR}, - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - version = #{version,jdbcType=INTEGER} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/SPGJobInfoDOMapper.xml b/server/infra/dao/src/main/resources/mapper/SPGJobInfoDOMapper.xml deleted file mode 100644 index 676c62dd..00000000 --- a/server/infra/dao/src/main/resources/mapper/SPGJobInfoDOMapper.xml +++ /dev/null @@ -1,370 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, gmt_create, gmt_modified, name, type, project_id, cron, status, ext_info, external_job_info_id - - - content - - - - - - delete from kg_spg_job_info - where id = #{id,jdbcType=BIGINT} - - - delete from kg_spg_job_info - - - - - - - SELECT LAST_INSERT_ID() - - insert into kg_spg_job_info (gmt_create, gmt_modified, name, - type, project_id, cron, - status, ext_info, external_job_info_id, - content) - values (#{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{name,jdbcType=VARCHAR}, - #{type,jdbcType=VARCHAR}, #{projectId,jdbcType=BIGINT}, #{cron,jdbcType=VARCHAR}, - #{status,jdbcType=VARCHAR}, #{extInfo,jdbcType=VARCHAR}, #{externalJobInfoId,jdbcType=VARCHAR}, - #{content,jdbcType=LONGVARCHAR}) - - - - SELECT LAST_INSERT_ID() - - insert into kg_spg_job_info - - - gmt_create, - - - gmt_modified, - - - name, - - - type, - - - project_id, - - - cron, - - - status, - - - ext_info, - - - external_job_info_id, - - - content, - - - - - #{gmtCreate,jdbcType=TIMESTAMP}, - - - #{gmtModified,jdbcType=TIMESTAMP}, - - - #{name,jdbcType=VARCHAR}, - - - #{type,jdbcType=VARCHAR}, - - - #{projectId,jdbcType=BIGINT}, - - - #{cron,jdbcType=VARCHAR}, - - - #{status,jdbcType=VARCHAR}, - - - #{extInfo,jdbcType=VARCHAR}, - - - #{externalJobInfoId,jdbcType=VARCHAR}, - - - #{content,jdbcType=LONGVARCHAR}, - - - - - - update kg_spg_job_info - - - id = #{record.id,jdbcType=BIGINT}, - - - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - - - name = #{record.name,jdbcType=VARCHAR}, - - - type = #{record.type,jdbcType=VARCHAR}, - - - project_id = #{record.projectId,jdbcType=BIGINT}, - - - cron = #{record.cron,jdbcType=VARCHAR}, - - - status = #{record.status,jdbcType=VARCHAR}, - - - ext_info = #{record.extInfo,jdbcType=VARCHAR}, - - - external_job_info_id = #{record.externalJobInfoId,jdbcType=VARCHAR}, - - - content = #{record.content,jdbcType=LONGVARCHAR}, - - - - - - - - update kg_spg_job_info - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - name = #{record.name,jdbcType=VARCHAR}, - type = #{record.type,jdbcType=VARCHAR}, - project_id = #{record.projectId,jdbcType=BIGINT}, - cron = #{record.cron,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=VARCHAR}, - ext_info = #{record.extInfo,jdbcType=VARCHAR}, - external_job_info_id = #{record.externalJobInfoId,jdbcType=VARCHAR}, - content = #{record.content,jdbcType=LONGVARCHAR} - - - - - - update kg_spg_job_info - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - name = #{record.name,jdbcType=VARCHAR}, - type = #{record.type,jdbcType=VARCHAR}, - project_id = #{record.projectId,jdbcType=BIGINT}, - cron = #{record.cron,jdbcType=VARCHAR}, - status = #{record.status,jdbcType=VARCHAR}, - ext_info = #{record.extInfo,jdbcType=VARCHAR}, - external_job_info_id = #{record.externalJobInfoId,jdbcType=VARCHAR} - - - - - - update kg_spg_job_info - - - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - - - name = #{name,jdbcType=VARCHAR}, - - - type = #{type,jdbcType=VARCHAR}, - - - project_id = #{projectId,jdbcType=BIGINT}, - - - cron = #{cron,jdbcType=VARCHAR}, - - - status = #{status,jdbcType=VARCHAR}, - - - ext_info = #{extInfo,jdbcType=VARCHAR}, - - - external_job_info_id = #{externalJobInfoId,jdbcType=VARCHAR}, - - - content = #{content,jdbcType=LONGVARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update kg_spg_job_info - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - name = #{name,jdbcType=VARCHAR}, - type = #{type,jdbcType=VARCHAR}, - project_id = #{projectId,jdbcType=BIGINT}, - cron = #{cron,jdbcType=VARCHAR}, - status = #{status,jdbcType=VARCHAR}, - ext_info = #{extInfo,jdbcType=VARCHAR}, - external_job_info_id = #{externalJobInfoId,jdbcType=VARCHAR}, - content = #{content,jdbcType=LONGVARCHAR} - where id = #{id,jdbcType=BIGINT} - - - update kg_spg_job_info - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - name = #{name,jdbcType=VARCHAR}, - type = #{type,jdbcType=VARCHAR}, - project_id = #{projectId,jdbcType=BIGINT}, - cron = #{cron,jdbcType=VARCHAR}, - status = #{status,jdbcType=VARCHAR}, - ext_info = #{extInfo,jdbcType=VARCHAR}, - external_job_info_id = #{externalJobInfoId,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mapper/SPGJobInstDOMapper.xml b/server/infra/dao/src/main/resources/mapper/SPGJobInstDOMapper.xml deleted file mode 100644 index 9c69023b..00000000 --- a/server/infra/dao/src/main/resources/mapper/SPGJobInstDOMapper.xml +++ /dev/null @@ -1,403 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - - - - - - - - and ${criterion.condition} - - - and ${criterion.condition} #{criterion.value} - - - and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} - - - and ${criterion.condition} - - #{listItem} - - - - - - - - - - - id, gmt_create, gmt_modified, job_id, type, project_id, status, start_time, end_time, - external_job_inst_id - - - result, progress, log_info - - - - - - delete from kg_spg_job_inst - where id = #{id,jdbcType=BIGINT} - - - delete from kg_spg_job_inst - - - - - - - SELECT LAST_INSERT_ID() - - insert into kg_spg_job_inst (gmt_create, gmt_modified, job_id, - type, project_id, status, - start_time, end_time, external_job_inst_id, - result, progress, log_info - ) - values (#{gmtCreate,jdbcType=TIMESTAMP}, #{gmtModified,jdbcType=TIMESTAMP}, #{jobId,jdbcType=BIGINT}, - #{type,jdbcType=VARCHAR}, #{projectId,jdbcType=BIGINT}, #{status,jdbcType=VARCHAR}, - #{startTime,jdbcType=TIMESTAMP}, #{endTime,jdbcType=TIMESTAMP}, #{externalJobInstId,jdbcType=VARCHAR}, - #{result,jdbcType=LONGVARCHAR}, #{progress,jdbcType=LONGVARCHAR}, #{logInfo,jdbcType=LONGVARCHAR} - ) - - - - SELECT LAST_INSERT_ID() - - insert into kg_spg_job_inst - - - gmt_create, - - - gmt_modified, - - - job_id, - - - type, - - - project_id, - - - status, - - - start_time, - - - end_time, - - - external_job_inst_id, - - - result, - - - progress, - - - log_info, - - - - - #{gmtCreate,jdbcType=TIMESTAMP}, - - - #{gmtModified,jdbcType=TIMESTAMP}, - - - #{jobId,jdbcType=BIGINT}, - - - #{type,jdbcType=VARCHAR}, - - - #{projectId,jdbcType=BIGINT}, - - - #{status,jdbcType=VARCHAR}, - - - #{startTime,jdbcType=TIMESTAMP}, - - - #{endTime,jdbcType=TIMESTAMP}, - - - #{externalJobInstId,jdbcType=VARCHAR}, - - - #{result,jdbcType=LONGVARCHAR}, - - - #{progress,jdbcType=LONGVARCHAR}, - - - #{logInfo,jdbcType=LONGVARCHAR}, - - - - - - update kg_spg_job_inst - - - id = #{record.id,jdbcType=BIGINT}, - - - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - - - job_id = #{record.jobId,jdbcType=BIGINT}, - - - type = #{record.type,jdbcType=VARCHAR}, - - - project_id = #{record.projectId,jdbcType=BIGINT}, - - - status = #{record.status,jdbcType=VARCHAR}, - - - start_time = #{record.startTime,jdbcType=TIMESTAMP}, - - - end_time = #{record.endTime,jdbcType=TIMESTAMP}, - - - external_job_inst_id = #{record.externalJobInstId,jdbcType=VARCHAR}, - - - result = #{record.result,jdbcType=LONGVARCHAR}, - - - progress = #{record.progress,jdbcType=LONGVARCHAR}, - - - log_info = #{record.logInfo,jdbcType=LONGVARCHAR}, - - - - - - - - update kg_spg_job_inst - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - job_id = #{record.jobId,jdbcType=BIGINT}, - type = #{record.type,jdbcType=VARCHAR}, - project_id = #{record.projectId,jdbcType=BIGINT}, - status = #{record.status,jdbcType=VARCHAR}, - start_time = #{record.startTime,jdbcType=TIMESTAMP}, - end_time = #{record.endTime,jdbcType=TIMESTAMP}, - external_job_inst_id = #{record.externalJobInstId,jdbcType=VARCHAR}, - result = #{record.result,jdbcType=LONGVARCHAR}, - progress = #{record.progress,jdbcType=LONGVARCHAR}, - log_info = #{record.logInfo,jdbcType=LONGVARCHAR} - - - - - - update kg_spg_job_inst - set id = #{record.id,jdbcType=BIGINT}, - gmt_create = #{record.gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{record.gmtModified,jdbcType=TIMESTAMP}, - job_id = #{record.jobId,jdbcType=BIGINT}, - type = #{record.type,jdbcType=VARCHAR}, - project_id = #{record.projectId,jdbcType=BIGINT}, - status = #{record.status,jdbcType=VARCHAR}, - start_time = #{record.startTime,jdbcType=TIMESTAMP}, - end_time = #{record.endTime,jdbcType=TIMESTAMP}, - external_job_inst_id = #{record.externalJobInstId,jdbcType=VARCHAR} - - - - - - update kg_spg_job_inst - - - gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - - - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - - - job_id = #{jobId,jdbcType=BIGINT}, - - - type = #{type,jdbcType=VARCHAR}, - - - project_id = #{projectId,jdbcType=BIGINT}, - - - status = #{status,jdbcType=VARCHAR}, - - - start_time = #{startTime,jdbcType=TIMESTAMP}, - - - end_time = #{endTime,jdbcType=TIMESTAMP}, - - - external_job_inst_id = #{externalJobInstId,jdbcType=VARCHAR}, - - - result = #{result,jdbcType=LONGVARCHAR}, - - - progress = #{progress,jdbcType=LONGVARCHAR}, - - - log_info = #{logInfo,jdbcType=LONGVARCHAR}, - - - where id = #{id,jdbcType=BIGINT} - - - update kg_spg_job_inst - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - job_id = #{jobId,jdbcType=BIGINT}, - type = #{type,jdbcType=VARCHAR}, - project_id = #{projectId,jdbcType=BIGINT}, - status = #{status,jdbcType=VARCHAR}, - start_time = #{startTime,jdbcType=TIMESTAMP}, - end_time = #{endTime,jdbcType=TIMESTAMP}, - external_job_inst_id = #{externalJobInstId,jdbcType=VARCHAR}, - result = #{result,jdbcType=LONGVARCHAR}, - progress = #{progress,jdbcType=LONGVARCHAR}, - log_info = #{logInfo,jdbcType=LONGVARCHAR} - where id = #{id,jdbcType=BIGINT} - - - update kg_spg_job_inst - set gmt_create = #{gmtCreate,jdbcType=TIMESTAMP}, - gmt_modified = #{gmtModified,jdbcType=TIMESTAMP}, - job_id = #{jobId,jdbcType=BIGINT}, - type = #{type,jdbcType=VARCHAR}, - project_id = #{projectId,jdbcType=BIGINT}, - status = #{status,jdbcType=VARCHAR}, - start_time = #{startTime,jdbcType=TIMESTAMP}, - end_time = #{endTime,jdbcType=TIMESTAMP}, - external_job_inst_id = #{externalJobInstId,jdbcType=VARCHAR} - where id = #{id,jdbcType=BIGINT} - - \ No newline at end of file diff --git a/server/infra/dao/src/main/resources/mybatis-generator-config.xml b/server/infra/dao/src/main/resources/mybatis-generator-config.xml index 87008581..00562b2d 100644 --- a/server/infra/dao/src/main/resources/mybatis-generator-config.xml +++ b/server/infra/dao/src/main/resources/mybatis-generator-config.xml @@ -68,14 +68,6 @@
- - - -
- - - -
@@ -100,21 +92,5 @@
- - - -
- - - -
- - - -
- - - -
diff --git a/server/initdb.sql b/server/initdb.sql deleted file mode 100644 index 0a1bb696..00000000 --- a/server/initdb.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Copyright 2023 Ant Group CO., Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except --- in compliance with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software distributed under the License --- is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express --- or implied. - - - -use openspg; - - - - diff --git a/server/lib/spgreasoner-local-0.0.1.jar b/server/lib/spgreasoner-local-0.0.1.jar deleted file mode 100644 index 1beb5a1a..00000000 Binary files a/server/lib/spgreasoner-local-0.0.1.jar and /dev/null differ diff --git a/server/pom.xml b/server/pom.xml index 533cf8aa..78a4815e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -40,20 +40,14 @@ api/http-server arks/sofaboot biz/common - biz/reasoner biz/schema - cloudext/interface/job-scheduler - cloudext/interface/computing - cloudext/impl/job-scheduler/local - cloudext/impl/computing/local common/model common/service core/schema/model core/schema/service - core/reasoner/model - core/reasoner/service infra/dao test + testdata @@ -96,36 +90,11 @@ biz-common ${project.version}
- - com.antgroup.openspg.server - biz-reasoner - ${project.version} - com.antgroup.openspg.server biz-schema ${project.version} - - com.antgroup.openspg.server - cloudext-interface-job-scheduler - ${project.version} - - - com.antgroup.openspg.server - cloudext-interface-computing - ${project.version} - - - com.antgroup.openspg.server - cloudext-impl-job-scheduler-local - ${project.version} - - - com.antgroup.openspg.server - cloudext-impl-computing-local - ${project.version} - com.antgroup.openspg.server common-service @@ -136,16 +105,6 @@ core-schema-service ${project.version} - - com.antgroup.openspg.server - core-reasoner-model - ${project.version} - - - com.antgroup.openspg.server - core-reasoner-service - ${project.version} - com.antgroup.openspg.server infra-dao diff --git a/server/test/pom.xml b/server/test/pom.xml index e99aa0e5..501f868a 100644 --- a/server/test/pom.xml +++ b/server/test/pom.xml @@ -22,6 +22,10 @@ test + + com.antgroup.openspg.server + core-schema-model + org.springframework spring-context diff --git a/server/testdata/pom.xml b/server/testdata/pom.xml new file mode 100644 index 00000000..474f884b --- /dev/null +++ b/server/testdata/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + com.antgroup.openspg.server + server-parent + 0.0.1-SNAPSHOT + + + testdata + + + + com.antgroup.openspg.server + core-schema-model + + + diff --git a/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/RiskMiningSchema.java b/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/RiskMiningSchema.java new file mode 100644 index 00000000..ee6032c9 --- /dev/null +++ b/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/RiskMiningSchema.java @@ -0,0 +1,113 @@ +package com.antgroup.openspg.server.testdata; + +import static com.antgroup.openspg.server.testdata.StandardTypes.CHINA_MOBILE; +import static com.antgroup.openspg.server.testdata.TestCommons.*; +import static com.antgroup.openspg.server.testdata.TestCommons.THING; + +import com.antgroup.openspg.core.schema.model.BasicInfo; +import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; +import com.antgroup.openspg.core.schema.model.type.*; +import com.google.common.collect.Lists; + +public class RiskMiningSchema { + + public static final String NAMESPACE = "RiskMining"; + + /** basic type */ + public static final BasicType LONG = new BasicType.LongBasicType(); + + public static final BasicType TEXT = new BasicType.TextBasicType(); + public static final BasicType DOUBLE = new BasicType.DoubleBasicType(); + + /** concept type */ + public static final ConceptType TAX_OF_RISK_USER = + new ConceptType( + new BasicInfo<>(newSPGTypeIdentifier("TaxOfRiskUser")), + THING, + Lists.newArrayList(), + Lists.newArrayList(), + new SPGTypeAdvancedConfig(), + new ConceptLayerConfig("isA", Lists.newArrayList()), + new ConceptTaxonomicConfig(newSPGTypeIdentifier("Person")), + null); + + public static final ConceptType TAX_OF_RISK_APP = + new ConceptType( + new BasicInfo<>(newSPGTypeIdentifier("TaxOfRiskApp")), + THING, + Lists.newArrayList(), + Lists.newArrayList(), + new SPGTypeAdvancedConfig(), + new ConceptLayerConfig("isA", Lists.newArrayList()), + new ConceptTaxonomicConfig(newSPGTypeIdentifier("App")), + null); + + /** entity type */ + public static final EntityType CERT = + new EntityType( + new BasicInfo<>(newSPGTypeIdentifier("Cert")), + THING, + Lists.newArrayList( + newProperty("id", "id", TEXT), + newProperty("name", "name", TEXT), + newProperty("certNum", "证书编号", TEXT)), + Lists.newArrayList(), + new SPGTypeAdvancedConfig()); + + public static final EntityType APP = + new EntityType( + new BasicInfo<>(newSPGTypeIdentifier("App")), + THING, + Lists.newArrayList( + newProperty("id", "id", TEXT), + newProperty("name", "name", TEXT), + newProperty("riskMark", "风险标记", TEXT), + newProperty("useCert", "使用证书", CERT), + newProperty("belongTo", "属于", TAX_OF_RISK_APP)), + Lists.newArrayList(), + new SPGTypeAdvancedConfig()); + + public static final EntityType COMPANY = + new EntityType( + new BasicInfo<>(newSPGTypeIdentifier("Company")), + THING, + Lists.newArrayList( + newProperty("id", "id", TEXT), + newProperty("name", "name", TEXT), + newProperty("hasPhone", "电话号码", CHINA_MOBILE)), + Lists.newArrayList(newRelation("hasCert", "拥有证书", CERT)), + new SPGTypeAdvancedConfig()); + + static { + COMPANY.getRelations().add(newRelation("holdShare", "持股", COMPANY)); + } + + public static final EntityType DEVICE = + new EntityType( + new BasicInfo<>(newSPGTypeIdentifier("Device")), + THING, + Lists.newArrayList( + newProperty("id", "id", TEXT), + newProperty("name", "name", TEXT), + newProperty("umid", "设备umid", TEXT), + newProperty("install", "安装", APP)), + Lists.newArrayList(), + new SPGTypeAdvancedConfig()); + + public static final EntityType PERSON = + new EntityType( + new BasicInfo<>(newSPGTypeIdentifier("Person")), + THING, + Lists.newArrayList( + newProperty("id", "id", TEXT), + newProperty("name", "name", TEXT), + newProperty("age", "年龄", LONG), + newProperty("hasPhone", "电话号码", CHINA_MOBILE), + newProperty("belongTo", "属于", TAX_OF_RISK_USER)), + Lists.newArrayList(), + new SPGTypeAdvancedConfig()); + + private static SPGTypeIdentifier newSPGTypeIdentifier(String identifier) { + return SPGTypeIdentifier.parse(NAMESPACE + "_" + identifier); + } +} diff --git a/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/StandardTypes.java b/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/StandardTypes.java new file mode 100644 index 00000000..a145a179 --- /dev/null +++ b/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/StandardTypes.java @@ -0,0 +1,29 @@ +package com.antgroup.openspg.server.testdata; + +import static com.antgroup.openspg.server.testdata.TestCommons.THING; + +import com.antgroup.openspg.core.schema.model.BasicInfo; +import com.antgroup.openspg.core.schema.model.constraint.RegularConstraint; +import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; +import com.antgroup.openspg.core.schema.model.type.SPGTypeAdvancedConfig; +import com.antgroup.openspg.core.schema.model.type.StandardType; +import com.google.common.collect.Lists; + +public class StandardTypes { + + public static final StandardType CHINA_MOBILE = + new StandardType( + new BasicInfo<>(newStandardIdentifier("ChinaMobile")), + THING, + Lists.newArrayList(), + Lists.newArrayList(), + new SPGTypeAdvancedConfig(), + Boolean.TRUE, + Lists.newArrayList( + new RegularConstraint( + "^((13[0-9])|(14[5,7,9])|(15([0-3]|[5-9]))|(16[5,6])|(17[0-8])|(18[0-9])|(19[1,5,8,9]))[0-9]{8}$"))); + + private static SPGTypeIdentifier newStandardIdentifier(String identifier) { + return SPGTypeIdentifier.parse("STD" + "_" + identifier); + } +} diff --git a/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/TestCommons.java b/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/TestCommons.java new file mode 100644 index 00000000..77ad2f88 --- /dev/null +++ b/server/testdata/src/main/java/com/antgroup/openspg/server/testdata/TestCommons.java @@ -0,0 +1,35 @@ +package com.antgroup.openspg.server.testdata; + +import com.antgroup.openspg.core.schema.model.BasicInfo; +import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier; +import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier; +import com.antgroup.openspg.core.schema.model.predicate.Property; +import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig; +import com.antgroup.openspg.core.schema.model.predicate.Relation; +import com.antgroup.openspg.core.schema.model.type.BaseSPGType; +import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo; +import java.util.ArrayList; + +public class TestCommons { + + public static final ParentTypeInfo THING = + new ParentTypeInfo(1L, 1L, SPGTypeIdentifier.parse("THING"), new ArrayList<>()); + + public static Property newProperty(String propertyName, String desc, BaseSPGType objectType) { + return new Property( + new BasicInfo<>(new PredicateIdentifier(propertyName), desc, desc), + null, + objectType.toRef(), + Boolean.FALSE, + new PropertyAdvancedConfig()); + } + + public static Relation newRelation(String propertyName, String desc, BaseSPGType objectType) { + return new Relation( + new BasicInfo<>(new PredicateIdentifier(propertyName), desc, desc), + null, + objectType.toRef(), + Boolean.FALSE, + new PropertyAdvancedConfig()); + } +}