yujunjun/openspg
1
0
Fork 0
mirror of https://github.com/OpenSPG/openspg.git synced 2025-12-11 15:21:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
baifuyu 2023-12-20 14:45:24 +08:00
parent b8fc13c2ab
commit 04a0dcc9cb
203 changed files with 399 additions and 12518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
builder/core/pom.xml
View File

@ -23,6 +23,14 @@
<artifactId>builder-core</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-local-runner</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-cloudext-warehouse</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>api-http-client</artifactId>
@ -35,10 +43,6 @@
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-graph-store-tugraph</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-table-store</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-search-engine</artifactId>
@ -47,13 +51,6 @@
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-search-engine-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.kg.reasoner</groupId>
<artifactId>local</artifactId>
<version>${kgreasoner-version}</version>
<scope>system</scope>
<systemPath>${project.basedir}/../../server/lib/spgreasoner-local-0.0.1.jar</systemPath>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>core-schema-model</artifactId>

14
builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/ReasonProcessor.java
View File

@ -1,10 +1,5 @@
package com.antgroup.openspg.builder.core.reason;
import com.antgroup.kg.reasoner.catalog.impl.OpenKgCatalog;
import com.antgroup.kg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.kg.reasoner.graphstate.GraphState;
import com.antgroup.kg.reasoner.graphstate.impl.CloudExtGraphState;
import com.antgroup.kg.reasoner.lube.catalog.Catalog;
import com.antgroup.openspg.builder.core.physical.process.BaseProcessor;
import com.antgroup.openspg.builder.core.reason.impl.CausalConceptReasoner;
import com.antgroup.openspg.builder.core.reason.impl.InductiveConceptReasoner;
@ -16,6 +11,11 @@ import com.antgroup.openspg.builder.model.record.BaseSPGRecord;
import com.antgroup.openspg.core.schema.model.semantic.DynamicTaxonomySemantic;
import com.antgroup.openspg.core.schema.model.semantic.LogicalCausationSemantic;
import com.antgroup.openspg.core.schema.model.type.ConceptList;
import com.antgroup.openspg.reasoner.catalog.impl.OpenSPGCatalog;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.openspg.reasoner.graphstate.GraphState;
import com.antgroup.openspg.reasoner.lube.catalog.Catalog;
import com.antgroup.openspg.reasoner.warehouse.cloudext.CloudExtGraphState;
import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo;
import com.google.common.collect.Lists;
import java.util.*;
@ -85,8 +85,8 @@ public class ReasonProcessor extends BaseProcessor<ReasonProcessor.ReasonerNodeC
}
private Catalog buildCatalog() {
OpenKgCatalog catalog =
new OpenKgCatalog(context.getProjectId(), null, context.getCatalog().getProjectSchema());
Catalog catalog =
new OpenSPGCatalog(context.getProjectId(), null, context.getCatalog().getProjectSchema());
catalog.init();
return catalog;
}

10
builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/ReasonerProcessorUtils.java
View File

@ -13,11 +13,6 @@
package com.antgroup.openspg.builder.core.reason;
import com.antgroup.kg.reasoner.common.graph.edge.IEdge;
import com.antgroup.kg.reasoner.common.graph.property.IProperty;
import com.antgroup.kg.reasoner.common.graph.vertex.IVertex;
import com.antgroup.kg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.kg.reasoner.local.model.LocalReasonerResult;
import com.antgroup.openspg.builder.core.runtime.BuilderCatalog;
import com.antgroup.openspg.builder.model.record.BaseAdvancedRecord;
import com.antgroup.openspg.builder.model.record.BaseSPGRecord;
@ -35,6 +30,11 @@ import com.antgroup.openspg.core.schema.model.semantic.SystemPredicateEnum;
import com.antgroup.openspg.core.schema.model.type.BaseSPGType;
import com.antgroup.openspg.core.schema.model.type.ConceptList;
import com.antgroup.openspg.core.schema.model.type.SPGTypeRef;
import com.antgroup.openspg.reasoner.common.graph.edge.IEdge;
import com.antgroup.openspg.reasoner.common.graph.property.IProperty;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;

12
builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/impl/CausalConceptReasoner.java
View File

@ -1,11 +1,5 @@
package com.antgroup.openspg.builder.core.reason.impl;
import com.antgroup.kg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.kg.reasoner.graphstate.GraphState;
import com.antgroup.kg.reasoner.local.KGReasonerLocalRunner;
import com.antgroup.kg.reasoner.local.model.LocalReasonerResult;
import com.antgroup.kg.reasoner.local.model.LocalReasonerTask;
import com.antgroup.kg.reasoner.lube.catalog.Catalog;
import com.antgroup.openspg.builder.core.reason.ConceptReasoner;
import com.antgroup.openspg.builder.core.reason.ReasonerProcessorUtils;
import com.antgroup.openspg.builder.core.runtime.BuilderCatalog;
@ -17,6 +11,12 @@ import com.antgroup.openspg.core.schema.model.semantic.DynamicTaxonomySemantic;
import com.antgroup.openspg.core.schema.model.semantic.LogicalCausationSemantic;
import com.antgroup.openspg.core.schema.model.semantic.SystemPredicateEnum;
import com.antgroup.openspg.core.schema.model.type.ConceptList;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.openspg.reasoner.graphstate.GraphState;
import com.antgroup.openspg.reasoner.lube.catalog.Catalog;
import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner;
import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult;
import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;

12
builder/core/src/main/java/com/antgroup/openspg/builder/core/reason/impl/InductiveConceptReasoner.java
View File

@ -1,17 +1,17 @@
package com.antgroup.openspg.builder.core.reason.impl;
import com.antgroup.kg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.kg.reasoner.graphstate.GraphState;
import com.antgroup.kg.reasoner.local.KGReasonerLocalRunner;
import com.antgroup.kg.reasoner.local.model.LocalReasonerResult;
import com.antgroup.kg.reasoner.local.model.LocalReasonerTask;
import com.antgroup.kg.reasoner.lube.catalog.Catalog;
import com.antgroup.openspg.builder.core.reason.ConceptReasoner;
import com.antgroup.openspg.builder.core.reason.ReasonerProcessorUtils;
import com.antgroup.openspg.builder.model.record.BaseAdvancedRecord;
import com.antgroup.openspg.builder.model.record.BaseRecord;
import com.antgroup.openspg.builder.model.record.BaseSPGRecord;
import com.antgroup.openspg.core.schema.model.semantic.DynamicTaxonomySemantic;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.openspg.reasoner.graphstate.GraphState;
import com.antgroup.openspg.reasoner.lube.catalog.Catalog;
import com.antgroup.openspg.reasoner.runner.local.KGReasonerLocalRunner;
import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerResult;
import com.antgroup.openspg.reasoner.runner.local.model.LocalReasonerTask;
import com.google.common.collect.Lists;
import java.util.List;
import lombok.Setter;

18
builder/core/src/test/groovy/com/antgroup/openspg/builder/core/runtime/impl/DefaultBuilderExecutorTest.groovy
View File

@ -9,22 +9,4 @@ import spock.lang.Specification
class DefaultBuilderExecutorTest extends Specification {
def eval1() {
given:
Pipeline pipeline1 = genPipeline1()
BuilderExecutor builderExecutor = new DefaultBuilderExecutor()
when:
def logicalPlan = LogicalPlan.parse(pipeline1)
def physicalPlan = PhysicalPlan.plan(logicalPlan)
then:
builderExecutor.eval()
}
private def genPipeline1() {
def node1 = new Node()
def node2 = new Node()
def node3 = new Node()
}
}

2
builder/model/src/test/groovy/com/antgroup/openspg/builder/model/pipeline/PipelineTest.groovy
View File

@ -59,6 +59,6 @@ class PipelineTest extends Specification {
Pipeline pipeline = new Pipeline([node1, node2, node3], [edge1, edge2]);
expect:
BuilderJsonUtils.serialize(pipeline) == '''{"nodes":[{"id":"1","name":"csv","nodeConfig":{"@type":"CSV_SOURCE","startRow":2,"url":"./data/App.csv","columns":["id","riskMark","useCert"],"type":"CSV_SOURCE"}},{"id":"2","name":"mapping","nodeConfig":{"@type":"SPG_TYPE_MAPPING","spgType":"RiskMining.App","mappingFilters":[],"mappingConfigs":[{"source":"id","target":"id"},{"source":"id","target":"name"},{"source":"riskMark","target":"riskMark"},{"source":"useCert","target":"userCert","normalizerConfig":{"@type":"OPERATOR","operatorConfig":{"filePath":"examples/riskmining/builder/operator/cert_link_operator.py","modulePath":"cert_link_operator","className":"CertLinkerOperator","method":"handle","uniqueKey":{}},"normalizerType":"OPERATOR"}}],"type":"SPG_TYPE_MAPPING"}},{"id":"3","name":"sink","nodeConfig":{"@type":"GRAPH_SINK","type":"GRAPH_SINK"}}],"edges":[{"from":"1","to":"2"},{"from":"2","to":"3"}]}'''
BuilderJsonUtils.serialize(pipeline) == '''{"nodes":[{"id":"1","name":"csv","nodeConfig":{"@type":"CSV_SOURCE","startRow":2,"url":"./data/App.csv","columns":["id","riskMark","useCert"],"type":"CSV_SOURCE"}},{"id":"2","name":"mapping","nodeConfig":{"@type":"SPG_TYPE_MAPPING","spgType":"RiskMining.App","mappingFilters":[],"mappingConfigs":[{"source":"id","target":"id"},{"source":"id","target":"name"},{"source":"riskMark","target":"riskMark"},{"source":"useCert","target":"userCert","normalizerConfig":{"@type":"OPERATOR","operatorConfig":{"filePath":"examples/riskmining/builder/operator/cert_link_operator.py","modulePath":"cert_link_operator","className":"CertLinkerOperator","method":"handle"},"normalizerType":"OPERATOR"}}],"type":"SPG_TYPE_MAPPING"}},{"id":"3","name":"sink","nodeConfig":{"@type":"GRAPH_SINK","type":"GRAPH_SINK"}}],"edges":[{"from":"1","to":"2"},{"from":"2","to":"3"}]}'''
}
}

36
cloudext/impl/object-store/local/pom.xml
View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-impl-object-store-local</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-object-store</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
</dependencies>
</project>

48
cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClient.java
View File

@ -1,48 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.objectstore.local;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient;
import com.antgroup.openspg.cloudext.interfaces.objectstore.cmd.ObjectStoreSaveCmd;
import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath;
import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo;
import java.io.File;
import java.io.IOException;
import lombok.Getter;
import org.apache.commons.io.FileUtils;
public class LocalObjectStoreClient implements ObjectStoreClient {
@Getter private final ObjectStoreConnectionInfo connInfo;
private final String localRootDir;
public LocalObjectStoreClient(ObjectStoreConnectionInfo connInfo) {
this.connInfo = connInfo;
this.localRootDir = (String) connInfo.getNotNullParam("localDir");
}
@Override
public ObjectStorePath save(ObjectStoreSaveCmd cmd) {
ObjectStorePath path = cmd.getPath();
File file = new File("./" + localRootDir + "/" + path.getRelativePath());
try {
FileUtils.copyInputStreamToFile(cmd.getInputStream(), file);
} catch (IOException e) {
throw new RuntimeException(e);
}
path.setAbsolutePath(file.getAbsolutePath());
return path;
}
}

36
cloudext/impl/object-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/objectstore/local/LocalObjectStoreClientDriver.java
View File

@ -1,36 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.objectstore.local;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriver;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriverManager;
import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo;
public class LocalObjectStoreClientDriver implements ObjectStoreClientDriver {
static {
ObjectStoreClientDriverManager.registerDriver(new LocalObjectStoreClientDriver());
}
@Override
public String driverScheme() {
return "local";
}
@Override
public ObjectStoreClient connect(ObjectStoreConnectionInfo connInfo) {
return new LocalObjectStoreClient(connInfo);
}
}

7
cloudext/impl/search-engine/elasticsearch/src/main/java/com/antgroup/openspg/cloudext/impl/searchengine/elasticsearch/util/ElasticSearchRecordUtils.java
View File

@ -32,7 +32,7 @@ import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query
import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.QueryGroup;
import com.antgroup.openspg.cloudext.interfaces.searchengine.model.request.query.TermQuery;
import com.antgroup.openspg.common.util.StringUtils;
import com.antgroup.openspg.server.api.facade.JSON;
import com.antgroup.openspg.server.api.facade.SchemaJsonUtils;
import com.dtflys.forest.http.ForestResponse;
import com.google.gson.reflect.TypeToken;
import java.util.ArrayList;
@ -93,7 +93,8 @@ public class ElasticSearchRecordUtils {
throw new RuntimeException("mGetIdxRecords error, errorMsg=" + response.getContent());
}
EsDocs esDocs = JSON.deserialize(response.getContent(), new TypeToken<EsDocs>() {}.getType());
EsDocs esDocs =
SchemaJsonUtils.deserialize(response.getContent(), new TypeToken<EsDocs>() {}.getType());
return esDocs.toIdxRecord();
}
@ -119,7 +120,7 @@ public class ElasticSearchRecordUtils {
if (StringUtils.isBlank(hits)) {
return Collections.emptyList();
}
EsHits esHits = JSON.deserialize(hits, new TypeToken<EsHits>() {}.getType());
EsHits esHits = SchemaJsonUtils.deserialize(hits, new TypeToken<EsHits>() {}.getType());
return esHits.toIdxRecord();
}

36
cloudext/impl/table-store/local/pom.xml
View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-impl-table-store-local</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-table-store</artifactId>
</dependency>
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
</dependency>
</dependencies>
</project>

75
cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableFileHandler.java
View File

@ -1,75 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.tablestore.local;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableFileHandler;
import com.antgroup.openspg.cloudext.interfaces.tablestore.model.TableRecord;
import com.opencsv.CSVWriter;
import java.util.Arrays;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;
public class LocalTableFileHandler implements TableFileHandler {
private final CSVWriter csvWriter;
private final String tableName;
private static final int DEFAULT_BUFFER_SIZE = 1000;
private final Queue<String[]> buffer;
public LocalTableFileHandler(CSVWriter csvWriter, String tableName) {
this.csvWriter = csvWriter;
this.tableName = tableName;
buffer = new ArrayBlockingQueue<>(DEFAULT_BUFFER_SIZE);
}
@Override
public String getTableName() {
return tableName;
}
@Override
public int write(TableRecord record) {
if (record.getValues() != null && record.getValues().length > 0) {
buffer.add(Arrays.stream(record.getValues()).map(Object::toString).toArray(String[]::new));
if (buffer.size() == DEFAULT_BUFFER_SIZE) {
flushBuffer();
}
return 1;
}
return 0;
}
@Override
public int batchWrite(List<TableRecord> records) {
for (TableRecord record : records) {
write(record);
}
return records.size();
}
private synchronized void flushBuffer() {
while (!buffer.isEmpty()) {
csvWriter.writeNext(buffer.poll());
}
}
@Override
public void close() throws Exception {
if (csvWriter != null) {
flushBuffer();
csvWriter.close();
}
}
}

59
cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClient.java
View File

@ -1,59 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.tablestore.local;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableFileHandler;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient;
import com.antgroup.openspg.cloudext.interfaces.tablestore.cmd.TableFileCreateCmd;
import com.antgroup.openspg.cloudext.interfaces.tablestore.model.ColumnMeta;
import com.antgroup.openspg.common.util.tuple.Tuple2;
import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo;
import com.opencsv.CSVWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import lombok.Getter;
public class LocalTableStoreClient implements TableStoreClient {
@Getter private final TableStoreConnectionInfo connInfo;
private final String localRootDir;
public LocalTableStoreClient(TableStoreConnectionInfo connInfo) {
this.connInfo = connInfo;
this.localRootDir = (String) connInfo.getNotNullParam("localDir");
new File("./" + localRootDir).mkdirs();
}
@Override
public TableFileHandler create(TableFileCreateCmd cmd) {
try {
Tuple2<CSVWriter, String> tuple2 = buildCsvWriter(cmd);
return new LocalTableFileHandler(tuple2.first, tuple2.second);
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
private Tuple2<CSVWriter, String> buildCsvWriter(TableFileCreateCmd cmd) throws IOException {
String filePath = "./" + localRootDir + "/" + cmd.getName() + ".csv";
CSVWriter csvWriter = new CSVWriter(new FileWriter(filePath));
String[] columns =
Arrays.stream(cmd.getColumns()).map(ColumnMeta::getName).toArray(String[]::new);
csvWriter.writeNext(columns);
return Tuple2.of(csvWriter, filePath);
}
}

36
cloudext/impl/table-store/local/src/main/java/com/antgroup/openspg/cloudext/impl/tablestore/local/LocalTableStoreClientDriver.java
View File

@ -1,36 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.tablestore.local;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriver;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriverManager;
import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo;
public class LocalTableStoreClientDriver implements TableStoreClientDriver {
static {
TableStoreClientDriverManager.registerDriver(new LocalTableStoreClientDriver());
}
@Override
public String driverScheme() {
return "local";
}
@Override
public TableStoreClient connect(TableStoreConnectionInfo connInfo) {
return new LocalTableStoreClient(connInfo);
}
}

16
cloudext/interface/graph-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/graphstore/model/lpg/record/VertexRecord.java
View File

@ -14,8 +14,11 @@
package com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record;
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.VertexType;
import com.antgroup.openspg.server.api.facade.ApiConstants;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
@ -44,4 +47,17 @@ public class VertexRecord extends BaseLPGRecord {
otherProperties.put(VertexType.ID, id);
return otherProperties;
}
public Map<String, TreeMap<Long, Object>> toPropertyMapWithIdAndVersion() {
Map<String, Object> otherProperties = toPropertyMapWithId();
Map<String, TreeMap<Long, Object>> results = new HashMap<>(otherProperties.size());
otherProperties.forEach(
(key, value) -> {
TreeMap<Long, Object> propertyVersion = new TreeMap<>();
propertyVersion.put(ApiConstants.DEFAULT_VERSION, value);
results.put(key, propertyVersion);
});
return results;
}
}

32
cloudext/interface/object-store/pom.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-interface-object-store</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg</groupId>
<artifactId>common-util</artifactId>
</dependency>
</dependencies>
</project>

23
cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClient.java
View File

@ -1,23 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.objectstore;
import com.antgroup.openspg.cloudext.interfaces.objectstore.cmd.ObjectStoreSaveCmd;
import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath;
import com.antgroup.openspg.common.util.cloudext.CloudExtClient;
public interface ObjectStoreClient extends CloudExtClient {
ObjectStorePath save(ObjectStoreSaveCmd cmd);
}

20
cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriver.java
View File

@ -1,20 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.objectstore;
import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver;
import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo;
public interface ObjectStoreClientDriver
extends CloudExtClientDriver<ObjectStoreClient, ObjectStoreConnectionInfo> {}

52
cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/ObjectStoreClientDriverManager.java
View File

@ -1,52 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.objectstore;
import com.antgroup.openspg.common.util.DriverManagerUtils;
import com.antgroup.openspg.server.common.model.datasource.connection.ObjectStoreConnectionInfo;
import com.antgroup.openspg.server.common.model.exception.CloudExtException;
import java.util.concurrent.CopyOnWriteArrayList;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ObjectStoreClientDriverManager {
private static final CopyOnWriteArrayList<ObjectStoreClientDriver> registeredDrivers =
new CopyOnWriteArrayList<>();
private ObjectStoreClientDriverManager() {}
static {
DriverManagerUtils.loadDrivers("cloudext.objectstore.drivers", ObjectStoreClientDriver.class);
log.info("object-store DriverManager initialized");
}
public static synchronized void registerDriver(ObjectStoreClientDriver driver) {
if (driver != null) {
registeredDrivers.addIfAbsent(driver);
} else {
throw new NullPointerException();
}
log.info("registerDriver: {}", driver);
}
public static ObjectStoreClient getClient(ObjectStoreConnectionInfo config) {
for (ObjectStoreClientDriver driver : registeredDrivers) {
if (driver.acceptsConfig(config)) {
return driver.connect(config);
}
}
throw CloudExtException.driverNotExist(config);
}
}

29
cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/cmd/ObjectStoreSaveCmd.java
View File

@ -1,29 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.objectstore.cmd;
import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath;
import com.antgroup.openspg.server.common.model.base.BaseCmd;
import java.io.InputStream;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class ObjectStoreSaveCmd extends BaseCmd {
private final ObjectStorePath path;
private final InputStream inputStream;
}

30
cloudext/interface/object-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/objectstore/model/ObjectStorePath.java
View File

@ -1,30 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.objectstore.model;
import com.antgroup.openspg.server.common.model.base.BaseValObj;
import lombok.Getter;
import lombok.Setter;
@Getter
public class ObjectStorePath extends BaseValObj {
private final String relativePath;
@Setter private String absolutePath;
public ObjectStorePath(String relativePath) {
this.relativePath = relativePath;
}
}

1
cloudext/interface/object-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriver
View File

@ -1 +0,0 @@
com.antgroup.openspg.cloudext.impl.objectstore.local.LocalObjectStoreClientDriver

32
cloudext/interface/table-store/pom.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-interface-table-store</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg</groupId>
<artifactId>common-util</artifactId>
</dependency>
</dependencies>
</project>

28
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableFileHandler.java
View File

@ -1,28 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore;
import com.antgroup.openspg.cloudext.interfaces.tablestore.model.TableRecord;
import java.util.List;
public interface TableFileHandler {
String getTableName();
int write(TableRecord record);
int batchWrite(List<TableRecord> records);
void close() throws Exception;
}

22
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClient.java
View File

@ -1,22 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore;
import com.antgroup.openspg.cloudext.interfaces.tablestore.cmd.TableFileCreateCmd;
import com.antgroup.openspg.common.util.cloudext.CloudExtClient;
public interface TableStoreClient extends CloudExtClient {
TableFileHandler create(TableFileCreateCmd cmd);
}

20
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriver.java
View File

@ -1,20 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore;
import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver;
import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo;
public interface TableStoreClientDriver
extends CloudExtClientDriver<TableStoreClient, TableStoreConnectionInfo> {}

52
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/TableStoreClientDriverManager.java
View File

@ -1,52 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore;
import com.antgroup.openspg.common.util.DriverManagerUtils;
import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo;
import com.antgroup.openspg.server.common.model.exception.CloudExtException;
import java.util.concurrent.CopyOnWriteArrayList;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TableStoreClientDriverManager {
private static final CopyOnWriteArrayList<TableStoreClientDriver> registeredDrivers =
new CopyOnWriteArrayList<>();
private TableStoreClientDriverManager() {}
static {
DriverManagerUtils.loadDrivers("cloudext.tablestore.drivers", TableStoreClientDriver.class);
log.info("table-store DriverManager initialized");
}
public static synchronized void registerDriver(TableStoreClientDriver driver) {
if (driver != null) {
registeredDrivers.addIfAbsent(driver);
} else {
throw new NullPointerException();
}
log.info("registerDriver: {}", driver);
}
public static TableStoreClient getClient(TableStoreConnectionInfo config) {
for (TableStoreClientDriver driver : registeredDrivers) {
if (driver.acceptsConfig(config)) {
return driver.connect(config);
}
}
throw CloudExtException.driverNotExist(config);
}
}

28
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/cmd/TableFileCreateCmd.java
View File

@ -1,28 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore.cmd;
import com.antgroup.openspg.cloudext.interfaces.tablestore.model.ColumnMeta;
import com.antgroup.openspg.server.common.model.base.BaseCmd;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class TableFileCreateCmd extends BaseCmd {
private final String name;
private final ColumnMeta[] columns;
}

25
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/ColumnMeta.java
View File

@ -1,25 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore.model;
import com.antgroup.openspg.server.common.model.base.BaseValObj;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class ColumnMeta extends BaseValObj {
private final String name;
}

25
cloudext/interface/table-store/src/main/java/com/antgroup/openspg/cloudext/interfaces/tablestore/model/TableRecord.java
View File

@ -1,25 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.tablestore.model;
import com.antgroup.openspg.server.common.model.base.BaseValObj;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class TableRecord extends BaseValObj {
private final Object[] values;
}

1
cloudext/interface/table-store/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriver
View File

@ -1 +0,0 @@
com.antgroup.openspg.cloudext.impl.tablestore.local.LocalTableStoreClientDriver

4
cloudext/pom.xml
View File

@ -36,12 +36,8 @@
<modules>
<module>interface/graph-store</module>
<module>interface/object-store</module>
<module>interface/table-store</module>
<module>interface/search-engine</module>
<module>impl/graph-store/tugraph</module>
<module>impl/object-store/local</module>
<module>impl/table-store/local</module>
<module>impl/search-engine/elasticsearch</module>
</modules>

4
common/util/src/main/java/com/antgroup/openspg/common/util/JsonUtils.java
View File

@ -13,7 +13,7 @@
package com.antgroup.openspg.common.util;
import com.antgroup.openspg.server.api.facade.JSON;
import com.antgroup.openspg.server.api.facade.SchemaJsonUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -22,7 +22,7 @@ public class JsonUtils {
public static String toJsonString(Object object) {
String result = null;
try {
result = JSON.serialize(object);
result = SchemaJsonUtils.serialize(object);
} catch (Exception e) {
log.error("toJsonString error, object={}", object, e);
result = StringUtils.toString(object);

30
pom.xml
View File

@ -69,6 +69,16 @@
<artifactId>testdata</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-local-runner</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-cloudext-warehouse</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>common-model</artifactId>
@ -109,31 +119,11 @@
<artifactId>cloudext-interface-search-engine</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-object-store</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-table-store</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-graph-store-tugraph</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-object-store-local</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-table-store-local</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-search-engine-elasticsearch</artifactId>

3
reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/IVertexId.java
View File

@ -35,6 +35,9 @@ public interface IVertexId extends Serializable, Comparable<IVertexId> {
/** get kgstate internal id */
long getInternalId();
/** get biz id */
String getBizId();
/** get vertex type id */
long getTypeId();

4
reasoner/common/src/main/java/com/antgroup/openspg/reasoner/common/graph/vertex/impl/VertexId.java
View File

@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import java.util.Comparator;
import lombok.Getter;
public class VertexId implements IVertexId {
/**
@ -33,6 +34,8 @@ public class VertexId implements IVertexId {
*/
private final byte[] vertexIdBytes;
@Getter private String bizId;
/** create vertex id */
public VertexId(long internalId, String type) {
this.vertexIdBytes = generateIdBytes(internalId, type);
@ -40,6 +43,7 @@ public class VertexId implements IVertexId {
public VertexId(String bizId, String type) {
this.vertexIdBytes = generateIdBytes(generateInternalId(bizId, type), type);
this.bizId = bizId;
}
public VertexId(byte[] bytes) {

13
reasoner/pom.xml
View File

@ -31,10 +31,11 @@
<module>lube-logical</module>
<module>lube-physical</module>
<module>warehouse/warehouse-common</module>
<module>warehouse/cloudext-warehouse</module>
<module>kgdsl-parser</module>
<module>runner/runner-common</module>
<module>catalog/openspg-catalog</module>
<module>runner/local-runner</module>
<module>catalog/openspg-catalog</module>
</modules>
<properties>
@ -89,11 +90,6 @@
<artifactId>reasoner-kgdsl-parser</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-warehouse-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-runner-common</artifactId>
@ -109,6 +105,11 @@
<artifactId>reasoner-warehouse-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-cloudext-warehouse</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-local-runner</artifactId>

2
reasoner/runner/runner-common/src/main/java/com/antgroup/openspg/reasoner/graphstate/impl/MemGraphState.java
View File

@ -49,7 +49,7 @@ public class MemGraphState implements GraphState<IVertexId> {
* vertex map key is s, value is vertex property vertex property is Map<key, Map<longVersion,
* value>>
*/
Map<IVertexId, IProperty> vertexMap = new HashMap<>();
protected Map<IVertexId, IProperty> vertexMap = new HashMap<>();
/** in edges map key is s, value is p_o_t */
Map<IVertexId, Set<Tuple3<String, IVertexId, Long>>> inEdgeMap = new HashMap<>();

32
reasoner/warehouse/cloudext-warehouse/pom.xml Normal file
View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>reasoner-cloudext-warehouse</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-runner-common</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.reasoner</groupId>
<artifactId>reasoner-common</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-graph-store</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-impl-graph-store-tugraph</artifactId>
</dependency>
</dependencies>
</project>

171
reasoner/warehouse/cloudext-warehouse/src/main/java/com/antgroup/openspg/reasoner/warehouse/cloudext/CloudExtGraphState.java Normal file
View File

@ -0,0 +1,171 @@
package com.antgroup.openspg.reasoner.warehouse.cloudext;
import com.antgroup.openspg.cloudext.interfaces.graphstore.BaseLPGGraphStoreClient;
import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.OneHopLPGRecordQuery;
import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.ScanLPGRecordQuery;
import com.antgroup.openspg.cloudext.interfaces.graphstore.cmd.VertexLPGRecordQuery;
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.EdgeRecord;
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord;
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.struct.GraphLPGRecordStruct;
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName;
import com.antgroup.openspg.reasoner.common.graph.edge.Direction;
import com.antgroup.openspg.reasoner.common.graph.edge.IEdge;
import com.antgroup.openspg.reasoner.common.graph.edge.impl.Edge;
import com.antgroup.openspg.reasoner.common.graph.property.IProperty;
import com.antgroup.openspg.reasoner.common.graph.property.IVersionProperty;
import com.antgroup.openspg.reasoner.common.graph.property.impl.EdgeProperty;
import com.antgroup.openspg.reasoner.common.graph.property.impl.VertexVersionProperty;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertex;
import com.antgroup.openspg.reasoner.common.graph.vertex.IVertexId;
import com.antgroup.openspg.reasoner.common.graph.vertex.impl.Vertex;
import com.antgroup.openspg.reasoner.common.graph.vertex.impl.VertexId;
import com.antgroup.openspg.reasoner.graphstate.impl.MemGraphState;
import com.antgroup.openspg.reasoner.graphstate.model.MergeTypeEnum;
import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo;
import java.util.*;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
@Slf4j
public class CloudExtGraphState extends MemGraphState {
private BaseLPGGraphStoreClient lpgGraphStoreClient;
@Override
public void init(Map<String, String> param) {
super.init(param);
GraphStoreConnectionInfo connInfo = new GraphStoreConnectionInfo();
connInfo.setScheme(param.getOrDefault("cloudext.graphstore.schema", "tugraph"));
connInfo.setParams(Collections.unmodifiableMap(param));
lpgGraphStoreClient =
(BaseLPGGraphStoreClient) GraphStoreClientDriverManager.getClient(connInfo);
}
@Override
public IVertex<IVertexId, IProperty> getVertex(IVertexId id, Long version) {
IVersionProperty iProperty = (IVersionProperty) vertexMap.get(id);
if (iProperty != null) {
return super.getVertex(id, version);
}
GraphLPGRecordStruct recordStruct =
(GraphLPGRecordStruct)
lpgGraphStoreClient.queryRecord(new VertexLPGRecordQuery(id.getBizId(), id.getType()));
if (recordStruct.isEmpty()) {
return null;
}
for (VertexRecord vertexRecord : recordStruct.getVertices()) {
VertexVersionProperty property =
new VertexVersionProperty(vertexRecord.toPropertyMapWithIdAndVersion());
return new Vertex<>(
new VertexId(vertexRecord.getId(), vertexRecord.getVertexType()), property);
}
return null;
}
@Override
public void mergeVertexProperty(
IVertexId id, Map<String, Object> property, MergeTypeEnum mergeType, Long version) {
IProperty iProperty = vertexMap.get(id);
if (iProperty == null) {
IVertex<IVertexId, IProperty> vertex = getVertex(id, version);
addVertex(vertex, version);
}
super.mergeVertexProperty(id, property, mergeType, version);
}
@Override
public void mergeEdgeProperty(
IVertexId s,
String p,
IVertexId o,
Long version,
Direction direction,
Map<String, Object> property,
MergeTypeEnum mergeType) {
throw new UnsupportedOperationException();
}
@Override
public Iterator<IVertex<IVertexId, IProperty>> getVertexIterator(Set<String> vertexTypes) {
if (CollectionUtils.isEmpty(vertexTypes)) {
return Collections.emptyIterator();
}
List<VertexRecord> vertexRecords = new ArrayList<>();
for (String vertexType : vertexTypes) {
GraphLPGRecordStruct recordStruct =
(GraphLPGRecordStruct)
lpgGraphStoreClient.queryRecord(new ScanLPGRecordQuery(vertexType, null));
vertexRecords.addAll(recordStruct.getVertices());
}
if (CollectionUtils.isEmpty(vertexRecords)) {
return Collections.emptyIterator();
}
return vertexRecords.stream()
.map(
vertexRecord -> {
VertexVersionProperty property =
new VertexVersionProperty(vertexRecord.toPropertyMapWithIdAndVersion());
return (IVertex<IVertexId, IProperty>)
new Vertex<IVertexId, IProperty>(
new VertexId(vertexRecord.getId(), vertexRecord.getVertexType()), property);
})
.iterator();
}
@Override
public List<IEdge<IVertexId, IProperty>> getEdges(
IVertexId vertexId,
Long startVersion,
Long endVersion,
Set<String> types,
Direction direction) {
List<IEdge<IVertexId, IProperty>> edges =
super.getEdges(vertexId, startVersion, endVersion, types, direction);
if (CollectionUtils.isNotEmpty(edges)) {
return edges;
}
GraphLPGRecordStruct recordStruct =
(GraphLPGRecordStruct)
lpgGraphStoreClient.queryRecord(
new OneHopLPGRecordQuery(
vertexId.getBizId(),
vertexId.getType(),
types.stream().map(EdgeTypeName::parse).collect(Collectors.toSet()),
com.antgroup.openspg.cloudext.interfaces.graphstore.model.Direction.valueOf(
direction.name())));
List<EdgeRecord> edgeRecords = recordStruct.getEdges();
List<IEdge<IVertexId, IProperty>> results = new ArrayList<>(edgeRecords.size());
for (EdgeRecord edgeRecord : edgeRecords) {
VertexId srcVertexId =
new VertexId(edgeRecord.getSrcId(), edgeRecord.getEdgeType().getStartVertexType());
VertexId dstVertexId =
new VertexId(edgeRecord.getDstId(), edgeRecord.getEdgeType().getEndVertexType());
if (Direction.IN.equals(direction)) {
// 当是IN边时，这里交换下src和dst，由于图存的方向和kgdsl需要的方向不一致
VertexId tmp = srcVertexId;
srcVertexId = dstVertexId;
dstVertexId = tmp;
}
results.add(
new Edge<>(
srcVertexId,
dstVertexId,
new EdgeProperty(edgeRecord.toPropertyMapWithId()),
0L,
direction,
edgeRecord.getEdgeType().toString()));
}
return results;
}
}

4
server/api/facade/pom.xml
View File

@ -28,10 +28,6 @@
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>core-schema-model</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>core-reasoner-model</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>

33
server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/JSON.java → server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/SchemaJsonUtils.java
View File

@ -41,25 +41,13 @@ import com.antgroup.openspg.core.schema.model.type.EntityType;
import com.antgroup.openspg.core.schema.model.type.EventType;
import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum;
import com.antgroup.openspg.core.schema.model.type.StandardType;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerResult;
import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult;
import com.antgroup.openspg.server.core.reasoner.model.service.JobReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.model.service.KgdslReasonerContent;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerContentTypeEnum;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerReceiptTypeEnum;
import com.antgroup.openspg.server.core.reasoner.model.service.SuccessReasonerResult;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.model.service.VertexReasonerContent;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.ToNumberPolicy;
import com.google.gson.typeadapters.RuntimeTypeAdapterFactory;
import java.lang.reflect.Type;
public class JSON {
public class SchemaJsonUtils {
public static final String DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static final String DEFAULT_TYPE_FIELD_NAME = "@type";
@ -103,32 +91,13 @@ public class JSON {
.registerSubtype(
OperatorIdentifier.class, SPGIdentifierTypeEnum.OPERATOR.name())
.recognizeSubtypes())
.registerTypeAdapterFactory(
RuntimeTypeAdapterFactory.of(BaseReasonerResult.class, DEFAULT_TYPE_FIELD_NAME)
.registerSubtype(FailureReasonerResult.class, JobInstStatusEnum.FAILURE.name())
.registerSubtype(SuccessReasonerResult.class, JobInstStatusEnum.SUCCESS.name())
.recognizeSubtypes())
// BaseConceptSemantic
.registerTypeAdapterFactory(
RuntimeTypeAdapterFactory.of(BaseConceptSemantic.class, DEFAULT_TYPE_FIELD_NAME)
.registerSubtype(DynamicTaxonomySemantic.class)
.registerSubtype(LogicalCausationSemantic.class)
.recognizeSubtypes())
// BaseReasonerContent
.registerTypeAdapterFactory(
RuntimeTypeAdapterFactory.of(BaseReasonerContent.class, DEFAULT_TYPE_FIELD_NAME)
.registerSubtype(
KgdslReasonerContent.class, ReasonerContentTypeEnum.KGDSL.name())
.registerSubtype(
VertexReasonerContent.class, ReasonerContentTypeEnum.VERTEX.name())
.recognizeSubtypes())
// BaseReasonerReceipt
.registerTypeAdapterFactory(
RuntimeTypeAdapterFactory.of(BaseReasonerReceipt.class, DEFAULT_TYPE_FIELD_NAME)
.registerSubtype(
TableReasonerReceipt.class, ReasonerReceiptTypeEnum.TABLE.name())
.registerSubtype(JobReasonerReceipt.class, ReasonerReceiptTypeEnum.JOB.name())
.recognizeSubtypes())
.setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE)
.setDateFormat(DATA_FORMAT)
.create();

51
server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerDslRunRequest.java
View File

@ -1,51 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.facade.dto.reasoner.request;
import com.antgroup.openspg.server.common.model.base.BaseRequest;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent;
import java.util.Map;
public class ReasonerDslRunRequest extends BaseRequest {
private Long projectId;
private BaseReasonerContent content;
private Map<String, Object> params;
public BaseReasonerContent getContent() {
return content;
}
public void setContent(BaseReasonerContent content) {
this.content = content;
}
public Long getProjectId() {
return projectId;
}
public void setProjectId(Long projectId) {
this.projectId = projectId;
}
public Map<String, Object> getParams() {
return params;
}
public void setParams(Map<String, Object> params) {
this.params = params;
}
}

41
server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInfoQuery.java
View File

@ -1,41 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.facade.dto.reasoner.request;
import com.antgroup.openspg.server.common.model.base.BaseQuery;
public class ReasonerJobInfoQuery extends BaseQuery {
private Long reasonerJobInfoId;
private String externalJobInfoId;
public Long getReasonerJobInfoId() {
return reasonerJobInfoId;
}
public ReasonerJobInfoQuery setReasonerJobInfoId(Long reasonerJobInfoId) {
this.reasonerJobInfoId = reasonerJobInfoId;
return this;
}
public String getExternalJobInfoId() {
return externalJobInfoId;
}
public ReasonerJobInfoQuery setExternalJobInfoId(String externalJobInfoId) {
this.externalJobInfoId = externalJobInfoId;
return this;
}
}

41
server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobInstQuery.java
View File

@ -1,41 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.facade.dto.reasoner.request;
import com.antgroup.openspg.server.common.model.base.BaseQuery;
public class ReasonerJobInstQuery extends BaseQuery {
private Long reasonerJobInstId;
private String externalJobInstId;
public Long getReasonerJobInstId() {
return reasonerJobInstId;
}
public ReasonerJobInstQuery setReasonerJobInstId(Long reasonerJobInstId) {
this.reasonerJobInstId = reasonerJobInstId;
return this;
}
public String getExternalJobInstId() {
return externalJobInstId;
}
public ReasonerJobInstQuery setExternalJobInstId(String externalJobInstId) {
this.externalJobInstId = externalJobInstId;
return this;
}
}

81
server/api/facade/src/main/java/com/antgroup/openspg/server/api/facade/dto/reasoner/request/ReasonerJobSubmitRequest.java
View File

@ -1,81 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.facade.dto.reasoner.request;
import com.antgroup.openspg.server.common.model.base.BaseRequest;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent;
import java.util.Map;
public class ReasonerJobSubmitRequest extends BaseRequest {
private String jobName;
private Long projectId;
private BaseReasonerContent content;
private String cron;
private String idempotentId;
private Map<String, Object> params;
public BaseReasonerContent getContent() {
return content;
}
public void setContent(BaseReasonerContent content) {
this.content = content;
}
public String getJobName() {
return jobName;
}
public void setJobName(String jobName) {
this.jobName = jobName;
}
public Long getProjectId() {
return projectId;
}
public void setProjectId(Long projectId) {
this.projectId = projectId;
}
public String getCron() {
return cron;
}
public void setCron(String cron) {
this.cron = cron;
}
public String getIdempotentId() {
return idempotentId;
}
public void setIdempotentId(String idempotentId) {
this.idempotentId = idempotentId;
}
public Map<String, Object> getParams() {
return params;
}
public void setParams(Map<String, Object> params) {
this.params = params;
}
}

10
server/api/http-client/src/main/java/com/antgroup/openspg/server/api/http/client/forest/GsonConvertor.java
View File

@ -13,7 +13,7 @@
package com.antgroup.openspg.server.api.http.client.forest;
import com.antgroup.openspg.server.api.facade.JSON;
import com.antgroup.openspg.server.api.facade.SchemaJsonUtils;
import com.dtflys.forest.converter.ConvertOptions;
import com.dtflys.forest.converter.json.ForestJsonConverter;
import com.dtflys.forest.http.ForestRequest;
@ -40,7 +40,7 @@ public class GsonConvertor implements ForestJsonConverter {
@Override
public String encodeToString(Object obj) {
return JSON.serialize(obj);
return SchemaJsonUtils.serialize(obj);
}
@Override
@ -76,7 +76,7 @@ public class GsonConvertor implements ForestJsonConverter {
if (obj instanceof CharSequence) {
return convertToJavaObject(obj.toString(), LinkedHashMap.class);
}
final Gson gson = JSON.gson;
final Gson gson = SchemaJsonUtils.gson;
final JsonElement jsonElement = gson.toJsonTree(obj);
return toMap(jsonElement.getAsJsonObject(), true);
}
@ -88,7 +88,7 @@ public class GsonConvertor implements ForestJsonConverter {
@Override
public String getDateFormat() {
return JSON.DATA_FORMAT;
return SchemaJsonUtils.DATA_FORMAT;
}
@Override
@ -96,7 +96,7 @@ public class GsonConvertor implements ForestJsonConverter {
if (StringUtils.isBlank(source)) {
return null;
}
return JSON.deserialize(source, targetType);
return SchemaJsonUtils.deserialize(source, targetType);
}
@Override

4
server/api/http-server/pom.xml
View File

@ -36,10 +36,6 @@
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>biz-common</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>biz-reasoner</artifactId>
</dependency>
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-ui</artifactId>

4
server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/WebMvcConfig.java
View File

@ -13,7 +13,7 @@
package com.antgroup.openspg.server.api.http.server;
import com.antgroup.openspg.server.api.facade.JSON;
import com.antgroup.openspg.server.api.facade.SchemaJsonUtils;
import java.util.Collections;
import java.util.List;
import org.springframework.context.annotation.Configuration;
@ -28,7 +28,7 @@ public class WebMvcConfig implements WebMvcConfigurer {
@Override
public void extendMessageConverters(List<HttpMessageConverter<?>> converters) {
GsonHttpMessageConverter gsonHttpMessageConverter = new GsonHttpMessageConverter();
gsonHttpMessageConverter.setGson(JSON.gson);
gsonHttpMessageConverter.setGson(SchemaJsonUtils.gson);
gsonHttpMessageConverter.setSupportedMediaTypes(
Collections.singletonList(MediaType.APPLICATION_JSON));
converters.add(0, gsonHttpMessageConverter);

56
server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ObjectStoreController.java
View File

@ -1,56 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.http.server.openapi;
import com.antgroup.openspg.server.api.facade.dto.common.request.ObjectStoreRequest;
import com.antgroup.openspg.server.api.facade.dto.common.response.ObjectStoreResponse;
import com.antgroup.openspg.server.api.http.server.HttpBizCallback;
import com.antgroup.openspg.server.api.http.server.HttpBizTemplate;
import com.antgroup.openspg.server.biz.common.ObjectStoreManager;
import java.io.IOException;
import java.io.InputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.multipart.MultipartFile;
@Controller
@RequestMapping("/public/v1/objectStore")
public class ObjectStoreController {
@Autowired private ObjectStoreManager objectStoreManager;
@RequestMapping(method = RequestMethod.POST, consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<Object> upload(ObjectStoreRequest request, MultipartFile file) {
return HttpBizTemplate.execute(
new HttpBizCallback<ObjectStoreResponse>() {
@Override
public void check() {}
@Override
public ObjectStoreResponse action() {
InputStream inputStream = null;
try {
inputStream = file.getInputStream();
} catch (IOException e) {
throw new RuntimeException(e);
}
return objectStoreManager.objectStore(request, inputStream);
}
});
}
}

83
server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/ReasonerController.java
View File

@ -1,83 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.http.server.openapi;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerDslRunRequest;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobSubmitRequest;
import com.antgroup.openspg.server.api.http.server.BaseController;
import com.antgroup.openspg.server.api.http.server.HttpBizCallback;
import com.antgroup.openspg.server.api.http.server.HttpBizTemplate;
import com.antgroup.openspg.server.biz.reasoner.ReasonerManager;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
@Controller
@RequestMapping("/public/v1/reasoner")
public class ReasonerController extends BaseController {
@Autowired private ReasonerManager reasonerManager;
@RequestMapping(value = "/runDsl", method = RequestMethod.POST)
public ResponseEntity<Object> runDsl(@RequestBody ReasonerDslRunRequest request) {
return HttpBizTemplate.execute(
new HttpBizCallback<BaseReasonerReceipt>() {
@Override
public void check() {}
@Override
public BaseReasonerReceipt action() {
return reasonerManager.runDsl(request);
}
});
}
@RequestMapping(value = "/submitJobInfo", method = RequestMethod.POST)
public ResponseEntity<Object> submitJobInfo(@RequestBody ReasonerJobSubmitRequest request) {
return HttpBizTemplate.execute(
new HttpBizCallback<BaseReasonerReceipt>() {
@Override
public void check() {}
@Override
public BaseReasonerReceipt action() {
return reasonerManager.submitJob(request);
}
});
}
@RequestMapping(value = "/queryJobInst", method = RequestMethod.GET)
public ResponseEntity<Object> queryJobInst(@RequestParam(required = false) Long jobInstId) {
return HttpBizTemplate.execute(
new HttpBizCallback<List<ReasonerJobInst>>() {
@Override
public void check() {}
@Override
public List<ReasonerJobInst> action() {
ReasonerJobInstQuery query = new ReasonerJobInstQuery();
query.setReasonerJobInstId(jobInstId);
return reasonerManager.queryJobInst(query);
}
});
}
}

49
server/api/http-server/src/main/java/com/antgroup/openspg/server/api/http/server/openapi/TableStoreController.java
View File

@ -1,49 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.api.http.server.openapi;
import com.antgroup.openspg.server.api.http.server.BaseController;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@Controller
@RequestMapping("/public/v1/tableStore")
public class TableStoreController extends BaseController {
@RequestMapping(path = "/download", method = RequestMethod.GET)
public ResponseEntity<Object> download(String fileName) {
File file = new File(fileName);
FileInputStream fileInputStream = null;
try {
fileInputStream = new FileInputStream(file);
} catch (FileNotFoundException e) {
return ResponseEntity.notFound().build();
}
InputStreamResource resource = new InputStreamResource(fileInputStream);
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fileName)
.contentLength(file.length())
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.body(resource);
}
}

5
server/arks/sofaboot/pom.xml
View File

@ -27,10 +27,6 @@
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>api-http-server</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>biz-reasoner</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>biz-schema</artifactId>
@ -71,7 +67,6 @@
<version>2.5.7</version>
<configuration>
<includeSystemScope>true</includeSystemScope>
<outputDirectory>../../dev/release/openspg/target</outputDirectory>
<classifier>executable</classifier>
</configuration>
<executions>

23
server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/ObjectStoreManager.java
View File

@ -1,23 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.biz.common;
import com.antgroup.openspg.server.api.facade.dto.common.request.ObjectStoreRequest;
import com.antgroup.openspg.server.api.facade.dto.common.response.ObjectStoreResponse;
import java.io.InputStream;
public interface ObjectStoreManager {
ObjectStoreResponse objectStore(ObjectStoreRequest request, InputStream file);
}

43
server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/ObjectStoreManagerImpl.java
View File

@ -1,43 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.biz.common.impl;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient;
import com.antgroup.openspg.cloudext.interfaces.objectstore.cmd.ObjectStoreSaveCmd;
import com.antgroup.openspg.cloudext.interfaces.objectstore.model.ObjectStorePath;
import com.antgroup.openspg.server.api.facade.dto.common.request.ObjectStoreRequest;
import com.antgroup.openspg.server.api.facade.dto.common.response.ObjectStoreResponse;
import com.antgroup.openspg.server.biz.common.ObjectStoreManager;
import com.antgroup.openspg.server.common.service.datasource.DataSourceService;
import java.io.InputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class ObjectStoreManagerImpl implements ObjectStoreManager {
@Autowired private DataSourceService dataSourceService;
@Override
public ObjectStoreResponse objectStore(ObjectStoreRequest request, InputStream file) {
ObjectStoreClient objectStoreClient = dataSourceService.buildSharedFileStoreClient();
ObjectStorePath filePath =
objectStoreClient.save(
new ObjectStoreSaveCmd(new ObjectStorePath(request.getName()), file));
return new ObjectStoreResponse()
.setRelativePath(filePath.getRelativePath())
.setAbsolutePath(filePath.getAbsolutePath());
}
}

4
server/biz/common/src/main/java/com/antgroup/openspg/server/biz/common/impl/SearchEngineManagerImpl.java
View File

@ -14,7 +14,7 @@
package com.antgroup.openspg.server.biz.common.impl;
import com.antgroup.openspg.cloudext.interfaces.searchengine.SearchEngineClient;
import com.antgroup.openspg.server.api.facade.JSON;
import com.antgroup.openspg.server.api.facade.SchemaJsonUtils;
import com.antgroup.openspg.server.api.facade.dto.common.request.SearchEngineIndexRequest;
import com.antgroup.openspg.server.api.facade.dto.common.response.SearchEngineIndexResponse;
import com.antgroup.openspg.server.biz.common.SearchEngineManager;
@ -38,6 +38,6 @@ public class SearchEngineManagerImpl implements SearchEngineManager {
(SearchEngineConnectionInfo) searchEngineClient.getConnInfo();
return new SearchEngineIndexResponse()
.setIndexName(convertedIndexName)
.setConnInfo(JSON.serialize(connInfo));
.setConnInfo(SchemaJsonUtils.serialize(connInfo));
}
}

43
server/biz/reasoner/pom.xml
View File

@ -1,43 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>server-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>biz-reasoner</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>api-http-client</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>common-service</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>core-reasoner-service</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
</dependencies>
</project>

31
server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/ReasonerManager.java
View File

@ -1,31 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.biz.reasoner;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerDslRunRequest;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobSubmitRequest;
import com.antgroup.openspg.server.core.reasoner.model.service.JobReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
import java.util.List;
public interface ReasonerManager {
TableReasonerReceipt runDsl(ReasonerDslRunRequest request);
JobReasonerReceipt submitJob(ReasonerJobSubmitRequest request);
List<ReasonerJobInst> queryJobInst(ReasonerJobInstQuery query);
}

96
server/biz/reasoner/src/main/java/com/antgroup/openspg/server/biz/reasoner/impl/ReasonerManagerImpl.java
View File

@ -1,96 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.biz.reasoner.impl;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd;
import com.antgroup.openspg.common.util.StringUtils;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerDslRunRequest;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobInstQuery;
import com.antgroup.openspg.server.api.facade.dto.reasoner.request.ReasonerJobSubmitRequest;
import com.antgroup.openspg.server.biz.reasoner.ReasonerManager;
import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo;
import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import com.antgroup.openspg.server.common.service.config.AppEnvConfig;
import com.antgroup.openspg.server.common.service.datasource.DataSourceService;
import com.antgroup.openspg.server.core.reasoner.model.service.JobReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInfoService;
import com.antgroup.openspg.server.core.reasoner.service.ReasonerJobInstService;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class ReasonerManagerImpl implements ReasonerManager {
@Autowired private AppEnvConfig appEnvConfig;
@Autowired private ReasonerJobInfoService reasonerJobInfoService;
@Autowired private ReasonerJobInstService reasonerJobInstService;
@Autowired private DataSourceService dataSourceService;
@Override
public TableReasonerReceipt runDsl(ReasonerDslRunRequest request) {
ComputingClient computingClient = dataSourceService.buildSharedComputingClient();
return computingClient.run(
new ReasonerJobRunCmd(
request.getProjectId(),
appEnvConfig.getSchemaUri(),
(GraphStoreConnectionInfo) dataSourceService.buildSharedKgStoreClient().getConnInfo(),
request.getContent()));
}
@Override
public JobReasonerReceipt submitJob(ReasonerJobSubmitRequest request) {
ReasonerJobInfo reasonerJobInfo =
new ReasonerJobInfo(
request.getJobName(),
request.getProjectId(),
request.getContent(),
request.getCron(),
JobInfoStateEnum.ENABLE,
request.getParams());
// create a reasoner job
Long reasonerJobInfoId = reasonerJobInfoService.create(reasonerJobInfo);
// if the cron expression is empty, create a reasoner job instance
Long reasonerJobInstId = null;
if (StringUtils.isBlank(request.getCron())) {
ReasonerJobInst reasonerJobInst =
new ReasonerJobInst(
reasonerJobInfoId,
reasonerJobInfo.getProjectId(),
JobInstStatusEnum.INIT,
null,
null,
null,
null,
null);
reasonerJobInstId = reasonerJobInstService.create(reasonerJobInfo, reasonerJobInst);
}
return new JobReasonerReceipt(reasonerJobInfoId, reasonerJobInstId);
}
@Override
public List<ReasonerJobInst> queryJobInst(ReasonerJobInstQuery query) {
return reasonerJobInstService.query(query);
}
}

47
server/cloudext/impl/computing/local/pom.xml
View File

@ -1,47 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>server-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-impl-computing-local</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>cloudext-interface-computing</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-table-store</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.kg.reasoner</groupId>
<artifactId>local</artifactId>
<version>${kgreasoner-version}</version>
<scope>system</scope>
<systemPath>${project.basedir}/../../../../lib/spgreasoner-local-0.0.1.jar</systemPath>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>core-reasoner-service</artifactId>
</dependency>
</dependencies>
</project>

62
server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClient.java
View File

@ -1,62 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.computing.local;
import com.antgroup.openspg.cloudext.impl.computing.local.impl.LocalReasonerExecutorImpl;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd;
import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LocalComputingClient implements ComputingClient {
private static final String BUILDER_NUMBER_OF_THREAD = "builder.nThreads";
@Getter private final ComputingConnectionInfo connInfo;
private final LocalReasonerExecutor localReasonerExecutor;
public LocalComputingClient(ComputingConnectionInfo connInfo) {
this.connInfo = connInfo;
localReasonerExecutor =
new LocalReasonerExecutorImpl(
(String) connInfo.getParamOrDefault(BUILDER_NUMBER_OF_THREAD, "*2"));
}
@Override
public ReasonerStatusWithProgress query(ReasonerJobProcessQuery query) {
return localReasonerExecutor.query(query);
}
@Override
public boolean canSubmit(ReasonerJobCanSubmitQuery query) {
return localReasonerExecutor.canSubmit(query);
}
@Override
public String submit(ReasonerJobSubmitCmd cmd) {
return localReasonerExecutor.submit(cmd);
}
@Override
public TableReasonerReceipt run(ReasonerJobRunCmd cmd) {
return localReasonerExecutor.run(cmd);
}
}

39
server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalComputingClientDriver.java
View File

@ -1,39 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.computing.local;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriver;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriverManager;
import com.antgroup.openspg.common.util.cloudext.CachedCloudExtClientDriver;
import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo;
public class LocalComputingClientDriver
extends CachedCloudExtClientDriver<ComputingClient, ComputingConnectionInfo>
implements ComputingClientDriver {
static {
ComputingClientDriverManager.registerDriver(new LocalComputingClientDriver());
}
@Override
public String driverScheme() {
return "local";
}
@Override
protected ComputingClient innerConnect(ComputingConnectionInfo connInfo) {
return new LocalComputingClient(connInfo);
}
}

32
server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/LocalReasonerExecutor.java
View File

@ -1,32 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.computing.local;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
public interface LocalReasonerExecutor {
ReasonerStatusWithProgress query(ReasonerJobProcessQuery query);
boolean canSubmit(ReasonerJobCanSubmitQuery query);
String submit(ReasonerJobSubmitCmd cmd);
TableReasonerReceipt run(ReasonerJobRunCmd cmd);
}

224
server/cloudext/impl/computing/local/src/main/java/com/antgroup/openspg/cloudext/impl/computing/local/impl/LocalReasonerExecutorImpl.java
View File

@ -1,224 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.computing.local.impl;
import com.antgroup.kg.reasoner.catalog.impl.KgSchemaConnectionInfo;
import com.antgroup.kg.reasoner.local.KGReasonerLocalRunner;
import com.antgroup.kg.reasoner.local.model.LocalReasonerResult;
import com.antgroup.kg.reasoner.local.model.LocalReasonerTask;
import com.antgroup.openspg.cloudext.impl.computing.local.LocalReasonerExecutor;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableFileHandler;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.tablestore.cmd.TableFileCreateCmd;
import com.antgroup.openspg.cloudext.interfaces.tablestore.model.ColumnMeta;
import com.antgroup.openspg.cloudext.interfaces.tablestore.model.TableRecord;
import com.antgroup.openspg.common.util.StringUtils;
import com.antgroup.openspg.common.util.thread.ThreadUtils;
import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import com.antgroup.openspg.server.core.reasoner.model.ReasonerException;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerResult;
import com.antgroup.openspg.server.core.reasoner.model.service.FailureReasonerResult;
import com.antgroup.openspg.server.core.reasoner.model.service.KgdslReasonerContent;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerProgress;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress;
import com.antgroup.openspg.server.core.reasoner.model.service.SuccessReasonerResult;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
import com.antgroup.openspg.server.core.reasoner.service.util.LocalRunnerUtils;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
@Slf4j
public class LocalReasonerExecutorImpl implements LocalReasonerExecutor {
// for reasoner job
private final ThreadPoolExecutor jobDriver;
private final ThreadPoolExecutor jobWorker;
private final Map<String, Future<ReasonerStatusWithProgress>> runningReasonerJobInst;
// for olap
private final ThreadPoolExecutor olapDriver;
private final ThreadPoolExecutor olapWorker;
public LocalReasonerExecutorImpl(String nThreads) {
jobDriver = ThreadUtils.newDaemonFixedThreadPool(1, "localReasonerDriver");
jobWorker =
ThreadUtils.newDaemonFixedThreadPool(ThreadUtils.nThreads(nThreads), "localReasonerWorker");
olapDriver =
ThreadUtils.newDaemonFixedThreadPool(ThreadUtils.nThreads("*1"), "olapReasonerDriver");
olapWorker =
ThreadUtils.newDaemonFixedThreadPool(ThreadUtils.nThreads(nThreads), "olapReasonerWorker");
runningReasonerJobInst = new ConcurrentHashMap<>();
}
@Override
public ReasonerStatusWithProgress query(ReasonerJobProcessQuery query) {
Future<ReasonerStatusWithProgress> future =
runningReasonerJobInst.get(query.getComputingJobInstId());
if (future == null) {
return null;
}
if (future.isDone()) {
// if finished, remove it from runningBuilderJobInst
runningReasonerJobInst.remove(query.getComputingJobInstId());
ReasonerStatusWithProgress progress = null;
try {
progress = future.get();
} catch (Throwable e) {
throw ReasonerException.reasonerError(e);
}
return progress;
} else {
return new ReasonerStatusWithProgress(JobInstStatusEnum.RUNNING);
}
}
@Override
public boolean canSubmit(ReasonerJobCanSubmitQuery query) {
return runningReasonerJobInst.isEmpty();
}
@Override
public String submit(ReasonerJobSubmitCmd cmd) {
ReasonerJobInst jobInst = cmd.getJobInst();
if (runningReasonerJobInst.isEmpty()) {
synchronized (runningReasonerJobInst) {
if (runningReasonerJobInst.isEmpty()) {
Future<ReasonerStatusWithProgress> future = doSubmit(cmd);
String computingJobInstId = String.valueOf(jobInst.getJobInstId());
runningReasonerJobInst.put(computingJobInstId, future);
return computingJobInstId;
}
}
}
return null;
}
@Override
public TableReasonerReceipt run(ReasonerJobRunCmd cmd) {
Future<TableReasonerReceipt> future =
olapDriver.submit(
() -> {
LocalReasonerResult localReasonerResult =
doRun(
cmd.getProjectId(),
cmd.getSchemaUrl(),
cmd.getConnInfo(),
cmd.getContent(),
olapWorker);
return buildReceipt(localReasonerResult);
});
try {
return future.get(3, TimeUnit.MINUTES);
} catch (InterruptedException | ExecutionException e) {
throw ReasonerException.reasonerError(e);
} catch (TimeoutException e) {
future.cancel(true);
throw ReasonerException.timeout(3);
}
}
private Future<ReasonerStatusWithProgress> doSubmit(ReasonerJobSubmitCmd cmd) {
return jobDriver.submit(
() -> {
try {
ReasonerJobInfo jobInfo = cmd.getJobInfo();
LocalReasonerResult localReasonerResult =
doRun(
jobInfo.getProjectId(),
cmd.getSchemaUrl(),
cmd.getGraphStoreConnInfo(),
jobInfo.getContent(),
jobWorker);
JobInstStatusEnum status = null;
BaseReasonerResult result = null;
ReasonerProgress progress = new ReasonerProgress();
if (StringUtils.isNotBlank(localReasonerResult.getErrMsg())) {
status = JobInstStatusEnum.FAILURE;
result = new FailureReasonerResult(localReasonerResult.getErrMsg());
} else {
status = JobInstStatusEnum.SUCCESS;
result = writeResult2TableFile(cmd, localReasonerResult);
}
return new ReasonerStatusWithProgress(status, result, progress);
} catch (Throwable e) {
throw ReasonerException.reasonerError(e);
}
});
}
private LocalReasonerResult doRun(
Long projectId,
String schemaUrl,
GraphStoreConnectionInfo connInfo,
BaseReasonerContent content,
ThreadPoolExecutor executor) {
KGReasonerLocalRunner localRunner = new KGReasonerLocalRunner();
LocalReasonerTask reasonerTask = new LocalReasonerTask();
reasonerTask.setThreadPoolExecutor(executor);
reasonerTask.setCatalog(
LocalRunnerUtils.buildCatalog(projectId, new KgSchemaConnectionInfo(schemaUrl, "")));
reasonerTask.setGraphState(LocalRunnerUtils.buildGraphState(connInfo));
reasonerTask.setDsl(((KgdslReasonerContent) content).getKgdsl());
return localRunner.run(reasonerTask);
}
private SuccessReasonerResult writeResult2TableFile(
ReasonerJobSubmitCmd cmd, LocalReasonerResult result) throws Exception {
TableStoreClient tableStoreClient =
TableStoreClientDriverManager.getClient(cmd.getTableStoreConnInfo());
if (CollectionUtils.isEmpty(result.getColumns())) {
return new SuccessReasonerResult(null);
}
String tableName = cmd.tableName();
TableFileHandler fileHandler =
tableStoreClient.create(
new TableFileCreateCmd(
tableName,
result.getColumns().stream().map(ColumnMeta::new).toArray(ColumnMeta[]::new)));
fileHandler.batchWrite(
result.getRows().stream().map(TableRecord::new).collect(Collectors.toList()));
fileHandler.close();
return new SuccessReasonerResult(fileHandler.getTableName());
}
private TableReasonerReceipt buildReceipt(LocalReasonerResult reasonerResult) {
if (StringUtils.isNotBlank(reasonerResult.getErrMsg())) {
throw ReasonerException.reasonerError(reasonerResult.getErrMsg());
}
return new TableReasonerReceipt(
reasonerResult.getColumns(),
reasonerResult.getRows().stream().map(Arrays::asList).collect(Collectors.toList()));
}
}

36
server/cloudext/impl/job-scheduler/local/pom.xml
View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>server-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-impl-job-scheduler-local</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>cloudext-interface-job-scheduler</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>common-service</artifactId>
</dependency>
</dependencies>
</project>

57
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClient.java
View File

@ -1,57 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.scheduler.JobExecuteScheduler;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInfoService;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInstService;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.SchedulerCallback;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo;
import com.antgroup.openspg.server.common.service.spring.SpringContextHolder;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
public class LocalJobSchedulerClient implements JobSchedulerClient {
@Getter private final JobSchedulerConnectionInfo connInfo;
@Getter private final List<SchedulerCallback> schedulerCallbacks;
@Setter private SchedulerJobInfoService jobInfoService;
@Setter private SchedulerJobInstService jobInstService;
public LocalJobSchedulerClient(JobSchedulerConnectionInfo connInfo) {
this.connInfo = connInfo;
jobInfoService = SpringContextHolder.getBean(SchedulerJobInfoService.class);
jobInstService = SpringContextHolder.getBean(SchedulerJobInstService.class);
schedulerCallbacks = SpringContextHolder.getBeans(SchedulerCallback.class);
new JobExecuteScheduler(jobInstService, schedulerCallbacks).init();
}
@Override
public String createJobInfo(SchedulerJobInfo jobInfo) {
return jobInfoService.create(jobInfo);
}
@Override
public String createJobInst(SchedulerJobInst jobInst) {
return jobInstService.create(jobInst);
}
@Override
public void close() throws Exception {}
}

39
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/LocalJobSchedulerClientDriver.java
View File

@ -1,39 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriver;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriverManager;
import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo;
public class LocalJobSchedulerClientDriver implements JobSchedulerClientDriver {
private static final JobSchedulerClient INSTANCE =
new LocalJobSchedulerClient(new JobSchedulerConnectionInfo().setScheme("local"));
static {
JobSchedulerClientDriverManager.registerDriver(new LocalJobSchedulerClientDriver());
}
@Override
public String driverScheme() {
return "local";
}
@Override
public JobSchedulerClient connect(JobSchedulerConnectionInfo connInfo) {
return INSTANCE;
}
}

31
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/cmd/SchedulerJobInstQuery.java
View File

@ -1,31 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd;
import com.antgroup.openspg.server.common.model.base.BaseQuery;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
@Setter
@Getter
@Accessors(chain = true)
public class SchedulerJobInstQuery extends BaseQuery {
private Set<JobInstStatusEnum> status;
private String orderBy;
}

21
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInfoRepository.java
View File

@ -1,21 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo;
public interface SchedulerJobInfoRepository {
String save(SchedulerJobInfo jobInfo);
}

28
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/repo/SchedulerJobInstRepository.java
View File

@ -1,28 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd.SchedulerJobInstQuery;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import java.util.List;
public interface SchedulerJobInstRepository {
String save(SchedulerJobInst jobInst);
List<SchedulerJobInst> query(SchedulerJobInstQuery query);
int updateStatus(String jobInstId, JobInstStatusEnum status);
}

21
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/BaseScheduler.java
View File

@ -1,21 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.scheduler;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Getter
public abstract class BaseScheduler {}

104
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/scheduler/JobExecuteScheduler.java
View File

@ -1,104 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.scheduler;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInstService;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.SchedulerCallback;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.CallbackResult;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import com.antgroup.openspg.common.util.thread.SPGThread;
import com.antgroup.openspg.common.util.thread.ThreadUtils;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
@Slf4j
public class JobExecuteScheduler extends BaseScheduler {
private final SchedulerJobInstService schedulerJobInstService;
private final Map<String, SchedulerCallback> schedulerCallbacks;
public JobExecuteScheduler(
SchedulerJobInstService schedulerJobInstService, List<SchedulerCallback> schedulerCallbacks) {
this.schedulerJobInstService = schedulerJobInstService;
this.schedulerCallbacks = new HashMap<>(JobTypeEnum.values().length);
for (SchedulerCallback schedulerCallback : schedulerCallbacks) {
for (JobTypeEnum jobType : schedulerCallback.accept()) {
this.schedulerCallbacks.put(jobType.name(), schedulerCallback);
}
}
}
public void init() {
log.info("init JobExecuteScheduler...");
new SPGThread(
"JobExecuteScheduler",
() -> {
while (true) {
try {
doExecute();
ThreadUtils.sleep(3000);
} catch (Throwable e) {
log.warn("JobExecuteScheduler execute fail", e);
}
}
})
.start();
}
private void doExecute() {
List<SchedulerJobInst> runningJobInsts = schedulerJobInstService.queryRunningJobInsts();
if (CollectionUtils.isNotEmpty(runningJobInsts)) {
processJobInsts(runningJobInsts);
return;
}
List<SchedulerJobInst> queuedJobInsts = schedulerJobInstService.queryToRunJobInsts();
if (CollectionUtils.isNotEmpty(queuedJobInsts)) {
processJobInsts(queuedJobInsts);
}
}
private void processJobInsts(List<SchedulerJobInst> jobInsts) {
for (SchedulerJobInst jobInst : jobInsts) {
log.info(
"polling jobType={} schedulerJobInstId={}", jobInst.getJobType(), jobInst.getJobInstId());
SchedulerCallback callback = schedulerCallbacks.get(jobInst.getJobType());
if (callback == null) {
updateStatus(jobInst, JobInstStatusEnum.FAILURE);
continue;
}
CallbackResult result = callback.polling(jobInst);
JobInstStatusEnum newStatus = result.getStatus();
updateStatus(jobInst, newStatus);
}
}
private void updateStatus(SchedulerJobInst jobInst, JobInstStatusEnum newStatus) {
if (!newStatus.equals(jobInst.getStatus())) {
log.info(
"update schedulerJobInstId={} from status={} to status={}",
jobInst.getJobInstId(),
jobInst.getStatus(),
newStatus);
schedulerJobInstService.updateStatus(jobInst.getJobInstId(), newStatus);
}
}
}

21
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInfoService.java
View File

@ -1,21 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo;
public interface SchedulerJobInfoService {
String create(SchedulerJobInfo jobInfo);
}

29
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/SchedulerJobInstService.java
View File

@ -1,29 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import java.util.List;
public interface SchedulerJobInstService {
String create(SchedulerJobInst jobInst);
List<SchedulerJobInst> queryRunningJobInsts();
List<SchedulerJobInst> queryToRunJobInsts();
void updateStatus(String jobInstId, JobInstStatusEnum status);
}

31
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInfoServiceImpl.java
View File

@ -1,31 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.impl;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo.SchedulerJobInfoRepository;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInfoService;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class SchedulerJobInfoServiceImpl implements SchedulerJobInfoService {
@Autowired private SchedulerJobInfoRepository jobInfoRepository;
@Override
public String create(SchedulerJobInfo jobInfo) {
return jobInfoRepository.save(jobInfo);
}
}

60
server/cloudext/impl/job-scheduler/local/src/main/java/com/antgroup/openspg/cloudext/impl/jobscheduler/local/service/impl/SchedulerJobInstServiceImpl.java
View File

@ -1,60 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.impl;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.cmd.SchedulerJobInstQuery;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.repo.SchedulerJobInstRepository;
import com.antgroup.openspg.cloudext.impl.jobscheduler.local.service.SchedulerJobInstService;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import com.google.common.collect.Sets;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class SchedulerJobInstServiceImpl implements SchedulerJobInstService {
@Autowired private SchedulerJobInstRepository schedulerJobInstRepository;
@Override
public String create(SchedulerJobInst jobInst) {
return schedulerJobInstRepository.save(jobInst);
}
@Override
public List<SchedulerJobInst> queryRunningJobInsts() {
SchedulerJobInstQuery jobInstQuery =
new SchedulerJobInstQuery()
.setStatus(Sets.newHashSet(JobInstStatusEnum.RUNNING_STATUS))
.setOrderBy("id asc");
return schedulerJobInstRepository.query(jobInstQuery);
}
@Override
public List<SchedulerJobInst> queryToRunJobInsts() {
SchedulerJobInstQuery jobInstQuery =
new SchedulerJobInstQuery()
.setStatus(Sets.newHashSet(JobInstStatusEnum.QUEUE, JobInstStatusEnum.INIT))
.setOrderBy("id asc");
return schedulerJobInstRepository.query(jobInstQuery);
}
@Override
public void updateStatus(String jobInstId, JobInstStatusEnum status) {
schedulerJobInstRepository.updateStatus(jobInstId, status);
}
}

36
server/cloudext/interface/computing/pom.xml
View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>server-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-interface-computing</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg</groupId>
<artifactId>common-util</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>core-reasoner-model</artifactId>
</dependency>
</dependencies>
</project>

37
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClient.java
View File

@ -1,37 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobCanSubmitQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobProcessQuery;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobRunCmd;
import com.antgroup.openspg.cloudext.interfaces.computing.cmd.ReasonerJobSubmitCmd;
import com.antgroup.openspg.common.util.cloudext.CloudExtClient;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerStatusWithProgress;
import com.antgroup.openspg.server.core.reasoner.model.service.TableReasonerReceipt;
public interface ComputingClient extends CloudExtClient {
/* ----------------------- *
| Reasoner Job |
* ----------------------- */
ReasonerStatusWithProgress query(ReasonerJobProcessQuery query);
boolean canSubmit(ReasonerJobCanSubmitQuery query);
String submit(ReasonerJobSubmitCmd cmd);
TableReasonerReceipt run(ReasonerJobRunCmd cmd);
}

20
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriver.java
View File

@ -1,20 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing;
import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver;
import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo;
public interface ComputingClientDriver
extends CloudExtClientDriver<ComputingClient, ComputingConnectionInfo> {}

52
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/ComputingClientDriverManager.java
View File

@ -1,52 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing;
import com.antgroup.openspg.common.util.DriverManagerUtils;
import com.antgroup.openspg.server.common.model.datasource.connection.ComputingConnectionInfo;
import com.antgroup.openspg.server.common.model.exception.CloudExtException;
import java.util.concurrent.CopyOnWriteArrayList;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ComputingClientDriverManager {
private static final CopyOnWriteArrayList<ComputingClientDriver> registeredDrivers =
new CopyOnWriteArrayList<>();
private ComputingClientDriverManager() {}
static {
DriverManagerUtils.loadDrivers("cloudext.computing.drivers", ComputingClientDriver.class);
log.info("computing DriverManager initialized");
}
public static synchronized void registerDriver(ComputingClientDriver driver) {
if (driver != null) {
registeredDrivers.addIfAbsent(driver);
} else {
throw new NullPointerException();
}
log.info("registerDriver: {}", driver);
}
public static ComputingClient getClient(ComputingConnectionInfo config) {
for (ComputingClientDriver driver : registeredDrivers) {
if (driver.acceptsConfig(config)) {
return driver.connect(config);
}
}
throw CloudExtException.driverNotExist(config);
}
}

18
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobCanSubmitQuery.java
View File

@ -1,18 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing.cmd;
import com.antgroup.openspg.server.common.model.base.BaseQuery;
public class ReasonerJobCanSubmitQuery extends BaseQuery {}

25
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobProcessQuery.java
View File

@ -1,25 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing.cmd;
import com.antgroup.openspg.server.common.model.base.BaseQuery;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class ReasonerJobProcessQuery extends BaseQuery {
private final String computingJobInstId;
}

33
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobRunCmd.java
View File

@ -1,33 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing.cmd;
import com.antgroup.openspg.server.common.model.base.BaseCmd;
import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo;
import com.antgroup.openspg.server.core.reasoner.model.service.BaseReasonerContent;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class ReasonerJobRunCmd extends BaseCmd {
private final Long projectId;
private final String schemaUrl;
private final GraphStoreConnectionInfo connInfo;
private final BaseReasonerContent content;
}

44
server/cloudext/interface/computing/src/main/java/com/antgroup/openspg/cloudext/interfaces/computing/cmd/ReasonerJobSubmitCmd.java
View File

@ -1,44 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.computing.cmd;
import com.antgroup.openspg.server.common.model.base.BaseCmd;
import com.antgroup.openspg.server.common.model.datasource.connection.GraphStoreConnectionInfo;
import com.antgroup.openspg.server.common.model.datasource.connection.TableStoreConnectionInfo;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInfo;
import com.antgroup.openspg.server.core.reasoner.model.service.ReasonerJobInst;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class ReasonerJobSubmitCmd extends BaseCmd {
private final ReasonerJobInst jobInst;
private final ReasonerJobInfo jobInfo;
private final GraphStoreConnectionInfo graphStoreConnInfo;
private final TableStoreConnectionInfo tableStoreConnInfo;
private final String schemaUrl;
private final Map<String, Object> params;
public String tableName() {
return String.format("spgreasoner_%s_%s_result", jobInfo.getJobName(), jobInst.getJobInstId());
}
}

1
server/cloudext/interface/computing/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriver
View File

@ -1 +0,0 @@
com.antgroup.openspg.cloudext.impl.computing.local.LocalComputingClientDriver

32
server/cloudext/interface/job-scheduler/pom.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>server-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>cloudext-interface-job-scheduler</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg</groupId>
<artifactId>common-util</artifactId>
</dependency>
</dependencies>
</project>

27
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClient.java
View File

@ -1,27 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInfo;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import com.antgroup.openspg.common.util.cloudext.CloudExtClient;
public interface JobSchedulerClient extends CloudExtClient {
String createJobInfo(SchedulerJobInfo jobInfo);
String createJobInst(SchedulerJobInst jobInst);
void close() throws Exception;
}

20
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriver.java
View File

@ -1,20 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler;
import com.antgroup.openspg.common.util.cloudext.CloudExtClientDriver;
import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo;
public interface JobSchedulerClientDriver
extends CloudExtClientDriver<JobSchedulerClient, JobSchedulerConnectionInfo> {}

52
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/JobSchedulerClientDriverManager.java
View File

@ -1,52 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler;
import com.antgroup.openspg.common.util.DriverManagerUtils;
import com.antgroup.openspg.server.common.model.datasource.connection.JobSchedulerConnectionInfo;
import com.antgroup.openspg.server.common.model.exception.CloudExtException;
import java.util.concurrent.CopyOnWriteArrayList;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class JobSchedulerClientDriverManager {
private static final CopyOnWriteArrayList<JobSchedulerClientDriver> registeredDrivers =
new CopyOnWriteArrayList<>();
private JobSchedulerClientDriverManager() {}
static {
DriverManagerUtils.loadDrivers("cloudext.jobscheduler.drivers", JobSchedulerClientDriver.class);
log.info("job-scheduler DriverManager initialized");
}
public static synchronized void registerDriver(JobSchedulerClientDriver driver) {
if (driver != null) {
registeredDrivers.addIfAbsent(driver);
} else {
throw new NullPointerException();
}
log.info("registerDriver: {}", driver);
}
public static JobSchedulerClient getClient(JobSchedulerConnectionInfo config) {
for (JobSchedulerClientDriver driver : registeredDrivers) {
if (driver.acceptsConfig(config)) {
return driver.connect(config);
}
}
throw CloudExtException.driverNotExist(config);
}
}

26
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/SchedulerCallback.java
View File

@ -1,26 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.CallbackResult;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.JobTypeEnum;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.model.SchedulerJobInst;
import java.util.Set;
public interface SchedulerCallback {
Set<JobTypeEnum> accept();
CallbackResult polling(SchedulerJobInst jobInst);
}

29
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/CallbackResult.java
View File

@ -1,29 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model;
import com.antgroup.openspg.server.common.model.job.BaseJobRunResult;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class CallbackResult implements Serializable {
private final JobInstStatusEnum status;
private final BaseJobRunResult result;
}

20
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/JobTypeEnum.java
View File

@ -1,20 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model;
public enum JobTypeEnum {
BUILDING,
REASONING,
;
}

37
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInfo.java
View File

@ -1,37 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model;
import com.antgroup.openspg.server.common.model.base.BaseValObj;
import com.antgroup.openspg.server.common.model.job.JobInfoStateEnum;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
@Getter
@AllArgsConstructor
public class SchedulerJobInfo extends BaseValObj {
@Setter private String jobId;
private final String jobName;
private final String jobType;
private final String cron;
private final JobInfoStateEnum status;
private final String idempotentId;
}

36
server/cloudext/interface/job-scheduler/src/main/java/com/antgroup/openspg/cloudext/interfaces/jobscheduler/model/SchedulerJobInst.java
View File

@ -1,36 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.cloudext.interfaces.jobscheduler.model;
import com.antgroup.openspg.server.common.model.base.BaseValObj;
import com.antgroup.openspg.server.common.model.job.JobInstStatusEnum;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public class SchedulerJobInst extends BaseValObj {
private final String jobInstId;
private final String jobId;
private final String jobType;
private final JobInstStatusEnum status;
private final String host;
private final String idempotentId;
}

1
server/cloudext/interface/job-scheduler/src/main/resources/META-INF/services/com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriver
View File

@ -1 +0,0 @@
com.antgroup.openspg.cloudext.impl.jobscheduler.local.LocalJobSchedulerClientDriver

16
server/common/service/pom.xml
View File

@ -36,26 +36,10 @@
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-graph-store</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-object-store</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-search-engine</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>cloudext-interface-job-scheduler</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>cloudext-interface-computing</artifactId>
</dependency>
<dependency>
<groupId>com.antgroup.openspg.cloudext</groupId>
<artifactId>cloudext-interface-table-store</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>

42
server/common/service/src/main/java/com/antgroup/openspg/server/common/service/datasource/DataSourceService.java
View File

@ -13,18 +13,10 @@
package com.antgroup.openspg.server.common.service.datasource;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClient;
import com.antgroup.openspg.cloudext.interfaces.computing.ComputingClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClient;
import com.antgroup.openspg.cloudext.interfaces.graphstore.GraphStoreClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClient;
import com.antgroup.openspg.cloudext.interfaces.jobscheduler.JobSchedulerClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClient;
import com.antgroup.openspg.cloudext.interfaces.objectstore.ObjectStoreClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.searchengine.SearchEngineClient;
import com.antgroup.openspg.cloudext.interfaces.searchengine.SearchEngineClientDriverManager;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClient;
import com.antgroup.openspg.cloudext.interfaces.tablestore.TableStoreClientDriverManager;
import com.antgroup.openspg.server.common.model.datasource.DataSource;
import com.antgroup.openspg.server.common.model.datasource.DataSourceUsageTypeEnum;
import com.antgroup.openspg.server.common.model.datasource.connection.*;
@ -44,43 +36,9 @@ public interface DataSourceService {
(GraphStoreConnectionInfo) graphStore.getConnectionInfo());
}
default ObjectStoreClient buildSharedOperatorStoreClient() {
DataSource objectStore =
getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.OPERATOR_STORE);
return ObjectStoreClientDriverManager.getClient(
(ObjectStoreConnectionInfo) objectStore.getConnectionInfo());
}
default ObjectStoreClient buildSharedFileStoreClient() {
DataSource objectStore =
getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.FILE_STORE);
return ObjectStoreClientDriverManager.getClient(
(ObjectStoreConnectionInfo) objectStore.getConnectionInfo());
}
default SearchEngineClient buildSharedSearchEngineClient() {
DataSource searchEngine = getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.SEARCH);
return SearchEngineClientDriverManager.getClient(
(SearchEngineConnectionInfo) searchEngine.getConnectionInfo());
}
default JobSchedulerClient buildSharedJobSchedulerClient() {
DataSource scheduler =
getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.JOB_SCHEDULER);
return JobSchedulerClientDriverManager.getClient(
(JobSchedulerConnectionInfo) scheduler.getConnectionInfo());
}
default ComputingClient buildSharedComputingClient() {
DataSource computing = getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.COMPUTING);
return ComputingClientDriverManager.getClient(
(ComputingConnectionInfo) computing.getConnectionInfo());
}
default TableStoreClient buildSharedTableStoreClient() {
DataSource computing =
getFirstDataSource(SHARED_PROJECT_ID, DataSourceUsageTypeEnum.TABLE_STORE);
return TableStoreClientDriverManager.getClient(
(TableStoreConnectionInfo) computing.getConnectionInfo());
}
}

31
server/core/reasoner/model/pom.xml
View File

@ -1,31 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2023 Ant Group CO., Ltd.
~
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
~ in compliance with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software distributed under the License
~ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
~ or implied.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>server-parent</artifactId>
<version>0.0.1-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>core-reasoner-model</artifactId>
<dependencies>
<dependency>
<groupId>com.antgroup.openspg.server</groupId>
<artifactId>common-model</artifactId>
</dependency>
</dependencies>
</project>

40
server/core/reasoner/model/src/main/java/com/antgroup/openspg/server/core/reasoner/model/ReasonerException.java
View File

@ -1,40 +0,0 @@
/*
* Copyright 2023 Ant Group CO., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied.
*/
package com.antgroup.openspg.server.core.reasoner.model;
import com.antgroup.openspg.server.common.model.exception.OpenSPGException;
public class ReasonerException extends OpenSPGException {
private ReasonerException(Throwable cause, String messagePattern, Object... args) {
super(cause, true, true, messagePattern, args);
}
private ReasonerException(String messagePattern, Object... args) {
this(null, messagePattern, args);
}
public static ReasonerException reasonerError(Throwable e) {
return new ReasonerException(e, e.getMessage());
}
public static ReasonerException reasonerError(String errorMsg) {
return new ReasonerException(errorMsg);
}
public static ReasonerException timeout(int minutes) {
return new ReasonerException(
"reasoner cannot finish in {} minutes, please submit reasoner job", minutes);
}
}

Some files were not shown because too many files have changed in this diff Show More