mirror of
https://github.com/OpenSPG/openspg.git
synced 2025-06-27 03:20:10 +00:00
fix(builder): add embedding model (#385)
This commit is contained in:
Andy
GitHub
parent
3f863f8d26
commit
0e687f74e5
@ -117,7 +117,6 @@ public class LLMNlExtractProcessor extends BasePythonProcessor<LLMNlExtractNodeC
|
||||
log.info("LLMNlExtractProcessor invoke Chunks: {}", names);
|
||||
List<Object> result =
|
||||
(List<Object>) operatorFactory.invoke(config.getOperatorConfig(), record);
|
||||
log.info("LLMNlExtractProcessor invoke result: {}", JSON.toJSONString(result));
|
||||
List<SubGraphRecord> records =
|
||||
JSON.parseObject(JSON.toJSONString(result), new TypeReference<List<SubGraphRecord>>() {});
|
||||
node.addTraceLog(
|
||||
|
||||
@ -47,9 +47,12 @@ public class VectorizerProcessor extends BasePythonProcessor<VectorizerProcessor
|
||||
node.addTraceLog("Start vectorizer processor...");
|
||||
List<BaseRecord> results = new ArrayList<>();
|
||||
SubGraphRecord subGraph = new SubGraphRecord(Lists.newArrayList(), Lists.newArrayList());
|
||||
SubGraphRecord outputs = new SubGraphRecord(Lists.newArrayList(), Lists.newArrayList());
|
||||
|
||||
for (BaseRecord record : inputs) {
|
||||
SubGraphRecord spgRecord = (SubGraphRecord) record;
|
||||
outputs.getResultNodes().addAll(spgRecord.getResultNodes());
|
||||
outputs.getResultEdges().addAll(spgRecord.getResultEdges());
|
||||
Map map = mapper.convertValue(spgRecord, Map.class);
|
||||
node.addTraceLog(
|
||||
"invoke vectorizer processor operator:%s", config.getOperatorConfig().getClassName());
|
||||
@ -69,7 +72,7 @@ public class VectorizerProcessor extends BasePythonProcessor<VectorizerProcessor
|
||||
}
|
||||
results.add(subGraph);
|
||||
node.addTraceLog("post vectorizer complete...");
|
||||
node.setOutputs(subGraph);
|
||||
node.setOutputs(outputs);
|
||||
node.setStatus(StatusEnum.FINISH);
|
||||
return results;
|
||||
}
|
||||
|
||||
@ -30,7 +30,6 @@ import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.Edge
|
||||
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.LPGPropertyRecord;
|
||||
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.record.VertexRecord;
|
||||
import com.antgroup.openspg.cloudext.interfaces.graphstore.model.lpg.schema.EdgeTypeName;
|
||||
import com.antgroup.openspg.reasoner.runner.local.impl.LocalRunnerThreadPool;
|
||||
import com.antgroup.openspg.server.common.model.project.Project;
|
||||
import com.google.common.collect.Lists;
|
||||
import java.util.ArrayList;
|
||||
@ -39,16 +38,23 @@ import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Consumer;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class Neo4jSinkWriter extends BaseSinkWriter<Neo4jSinkNodeConfig> {
|
||||
|
||||
private static final int NUM_THREADS = 10;
|
||||
|
||||
private ExecuteNode node;
|
||||
private Neo4jStoreClient client;
|
||||
private Project project;
|
||||
private static final String DOT = ".";
|
||||
ExecutorService nodeExecutor;
|
||||
ExecutorService edgeExecutor;
|
||||
|
||||
public Neo4jSinkWriter(String id, String name, Neo4jSinkNodeConfig config) {
|
||||
super(id, name, config);
|
||||
@ -63,6 +69,22 @@ public class Neo4jSinkWriter extends BaseSinkWriter<Neo4jSinkNodeConfig> {
|
||||
}
|
||||
client = new Neo4jStoreClient(context.getGraphStoreUrl());
|
||||
project = JSON.parseObject(context.getProject(), Project.class);
|
||||
nodeExecutor =
|
||||
new ThreadPoolExecutor(
|
||||
NUM_THREADS,
|
||||
NUM_THREADS,
|
||||
2 * 60L,
|
||||
TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(1000),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
edgeExecutor =
|
||||
new ThreadPoolExecutor(
|
||||
NUM_THREADS,
|
||||
NUM_THREADS,
|
||||
2 * 60L,
|
||||
TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(1000),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -90,8 +112,6 @@ public class Neo4jSinkWriter extends BaseSinkWriter<Neo4jSinkNodeConfig> {
|
||||
public void writeToNeo4j(SubGraphRecord subGraphRecord) {
|
||||
subGraphRecord.getResultNodes().forEach(node -> convertProperties(node.getProperties()));
|
||||
subGraphRecord.getResultEdges().forEach(edge -> convertProperties(edge.getProperties()));
|
||||
|
||||
ExecutorService nodeExecutor = LocalRunnerThreadPool.getThreadPoolExecutor(null);
|
||||
try {
|
||||
node.addTraceLog("Start Writer Nodes processor...");
|
||||
List<Future<Void>> nodeFutures =
|
||||
@ -102,8 +122,6 @@ public class Neo4jSinkWriter extends BaseSinkWriter<Neo4jSinkNodeConfig> {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new RuntimeException("Error during node upsert", e);
|
||||
}
|
||||
|
||||
ExecutorService edgeExecutor = LocalRunnerThreadPool.getThreadPoolExecutor(null);
|
||||
try {
|
||||
node.addTraceLog("Start Writer Edges processor...");
|
||||
List<Future<Void>> edgeFutures =
|
||||
|
||||
@ -14,9 +14,11 @@ package com.antgroup.openspg.common.util.neo4j;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import org.neo4j.driver.AuthTokens;
|
||||
import org.neo4j.driver.Config;
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.neo4j.driver.GraphDatabase;
|
||||
|
||||
@ -32,7 +34,14 @@ public class Neo4jDriverManager {
|
||||
if (instanceMap.get(uniqueKey) == null) {
|
||||
Driver driver;
|
||||
try {
|
||||
driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password));
|
||||
Config config =
|
||||
Config.builder()
|
||||
.withMaxConnectionPoolSize(200)
|
||||
.withMaxConnectionLifetime(2, TimeUnit.HOURS)
|
||||
.withMaxTransactionRetryTime(300, TimeUnit.SECONDS)
|
||||
.withConnectionAcquisitionTimeout(300, TimeUnit.SECONDS)
|
||||
.build();
|
||||
driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password), config);
|
||||
driver.verifyConnectivity();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("init Neo4j Client failed :" + uri, e);
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
version: "3.7"
|
||||
services:
|
||||
server:
|
||||
restart: always
|
||||
|
||||
@ -10,8 +10,6 @@
|
||||
# or implied.
|
||||
|
||||
docker buildx build -f Dockerfile --platform linux/arm64/v8,linux/amd64 --push \
|
||||
-t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-mysql:0.0.3 \
|
||||
-t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-mysql:latest \
|
||||
-t openspg/openspg-mysql:0.0.3 \
|
||||
-t openspg/openspg-mysql:0.5 \
|
||||
-t openspg/openspg-mysql:latest \
|
||||
.
|
||||
|
||||
@ -234,7 +234,6 @@ CREATE TABLE `kg_ontology_ext` (
|
||||
) AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8mb4 COMMENT = 'schema的扩展属性';
|
||||
|
||||
INSERT INTO kg_biz_domain (`id`,`gmt_create`,`gmt_modified`,`name`,`status`,`description`,`global_config`) VALUES(1,'2023-09-01 00:00:00','2023-09-01 00:00:00','defaultTenant','VALID','',null);
|
||||
INSERT INTO kg_project_info (`id`,`name`,`description`,`status`,`gmt_create`,`gmt_modified`,`namespace`,`biz_domain_id`) VALUES(1,'defaultProject','defaultProject','VALID','2023-09-01 00:00:00','2023-09-01 00:00:00','DEFAULT',1);
|
||||
|
||||
INSERT INTO kg_ontology_entity (`id`,`original_id`,`name`,`name_zh`,`entity_category`,`layer`,`description`,`description_zh`,`status`,`with_index`,`scope`,`version`,`version_status`,`gmt_create`,`gmt_modified`,`transformer_id`,`operator_config`,`config`,`unique_name`) VALUES(1,1,'Thing','事物','ADVANCED','EXTENSION','Base class for all schema types, all of which inherit the type either directly or indirectly','所有schema类型的基类，它们都直接或者间接继承该类型','1','TRUE','PUBLIC',44,'ONLINE','2023-09-01 00:00:00','2023-09-01 00:00:00',0,null,null,'Thing');
|
||||
INSERT INTO kg_ontology_entity (`id`,`original_id`,`name`,`name_zh`,`entity_category`,`layer`,`description`,`description_zh`,`status`,`with_index`,`scope`,`version`,`version_status`,`gmt_create`,`gmt_modified`,`transformer_id`,`operator_config`,`config`,`unique_name`) VALUES(2,2,'Text','文本','BASIC','CORE','文本','基本数据类型-文本','1','TRUE','PUBLIC',0,'ONLINE','2023-09-01 00:00:00','2023-09-01 00:00:00',0,null,'{"constrains":[{"id":"REQUIRE","name":"Required","nameZh":"值非空","value":null},{"id":"UNIQUE","name":"Unique","nameZh":"值唯一","value":null},{"id":"ENUM","name":"Enum","nameZh":"枚举","value":null},{"id":"MULTIVALUE","name":"Multi value","nameZh":"多值","value":null},{"id":"REGULAR","name":"Regular match","nameZh":"正则匹配","value":null}]}','Text');
|
||||
|
||||
@ -43,13 +43,10 @@ RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then \
|
||||
RUN python3 -m venv /openspg_venv && \
|
||||
. /openspg_venv/bin/activate && \
|
||||
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture) && \
|
||||
pip3 install openspg-kag==0.0.3.20241022.2 -i https://artifacts.antgroup-inc.cn/artifact/repositories/simple-dev/ && \
|
||||
pip3 install openspg-kag==0.5.1 && \
|
||||
pip3 install pemja==0.4.0 && \
|
||||
pip3 install -U "http://alps-common.oss-cn-hangzhou-zmf.aliyuncs.com/nscommon/shiji/nscommon-0.0.1.tar.gz" &&\
|
||||
echo "if (tty -s); then \n . /openspg_venv/bin/activate \nfi" >> ~/.bashrc
|
||||
|
||||
ADD openspg/dev/release/python/lib/builder*.jar /openspg_venv/lib/python3.8/site-packages/knext/builder/lib
|
||||
ADD openspg/dev/release/python/lib/reasoner*.jar /openspg_venv/lib/python3.8/site-packages/knext/reasoner/lib
|
||||
|
||||
COPY openspg/ /openspg
|
||||
#RUN git clone --depth=1 https://github.com/OpenSPG/openspg.git
|
||||
RUN git clone --depth=1 https://github.com/OpenSPG/KAG.git
|
||||
@ -10,7 +10,7 @@
|
||||
# or implied.
|
||||
|
||||
IMAGE="spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-python"
|
||||
VERSION="kag1"
|
||||
VERSION="0.5"
|
||||
cd ../../../../
|
||||
docker build -f openspg/dev/release/python/Dockerfile --platform linux/arm64/v8 --push \
|
||||
-t ${IMAGE}:${VERSION}-arm64 \
|
||||
@ -31,4 +31,18 @@ docker manifest create \
|
||||
${IMAGE}:${VERSION}-amd64 \
|
||||
${IMAGE}:${VERSION}-arm64
|
||||
|
||||
docker manifest push ${IMAGE}:${VERSION}
|
||||
docker manifest push ${IMAGE}:${VERSION}
|
||||
|
||||
if docker manifest inspect ${IMAGE}:${LATEST} &> /dev/null; then
|
||||
echo "Manifest already exists, removing it..."
|
||||
docker manifest rm ${IMAGE}:${LATEST}
|
||||
else
|
||||
echo "Manifest does not exist, proceeding with creation and push."
|
||||
fi
|
||||
|
||||
docker manifest create \
|
||||
${IMAGE}:${LATEST} \
|
||||
${IMAGE}:${VERSION}-amd64 \
|
||||
${IMAGE}:${VERSION}-arm64
|
||||
|
||||
docker manifest push ${IMAGE}:${LATEST}
|
||||
@ -11,8 +11,6 @@
|
||||
|
||||
# for amd64
|
||||
docker build -f Dockerfile --platform linux/amd64 --push \
|
||||
-t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-python:0.0.3 \
|
||||
-t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-python:latest \
|
||||
-t openspg/openspg-python:0.0.3 \
|
||||
-t openspg/openspg-python:0.5 \
|
||||
-t openspg/openspg-python:latest \
|
||||
.
|
||||
|
||||
@ -10,8 +10,6 @@
|
||||
# or implied.
|
||||
|
||||
docker buildx build -f Dockerfile --platform linux/arm64/v8,linux/amd64 --push \
|
||||
-t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-server:0.0.3 \
|
||||
-t spg-registry.cn-hangzhou.cr.aliyuncs.com/spg/openspg-server:latest \
|
||||
-t openspg/openspg-server:0.0.3 \
|
||||
-t openspg/openspg-server:0.5 \
|
||||
-t openspg/openspg-server:latest \
|
||||
.
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
version: "3.7"
|
||||
services:
|
||||
mysql:
|
||||
restart: always
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
/*
|
||||
* Copyright 2023 OpenSPG Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
|
||||
* in compliance with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied.
|
||||
*/
|
||||
|
||||
package com.antgroup.openspg.reasoner.udf.builtin.udf;
|
||||
|
||||
import com.antgroup.openspg.reasoner.udf.model.UdfDefine;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public class SplitPart {
|
||||
@UdfDefine(name = "split_part", compatibleName = "SplitPart")
|
||||
public String split(String str, String separator, Integer partNum) throws Exception {
|
||||
String[] strings = StringUtils.split(str, separator);
|
||||
int length = strings.length;
|
||||
if (partNum < 0) {
|
||||
partNum = length - 1;
|
||||
}
|
||||
if (partNum >= length) {
|
||||
return "";
|
||||
}
|
||||
return strings[partNum];
|
||||
}
|
||||
}
|
||||
@ -169,23 +169,6 @@ public class UdfTest {
|
||||
Assert.assertEquals(rst2, "岁");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSplitPart() {
|
||||
UdfMng mng = UdfMngFactory.getUdfMng();
|
||||
IUdfMeta udfMeta =
|
||||
mng.getUdfMeta(
|
||||
"split_part",
|
||||
Lists.newArrayList(KTString$.MODULE$, KTString$.MODULE$, KTInteger$.MODULE$));
|
||||
Object rst1 = udfMeta.invoke("Hello,World!", ",", 0);
|
||||
Assert.assertEquals("Hello", rst1);
|
||||
Object rst2 = udfMeta.invoke("Hello,Ni,Hao", ",", -1);
|
||||
Assert.assertEquals("Hao", rst2);
|
||||
Object rst3 = udfMeta.invoke("A省B市C村XXX", "村", 0);
|
||||
Assert.assertEquals("A省B市C", rst3);
|
||||
Object rst4 = udfMeta.invoke("A省B市C村XXX", "村", 5);
|
||||
Assert.assertEquals("", rst4);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCast() {
|
||||
UdfMng mng = UdfMngFactory.getUdfMng();
|
||||
|
||||
@ -13,20 +13,6 @@
|
||||
|
||||
package com.antgroup.openspg.server.api.http.server.openapi;
|
||||
|
||||
import com.antgroup.openspg.common.util.StringUtils;
|
||||
import com.antgroup.openspg.core.schema.model.BasicInfo;
|
||||
import com.antgroup.openspg.core.schema.model.alter.AlterOperationEnum;
|
||||
import com.antgroup.openspg.core.schema.model.alter.SchemaDraft;
|
||||
import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier;
|
||||
import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.IndexTypeEnum;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.Property;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig;
|
||||
import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType;
|
||||
import com.antgroup.openspg.core.schema.model.type.EntityType;
|
||||
import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo;
|
||||
import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum;
|
||||
import com.antgroup.openspg.core.schema.model.type.SPGTypeRef;
|
||||
import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectCreateRequest;
|
||||
import com.antgroup.openspg.server.api.facade.dto.common.request.ProjectQueryRequest;
|
||||
import com.antgroup.openspg.server.api.facade.dto.schema.request.SchemaAlterRequest;
|
||||
@ -35,10 +21,7 @@ import com.antgroup.openspg.server.api.http.server.HttpBizCallback;
|
||||
import com.antgroup.openspg.server.api.http.server.HttpBizTemplate;
|
||||
import com.antgroup.openspg.server.biz.common.ProjectManager;
|
||||
import com.antgroup.openspg.server.common.model.project.Project;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Controller;
|
||||
@ -68,7 +51,8 @@ public class ProjectController extends BaseController {
|
||||
if (request.getAutoSchema() == null || Boolean.TRUE.equals(request.getAutoSchema())) {
|
||||
SchemaAlterRequest request = new SchemaAlterRequest();
|
||||
request.setProjectId(project.getId());
|
||||
request.setSchemaDraft(getDefaultSchemaDraft(project.getNamespace()));
|
||||
request.setSchemaDraft(
|
||||
SchemaController.getDefaultSchemaDraft(project.getNamespace()));
|
||||
schemaController.alterSchema(request);
|
||||
}
|
||||
return project;
|
||||
@ -76,73 +60,6 @@ public class ProjectController extends BaseController {
|
||||
});
|
||||
}
|
||||
|
||||
private SchemaDraft getDefaultSchemaDraft(String namespace) {
|
||||
SchemaDraft sd = new SchemaDraft();
|
||||
List<BaseAdvancedType> alterSpgTypes = getDefaultSchema(namespace);
|
||||
sd.setAlterSpgTypes(alterSpgTypes);
|
||||
return sd;
|
||||
}
|
||||
|
||||
private List<BaseAdvancedType> getDefaultSchema(String namespace) {
|
||||
List<BaseAdvancedType> schemaTypes = Lists.newArrayList();
|
||||
|
||||
Map<String, IndexTypeEnum> chunkProperties = Maps.newHashMap();
|
||||
chunkProperties.put("content", IndexTypeEnum.TEXT_AND_VECTOR);
|
||||
|
||||
Map<String, IndexTypeEnum> properties = Maps.newHashMap();
|
||||
properties.put("desc", IndexTypeEnum.TEXT_AND_VECTOR);
|
||||
properties.put("semanticType", IndexTypeEnum.TEXT);
|
||||
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Chunk", "文本块", chunkProperties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "ArtificialObject", "人造物体", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Astronomy", "天文学", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Building", "建筑", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Creature", "生物", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Concept", "概念", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Date", "日期", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "GeographicLocation", "地理位置", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Keyword", "关键词", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Medicine", "药物", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "NaturalScience", "自然科学", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Organization", "组织机构", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Person", "人物", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Transport", "运输", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Works", "作品", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Event", "事件", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Others", "其他", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "SemanticConcept", "语义概念", properties));
|
||||
return schemaTypes;
|
||||
}
|
||||
|
||||
private EntityType getBaseSPGType(
|
||||
String namespace, String label, String nameZh, Map<String, IndexTypeEnum> properties) {
|
||||
List<Property> propertyList = Lists.newArrayList();
|
||||
for (String pro : properties.keySet()) {
|
||||
|
||||
Property property =
|
||||
new Property(
|
||||
new BasicInfo<>(new PredicateIdentifier(pro), pro, StringUtils.EMPTY),
|
||||
null,
|
||||
new SPGTypeRef(
|
||||
new BasicInfo<>(new SPGTypeIdentifier(null, "Text"), "文本", "文本"),
|
||||
SPGTypeEnum.BASIC_TYPE),
|
||||
false,
|
||||
new PropertyAdvancedConfig().setIndexType(properties.get(pro)));
|
||||
property.setAlterOperation(AlterOperationEnum.CREATE);
|
||||
propertyList.add(property);
|
||||
}
|
||||
|
||||
EntityType entityType =
|
||||
new EntityType(
|
||||
new BasicInfo(new SPGTypeIdentifier(namespace, label), nameZh, StringUtils.EMPTY),
|
||||
ParentTypeInfo.THING,
|
||||
propertyList,
|
||||
null,
|
||||
null);
|
||||
entityType.setAlterOperation(AlterOperationEnum.CREATE);
|
||||
return entityType;
|
||||
}
|
||||
|
||||
@RequestMapping(method = RequestMethod.GET)
|
||||
public ResponseEntity<Object> query(
|
||||
@RequestParam(required = false) Long tenantId,
|
||||
|
||||
@ -13,23 +13,37 @@
|
||||
|
||||
package com.antgroup.openspg.server.api.http.server.openapi;
|
||||
|
||||
import com.antgroup.openspg.common.util.StringUtils;
|
||||
import com.antgroup.openspg.core.schema.model.BasicInfo;
|
||||
import com.antgroup.openspg.core.schema.model.SchemaExtInfo;
|
||||
import com.antgroup.openspg.core.schema.model.alter.AlterOperationEnum;
|
||||
import com.antgroup.openspg.core.schema.model.alter.SchemaDraft;
|
||||
import com.antgroup.openspg.core.schema.model.identifier.PredicateIdentifier;
|
||||
import com.antgroup.openspg.core.schema.model.identifier.SPGTypeIdentifier;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.IndexTypeEnum;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.Property;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.Relation;
|
||||
import com.antgroup.openspg.core.schema.model.predicate.SubProperty;
|
||||
import com.antgroup.openspg.core.schema.model.semantic.PredicateSemantic;
|
||||
import com.antgroup.openspg.core.schema.model.type.BaseAdvancedType;
|
||||
import com.antgroup.openspg.core.schema.model.type.BaseSPGType;
|
||||
import com.antgroup.openspg.core.schema.model.type.EntityType;
|
||||
import com.antgroup.openspg.core.schema.model.type.ParentTypeInfo;
|
||||
import com.antgroup.openspg.core.schema.model.type.ProjectSchema;
|
||||
import com.antgroup.openspg.core.schema.model.type.SPGTypeAdvancedConfig;
|
||||
import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum;
|
||||
import com.antgroup.openspg.core.schema.model.type.SPGTypeRef;
|
||||
import com.antgroup.openspg.server.api.facade.dto.schema.request.*;
|
||||
import com.antgroup.openspg.server.api.http.server.BaseController;
|
||||
import com.antgroup.openspg.server.api.http.server.HttpBizCallback;
|
||||
import com.antgroup.openspg.server.api.http.server.HttpBizTemplate;
|
||||
import com.antgroup.openspg.server.biz.common.util.AssertUtils;
|
||||
import com.antgroup.openspg.server.biz.schema.SchemaManager;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@ -45,6 +59,73 @@ public class SchemaController extends BaseController {
|
||||
|
||||
@Autowired private SchemaManager schemaManager;
|
||||
|
||||
public static SchemaDraft getDefaultSchemaDraft(String namespace) {
|
||||
SchemaDraft sd = new SchemaDraft();
|
||||
List<BaseAdvancedType> alterSpgTypes = getDefaultSchema(namespace);
|
||||
sd.setAlterSpgTypes(alterSpgTypes);
|
||||
return sd;
|
||||
}
|
||||
|
||||
public static List<BaseAdvancedType> getDefaultSchema(String namespace) {
|
||||
List<BaseAdvancedType> schemaTypes = Lists.newArrayList();
|
||||
|
||||
Map<String, IndexTypeEnum> chunkProperties = Maps.newHashMap();
|
||||
chunkProperties.put("content", IndexTypeEnum.TEXT_AND_VECTOR);
|
||||
|
||||
Map<String, IndexTypeEnum> properties = Maps.newHashMap();
|
||||
properties.put("desc", IndexTypeEnum.TEXT_AND_VECTOR);
|
||||
properties.put("semanticType", IndexTypeEnum.TEXT);
|
||||
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Chunk", "文本块", chunkProperties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "ArtificialObject", "人造物体", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Astronomy", "天文学", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Building", "建筑", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Creature", "生物", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Concept", "概念", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Date", "日期", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "GeographicLocation", "地理位置", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Keyword", "关键词", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Medicine", "药物", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "NaturalScience", "自然科学", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Organization", "组织机构", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Person", "人物", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Transport", "运输", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Works", "作品", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Event", "事件", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "Others", "其他", properties));
|
||||
schemaTypes.add(getBaseSPGType(namespace, "SemanticConcept", "语义概念", properties));
|
||||
return schemaTypes;
|
||||
}
|
||||
|
||||
public static EntityType getBaseSPGType(
|
||||
String namespace, String label, String nameZh, Map<String, IndexTypeEnum> properties) {
|
||||
List<Property> propertyList = Lists.newArrayList();
|
||||
for (String pro : properties.keySet()) {
|
||||
|
||||
Property property =
|
||||
new Property(
|
||||
new BasicInfo<>(new PredicateIdentifier(pro), pro, StringUtils.EMPTY),
|
||||
null,
|
||||
new SPGTypeRef(
|
||||
new BasicInfo<>(new SPGTypeIdentifier(null, "Text"), "文本", "文本"),
|
||||
SPGTypeEnum.BASIC_TYPE),
|
||||
false,
|
||||
new PropertyAdvancedConfig().setIndexType(properties.get(pro)));
|
||||
property.setAlterOperation(AlterOperationEnum.CREATE);
|
||||
propertyList.add(property);
|
||||
}
|
||||
|
||||
EntityType entityType =
|
||||
new EntityType(
|
||||
new BasicInfo(new SPGTypeIdentifier(namespace, label), nameZh, StringUtils.EMPTY),
|
||||
ParentTypeInfo.THING,
|
||||
propertyList,
|
||||
null,
|
||||
null);
|
||||
entityType.setAlterOperation(AlterOperationEnum.CREATE);
|
||||
return entityType;
|
||||
}
|
||||
|
||||
@RequestMapping(value = "/alterSchema", method = RequestMethod.POST)
|
||||
@ResponseBody
|
||||
public ResponseEntity<Object> alterSchema(@RequestBody SchemaAlterRequest request) {
|
||||
|
||||
@ -35,5 +35,9 @@
|
||||
<groupId>com.antgroup.openspg.cloudext</groupId>
|
||||
<artifactId>cloudext-impl-graph-store-neo4j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>pemja</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@ -44,6 +44,7 @@ public class ProjectManagerImpl implements ProjectManager {
|
||||
@Override
|
||||
public Project create(ProjectCreateRequest request) {
|
||||
JSONObject config = setDatabase(request.getConfig(), request.getNamespace());
|
||||
createNeo4jDatabase(request.getNamespace(), config);
|
||||
Project project =
|
||||
new Project(
|
||||
null,
|
||||
@ -52,7 +53,6 @@ public class ProjectManagerImpl implements ProjectManager {
|
||||
request.getNamespace(),
|
||||
request.getTenantId(),
|
||||
config.toJSONString());
|
||||
createNeo4jDatabase(project.getNamespace(), config);
|
||||
Long projectId = projectRepository.save(project);
|
||||
project.setId(projectId);
|
||||
return project;
|
||||
@ -62,6 +62,7 @@ public class ProjectManagerImpl implements ProjectManager {
|
||||
public Project update(ProjectCreateRequest request) {
|
||||
Project project = projectRepository.queryById(request.getId());
|
||||
JSONObject config = setDatabase(request.getConfig(), project.getNamespace());
|
||||
config = setVectorDimensions(config, project);
|
||||
Project update = new Project(request.getId(), null, null, null, null, config.toJSONString());
|
||||
return projectRepository.update(update);
|
||||
}
|
||||
@ -83,6 +84,30 @@ public class ProjectManagerImpl implements ProjectManager {
|
||||
return config;
|
||||
}
|
||||
|
||||
private JSONObject setVectorDimensions(JSONObject config, Project project) {
|
||||
JSONObject oldConfig = JSONObject.parseObject(project.getConfig());
|
||||
String vectorDimensions = null;
|
||||
if (oldConfig.containsKey(CommonConstants.VECTORIZER)) {
|
||||
vectorDimensions =
|
||||
oldConfig
|
||||
.getJSONObject(CommonConstants.VECTORIZER)
|
||||
.getString(CommonConstants.VECTOR_DIMENSIONS);
|
||||
}
|
||||
if (StringUtils.isBlank(vectorDimensions)) {
|
||||
return config;
|
||||
}
|
||||
if (config.containsKey(CommonConstants.VECTORIZER)) {
|
||||
config
|
||||
.getJSONObject(CommonConstants.VECTORIZER)
|
||||
.put(CommonConstants.VECTOR_DIMENSIONS, vectorDimensions);
|
||||
} else {
|
||||
JSONObject graphStore = new JSONObject();
|
||||
graphStore.put(CommonConstants.VECTOR_DIMENSIONS, vectorDimensions);
|
||||
config.put(CommonConstants.VECTORIZER, graphStore);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Project queryById(Long projectId) {
|
||||
return projectRepository.queryById(projectId);
|
||||
@ -104,9 +129,13 @@ public class ProjectManagerImpl implements ProjectManager {
|
||||
}
|
||||
if (graphStore.containsKey(Neo4jConstants.USER)) {
|
||||
user = graphStore.getString(Neo4jConstants.USER);
|
||||
} else {
|
||||
graphStore.put(Neo4jConstants.USER, user);
|
||||
}
|
||||
if (graphStore.containsKey(Neo4jConstants.PASSWORD)) {
|
||||
password = graphStore.getString(Neo4jConstants.PASSWORD);
|
||||
} else {
|
||||
graphStore.put(Neo4jConstants.PASSWORD, password);
|
||||
}
|
||||
|
||||
Neo4jAdminUtils driver = new Neo4jAdminUtils(host, user, password, database);
|
||||
|
||||
@ -56,9 +56,6 @@ public class Neo4jSyncer extends BaseSchemaSyncer {
|
||||
if (StringUtils.isNotBlank(config)) {
|
||||
JSONObject vectorizerConfig =
|
||||
JSON.parseObject(config).getJSONObject(CommonConstants.VECTORIZER);
|
||||
if (vectorizerConfig == null) {
|
||||
return Neo4jCommonUtils.DEFAULT_VECTOR_DIMENSIONS;
|
||||
}
|
||||
Integer vectorDimensions = vectorizerConfig.getInteger(CommonConstants.VECTOR_DIMENSIONS);
|
||||
if (vectorDimensions != null) return vectorDimensions;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user