From f18648c86bb9a7fdcf42778eef65afa2d48552ca Mon Sep 17 00:00:00 2001 From: baifuyu Date: Mon, 11 Dec 2023 20:46:18 +0800 Subject: [PATCH] bugfix --- .../operator/PythonOperatorFactory.java | 2 +- .../operator/protocol/InvokeResult.java | 4 ++-- .../process/UserDefinedExtractProcessor.java | 2 +- .../impl/BasicPropertyNormalizer.java | 2 +- .../impl/PropertyOperatorNormalizer.java | 21 ++++++++++++------- .../impl/PropertySearchNormalizer.java | 8 ++++++- .../record/property/SPGSubPropertyRecord.java | 5 +++++ .../runner/local/LocalBuilderMain.java | 11 ++++++++-- .../builder/operator/cert_link_operator.py | 6 +++--- python/knext/operator/spg_record.py | 4 ++-- .../core/schema/model/predicate/Property.java | 5 +++++ 11 files changed, 50 insertions(+), 20 deletions(-) diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/PythonOperatorFactory.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/PythonOperatorFactory.java index 0726b1c2..6e4adee4 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/PythonOperatorFactory.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/PythonOperatorFactory.java @@ -30,7 +30,7 @@ public class PythonOperatorFactory implements OperatorFactory { private static PythonInterpreter newPythonInterpreter(BuilderContext context) { String pythonExec = context.getPythonExec(); String[] pythonPaths = - (context.getPythonPaths() != null ? context.getPythonPaths().split(",") : null); + (context.getPythonPaths() != null ? context.getPythonPaths().split(";") : null); PythonInterpreterConfig.PythonInterpreterConfigBuilder builder = PythonInterpreterConfig.newBuilder(); diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/protocol/InvokeResult.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/protocol/InvokeResult.java index 7818757f..b5bf1df4 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/protocol/InvokeResult.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/operator/protocol/InvokeResult.java @@ -23,9 +23,9 @@ import lombok.Setter; public class InvokeResult { private String spgTypeName; - private Map props; + private Map properties; public String getId() { - return props.get("id"); + return properties.get("id"); } } diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/UserDefinedExtractProcessor.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/UserDefinedExtractProcessor.java index e0805352..7b3d73f8 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/UserDefinedExtractProcessor.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/physical/process/UserDefinedExtractProcessor.java @@ -55,7 +55,7 @@ public class UserDefinedExtractProcessor for (InvokeResult data : invokeResultWrapper.getData()) { results.add( new BuilderRecord( - null, SPGTypeIdentifier.parse(data.getSpgTypeName()), data.getProps())); + null, SPGTypeIdentifier.parse(data.getSpgTypeName()), data.getProperties())); } } return results; diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/BasicPropertyNormalizer.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/BasicPropertyNormalizer.java index b466550b..cb41613c 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/BasicPropertyNormalizer.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/BasicPropertyNormalizer.java @@ -35,7 +35,7 @@ public class BasicPropertyNormalizer implements PropertyNormalizer { break; } } catch (NumberFormatException e) { - throw new PropertyNormalizeException(e, ""); + throw new PropertyNormalizeException(e, "{} normalize error", rawValue); } record.getValue().setSingleStd(stdValue); } diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertyOperatorNormalizer.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertyOperatorNormalizer.java index 4609828b..1b4628e9 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertyOperatorNormalizer.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertyOperatorNormalizer.java @@ -14,8 +14,10 @@ import com.antgroup.openspg.common.util.StringUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.*; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +@Slf4j @SuppressWarnings("unchecked") public class PropertyOperatorNormalizer implements PropertyNormalizer { @@ -40,13 +42,18 @@ public class PropertyOperatorNormalizer implements PropertyNormalizer { List ids = new ArrayList<>(rawValues.size()); for (String rawValue : rawValues) { - Map result = - (Map) - operatorFactory.invoke( - normalizerConfig.getOperatorConfig(), rawValue, new HashMap<>(0)); - InvokeResultWrapper> invokeResultWrapper = - mapper.convertValue( - result, new TypeReference>>() {}); + InvokeResultWrapper> invokeResultWrapper = null; + try { + Map result = + (Map) + operatorFactory.invoke( + normalizerConfig.getOperatorConfig(), rawValue, new HashMap<>(0)); + invokeResultWrapper = + mapper.convertValue( + result, new TypeReference>>() {}); + } catch (Exception e) { + throw new PropertyNormalizeException(e, "{} normalize error", rawValue); + } if (invokeResultWrapper == null || CollectionUtils.isEmpty(invokeResultWrapper.getData())) { continue; diff --git a/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertySearchNormalizer.java b/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertySearchNormalizer.java index ebab51e0..399149aa 100644 --- a/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertySearchNormalizer.java +++ b/builder/core/src/main/java/com/antgroup/openspg/builder/core/property/impl/PropertySearchNormalizer.java @@ -39,7 +39,13 @@ public class PropertySearchNormalizer implements PropertyNormalizer { if (!objectTypeRef.isEntityType() && !objectTypeRef.isConceptType()) { ids.add(rawValue); } else { - ids.add(search(objectTypeRef, rawValue)); + String searchResult = null; + try { + searchResult = search(objectTypeRef, rawValue); + } catch (Exception e) { + throw new PropertyNormalizeException(e, "{} normalize error", rawValue); + } + ids.add(searchResult); } } record.getValue().setStds(Collections.singletonList(ids)); diff --git a/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/property/SPGSubPropertyRecord.java b/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/property/SPGSubPropertyRecord.java index d36baa64..fea87b07 100644 --- a/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/property/SPGSubPropertyRecord.java +++ b/builder/model/src/main/java/com/antgroup/openspg/builder/model/record/property/SPGSubPropertyRecord.java @@ -61,4 +61,9 @@ public class SPGSubPropertyRecord extends BasePropertyRecord implements WithSPGT public SPGTypeEnum getSpgTypeEnum() { return getSubProperty().getObjectTypeRef().getSpgTypeEnum(); } + + @Override + public String toString() { + return subProperty.getName(); + } } diff --git a/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/LocalBuilderMain.java b/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/LocalBuilderMain.java index 74d3955a..b23345af 100644 --- a/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/LocalBuilderMain.java +++ b/builder/runner/local/src/main/java/com/antgroup/openspg/builder/runner/local/LocalBuilderMain.java @@ -21,9 +21,11 @@ import com.antgroup.openspg.server.api.facade.dto.schema.request.ProjectSchemaRe import com.antgroup.openspg.server.api.http.client.HttpSchemaFacade; import com.antgroup.openspg.server.api.http.client.util.ConnectionInfo; import com.antgroup.openspg.server.api.http.client.util.HttpClientBootstrap; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.cli.*; import org.slf4j.LoggerFactory; +@Slf4j public class LocalBuilderMain { private static final String PROJECT_ID_OPTION = "projectId"; @@ -36,9 +38,14 @@ public class LocalBuilderMain { private static final String ALTER_OPERATION_OPTION = "alterOperation"; private static final String LOG_FILE_OPTION = "logFile"; - public static void main(String[] args) throws Exception { + public static void main(String[] args) { CommandLine commandLine = parseArgs(args); - run(commandLine); + try { + run(commandLine); + } catch (Throwable e) { + log.error("unknown exception.", e); + System.exit(0); + } } public static CommandLine parseArgs(String[] args) { diff --git a/python/knext/examples/riskmining/builder/operator/cert_link_operator.py b/python/knext/examples/riskmining/builder/operator/cert_link_operator.py index 61e0a655..264bd176 100644 --- a/python/knext/examples/riskmining/builder/operator/cert_link_operator.py +++ b/python/knext/examples/riskmining/builder/operator/cert_link_operator.py @@ -14,7 +14,7 @@ from typing import List from knext.operator.op import LinkOp from knext.client.search import SearchClient -from operator.spg_record import SPGRecord +from knext.operator.spg_record import SPGRecord class CertLinkerOperator(LinkOp): @@ -28,8 +28,8 @@ class CertLinkerOperator(LinkOp): has_cert = property query = {"match": {"certNum": has_cert}} recall_certs = self.search_client.search(query, start=0, size=10) - if recall_certs is not None: + if recall_certs is not None and len(recall_certs) > 0: return [ SPGRecord('RiskMining.Cert', {'id': recall_certs[0].doc_id}) ] - return [SPGRecord('RiskMining.Cert', {})] + return [SPGRecord('RiskMining.Cert', {'id': property})] diff --git a/python/knext/operator/spg_record.py b/python/knext/operator/spg_record.py index 89364a41..2cd0c158 100644 --- a/python/knext/operator/spg_record.py +++ b/python/knext/operator/spg_record.py @@ -122,14 +122,14 @@ class SPGRecord: def to_dict(self): """Returns the model properties as a dict""" return { - "SPGTypeName": self.spg_type_name, + "spgTypeName": self.spg_type_name, "properties": self.properties, } @classmethod def from_dict(cls, input: Dict[str, Any]): """Returns the model from a dict""" - return cls(input.get("SPGTypeName"), input.get("properties")) + return cls(input.get("spgTypeName"), input.get("properties")) def __repr__(self): """For `print` and `pprint`""" diff --git a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/Property.java b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/Property.java index f84e6daf..b53738a3 100644 --- a/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/Property.java +++ b/server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/predicate/Property.java @@ -117,4 +117,9 @@ public class Property extends BaseSpoTriple this.getProjectId(), this.getOntologyId()); } + + @Override + public String toString() { + return getName(); + } }