mirror of
https://github.com/OpenSPG/KAG.git
synced 2025-11-20 04:17:44 +00:00
* add path find * fix find path * spg guided relation extraction * fix dict parse with same key * rename graphalgoclient to graphclient * rename graphalgoclient to graphclient * file reader supports http url * add checkpointer class * parser supports checkpoint * add build * remove incorrect logs * remove logs * update examples * update chain checkpointer * vectorizer batch size set to 32 * add a zodb backended checkpointer * add a zodb backended checkpointer * fix zodb based checkpointer * add thread for zodb IO * fix(common): resolve mutlithread conflict in zodb IO * fix(common): load existing zodb checkpoints * update examples * update examples * fix zodb writer * add docstring * fix jieba version mismatch * commit kag_config-tc.yaml 1、rename type to register_name 2、put a uniqe & specific name to register_name 3、rename reader to scanner 4、rename parser to reader 5、rename num_parallel to num_parallel_file, rename chain_level_num_paralle to num_parallel_chain_of_file 6、rename kag_extractor to schema_free_extractor, schema_base_extractor to schema_constraint_extractor 7、pre-define llm & vectorize_model and refer them in the yaml file Issues to be resolved: 1、examples of event extract & spg extract 2、statistic of indexer, such as nums of nodes & edges extracted, ratio of llm invoke. 3、Exceptions such as Debt, account does not exist should be thrown in llm invoke. 4、conf of solver need to be re-examined. * commit kag_config-tc.yaml 1、rename type to register_name 2、put a uniqe & specific name to register_name 3、rename reader to scanner 4、rename parser to reader 5、rename num_parallel to num_parallel_file, rename chain_level_num_paralle to num_parallel_chain_of_file 6、rename kag_extractor to schema_free_extractor, schema_base_extractor to schema_constraint_extractor 7、pre-define llm & vectorize_model and refer them in the yaml file Issues to be resolved: 1、examples of event extract & spg extract 2、statistic of indexer, such as nums of nodes & edges extracted, ratio of llm invoke. 3、Exceptions such as Debt, account does not exist should be thrown in llm invoke. 4、conf of solver need to be re-examined. * 1、fix bug in base_table_splitter * 1、fix bug in base_table_splitter * 1、fix bug in default_chain * 增加solver * add kag * update outline splitter * add main test * add op * code refactor * add tools * fix outline splitter * fix outline prompt * graph api pass * commit with page rank * add search api and graph api * add markdown report * fix vectorizer num batch compute * add retry for vectorize model call * update markdown reader * update markdown reader * update pdf reader * raise extractor failure * add default expr * add log * merge jc reader features * rm import * add build * fix zodb based checkpointer * add thread for zodb IO * fix(common): resolve mutlithread conflict in zodb IO * fix(common): load existing zodb checkpoints * update examples * update examples * fix zodb writer * add docstring * fix jieba version mismatch * commit kag_config-tc.yaml 1、rename type to register_name 2、put a uniqe & specific name to register_name 3、rename reader to scanner 4、rename parser to reader 5、rename num_parallel to num_parallel_file, rename chain_level_num_paralle to num_parallel_chain_of_file 6、rename kag_extractor to schema_free_extractor, schema_base_extractor to schema_constraint_extractor 7、pre-define llm & vectorize_model and refer them in the yaml file Issues to be resolved: 1、examples of event extract & spg extract 2、statistic of indexer, such as nums of nodes & edges extracted, ratio of llm invoke. 3、Exceptions such as Debt, account does not exist should be thrown in llm invoke. 4、conf of solver need to be re-examined. * commit kag_config-tc.yaml 1、rename type to register_name 2、put a uniqe & specific name to register_name 3、rename reader to scanner 4、rename parser to reader 5、rename num_parallel to num_parallel_file, rename chain_level_num_paralle to num_parallel_chain_of_file 6、rename kag_extractor to schema_free_extractor, schema_base_extractor to schema_constraint_extractor 7、pre-define llm & vectorize_model and refer them in the yaml file Issues to be resolved: 1、examples of event extract & spg extract 2、statistic of indexer, such as nums of nodes & edges extracted, ratio of llm invoke. 3、Exceptions such as Debt, account does not exist should be thrown in llm invoke. 4、conf of solver need to be re-examined. * 1、fix bug in base_table_splitter * 1、fix bug in base_table_splitter * 1、fix bug in default_chain * update outline splitter * add main test * add markdown report * code refactor * fix outline splitter * fix outline prompt * update markdown reader * fix vectorizer num batch compute * add retry for vectorize model call * update markdown reader * raise extractor failure * rm parser * run pipeline * add config option of whether to perform llm config check, default to false * fix * recover pdf reader * several components can be null for default chain * 支持完整qa运行 * add if * remove unused code * 使用chunk兜底 * excluded source relation to choose * add generate * default recall 10 * add local memory * 排除相似边 * 增加保护 * 修复并发问题 * add debug logger * 支持topk参数化 * 支持chunk截断和调整spo select 的prompt * 增加查询请求保护 * 增加force_chunk配置 * fix entity linker algorithm * 增加sub query改写 * fix md reader dup in test * fix * merge knext to kag parallel * fix package * 修复指标下跌问题 * scanner update * scanner update * add doc and update example scripts * fix * add bridge to spg server * add format * fix bridge * update conf for baike * disable ckpt for spg server runner * llm invoke error default raise exceptions * chore(version): bump version to X.Y.Z * update default response generation prompt * add method getSummarizationMetrics * fix(common): fix project conf empty error * fix typo * 增加上报信息 * 修改main solver * postprocessor support spg server * 修改solver支持名 * fix language * 修改chunker接口,增加openapi * rename vectorizer to vectorize_model in spg server config * generate_random_string start with gen * add knext llm vector checker * add knext llm vector checker * add knext llm vector checker * solver移除默认值 * udpate yaml and register_name for baike * udpate yaml and register_name for baike * remove config key check * 修复llmmodule * fix knext project * udpate yaml and register_name for examples * udpate yaml and register_name for examples * Revert "udpate yaml and register_name for examples" This reverts commit b3fa5ca9ba749e501133ac67bd8746027ab839d9. * update register name * fix * fix * support multiple resigter names * update component * update reader register names (#183) * fix markdown reader * fix llm client for retry * feat(common): add processed chunk id checkpoint (#185) * update reader register names * add processed chunk id checkpoint * feat(example): add example config (#186) * update reader register names * add processed chunk id checkpoint * add example config file * add max_workers parameter for getSummarizationMetrics to make it faster * add csqa data generation script generate_data.py * commit generated csqa builder and solver data * add csqa basic project files * adjust split_length and num_threads_per_chain to match lightrag settings * ignore ckpt dirs * add csqa evaluation script eval.py * save evaluation scripts summarization_metrics.py and factual_correctness.py * save LightRAG output csqa_lightrag_answers.json * ignore KAG output csqa_kag_answers.json * add README.md for CSQA * fix(solver): fix solver pipeline conf (#191) * update reader register names * add processed chunk id checkpoint * add example config file * update solver pipeline config * fix project create * update links and file paths * reformat csqa kag_config.yaml * reformat csqa python files * reformat getSummarizationMetrics and compare_summarization_answers * fix(solver): fix solver config (#192) * update reader register names * add processed chunk id checkpoint * add example config file * update solver pipeline config * fix project create * fix main solver conf * add except * fix typo in csqa README.md * feat(conf): support reinitialize config for call from java side (#199) * update reader register names * add processed chunk id checkpoint * add example config file * update solver pipeline config * fix project create * fix main solver conf * support reinitialize config for java call * revert default response generation prompt * update project list * add README.md for the hotpotqa, 2wiki and musique examples * 增加spo检索 * turn off kag config dump by default * turn off knext schema dump by default * add .gitignore and fix kag_config.yaml * add README.md for the medicine example * add README.md for the supplychain example * bugfix for risk mining * use exact out * refactor(solver): format solver code (#205) * update reader register names * add processed chunk id checkpoint * add example config file * update solver pipeline config * fix project create * fix main solver conf * support reinitialize config for java call * black format --------- Co-authored-by: peilong <peilong.zpl@antgroup.com> Co-authored-by: 锦呈 <zhangxinhong.zxh@antgroup.com> Co-authored-by: zhengke.gzk <zhengke.gzk@antgroup.com> Co-authored-by: huaidong.xhd <huaidong.xhd@antgroup.com>
364 lines
10 KiB
Python
364 lines
10 KiB
Python
# coding: utf-8
|
|
# Copyright 2023 OpenSPG Authors
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
|
|
# in compliance with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
# or implied.
|
|
|
|
|
|
"""
|
|
knext
|
|
|
|
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) # noqa: E501
|
|
|
|
The version of the OpenAPI document: 1.0.0
|
|
Generated by: https://openapi-generator.tech
|
|
"""
|
|
|
|
|
|
import pprint
|
|
import re # noqa: F401
|
|
|
|
import six
|
|
|
|
from knext.common.rest.configuration import Configuration
|
|
|
|
|
|
class Property(object):
|
|
"""NOTE: This class is auto generated by OpenAPI Generator.
|
|
Ref: https://openapi-generator.tech
|
|
|
|
Do not edit the class manually.
|
|
"""
|
|
|
|
"""
|
|
Attributes:
|
|
openapi_types (dict): The key is attribute name
|
|
and the value is attribute type.
|
|
attribute_map (dict): The key is attribute name
|
|
and the value is json key in definition.
|
|
"""
|
|
openapi_types = {
|
|
"basic_info": "SubPropertyBasicInfo",
|
|
"subject_type_ref": "SpgTypeRef",
|
|
"object_type_ref": "SpgTypeRef",
|
|
"inherited": "bool",
|
|
"advanced_config": "PropertyAdvancedConfig",
|
|
"project_id": "int",
|
|
"ontology_id": "OntologyId",
|
|
"alter_operation": "str",
|
|
"ext_info": "object",
|
|
}
|
|
|
|
attribute_map = {
|
|
"basic_info": "basicInfo",
|
|
"subject_type_ref": "subjectTypeRef",
|
|
"object_type_ref": "objectTypeRef",
|
|
"inherited": "inherited",
|
|
"advanced_config": "advancedConfig",
|
|
"project_id": "projectId",
|
|
"ontology_id": "ontologyId",
|
|
"alter_operation": "alterOperation",
|
|
"ext_info": "extInfo",
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
basic_info=None,
|
|
subject_type_ref=None,
|
|
object_type_ref=None,
|
|
inherited=None,
|
|
advanced_config=None,
|
|
project_id=None,
|
|
ontology_id=None,
|
|
alter_operation=None,
|
|
ext_info=None,
|
|
local_vars_configuration=None,
|
|
): # noqa: E501
|
|
"""Property - a model defined in OpenAPI""" # noqa: E501
|
|
if local_vars_configuration is None:
|
|
local_vars_configuration = Configuration()
|
|
self.local_vars_configuration = local_vars_configuration
|
|
|
|
self._basic_info = None
|
|
self._subject_type_ref = None
|
|
self._object_type_ref = None
|
|
self._inherited = None
|
|
self._advanced_config = None
|
|
self._project_id = None
|
|
self._ontology_id = None
|
|
self._alter_operation = None
|
|
self._ext_info = None
|
|
self.discriminator = None
|
|
|
|
if basic_info is not None:
|
|
self.basic_info = basic_info
|
|
if subject_type_ref is not None:
|
|
self.subject_type_ref = subject_type_ref
|
|
if object_type_ref is not None:
|
|
self.object_type_ref = object_type_ref
|
|
if inherited is not None:
|
|
self.inherited = inherited
|
|
if advanced_config is not None:
|
|
self.advanced_config = advanced_config
|
|
if project_id is not None:
|
|
self.project_id = project_id
|
|
if ontology_id is not None:
|
|
self.ontology_id = ontology_id
|
|
if alter_operation is not None:
|
|
self.alter_operation = alter_operation
|
|
if ext_info is not None:
|
|
self.ext_info = ext_info
|
|
|
|
@property
|
|
def basic_info(self):
|
|
"""Gets the basic_info of this Property. # noqa: E501
|
|
|
|
|
|
:return: The basic_info of this Property. # noqa: E501
|
|
:rtype: SubPropertyBasicInfo
|
|
"""
|
|
return self._basic_info
|
|
|
|
@basic_info.setter
|
|
def basic_info(self, basic_info):
|
|
"""Sets the basic_info of this Property.
|
|
|
|
|
|
:param basic_info: The basic_info of this Property. # noqa: E501
|
|
:type: SubPropertyBasicInfo
|
|
"""
|
|
|
|
self._basic_info = basic_info
|
|
|
|
@property
|
|
def subject_type_ref(self):
|
|
"""Gets the subject_type_ref of this Property. # noqa: E501
|
|
|
|
|
|
:return: The subject_type_ref of this Property. # noqa: E501
|
|
:rtype: SpgTypeRef
|
|
"""
|
|
return self._subject_type_ref
|
|
|
|
@subject_type_ref.setter
|
|
def subject_type_ref(self, subject_type_ref):
|
|
"""Sets the subject_type_ref of this Property.
|
|
|
|
|
|
:param subject_type_ref: The subject_type_ref of this Property. # noqa: E501
|
|
:type: SpgTypeRef
|
|
"""
|
|
|
|
self._subject_type_ref = subject_type_ref
|
|
|
|
@property
|
|
def object_type_ref(self):
|
|
"""Gets the object_type_ref of this Property. # noqa: E501
|
|
|
|
|
|
:return: The object_type_ref of this Property. # noqa: E501
|
|
:rtype: SpgTypeRef
|
|
"""
|
|
return self._object_type_ref
|
|
|
|
@object_type_ref.setter
|
|
def object_type_ref(self, object_type_ref):
|
|
"""Sets the object_type_ref of this Property.
|
|
|
|
|
|
:param object_type_ref: The object_type_ref of this Property. # noqa: E501
|
|
:type: SpgTypeRef
|
|
"""
|
|
|
|
self._object_type_ref = object_type_ref
|
|
|
|
@property
|
|
def inherited(self):
|
|
"""Gets the inherited of this Property. # noqa: E501
|
|
|
|
|
|
:return: The inherited of this Property. # noqa: E501
|
|
:rtype: bool
|
|
"""
|
|
return self._inherited
|
|
|
|
@inherited.setter
|
|
def inherited(self, inherited):
|
|
"""Sets the inherited of this Property.
|
|
|
|
|
|
:param inherited: The inherited of this Property. # noqa: E501
|
|
:type: bool
|
|
"""
|
|
|
|
self._inherited = inherited
|
|
|
|
@property
|
|
def advanced_config(self):
|
|
"""Gets the advanced_config of this Property. # noqa: E501
|
|
|
|
|
|
:return: The advanced_config of this Property. # noqa: E501
|
|
:rtype: PropertyAdvancedConfig
|
|
"""
|
|
return self._advanced_config
|
|
|
|
@advanced_config.setter
|
|
def advanced_config(self, advanced_config):
|
|
"""Sets the advanced_config of this Property.
|
|
|
|
|
|
:param advanced_config: The advanced_config of this Property. # noqa: E501
|
|
:type: PropertyAdvancedConfig
|
|
"""
|
|
|
|
self._advanced_config = advanced_config
|
|
|
|
@property
|
|
def project_id(self):
|
|
"""Gets the project_id of this Property. # noqa: E501
|
|
|
|
|
|
:return: The project_id of this Property. # noqa: E501
|
|
:rtype: int
|
|
"""
|
|
return self._project_id
|
|
|
|
@project_id.setter
|
|
def project_id(self, project_id):
|
|
"""Sets the project_id of this Property.
|
|
|
|
|
|
:param project_id: The project_id of this Property. # noqa: E501
|
|
:type: int
|
|
"""
|
|
|
|
self._project_id = project_id
|
|
|
|
@property
|
|
def ontology_id(self):
|
|
"""Gets the ontology_id of this Property. # noqa: E501
|
|
|
|
|
|
:return: The ontology_id of this Property. # noqa: E501
|
|
:rtype: OntologyId
|
|
"""
|
|
return self._ontology_id
|
|
|
|
@ontology_id.setter
|
|
def ontology_id(self, ontology_id):
|
|
"""Sets the ontology_id of this Property.
|
|
|
|
|
|
:param ontology_id: The ontology_id of this Property. # noqa: E501
|
|
:type: OntologyId
|
|
"""
|
|
|
|
self._ontology_id = ontology_id
|
|
|
|
@property
|
|
def alter_operation(self):
|
|
"""Gets the alter_operation of this Property. # noqa: E501
|
|
|
|
|
|
:return: The alter_operation of this Property. # noqa: E501
|
|
:rtype: str
|
|
"""
|
|
return self._alter_operation
|
|
|
|
@alter_operation.setter
|
|
def alter_operation(self, alter_operation):
|
|
"""Sets the alter_operation of this Property.
|
|
|
|
|
|
:param alter_operation: The alter_operation of this Property. # noqa: E501
|
|
:type: str
|
|
"""
|
|
allowed_values = ["CREATE", "UPDATE", "DELETE"] # noqa: E501
|
|
if (
|
|
self.local_vars_configuration.client_side_validation
|
|
and alter_operation not in allowed_values
|
|
): # noqa: E501
|
|
raise ValueError(
|
|
"Invalid value for `alter_operation` ({0}), must be one of {1}".format( # noqa: E501
|
|
alter_operation, allowed_values
|
|
)
|
|
)
|
|
|
|
self._alter_operation = alter_operation
|
|
|
|
@property
|
|
def ext_info(self):
|
|
"""Gets the ext_info of this Property. # noqa: E501
|
|
|
|
|
|
:return: The ext_info of this Property. # noqa: E501
|
|
:rtype: object
|
|
"""
|
|
return self._ext_info
|
|
|
|
@ext_info.setter
|
|
def ext_info(self, ext_info):
|
|
"""Sets the ext_info of this Property.
|
|
|
|
|
|
:param ext_info: The ext_info of this Property. # noqa: E501
|
|
:type: object
|
|
"""
|
|
|
|
self._ext_info = ext_info
|
|
|
|
def to_dict(self):
|
|
"""Returns the model properties as a dict"""
|
|
result = {}
|
|
|
|
for attr, _ in six.iteritems(self.openapi_types):
|
|
value = getattr(self, attr)
|
|
if isinstance(value, list):
|
|
result[attr] = list(
|
|
map(lambda x: x.to_dict() if hasattr(x, "to_dict") else x, value)
|
|
)
|
|
elif hasattr(value, "to_dict"):
|
|
result[attr] = value.to_dict()
|
|
elif isinstance(value, dict):
|
|
result[attr] = dict(
|
|
map(
|
|
lambda item: (item[0], item[1].to_dict())
|
|
if hasattr(item[1], "to_dict")
|
|
else item,
|
|
value.items(),
|
|
)
|
|
)
|
|
else:
|
|
result[attr] = value
|
|
|
|
return result
|
|
|
|
def to_str(self):
|
|
"""Returns the string representation of the model"""
|
|
return pprint.pformat(self.to_dict())
|
|
|
|
def __repr__(self):
|
|
"""For `print` and `pprint`"""
|
|
return self.to_str()
|
|
|
|
def __eq__(self, other):
|
|
"""Returns true if both objects are equal"""
|
|
if not isinstance(other, Property):
|
|
return False
|
|
|
|
return self.to_dict() == other.to_dict()
|
|
|
|
def __ne__(self, other):
|
|
"""Returns true if both objects are not equal"""
|
|
if not isinstance(other, Property):
|
|
return True
|
|
|
|
return self.to_dict() != other.to_dict()
|