Michał Martyniak 2d1923ac7e
Better element IDs - deterministic and document-unique hashes (#2673)
Part two of: https://github.com/Unstructured-IO/unstructured/pull/2842

Main changes compared to part one:
* hash computation includes element's sequence number on page, page
number, document filename and its text
* there are more test for deterministic behavior of IDs returned by
partitioning functions + their uniqueness (guaranteed at the document
level, and high probability across multiple documents)

This PR addresses the following issue:
https://github.com/Unstructured-IO/unstructured/issues/2461
2024-04-24 00:05:20 -07:00

222 lines
6.9 KiB
JSON

[
{
"element_id": "acc232c0abe9dbf57330753b7c6fe070",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "IssueID_IssueKey:10012 JCTP2-10",
"type": "Title"
},
{
"element_id": "09af46b99580c84162d7b24aad94d9f8",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "ProjectID_Key:JCTP2 Jira Connector Test Project 2",
"type": "Title"
},
{
"element_id": "c5b9769a825c1961aa260d7317acd3e6",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "IssueType:Bug",
"type": "Title"
},
{
"element_id": "734982b4bcfd585ebab75fbc2643129b",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Status:To Do",
"type": "Title"
},
{
"element_id": "0c9a7662744b1416b2573beafb132748",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Priority:{'self': 'https://unstructured-jira-connector-test.atlassian.net/rest/api/2/priority/3', 'iconUrl': 'https://unstructured-jira-connector-test.atlassian.net/images/icons/priorities/medium.svg', 'name': 'Medium', 'id': '3'}",
"type": "Title"
},
{
"element_id": "6338791425596a06c1f6a9a0825bdf37",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "AssigneeID_Name:{} {}",
"type": "Title"
},
{
"element_id": "04f313ea24554203330b0047ae9e7903",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "ReporterAdr_Name:devops+jira-connector@unstructured.io Unstructured Devops",
"type": "Title"
},
{
"element_id": "d5c81ea3bd283b099c857307f0ae53db",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Labels:",
"type": "Title"
},
{
"element_id": "b755d3d8d5304be988e52203016dc492",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Components:",
"type": "Title"
},
{
"element_id": "e23576c462b54c896c6427bf4f09b5bf",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Test Epic 1 Bug 1",
"type": "Title"
},
{
"element_id": "0fc63dbf4a6be97a0e07400d365e2a0c",
"metadata": {
"data_source": {
"date_created": "2023-08-22T11:36:09.310000+00:00",
"date_modified": "2023-08-29T11:46:13.812000+00:00",
"record_locator": {
"base_url": "https://unstructured-jira-connector-test.atlassian.net",
"issue_key": "JCTP2-10"
},
"url": "https://unstructured-jira-connector-test.atlassian.net/browse/JCTP2-10"
},
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "{}",
"type": "UncategorizedText"
}
]