yujunjun/unstructured
1
0
Fork 0
mirror of https://github.com/Unstructured-IO/unstructured.git synced 2025-08-15 04:08:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

bug: update sharepoint-with-permissions test to fix CI (#2589)

Browse Source 
Adding `metadata.data_source.permissions_data` to
sharepoint-with-permissions.sh --metadata-exclude to prevent sharepoint
deprecation warning from ruining test.

Updating expected-structured-output

As per Ahmet's comment. We do want to check sharepoint permissions
metadata at some point. But that will take a separate type of test. A
file diff test is too unstable. Permissions checking will be later down
the road.
This commit is contained in:
David Potter 2024-03-06 09:15:36 -08:00 committed by GitHub
parent dc376053dd
commit 1ca90d209a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 8 additions and 1907 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -1,4 +1,4 @@
## 0.12.6-dev2
## 0.12.6-dev3
### Enhancements
@ -13,6 +13,7 @@
* **Incorporate the `install-pandoc` Makefile recipe** into relevant stages of CI workflow, ensuring it is a version that supports RTF input files.
* **Fix Google Drive source key** Allow passing string for source connector key.
* **Fix table structure evaluations calculations** Replaced special value `-1.0` with `np.nan` and corrected rows filtering of files metrics basing on that.
* **Fix Sharepoint-with-permissions test** Ignore permissions metadata, update test.
## 0.12.5

600
test_unstructured_ingest/expected-structured-output/Sharepoint-with-permissions/Shared Documents/fake-text.txt.json
View File

@ -5,106 +5,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:55+00:00",
"date_modified": "2023-06-16T05:04:55+00:00",
"permissions_data": [
{
"grantedToIdentities": [],
"grantedToIdentitiesV2": [],
"hasPassword": false,
"id": "5896ac32-9e81-4c8b-ba83-0e25d1fa6db9",
"link": {
"preventsDownload": false,
"scope": "anonymous",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:t:/g/EcqAD4i_ztBItjmutnGzxDEBvTqVInhkhZ6z1DpdCi5xOg"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dDovZy9FY3FBRDRpX3p0Qkl0am11dG5HenhERUJ2VHFWSW5oa2haNnoxRHBkQ2k1eE9n"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/fake-text.txt",
"site_url": "https://unstructuredio.sharepoint.com"
@ -127,106 +27,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:55+00:00",
"date_modified": "2023-06-16T05:04:55+00:00",
"permissions_data": [
{
"grantedToIdentities": [],
"grantedToIdentitiesV2": [],
"hasPassword": false,
"id": "5896ac32-9e81-4c8b-ba83-0e25d1fa6db9",
"link": {
"preventsDownload": false,
"scope": "anonymous",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:t:/g/EcqAD4i_ztBItjmutnGzxDEBvTqVInhkhZ6z1DpdCi5xOg"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dDovZy9FY3FBRDRpX3p0Qkl0am11dG5HenhERUJ2VHFWSW5oa2haNnoxRHBkQ2k1eE9n"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/fake-text.txt",
"site_url": "https://unstructuredio.sharepoint.com"
@ -249,106 +49,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:55+00:00",
"date_modified": "2023-06-16T05:04:55+00:00",
"permissions_data": [
{
"grantedToIdentities": [],
"grantedToIdentitiesV2": [],
"hasPassword": false,
"id": "5896ac32-9e81-4c8b-ba83-0e25d1fa6db9",
"link": {
"preventsDownload": false,
"scope": "anonymous",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:t:/g/EcqAD4i_ztBItjmutnGzxDEBvTqVInhkhZ6z1DpdCi5xOg"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dDovZy9FY3FBRDRpX3p0Qkl0am11dG5HenhERUJ2VHFWSW5oa2haNnoxRHBkQ2k1eE9n"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/fake-text.txt",
"site_url": "https://unstructuredio.sharepoint.com"
@ -371,106 +71,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:55+00:00",
"date_modified": "2023-06-16T05:04:55+00:00",
"permissions_data": [
{
"grantedToIdentities": [],
"grantedToIdentitiesV2": [],
"hasPassword": false,
"id": "5896ac32-9e81-4c8b-ba83-0e25d1fa6db9",
"link": {
"preventsDownload": false,
"scope": "anonymous",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:t:/g/EcqAD4i_ztBItjmutnGzxDEBvTqVInhkhZ6z1DpdCi5xOg"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dDovZy9FY3FBRDRpX3p0Qkl0am11dG5HenhERUJ2VHFWSW5oa2haNnoxRHBkQ2k1eE9n"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/fake-text.txt",
"site_url": "https://unstructuredio.sharepoint.com"
@ -493,106 +93,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:55+00:00",
"date_modified": "2023-06-16T05:04:55+00:00",
"permissions_data": [
{
"grantedToIdentities": [],
"grantedToIdentitiesV2": [],
"hasPassword": false,
"id": "5896ac32-9e81-4c8b-ba83-0e25d1fa6db9",
"link": {
"preventsDownload": false,
"scope": "anonymous",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:t:/g/EcqAD4i_ztBItjmutnGzxDEBvTqVInhkhZ6z1DpdCi5xOg"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dDovZy9FY3FBRDRpX3p0Qkl0am11dG5HenhERUJ2VHFWSW5oa2haNnoxRHBkQ2k1eE9n"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/fake-text.txt",
"site_url": "https://unstructuredio.sharepoint.com"
@ -615,106 +115,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:55+00:00",
"date_modified": "2023-06-16T05:04:55+00:00",
"permissions_data": [
{
"grantedToIdentities": [],
"grantedToIdentitiesV2": [],
"hasPassword": false,
"id": "5896ac32-9e81-4c8b-ba83-0e25d1fa6db9",
"link": {
"preventsDownload": false,
"scope": "anonymous",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:t:/g/EcqAD4i_ztBItjmutnGzxDEBvTqVInhkhZ6z1DpdCi5xOg"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dDovZy9FY3FBRDRpX3p0Qkl0am11dG5HenhERUJ2VHFWSW5oa2haNnoxRHBkQ2k1eE9n"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/fake-text.txt",
"site_url": "https://unstructuredio.sharepoint.com"

88
test_unstructured_ingest/expected-structured-output/Sharepoint-with-permissions/Shared Documents/ideas-page.html.json
View File

@ -5,94 +5,6 @@
"data_source": {
"date_created": "2023-06-16T05:04:47+00:00",
"date_modified": "2023-06-16T05:04:47+00:00",
"permissions_data": [
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"inheritedFrom": {},
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"inheritedFrom": {},
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/ideas-page.html",
"site_url": "https://unstructuredio.sharepoint.com"

864
test_unstructured_ingest/expected-structured-output/Sharepoint-with-permissions/Shared Documents/permissions-fake-text.docx.json
View File

@ -5,150 +5,6 @@
"data_source": {
"date_created": "2023-10-27T11:06:24+00:00",
"date_modified": "2023-10-27T11:07:34+00:00",
"permissions_data": [
{
"grantedToIdentities": [
{
"user": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"user": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"grantedToIdentitiesV2": [
{
"siteUser": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "21",
"loginName": "i:0#.f|membership|ahmet_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"siteUser": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "20",
"loginName": "i:0#.f|membership|ryan_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"hasPassword": false,
"id": "4c35a2c4-3e49-43a3-9f76-3ac1ff36e9b0",
"link": {
"preventsDownload": false,
"scope": "users",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:w:/g/EWvc96L2vsJCtlsIisxZ78oBBZ0iCAlip9YtOqmtcYJPqA"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dzovZy9FV3ZjOTZMMnZzSkN0bHNJaXN4Wjc4b0JCWjBpQ0FsaXA5WXRPcW10Y1lKUHFB"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/permissions-fake-text.docx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -171,150 +27,6 @@
"data_source": {
"date_created": "2023-10-27T11:06:24+00:00",
"date_modified": "2023-10-27T11:07:34+00:00",
"permissions_data": [
{
"grantedToIdentities": [
{
"user": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"user": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"grantedToIdentitiesV2": [
{
"siteUser": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "21",
"loginName": "i:0#.f|membership|ahmet_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"siteUser": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "20",
"loginName": "i:0#.f|membership|ryan_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"hasPassword": false,
"id": "4c35a2c4-3e49-43a3-9f76-3ac1ff36e9b0",
"link": {
"preventsDownload": false,
"scope": "users",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:w:/g/EWvc96L2vsJCtlsIisxZ78oBBZ0iCAlip9YtOqmtcYJPqA"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dzovZy9FV3ZjOTZMMnZzSkN0bHNJaXN4Wjc4b0JCWjBpQ0FsaXA5WXRPcW10Y1lKUHFB"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/permissions-fake-text.docx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -337,150 +49,6 @@
"data_source": {
"date_created": "2023-10-27T11:06:24+00:00",
"date_modified": "2023-10-27T11:07:34+00:00",
"permissions_data": [
{
"grantedToIdentities": [
{
"user": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"user": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"grantedToIdentitiesV2": [
{
"siteUser": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "21",
"loginName": "i:0#.f|membership|ahmet_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"siteUser": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "20",
"loginName": "i:0#.f|membership|ryan_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"hasPassword": false,
"id": "4c35a2c4-3e49-43a3-9f76-3ac1ff36e9b0",
"link": {
"preventsDownload": false,
"scope": "users",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:w:/g/EWvc96L2vsJCtlsIisxZ78oBBZ0iCAlip9YtOqmtcYJPqA"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dzovZy9FV3ZjOTZMMnZzSkN0bHNJaXN4Wjc4b0JCWjBpQ0FsaXA5WXRPcW10Y1lKUHFB"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/permissions-fake-text.docx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -503,150 +71,6 @@
"data_source": {
"date_created": "2023-10-27T11:06:24+00:00",
"date_modified": "2023-10-27T11:07:34+00:00",
"permissions_data": [
{
"grantedToIdentities": [
{
"user": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"user": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"grantedToIdentitiesV2": [
{
"siteUser": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "21",
"loginName": "i:0#.f|membership|ahmet_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"siteUser": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "20",
"loginName": "i:0#.f|membership|ryan_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"hasPassword": false,
"id": "4c35a2c4-3e49-43a3-9f76-3ac1ff36e9b0",
"link": {
"preventsDownload": false,
"scope": "users",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:w:/g/EWvc96L2vsJCtlsIisxZ78oBBZ0iCAlip9YtOqmtcYJPqA"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dzovZy9FV3ZjOTZMMnZzSkN0bHNJaXN4Wjc4b0JCWjBpQ0FsaXA5WXRPcW10Y1lKUHFB"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/permissions-fake-text.docx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -669,150 +93,6 @@
"data_source": {
"date_created": "2023-10-27T11:06:24+00:00",
"date_modified": "2023-10-27T11:07:34+00:00",
"permissions_data": [
{
"grantedToIdentities": [
{
"user": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"user": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"grantedToIdentitiesV2": [
{
"siteUser": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "21",
"loginName": "i:0#.f|membership|ahmet_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"siteUser": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "20",
"loginName": "i:0#.f|membership|ryan_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"hasPassword": false,
"id": "4c35a2c4-3e49-43a3-9f76-3ac1ff36e9b0",
"link": {
"preventsDownload": false,
"scope": "users",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:w:/g/EWvc96L2vsJCtlsIisxZ78oBBZ0iCAlip9YtOqmtcYJPqA"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dzovZy9FV3ZjOTZMMnZzSkN0bHNJaXN4Wjc4b0JCWjBpQ0FsaXA5WXRPcW10Y1lKUHFB"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/permissions-fake-text.docx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -835,150 +115,6 @@
"data_source": {
"date_created": "2023-10-27T11:06:24+00:00",
"date_modified": "2023-10-27T11:07:34+00:00",
"permissions_data": [
{
"grantedToIdentities": [
{
"user": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"user": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"grantedToIdentitiesV2": [
{
"siteUser": {
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "21",
"loginName": "i:0#.f|membership|ahmet_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ahmet",
"email": "ahmet@unstructured.io",
"id": "7dbe8da8-9ee8-4211-869e-c4567ba4eeaa"
}
},
{
"siteUser": {
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "20",
"loginName": "i:0#.f|membership|ryan_unstructured.io#ext#@unstructuredio.onmicrosoft.com"
},
"user": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "ryan",
"email": "ryan@unstructured.io",
"id": "7ed7168e-5add-48e7-a2b8-ff024f91c4bf"
}
}
],
"hasPassword": false,
"id": "4c35a2c4-3e49-43a3-9f76-3ac1ff36e9b0",
"link": {
"preventsDownload": false,
"scope": "users",
"type": "edit",
"webUrl": "https://unstructuredio.sharepoint.com/:w:/g/EWvc96L2vsJCtlsIisxZ78oBBZ0iCAlip9YtOqmtcYJPqA"
},
"roles": [
"write"
],
"shareId": "u!aHR0cHM6Ly91bnN0cnVjdHVyZWRpby5zaGFyZXBvaW50LmNvbS86dzovZy9FV3ZjOTZMMnZzSkN0bHNJaXN4Wjc4b0JCWjBpQ0FsaXA5WXRPcW10Y1lKUHFB"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/permissions-fake-text.docx",
"site_url": "https://unstructuredio.sharepoint.com"

352
test_unstructured_ingest/expected-structured-output/Sharepoint-with-permissions/Shared Documents/stanley-cups.xlsx.json
View File

@ -5,94 +5,6 @@
"data_source": {
"date_created": "2023-06-16T05:05:05+00:00",
"date_modified": "2023-06-16T05:05:05+00:00",
"permissions_data": [
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"inheritedFrom": {},
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"inheritedFrom": {},
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/stanley-cups.xlsx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -117,94 +29,6 @@
"data_source": {
"date_created": "2023-06-16T05:05:05+00:00",
"date_modified": "2023-06-16T05:05:05+00:00",
"permissions_data": [
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"inheritedFrom": {},
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"inheritedFrom": {},
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/stanley-cups.xlsx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -229,94 +53,6 @@
"data_source": {
"date_created": "2023-06-16T05:05:05+00:00",
"date_modified": "2023-06-16T05:05:05+00:00",
"permissions_data": [
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"inheritedFrom": {},
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"inheritedFrom": {},
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/stanley-cups.xlsx",
"site_url": "https://unstructuredio.sharepoint.com"
@ -341,94 +77,6 @@
"data_source": {
"date_created": "2023-06-16T05:05:05+00:00",
"date_modified": "2023-06-16T05:05:05+00:00",
"permissions_data": [
{
"grantedTo": {
"user": {
"displayName": "Communication site Owners"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Owners",
"id": "3",
"loginName": "Communication site Owners"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE93bmVycw"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Visitors"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Visitors",
"id": "4",
"loginName": "Communication site Visitors"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz",
"inheritedFrom": {},
"roles": [
"read"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIFZpc2l0b3Jz"
},
{
"grantedTo": {
"user": {
"displayName": "Communication site Members"
}
},
"grantedToV2": {
"siteGroup": {
"displayName": "Communication site Members",
"id": "5",
"loginName": "Communication site Members"
}
},
"id": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM",
"inheritedFrom": {},
"roles": [
"write"
],
"shareId": "Q29tbXVuaWNhdGlvbiBzaXRlIE1lbWJlcnM"
},
{
"grantedTo": {
"user": {
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"grantedToV2": {
"group": {
"@odata.type": "#microsoft.graph.sharePointIdentity",
"displayName": "Global Administrator",
"id": "fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
},
"siteUser": {
"displayName": "Global Administrator",
"id": "7",
"loginName": "c:0t.c|tenant|fc095592-c25a-4ac7-b2a1-31e0d01e4b51"
}
},
"id": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE",
"inheritedFrom": {},
"roles": [
"owner"
],
"shareId": "YzowdC5jfHRlbmFudHxmYzA5NTU5Mi1jMjVhLTRhYzctYjJhMS0zMWUwZDAxZTRiNTE"
}
],
"record_locator": {
"server_path": "/Shared Documents/stanley-cups.xlsx",
"site_url": "https://unstructuredio.sharepoint.com"

6
test_unstructured_ingest/src/sharepoint-with-permissions.sh
View File

@ -1,5 +1,8 @@
#!/usr/bin/env bash
# Original intent of this test was to diff the permissions metadata. Unfortunately this is unstable.
# Since we still want to test the permission login functionality, we will test the permissions metadata when we get there.
set -e
SRC_PATH=$(dirname "$(realpath "$0")")
@ -35,11 +38,12 @@ if [ -z "$SHAREPOINT_PERMISSIONS_APP_ID" ] || [ -z "$SHAREPOINT_PERMISSIONS_APP_
fi
# excluding metadata.last_modified since this will always update as date processed because the Sharepoint connector creates documents on the fly
# excluding metadata.data_source.permissions_data since the api has deprecation warnings. Will want to do a separate test for permissions data
RUN_SCRIPT=${RUN_SCRIPT:-./unstructured/ingest/main.py}
PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
sharepoint \
--download-dir "$DOWNLOAD_DIR" \
--metadata-exclude file_directory,metadata.data_source.date_processed,metadata.last_modified,metadata.detection_class_prob,metadata.parent_id,metadata.category_depth \
--metadata-exclude file_directory,metadata.data_source.date_processed,metadata.last_modified,metadata.detection_class_prob,metadata.parent_id,metadata.category_depth,metadata.data_source.permissions_data \
--num-processes "$max_processes" \
--strategy hi_res \
--preserve-downloads \

2
unstructured/__version__.py
View File

@ -1 +1 @@
__version__ = "0.12.6-dev2" # pragma: no cover
__version__ = "0.12.6-dev3" # pragma: no cover