From 17f3d909d9d5868ca88d0bda5be744c46bad07ed Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 26 Jun 2023 09:56:49 +0200 Subject: [PATCH] add file providers docs --- .../00-overview.md | 8 ++- .../01-source-providers.md | 0 .../02-destination-providers.md | 0 .../_category_.json | 2 +- .../02-file-providers/00-overview.md | 15 +++++ .../02-file-providers/01-file-structure.md | 57 +++++++++++++++++++ .../02-file-providers/02-source.md | 35 ++++++++++++ .../02-file-providers/03-destination.md | 34 +++++++++++ .../_category_.json | 0 .../00-overview.md | 0 .../01-file-structure.md | 0 .../02-source.md | 0 .../03-destination.md | 0 .../_category_.json | 0 .../00-overview.md | 0 .../01-file-structure.md | 0 .../02-source.md | 0 .../03-destination.md | 0 .../_category_.json | 0 .../05-remote-strapi-providers/00-overview.md | 13 ----- .../01-file-structure.md | 11 ---- .../05-remote-strapi-providers/02-source.md | 13 ----- .../03-destination.md | 15 ----- .../src/file/providers/destination/index.ts | 12 ++-- .../src/file/providers/source/index.ts | 8 +-- 25 files changed, 157 insertions(+), 66 deletions(-) rename docs/docs/docs/01-core/data-transfer/{01-data-providers => 01-providers}/00-overview.md (79%) rename docs/docs/docs/01-core/data-transfer/{01-data-providers => 01-providers}/01-source-providers.md (100%) rename docs/docs/docs/01-core/data-transfer/{01-data-providers => 01-providers}/02-destination-providers.md (100%) rename docs/docs/docs/01-core/data-transfer/{01-data-providers => 01-providers}/_category_.json (55%) create mode 100644 docs/docs/docs/01-core/data-transfer/02-file-providers/00-overview.md create mode 100644 docs/docs/docs/01-core/data-transfer/02-file-providers/01-file-structure.md create mode 100644 docs/docs/docs/01-core/data-transfer/02-file-providers/02-source.md create mode 100644 docs/docs/docs/01-core/data-transfer/02-file-providers/03-destination.md rename docs/docs/docs/01-core/data-transfer/{03-file-providers => 02-file-providers}/_category_.json (100%) rename docs/docs/docs/01-core/data-transfer/{03-file-providers => 03-local-strapi-providers}/00-overview.md (100%) rename docs/docs/docs/01-core/data-transfer/{03-file-providers => 03-local-strapi-providers}/01-file-structure.md (100%) rename docs/docs/docs/01-core/data-transfer/{03-file-providers => 03-local-strapi-providers}/02-source.md (100%) rename docs/docs/docs/01-core/data-transfer/{03-file-providers => 03-local-strapi-providers}/03-destination.md (100%) rename docs/docs/docs/01-core/data-transfer/{04-local-strapi-providers => 03-local-strapi-providers}/_category_.json (100%) rename docs/docs/docs/01-core/data-transfer/{04-local-strapi-providers => 04-remote-strapi-providers}/00-overview.md (100%) rename docs/docs/docs/01-core/data-transfer/{04-local-strapi-providers => 04-remote-strapi-providers}/01-file-structure.md (100%) rename docs/docs/docs/01-core/data-transfer/{04-local-strapi-providers => 04-remote-strapi-providers}/02-source.md (100%) rename docs/docs/docs/01-core/data-transfer/{04-local-strapi-providers => 04-remote-strapi-providers}/03-destination.md (100%) rename docs/docs/docs/01-core/data-transfer/{05-remote-strapi-providers => 04-remote-strapi-providers}/_category_.json (100%) delete mode 100644 docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/00-overview.md delete mode 100644 docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/01-file-structure.md delete mode 100644 docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/02-source.md delete mode 100644 docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/03-destination.md diff --git a/docs/docs/docs/01-core/data-transfer/01-data-providers/00-overview.md b/docs/docs/docs/01-core/data-transfer/01-providers/00-overview.md similarity index 79% rename from docs/docs/docs/01-core/data-transfer/01-data-providers/00-overview.md rename to docs/docs/docs/01-core/data-transfer/01-providers/00-overview.md index 9f19218ca7..970658deec 100644 --- a/docs/docs/docs/01-core/data-transfer/01-data-providers/00-overview.md +++ b/docs/docs/docs/01-core/data-transfer/01-providers/00-overview.md @@ -1,14 +1,14 @@ --- -title: Destination Providers +title: Data Transfer Providers tags: - providers - data-transfer - experimental --- -# Data Providers +# Data Transfer Providers -Data providers are the interfaces for streaming data during a transfer. +Data transfer providers are the interfaces for streaming data during a transfer. [Source providers](./01-source-providers.md) provide read streams for each stage in the transfer. @@ -20,6 +20,8 @@ Strapi provides both source and destination providers for the following: - [Local Strapi](../04-local-strapi-providers/00-overview.md): a connection to a local Strapi project which uses its configured database connection to manage data - [Remote Strapi](../05-remote-strapi-providers/00-overview.md): a wrapper of local Strapi provider that adds a websocket interface to a running remote (network) instance of Strapi +Each provider must provide the same interface for transferring data, but will usually include its own unique set of options to be passed in when initializing the provider. + ## Creating your own providers To create your own providers, you must implement the interface(s) defined in ISourceProvider and IDestinationProvider found in `packages/core/data-transfer/types/providers.d.ts`. diff --git a/docs/docs/docs/01-core/data-transfer/01-data-providers/01-source-providers.md b/docs/docs/docs/01-core/data-transfer/01-providers/01-source-providers.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/01-data-providers/01-source-providers.md rename to docs/docs/docs/01-core/data-transfer/01-providers/01-source-providers.md diff --git a/docs/docs/docs/01-core/data-transfer/01-data-providers/02-destination-providers.md b/docs/docs/docs/01-core/data-transfer/01-providers/02-destination-providers.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/01-data-providers/02-destination-providers.md rename to docs/docs/docs/01-core/data-transfer/01-providers/02-destination-providers.md diff --git a/docs/docs/docs/01-core/data-transfer/01-data-providers/_category_.json b/docs/docs/docs/01-core/data-transfer/01-providers/_category_.json similarity index 55% rename from docs/docs/docs/01-core/data-transfer/01-data-providers/_category_.json rename to docs/docs/docs/01-core/data-transfer/01-providers/_category_.json index 5e765d1948..0754ae334b 100644 --- a/docs/docs/docs/01-core/data-transfer/01-data-providers/_category_.json +++ b/docs/docs/docs/01-core/data-transfer/01-providers/_category_.json @@ -1,5 +1,5 @@ { - "label": "Source Providers", + "label": "Data Transfer Providers", "collapsible": true, "collapsed": true } diff --git a/docs/docs/docs/01-core/data-transfer/02-file-providers/00-overview.md b/docs/docs/docs/01-core/data-transfer/02-file-providers/00-overview.md new file mode 100644 index 0000000000..3a330482e1 --- /dev/null +++ b/docs/docs/docs/01-core/data-transfer/02-file-providers/00-overview.md @@ -0,0 +1,15 @@ +--- +title: File Providers +tags: + - experimental + - providers + - import + - export + - data-transfer +--- + +# Strapi Data File Providers + +Strapi data file providers transfer data to or from a [Strapi Data File](./01-file-structure.md). + +The files are optionally compressed and/or encrypted using a given key (password). diff --git a/docs/docs/docs/01-core/data-transfer/02-file-providers/01-file-structure.md b/docs/docs/docs/01-core/data-transfer/02-file-providers/01-file-structure.md new file mode 100644 index 0000000000..fdd8093f7b --- /dev/null +++ b/docs/docs/docs/01-core/data-transfer/02-file-providers/01-file-structure.md @@ -0,0 +1,57 @@ +--- +title: Strapi File Structure +tags: + - providers + - data-transfer + - experimental +--- + +# Strapi File Structure + +The Strapi file providers expect a .tar file (optionally compressed with gzip and/or encrypted with 'aes-128-ecb') that internally uses POSIX style file paths with the following structure: + +``` +./ +configuration +entities +links +metadata.json +schemas + +./configuration: +configuration_00001.jsonl + +./entities: +entities_00001.jsonl + +./links: +links_00001.jsonl + +./schemas: +schemas_00001.jsonl +``` + +## metadata.json + +This file provides metadata about the original source of the data. At minimum, it should include a createdAt timestamp and the version of Strapi that the file was created with (for compatibility checks). + +```json +{ + "createdAt": "2023-06-26T07:31:20.062Z", + "strapi": { + "version": "4.11.2" + } +} +``` + +## A directory for each stage of data + +There should also be a directory for each stage of data that includes sequentially numbered JSON Lines (.jsonl) files + +The files are named in the format: `{stage}\{stage}_{5-digit sequence number}.jsonl` + +Any number of files may be provided for each stage, as long as the sequence numbers are in order. That is, after first reading 00001, the file source provider will attempt to read file 00002 and if it is not found, it will consider the stage complete. + +### JSONL files + +JSON Lines files are essentially JSON files, except that newline characters are used to delimit the JSON objects. This allows the provider to read in a single line at a time, rather than loading the entire file into memory, minimizing RAM usage during a transfer and allowing files containing any amount of data. diff --git a/docs/docs/docs/01-core/data-transfer/02-file-providers/02-source.md b/docs/docs/docs/01-core/data-transfer/02-file-providers/02-source.md new file mode 100644 index 0000000000..3cc408f3c2 --- /dev/null +++ b/docs/docs/docs/01-core/data-transfer/02-file-providers/02-source.md @@ -0,0 +1,35 @@ +--- +title: Strapi File +tags: + - providers + - data-transfer + - experimental +--- + +# Strapi File Source Provider + +This provider will open and read a Strapi Data File as a data source. + +## Provider Options + +The accepted options are defined in `ILocalFileSourceProviderOptions`. + +```typescript + file: { + path: string; // the file to load + }; + + encryption: { + enabled: boolean; // if the file is encrypted (and should be decrypted) + key?: string; // the key to decrypt the file + }; + + compression: { + enabled: boolean; // if the file in compressed (and should be decompressed) + }; +``` + +Note: When the Strapi CLI attempts to import a file, the options for compression and encryption are set based on the extension of the file being loaded, eg a file with the .gz extension will have the "compress" option set, and a file that includes the .enc extension will have the "encrypt" option set. + +When using the transfer engine programmatically, you may make the determination whether the file being loaded should be decrypted or compressed by setting +those options. diff --git a/docs/docs/docs/01-core/data-transfer/02-file-providers/03-destination.md b/docs/docs/docs/01-core/data-transfer/02-file-providers/03-destination.md new file mode 100644 index 0000000000..0b5b6e9113 --- /dev/null +++ b/docs/docs/docs/01-core/data-transfer/02-file-providers/03-destination.md @@ -0,0 +1,34 @@ +--- +title: Strapi File +tags: + - providers + - data-transfer + - experimental +--- + +# Strapi File Destination Provider + +This provider will output a Strapi Data File. + +Note: this destination provider does not provide a schema or metadata, and will therefore never report a schema match error or version validation error + +## Provider Options + +The accepted options are defined in `ILocalFileDestinationProviderOptions`. + +```typescript + encryption: { + enabled: boolean; // if the file should be decrypted + key?: string; // the key to use when enabled is true + }; + + compression: { + enabled: boolean; // if the file should be compressed with gzip + }; + + file: { + path: string; // the filename to create + maxSize?: number; // the max size of a single backup file + maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file + }; +``` diff --git a/docs/docs/docs/01-core/data-transfer/03-file-providers/_category_.json b/docs/docs/docs/01-core/data-transfer/02-file-providers/_category_.json similarity index 100% rename from docs/docs/docs/01-core/data-transfer/03-file-providers/_category_.json rename to docs/docs/docs/01-core/data-transfer/02-file-providers/_category_.json diff --git a/docs/docs/docs/01-core/data-transfer/03-file-providers/00-overview.md b/docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/00-overview.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/03-file-providers/00-overview.md rename to docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/00-overview.md diff --git a/docs/docs/docs/01-core/data-transfer/03-file-providers/01-file-structure.md b/docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/01-file-structure.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/03-file-providers/01-file-structure.md rename to docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/01-file-structure.md diff --git a/docs/docs/docs/01-core/data-transfer/03-file-providers/02-source.md b/docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/02-source.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/03-file-providers/02-source.md rename to docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/02-source.md diff --git a/docs/docs/docs/01-core/data-transfer/03-file-providers/03-destination.md b/docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/03-destination.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/03-file-providers/03-destination.md rename to docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/03-destination.md diff --git a/docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/_category_.json b/docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/_category_.json similarity index 100% rename from docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/_category_.json rename to docs/docs/docs/01-core/data-transfer/03-local-strapi-providers/_category_.json diff --git a/docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/00-overview.md b/docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/00-overview.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/00-overview.md rename to docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/00-overview.md diff --git a/docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/01-file-structure.md b/docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/01-file-structure.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/01-file-structure.md rename to docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/01-file-structure.md diff --git a/docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/02-source.md b/docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/02-source.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/02-source.md rename to docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/02-source.md diff --git a/docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/03-destination.md b/docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/03-destination.md similarity index 100% rename from docs/docs/docs/01-core/data-transfer/04-local-strapi-providers/03-destination.md rename to docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/03-destination.md diff --git a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/_category_.json b/docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/_category_.json similarity index 100% rename from docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/_category_.json rename to docs/docs/docs/01-core/data-transfer/04-remote-strapi-providers/_category_.json diff --git a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/00-overview.md b/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/00-overview.md deleted file mode 100644 index ccc1428f00..0000000000 --- a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/00-overview.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: File Providers -tags: - - experimental - - providers - - import - - export - - data-transfer ---- - -# Strapi Data File Providers - -**TODO** diff --git a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/01-file-structure.md b/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/01-file-structure.md deleted file mode 100644 index 9f838bc403..0000000000 --- a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/01-file-structure.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Strapi File -tags: - - providers - - data-transfer - - experimental ---- - -# Strapi File Structure - -**TODO** diff --git a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/02-source.md b/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/02-source.md deleted file mode 100644 index 2b664c07ed..0000000000 --- a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/02-source.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Strapi File -tags: - - providers - - data-transfer - - experimental ---- - -# Strapi File Source Provider - -This provider will output a Strapi Data File - -**TODO** diff --git a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/03-destination.md b/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/03-destination.md deleted file mode 100644 index 9398ac4053..0000000000 --- a/docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/03-destination.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Strapi File -tags: - - providers - - data-transfer - - experimental ---- - -# Strapi File Destination Provider - -This provider will output a Strapi Data File - -Note: this destination provider does not provide a schema or metadata, and will therefore never report a schema match error or version validation error - -**TODO** diff --git a/packages/core/data-transfer/src/file/providers/destination/index.ts b/packages/core/data-transfer/src/file/providers/destination/index.ts index 1f6ebd7ebd..37e93ca53f 100644 --- a/packages/core/data-transfer/src/file/providers/destination/index.ts +++ b/packages/core/data-transfer/src/file/providers/destination/index.ts @@ -20,18 +20,18 @@ import { ProviderTransferError } from '../../../errors/providers'; export interface ILocalFileDestinationProviderOptions { encryption: { - enabled: boolean; - key?: string; + enabled: boolean; // if the file should be decrypted + key?: string; // the key to use when enabled is true }; compression: { - enabled: boolean; + enabled: boolean; // if the file should be compressed with gzip }; file: { - path: string; - maxSize?: number; - maxSizeJsonl?: number; + path: string; // the filename to create + maxSize?: number; // the max size of a single backup file + maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file }; } diff --git a/packages/core/data-transfer/src/file/providers/source/index.ts b/packages/core/data-transfer/src/file/providers/source/index.ts index 80c993f1cc..18d46ebdf4 100644 --- a/packages/core/data-transfer/src/file/providers/source/index.ts +++ b/packages/core/data-transfer/src/file/providers/source/index.ts @@ -29,16 +29,16 @@ const METADATA_FILE_PATH = 'metadata.json'; */ export interface ILocalFileSourceProviderOptions { file: { - path: string; + path: string; // the file to load }; encryption: { - enabled: boolean; - key?: string; + enabled: boolean; // if the file is encrypted (and should be decrypted) + key?: string; // the key to decrypt the file }; compression: { - enabled: boolean; + enabled: boolean; // if the file in compressed (and should be decompressed) }; }