add file providers docs

docs/docs/docs/01-core/data-transfer/01-providers/00-overview.md

@@ -1,14 +1,14 @@
 ---
-title: Destination Providers
+title: Data Transfer Providers
 tags:
   - providers
   - data-transfer
   - experimental
 ---
 
-# Data Providers
+# Data Transfer Providers
 
-Data providers are the interfaces for streaming data during a transfer.
+Data transfer providers are the interfaces for streaming data during a transfer.
 
 [Source providers](./01-source-providers.md) provide read streams for each stage in the transfer.
 
@@ -20,6 +20,8 @@ Strapi provides both source and destination providers for the following:
 - [Local Strapi](../04-local-strapi-providers/00-overview.md): a connection to a local Strapi project which uses its configured database connection to manage data
 - [Remote Strapi](../05-remote-strapi-providers/00-overview.md): a wrapper of local Strapi provider that adds a websocket interface to a running remote (network) instance of Strapi
 
+Each provider must provide the same interface for transferring data, but will usually include its own unique set of options to be passed in when initializing the provider.
+
 ## Creating your own providers
 
 To create your own providers, you must implement the interface(s) defined in ISourceProvider and IDestinationProvider found in `packages/core/data-transfer/types/providers.d.ts`.

docs/docs/docs/01-core/data-transfer/01-providers/_category_.json

@@ -1,5 +1,5 @@
 {
-  "label": "Source Providers",
+  "label": "Data Transfer Providers",
   "collapsible": true,
   "collapsed": true
 }

docs/docs/docs/01-core/data-transfer/02-file-providers/00-overview.md

@@ -0,0 +1,15 @@
+---
+title: File Providers
+tags:
+  - experimental
+  - providers
+  - import
+  - export
+  - data-transfer
+---
+
+# Strapi Data File Providers
+
+Strapi data file providers transfer data to or from a [Strapi Data File](./01-file-structure.md).
+
+The files are optionally compressed and/or encrypted using a given key (password).

docs/docs/docs/01-core/data-transfer/02-file-providers/01-file-structure.md

@@ -0,0 +1,57 @@
+---
+title: Strapi File Structure
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi File Structure
+
+The Strapi file providers expect a .tar file (optionally compressed with gzip and/or encrypted with 'aes-128-ecb') that internally uses POSIX style file paths with the following structure:
+
+```
+./
+configuration
+entities
+links
+metadata.json
+schemas
+
+./configuration:
+configuration_00001.jsonl
+
+./entities:
+entities_00001.jsonl
+
+./links:
+links_00001.jsonl
+
+./schemas:
+schemas_00001.jsonl
+```
+
+## metadata.json
+
+This file provides metadata about the original source of the data. At minimum, it should include a createdAt timestamp and the version of Strapi that the file was created with (for compatibility checks).
+
+```json
+{
+  "createdAt": "2023-06-26T07:31:20.062Z",
+  "strapi": {
+    "version": "4.11.2"
+  }
+}
+```
+
+## A directory for each stage of data
+
+There should also be a directory for each stage of data that includes sequentially numbered JSON Lines (.jsonl) files
+
+The files are named in the format: `{stage}\{stage}_{5-digit sequence number}.jsonl`
+
+Any number of files may be provided for each stage, as long as the sequence numbers are in order. That is, after first reading 00001, the file source provider will attempt to read file 00002 and if it is not found, it will consider the stage complete.
+
+### JSONL files
+
+JSON Lines files are essentially JSON files, except that newline characters are used to delimit the JSON objects. This allows the provider to read in a single line at a time, rather than loading the entire file into memory, minimizing RAM usage during a transfer and allowing files containing any amount of data.

docs/docs/docs/01-core/data-transfer/02-file-providers/02-source.md

@@ -0,0 +1,35 @@
+---
+title: Strapi File
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi File Source Provider
+
+This provider will open and read a Strapi Data File as a data source.
+
+## Provider Options
+
+The accepted options are defined in `ILocalFileSourceProviderOptions`.
+
+```typescript
+  file: {
+    path: string; // the file to load
+  };
+
+  encryption: {
+    enabled: boolean; // if the file is encrypted (and should be decrypted)
+    key?: string; // the key to decrypt the file
+  };
+
+  compression: {
+    enabled: boolean; // if the file in compressed (and should be decompressed)
+  };
+```
+
+Note: When the Strapi CLI attempts to import a file, the options for compression and encryption are set based on the extension of the file being loaded, eg a file with the .gz extension will have the "compress" option set, and a file that includes the .enc extension will have the "encrypt" option set.
+
+When using the transfer engine programmatically, you may make the determination whether the file being loaded should be decrypted or compressed by setting
+those options.

docs/docs/docs/01-core/data-transfer/02-file-providers/03-destination.md

@@ -0,0 +1,34 @@
+---
+title: Strapi File
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi File Destination Provider
+
+This provider will output a Strapi Data File.
+
+Note: this destination provider does not provide a schema or metadata, and will therefore never report a schema match error or version validation error
+
+## Provider Options
+
+The accepted options are defined in `ILocalFileDestinationProviderOptions`.
+
+```typescript
+  encryption: {
+    enabled: boolean; // if the file should be decrypted
+    key?: string; // the key to use when enabled is true
+  };
+
+  compression: {
+    enabled: boolean; // if the file should be compressed with gzip
+  };
+
+  file: {
+    path: string; // the filename to create
+    maxSize?: number; // the max size of a single backup file
+    maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file
+  };
+```

docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/00-overview.md

@@ -1,13 +0,0 @@
----
-title: File Providers
-tags:
-  - experimental
-  - providers
-  - import
-  - export
-  - data-transfer
----
-
-# Strapi Data File Providers
-
-**TODO**

docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/01-file-structure.md

@@ -1,11 +0,0 @@
----
-title: Strapi File
-tags:
-  - providers
-  - data-transfer
-  - experimental
----
-
-# Strapi File Structure
-
-**TODO**

docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/02-source.md

@@ -1,13 +0,0 @@
----
-title: Strapi File
-tags:
-  - providers
-  - data-transfer
-  - experimental
----
-
-# Strapi File Source Provider
-
-This provider will output a Strapi Data File
-
-**TODO**

docs/docs/docs/01-core/data-transfer/05-remote-strapi-providers/03-destination.md

@@ -1,15 +0,0 @@
----
-title: Strapi File
-tags:
-  - providers
-  - data-transfer
-  - experimental
----
-
-# Strapi File Destination Provider
-
-This provider will output a Strapi Data File
-
-Note: this destination provider does not provide a schema or metadata, and will therefore never report a schema match error or version validation error
-
-**TODO**

packages/core/data-transfer/src/file/providers/destination/index.ts

@@ -20,18 +20,18 @@ import { ProviderTransferError } from '../../../errors/providers';
 
 export interface ILocalFileDestinationProviderOptions {
   encryption: {
-    enabled: boolean;
-    key?: string;
+    enabled: boolean; // if the file should be decrypted
+    key?: string; // the key to use when enabled is true
   };
 
   compression: {
-    enabled: boolean;
+    enabled: boolean; // if the file should be compressed with gzip
   };
 
   file: {
-    path: string;
-    maxSize?: number;
-    maxSizeJsonl?: number;
+    path: string; // the filename to create
+    maxSize?: number; // the max size of a single backup file
+    maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file
   };
 }
 

packages/core/data-transfer/src/file/providers/source/index.ts

@@ -29,16 +29,16 @@ const METADATA_FILE_PATH = 'metadata.json';
  */
 export interface ILocalFileSourceProviderOptions {
   file: {
-    path: string;
+    path: string; // the file to load
   };
 
   encryption: {
-    enabled: boolean;
-    key?: string;
+    enabled: boolean; // if the file is encrypted (and should be decrypted)
+    key?: string; // the key to decrypt the file
   };
 
   compression: {
-    enabled: boolean;
+    enabled: boolean; // if the file in compressed (and should be decompressed)
   };
 }