Merge pull request #16953 from strapi/chore/data-transfer-dev-docs

docs/docs/docs/01-core/data-transfer/00-intro.md

@@ -0,0 +1,19 @@
+---
+title: Introduction
+tags:
+  - data-transfer
+  - experimental
+---
+
+# Data Transfer
+
+This section is an overview of all the features related to the data-transfer package:
+
+```mdx-code-block
+import DocCardList from '@theme/DocCardList';
+import { useCurrentSidebarCategory } from '@docusaurus/theme-common';
+
+<DocCardList items={useCurrentSidebarCategory().items} />
+```
+
+Note: The data-transfer package is written in Typescript and any additions or changes must include all necessary typings.

docs/docs/docs/01-core/data-transfer/01-engine.md

@@ -0,0 +1,380 @@
+---
+title: Transfer Engine
+description: Conceptual guide to the data transfer engine
+tags:
+  - data-transfer
+  - experimental
+---
+
+The transfer engine manages the data transfer process by facilitating communication between a source provider and a destination provider.
+
+## Code location
+
+`packages/core/data-transfer/src/engine/index.ts`
+
+## The transfer process
+
+A transfer starts by bootstrapping and initializing itself and the providers. That is the stage where providers attempt to make any necessary connections to files, databases, websockets, etc.
+
+After that, the integrity check between the source and destination is run, which validates the requirements set by the chosen schemaStrategy and versionStrategy.
+
+Note: Schema differences during this stage can be resolved programmatically by adding an `onSchemaDiff` handler. However, be aware that this interface is likely to change to a more generic engine handler (such as `engine.on('schemaDiff', handler)`) before this feature is stable.
+
+Once the integrity check has passed, the transfer begins by opening streams from the source to the destination one stage at a time. The following is a list of the stages in the order they are run:
+
+1. schemas - content type schemas. Note: with all built-in Strapi destination providers, only the Strapi file provider makes use of this data
+2. entities - all entities (including components, dynamic zones, and media data but not media files) _without their relations_
+3. assets - the files from the /uploads folder
+4. links - the relations between entities
+5. configuration - the Strapi project configuration data
+
+Once all stages have been completed, the transfer waits for all providers to close and then emits a finish event and the transfer completes.
+
+## Setting up the transfer engine
+
+A transfer engine object is created by using `createTransferEngine`, which accepts a [source provider](./02-providers/01-source-providers.md), a [destination provider](./02-providers/02-destination-providers.md), and an options object.
+
+Note: By default, a transfer engine will transfer ALL data, including admin data, api tokens, etc. Transform filters must be used if you wish to exclude, as seen in the example below. An array called `DEFAULT_IGNORED_CONTENT_TYPES` is available from @strapi/data-transfer containing the uids that are excluded by default from the import, export, and transfer commands. If you intend to transfer admin data, be aware that this behavior will likely change in the future to automatically exclude the entire `admin::` uid namespace and will instead require them to be explicitly included.
+
+```typescript
+const engine = createTransferEngine(source, destination, options);
+```
+
+### Engine Options
+
+An example using every available option:
+
+```typescript
+const options = {
+  versionStrategy: 'ignore', // see versionStragy documentation
+  schemaStrategy: 'strict', // see schemaStragey documentation
+  exclude: [], // exclude these classifications of data; see CLI documentation of `--exclude` for list
+  only: [], // transfer only these classifications of data; see CLI documentation of `--only` for list
+  throttle: 0, // add a delay of this many millseconds between each item transferred
+
+  // the keys of `transforms` are the stage names for which they are run
+  transforms: {
+    links: [
+      {
+        // exclude all relations to ignored content types
+        filter(link) {
+          return (
+            !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) &&
+            !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type)
+          );
+        },
+      },
+      // Note: map exists for links but is not recommended
+    ],
+    entities: [
+      {
+        // exclude all ignored content types
+        filter(entity) {
+          return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type);
+        },
+      },
+      {
+        map(entity) {
+          // remove somePrivateField from privateThing entities
+          if (entity.type === 'api::privateThing.privateThing') {
+            entity.somePrivateField = undefined;
+          }
+
+          return entity;
+        },
+      },
+    ],
+  },
+};
+```
+
+#### versionStrategy
+
+The following `versionStrategy` values may be used:
+
+`'ignore'` - allow transfer between any versions of Strapi
+`'exact'` - require an exact version match (including tags such as -alpha and -beta)
+`'major'` - require only the semver major version to match (but allow minor, patch, and tag to differ)
+`'minor'` - require only the semver major and minor versions to match (but allow patch to differ)
+`'patch'` - require only the semver major, minor, and patch version to match (but allow tag differences such as -alpha and -beta)
+
+The default `versionStrategy` used when one is not provided is `'ignore'`.
+
+#### schemaStrategy
+
+The follow `schemaStrategy` values may be used:
+
+`'ignore'` - bypass schema validation (transfer will attempt to run but throw errors on incompatible data type inserts)
+`'strict'` - disallow mismatches that are expected to cause errors in the transfer, but allow certain non-data fields in the schema to differ
+`'exact'` - schema must be identical with no changes
+
+Note: The "strict" schema strategy is defined as "anything expected to cause errors in the transfer" and is the default method for the import, export, and transfer CLI commands. Therefore, the technical functionality will always be subject to change. If you need to find the definition for the current version of Strapi, see `packages/core/data-transfer/src/engine/validation/schemas/index.ts`
+
+The default `schemaStrategy` used when one is not provided is `'strict'`.
+
+##### Handling Schema differences
+
+When a schema diff is discovered with a given schemaStrategy, an error is throw. However, before throwing the error the engine checks to see if there are any schema diff handlers set via `engine.onSchemaDiff(handler)` which allows errors to be bypassed (for example, by prompting the user if they wish to proceed).
+
+A diff handler is an optionally asynchronous middleware function that accepts a `context` and a `next` parameter.
+
+`context` is an object of type `SchemaDiffHandlerContext`
+
+```typescript
+// type Diff can be found in /packages/core/data-transfer/src/utils/json.ts
+type SchemaDiffHandlerContext = {
+  ignoredDiffs: Record<string, Diff[]>;
+  diffs: Record<string, Diff[]>;
+  source: ISourceProvider;
+  destination: IDestinationProvider;
+};
+```
+
+`next` is a function that is called, passing the modified `context` object, to proceed to the next middleware function.
+
+```typescript
+const diffHandler = async (context, next) => {
+  const ignoreThese = {};
+  // loop through the diffs
+  Object.entries(context.diffs).forEach(([uid, diffs]) => {
+    for (const [i, diff] of diffs) {
+      // get the path of the diff in the schema
+      const path = [uid].concat(diff.path).join('.');
+
+      // Allow a diff on the country schema displayName
+      if (path === 'api::country.country.info.displayName') {
+        if (!isArray(context.ignoredDiffs[uid])) {
+          context.ignoredDiffs[uid] = [];
+        }
+        context.ignoredDiffs[uid][i] = diff;
+      }
+    }
+  });
+
+  return next(context);
+};
+
+engine.onSchemaDiff(diffHandler);
+```
+
+After all the schemaDiffHandler middlewares have been run, another diff is run between `context.ignoredDiffs` and `context.diffs` and any remaining diffs that have not been ignored are thrown as fatal errors and the engine will abort the transfer.
+
+### Progress Tracking events
+
+The transfer engine allows tracking the progress of your transfer either directly with the engine.progress.data object, or with listeners using the engine.progress.stream PassThrough stream. The engine.progress.data object definition is of type TransferProgress.
+
+Here is an example that logs a message at the beginning and end of each stage, as well as a message after each item has been transferred
+
+```typescript
+const progress = engine.progress.stream;
+
+progress.on(`stage::start`, ({ stage, data }) => {
+  console.log(`${stage} has started at ${data[stage].startTime}`);
+});
+
+progress.on('stage::finish', ({ stage, data }) => {
+  console.log(`${stage} has finished at ${data[stage].endTime}`);
+});
+
+progress.on('stage::progress', ({ stage, data }) => {
+  console.log('Transferred ${data[stage].bytes} bytes / ${data[stage].count} entities');
+});
+```
+
+Note: There is currently no way for a source provider to give a "total" number of records expected to be transferred, but it is expected in a future update.
+
+The following events are available:
+
+`stage::start` - at the start of each stage
+`stage::finish` - at the end of each stage
+`stage::progress` - after each entitity in that stage has been transferred
+`stage::skip` - when an entire stage is skipped (eg, when 'only' or 'exclude' are used)
+`stage::error` - when there is an error thrown during a stage
+`transfer::init` - at the very beginning of engine.transfer()
+`transfer::start` - after bootstrapping and initializing the providers, when the transfer is about to start
+`transfer::finish` - when the transfer has finished
+`transfer::error` - when there is an error thrown during the transfer
+
+### Diagnostics events
+
+The engine includes a diagnostics reporter which can be used to listen for diagnostics information (debug messages, errors, etc).
+
+Here is an example for creating a diagnostics listener:
+
+```typescript
+// listener function
+const diagnosticListener: DiagnosticListener = (data: GenericDiagnostic) => {
+  // handle the diagnostics event, for example with custom logging
+};
+
+// add a generic listener
+engine.diagnostics.onDiagnostic(diagnosticsListener);
+
+// add an error listener
+engine.diagnostics.on('error', diagnosticListener);
+
+// add a warning listener
+engine.diagnostics.on('warning', diagnosticListener);
+```
+
+To emit your own diagnostics event:
+
+```typescript
+const event: ErrorDiagnostic = {
+  kind: 'error',
+  details: {
+    message: 'Your diagnostics message'
+    createdAt: new Date(),
+  },
+  name: 'yourError',
+  severity: 'fatal',
+  error: new Error('your error message')
+}
+
+engine.diagnostics.report(event);
+```
+
+Here is an excerpt of the relevant types used in the previous examples:
+
+```typescript
+// engine/diagnostic.ts
+// format of the data sent to the listener
+export type GenericDiagnostic<K extends DiagnosticKind, T = unknown> = {
+  kind: K;
+  details: {
+    message: string;
+    createdAt: Date;
+  } & T;
+};
+
+export type DiagnosticKind = 'error' | 'warning' | 'info';
+
+export type Diagnostic = ErrorDiagnostic | WarningDiagnostic | InfoDiagnostic;
+
+export type ErrorDiagnosticSeverity = 'fatal' | 'error' | 'silly';
+
+export type ErrorDiagnostic = GenericDiagnostic<
+  'error',
+  {
+    name: string;
+    severity: ErrorDiagnosticSeverity;
+    error: Error;
+  }
+>;
+
+export type WarningDiagnostic = GenericDiagnostic<
+  'warning',
+  {
+    origin?: string;
+  }
+>;
+
+export type InfoDiagnostic<T = unknown> = GenericDiagnostic<
+  'info',
+  {
+    params?: T;
+  }
+>;
+```
+
+### Transforms
+
+Transforms allow you to manipulate the data that is sent from the source before it reaches the destination.
+
+## Filter (excluding data)
+
+Filters can be used to exclude data sent from the source before it is streamed to the destination. They are methods that accept an entity, link, schema, etc and return `true` to keep the entity and `false` to remove it.
+
+Here is an example that filters out all entities with an id higher than 100:
+
+```typescript
+const options = {
+  ...otherOptions,
+  transforms: {
+    entities: [
+      {
+        // exclude all ignored admin content types
+        filter(entity) {
+          return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type);
+        },
+      },
+      {
+        // exclude all entities with an id higher than 100
+        filter(entity) {
+          return Number(entity.id) <= 100;
+        },
+      },
+    ],
+    links: [
+      {
+        // exclude all relations to ignored content types
+        filter(link) {
+          return (
+            !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) &&
+            !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type)
+          );
+        },
+      },
+      {
+        // remember to exclude links as well or else an error will be thrown when attempting to link an entity we filtered
+        filter(entity) {
+          return Number(link.left.id) <= 100 || Number(link.right.id) <= 100)
+        },
+      },
+    ],
+  },
+};
+```
+
+## Map (modifying data)
+
+Maps can be used to modify data sent from the source before it is streamed to the destination. They are methods that accept an entity, link, schema, etc and return the modified version of the object.
+
+This can be used, for example, to sanitize data between environments.
+
+Here is an example that removes a field called `somePrivateField` from a content type `privateThing`.
+
+```typescript
+const options = {
+  ...otherOptions,
+  transforms: {
+    entities: [
+      {
+        // exclude all ignored content types
+        filter(entity) {
+          return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type);
+        },
+      },
+      {
+        map(entity) {
+          // remove somePrivateField from privateThing entities
+          if (entity.type === 'api::privateThing.privateThing') {
+            entity.somePrivateField = undefined;
+          }
+
+          return entity;
+        },
+      },
+    ],
+  },
+};
+```
+
+By mapping schemas as well as entities, it's even possible (although complex!) to modify data structures between source and destination.
+
+## Running a transfer
+
+Running a transfer simply involves calling the asynchrounous engine.transfer() method.
+
+```typescript
+const engine = createTransferEngine(source, destination, options);
+try {
+  await engine.transfer();
+} catch (e) {
+  console.error('Something went wrong: ', e?.message);
+}
+```
+
+Be aware that engine.transfer() throws on any fatal errors it encounters.
+
+Note: The transfer engine (and the providers) current only support a single `engine.transfer()` and must be re-instantiated if intended to run multiple times. In the future it is expected to allow them to be used for multiple transfers in a row, but that usage is untested and will result in unpredictable behavior.

docs/docs/docs/01-core/data-transfer/02-providers/00-overview.md

@@ -0,0 +1,35 @@
+---
+title: Introduction
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Data Transfer Providers
+
+Data transfer providers are the interfaces for streaming data during a transfer.
+
+[Source providers](./01-source-providers.md) provide read streams for each stage in the transfer.
+
+[Destination providers](./02-destination-providers.md) provide write streams for each stage in the transfer.
+
+Strapi provides both source and destination providers for the following:
+
+- [Strapi file](./03-strapi-file/00-overview.md): a standardized file format designed for the transfer process
+- [Local Strapi](./04-local-strapi/00-overview.md): a connection to a local Strapi project which uses its configured database connection to manage data
+- [Remote Strapi](./05-remote-strapi/00-overview.md): a wrapper of local Strapi provider that adds a websocket interface to a running remote (network) instance of Strapi
+
+Each provider must provide the same interface for transferring data, but will usually include its own unique set of options to be passed in when initializing the provider.
+
+## Creating your own providers
+
+To create your own providers, you must implement the interface(s) defined in `ISourceProvider` and `IDestinationProvider` found in `packages/core/data-transfer/types/providers.d.ts`.
+
+It is not necessary to create both a source and destination provider, only the part necessary for your use.
+
+For examples, see the existing providers such as the local Strapi provider.
+
+## Asset Transfers
+
+Currently, all of the data-transfer providers only handle local media assets (the `/upload` folder). Provider media is currently in development. Therefore, everything related to asset transfers -- including Strapi file structure, restore strategy, and rollback for assets -- is currently treated as `unstable` and likely to change in the near future.

docs/docs/docs/01-core/data-transfer/02-providers/01-source-providers.md

@@ -0,0 +1,17 @@
+---
+title: Source Providers
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Source Providers
+
+## Source provider structure
+
+A source provider must implement the interface ISourceProvider found in `packages/core/data-transfer/types/providers.d.ts`.
+
+In short, it provides a set of create{_stage_}ReadStream() methods for each stage that provide a Readable stream, which will retrieve its data (ideally from its own stream) and then perform a `stream.write(entity)` for each entity, link (relation), asset (file), configuration entity, or content type schema depending on the stage.
+
+When each stage's stream has finished sending all the data, the stream must be closed before the transfer engine will continue to the next stage.

docs/docs/docs/01-core/data-transfer/02-providers/02-destination-providers.md

@@ -0,0 +1,15 @@
+---
+title: Destination Providers
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Destination Providers
+
+## Destination provider structure
+
+A destination provider must implement the interface IDestinationProvider found in `packages/core/data-transfer/types/providers.d.ts`.
+
+In short, it provides a set of create{_stage_}WriteStream() methods for each stage that provide a Writable stream, which will be passed each entity, link (relation), asset (file), configuration entity, or content type schema (depending on the stage) piped from the Readable source provider stream.

docs/docs/docs/01-core/data-transfer/02-providers/03-strapi-file/00-overview.md

@@ -0,0 +1,15 @@
+---
+title: Overview
+tags:
+  - experimental
+  - providers
+  - import
+  - export
+  - data-transfer
+---
+
+# Strapi Data File Providers
+
+Strapi data file providers transfer data to or from a [Strapi Data File](./01-file-structure.md).
+
+The files are optionally compressed and/or encrypted using a given key (password).

docs/docs/docs/01-core/data-transfer/02-providers/03-strapi-file/01-file-structure.md

@@ -0,0 +1,57 @@
+---
+title: Strapi File Structure
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi File Structure
+
+The Strapi file providers expect a .tar file (optionally compressed with gzip and/or encrypted with 'aes-128-ecb') that internally uses POSIX style file paths with the following structure:
+
+```
+./
+configuration
+entities
+links
+metadata.json
+schemas
+
+./configuration:
+configuration_00001.jsonl
+
+./entities:
+entities_00001.jsonl
+
+./links:
+links_00001.jsonl
+
+./schemas:
+schemas_00001.jsonl
+```
+
+## metadata.json
+
+This file provides metadata about the original source of the data. At minimum, it should include a createdAt timestamp and the version of Strapi that the file was created with (for compatibility checks).
+
+```json
+{
+  "createdAt": "2023-06-26T07:31:20.062Z",
+  "strapi": {
+    "version": "4.11.2"
+  }
+}
+```
+
+## A directory for each stage of data
+
+There should also be a directory for each stage of data that includes sequentially numbered JSON Lines (.jsonl) files
+
+The files are named in the format: `{stage}\{stage}_{5-digit sequence number}.jsonl`
+
+Any number of files may be provided for each stage, as long as the sequence numbers are in order. That is, after first reading 00001, the file source provider will attempt to read file 00002 and if it is not found, it will consider the stage complete.
+
+### JSONL files
+
+[JSON Lines](https://jsonlines.org/) files are essentially JSON files, except that newline characters are used to delimit the JSON objects. This allows the provider to read in a single line at a time, rather than loading the entire file into memory, minimizing RAM usage during a transfer and allowing files containing any amount of data.

docs/docs/docs/01-core/data-transfer/02-providers/03-strapi-file/02-source.md

@@ -0,0 +1,35 @@
+---
+title: Source
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi File Source Provider
+
+This provider will open and read a Strapi Data File as a data source.
+
+## Provider Options
+
+The accepted options are defined in `ILocalFileSourceProviderOptions`.
+
+```typescript
+  file: {
+    path: string; // the file to load
+  };
+
+  encryption: {
+    enabled: boolean; // if the file is encrypted (and should be decrypted)
+    key?: string; // the key to decrypt the file
+  };
+
+  compression: {
+    enabled: boolean; // if the file is compressed (and should be decompressed)
+  };
+```
+
+Note: When the Strapi CLI attempts to import a file, the options for compression and encryption are set based on the extension of the file being loaded, eg a file with the .gz extension will have the "compress" option set, and a file that includes the .enc extension will have the "encrypt" option set.
+
+When using the transfer engine programmatically, you may make the determination whether the file being loaded should be decrypted or compressed by setting
+those options.

docs/docs/docs/01-core/data-transfer/02-providers/03-strapi-file/03-destination.md

@@ -0,0 +1,34 @@
+---
+title: Destination
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi File Destination Provider
+
+This provider will output a Strapi Data File.
+
+Note: this destination provider does not provide a schema or metadata, and will therefore never report a schema match error or version validation error
+
+## Provider Options
+
+The accepted options are defined in `ILocalFileDestinationProviderOptions`.
+
+```typescript
+  encryption: {
+    enabled: boolean; // if the file should be encrypted
+    key?: string; // the key to use when encryption.enabled is true
+  };
+
+  compression: {
+    enabled: boolean; // if the file should be compressed with gzip
+  };
+
+  file: {
+    path: string; // the filename to create
+    maxSize?: number; // the max size of a single backup file
+    maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file
+  };
+```

docs/docs/docs/01-core/data-transfer/02-providers/03-strapi-file/_category_.json

@@ -0,0 +1,5 @@
+{
+  "label": "File Providers",
+  "collapsible": true,
+  "collapsed": true
+}

docs/docs/docs/01-core/data-transfer/02-providers/04-local-strapi/00-overview.md

@@ -0,0 +1,17 @@
+---
+title: Overview
+tags:
+  - experimental
+  - providers
+  - import
+  - export
+  - data-transfer
+---
+
+# Local Strapi Providers
+
+The local Strapi provider allows using the local Strapi instance (the same project that the data transfer engine is being run from) as a data source.
+
+Creating a local Strapi data provider requires passing in an initialized `strapi` server object to interact with that server's Entity Service and Query Engine to manage the data. Therefore if the local Strapi project cannot be started (due to errors), the providers cannot be used.
+
+**Important**: When a transfer completes, the `strapi` object passed in is shut down automatically based on the `autoDestroy` option. If you are running a transfer via an external script, it is recommended to use `autoDestroy: true` to ensure it is shut down properly, but if you are running a transfer within a currently running Strapi instance you should set `autoDestroy: false` or your Strapi instance will be shut down at the end of the transfer.

docs/docs/docs/01-core/data-transfer/02-providers/04-local-strapi/01-source.md

@@ -0,0 +1,21 @@
+---
+title: Source
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Local Strapi Source Provider
+
+This provider will retrieve data from an initialized `strapi` instance using its Entity Service and Query Engine.
+
+## Provider Options
+
+The accepted options are defined in `ILocalFileSourceProviderOptions`.
+
+```typescript
+  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>; // return an initialized instance of Strapi
+
+  autoDestroy?: boolean; // shut down the instance returned by getStrapi() at the end of the transfer
+```

docs/docs/docs/01-core/data-transfer/02-providers/04-local-strapi/02-destination.md

@@ -0,0 +1,56 @@
+---
+title: Destination
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Local Strapi Destination Provider
+
+This provider will insert data into an initialized `strapi` instance using its Entity Service and Query Engine.
+
+## Provider Options
+
+The accepted options are defined in `ILocalFileSourceProviderOptions`.
+
+```typescript
+  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>; // return an initialized instance of Strapi
+  autoDestroy?: boolean; // shut down the instance returned by getStrapi() at the end of the transfer
+  restore?: restore.IRestoreOptions; // the options to use when strategy is 'restore'
+  strategy: 'restore'; // conflict management strategy; only the restore strategy is available at this time
+```
+
+`strategy` defines the conflict management strategy used. Currently, only `"restore"` is available as an option.
+
+### Restore
+
+A conflict management strategy of "restore" deletes all existing Strapi data before a transfer to avoid any conflicts.
+
+The following restore options are available:
+
+```typecript
+export interface IRestoreOptions {
+  assets?: boolean; // delete media library files before transfer
+  configuration?: {
+    webhook?: boolean; // delete webhooks before transfer
+    coreStore?: boolean; // delete core store before transfer
+  };
+  entities?: {
+    include?: string[]; // only delete these stage entities before transfer
+    exclude?: string[]; // exclude these stage entities from deletion
+    filters?: ((contentType: ContentTypeSchema) => boolean)[]; // custom filters to exclude a content type from deletion
+    params?: { [uid: string]: unknown }; // params object passed to deleteMany before transfer for custom deletions
+  };
+}
+```
+
+### Rollbacks
+
+This local Strapi destination provider automatically provides a rollback mechanism on error.
+
+For Strapi data, that is done with a database transaction wrapped around the restore and the insertion of data and committing on succes and rolling back on failure.
+
+For Strapi assets (ie, the media library files) this is done by attempting to temporarily move the existing assets to a backup directory to `uploads_backup_{timestamp}`, and then deleting it on success, or deleting the failed import files and putting the backup back into place on failure. In some cases of failure, it may be impossible to move the backup files back into place, so you will need to manually restore the backup assets files.
+
+Note: Because of the need for write access, environments without filesystem permissions to move the assets folder (common for virtual environments where /uploads is mounted as a read-only drive) will be unable to include assets in a transfer and the asset stage must be excluded in order to run the transfer.

docs/docs/docs/01-core/data-transfer/02-providers/04-local-strapi/_category_.json

@@ -0,0 +1,5 @@
+{
+  "label": "Local Strapi Providers",
+  "collapsible": true,
+  "collapsed": true
+}

docs/docs/docs/01-core/data-transfer/02-providers/05-remote-strapi/00-overview.md

@@ -0,0 +1,21 @@
+---
+title: Overview
+tags:
+  - experimental
+  - providers
+  - import
+  - export
+  - data-transfer
+---
+
+# Remote Strapi Providers
+
+Remote Strapi providers connect to an instance of Strapi over a network using a websocket.
+
+Internally, the remote Strapi providers map websocket requests to a local Strapi provider of the instance it is running in.
+
+In order to use remote transfer providers, the remote Strapi server must have a value for `transfer.token.salt` configured in `config/admin.js` and the remote transfer feature must not be disabled.
+
+## Disabling Remote Transfers
+
+If desired, the remote transfer feature of a server can be completely disabled by setting the environment variable `STRAPI_DISABLE_REMOTE_DATA_TRANSFER` to true.

docs/docs/docs/01-core/data-transfer/02-providers/05-remote-strapi/01-websocket.md

@@ -0,0 +1,59 @@
+---
+title: Websocket
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Remote Provider Websocket
+
+When the data transfer feature is enabled for a Strapi server (an `admin.transfer.token.salt` config value has been set and `STRAPI_DISABLE_REMOTE_DATA_TRANSFER` is not set to true), Strapi will create websocket servers available on the routes `/admin/transfer/runner/pull` and `/admin/transfer/runner/push`.
+
+Opening a websocket connection on those routes requires a valid transfer token as a bearer token in the Authorization header.
+
+Please see the `bootstrap()` method of the remote providers for an example of how to make the initial connection to the Strapi websocket.
+
+## Websocket Messages / Dispatcher
+
+The remote websocket server only accepts specific websocket messages which we call transfer commands. These commands must also be sent in a specific order, and an error messages will be returned if an unexpected message is received by the server.
+
+A message dispatcher object should be created to send messages to the server. See `packages/core/data-transfer/src/strapi/providers/utils.ts` for more inofrmation on the dispatcher.
+
+The dispatcher includes
+
+### dispatchCommand
+
+Accepts "commands" used for opening and closing a transfer.
+
+Allows the following `command` values:
+
+- `init`: for initializing a connection. Returns a transferID that must be sent with all future messages in this transfer
+- `end`: for ending a connection
+
+### dispatchTransferStep
+
+Used for switching between stages of a transfer and streaming the actual data of a transfer.
+
+Accepts the following `action` values:
+
+- `start`: sent with a `step` value for the name of the step/stage
+  - any number of `stream`: sent with a `step` value and the `data` being sent (ie, an entity)
+- `end`: sent with a `step` value for the step being ended
+
+### dispatchTransferAction
+
+Used for triggering 'actions' on the server equivalent to the local providers.
+
+- `bootstrap`
+- `getMetadata`
+- `beforeTransfer`
+- `getSchemas`
+- `rollback` (destination only)
+- `close`: for completing a transfer (but doesn't close the connection)
+
+See `packages/core/data-transfer/dist/strapi/remote/handlers/push.d.ts` and `packages/core/data-transfer/dist/strapi/remote/handlers/push.d.ts` for complete and precise definitions of the messages that must be sent.
+
+## Message Timeouts and Retries
+
+Because the transfer relies on a message->response protocol, if the websocket server is unable to send a reply, for example due to network instability, the connection would halt. For this reason, each provider's options includes `retryMessageOptions` which attempt to resend a message after a given timeout is reached and a max retry option to abort the transfer after a given number of failed retry attempts.

docs/docs/docs/01-core/data-transfer/02-providers/05-remote-strapi/02-source.md

@@ -0,0 +1,34 @@
+---
+title: Source
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi Remote Source Provider
+
+The Strapi remote source provider connects to a remote Strapi websocket server and sends messages to move between stages and pull data.
+
+## Provider Options
+
+The remote source provider accepts `url`, `auth`, and `retryMessageOptions` described below.
+
+```typescript
+interface ITransferTokenAuth {
+  type: 'token';
+  token: string;
+}
+
+export interface IRemoteStrapiDestinationProviderOptions
+  extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
+  url: URL;
+  auth?: ITransferTokenAuth;
+  retryMessageOptions?: {
+    retryMessageTimeout: number; // milliseconds to wait for a response from a message
+    retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
+  };
+}
+```
+
+Note: `url` must include the protocol `https` or `http` which will then be converted to `wss` or `ws` to make the connection. A secure connection is strongly recommended, especially given the high access level that the transfer token provides.

docs/docs/docs/01-core/data-transfer/02-providers/05-remote-strapi/03-destination.md

@@ -0,0 +1,34 @@
+---
+title: Destination
+tags:
+  - providers
+  - data-transfer
+  - experimental
+---
+
+# Strapi Remote Destination Provider
+
+The Strapi remote destination provider connects to a remote Strapi websocket server and sends messages to move between stages and push data.
+
+## Provider Options
+
+The remote destination provider accepts the same `restore` and `strategy` options from local Strapi destination provider, plus `url`, `auth`, and `retryMessageOptions` described below.
+
+```typescript
+interface ITransferTokenAuth {
+  type: 'token'; // the name of the auth strategy
+  token: string; // the transfer token
+}
+
+export interface IRemoteStrapiDestinationProviderOptions
+  extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
+  url: URL; // the url of the remote Strapi admin
+  auth?: ITransferTokenAuth;
+  retryMessageOptions?: {
+    retryMessageTimeout: number; // milliseconds to wait for a response from a message
+    retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
+  };
+}
+```
+
+Note: `url` must include the protocol `https` or `http` which will then be converted to `wss` or `ws` to make the connection. A secure connection is strongly recommended, especially given the high access level that the transfer token provides.

docs/docs/docs/01-core/data-transfer/02-providers/05-remote-strapi/_category_.json

@@ -0,0 +1,5 @@
+{
+  "label": "Remote Strapi Providers",
+  "collapsible": true,
+  "collapsed": true
+}

docs/docs/docs/01-core/data-transfer/02-providers/_category_.json

@@ -0,0 +1,5 @@
+{
+  "label": "Providers",
+  "collapsible": true,
+  "collapsed": true
+}

docs/docs/docs/01-core/data-transfer/_category_.json

@@ -0,0 +1,5 @@
+{
+  "label": "Data Transfer",
+  "collapsible": true,
+  "collapsed": true
+}

packages/core/data-transfer/src/engine/diagnostic.ts

@@ -15,7 +15,7 @@ export type GenericDiagnostic<K extends DiagnosticKind, T = unknown> = {
 export type DiagnosticKind = 'error' | 'warning' | 'info';
 
 export type DiagnosticListener<T extends DiagnosticKind = DiagnosticKind> = (
-  diagnostic: { kind: T } & Diagnostic extends infer U ? U : 'foo'
+  diagnostic: { kind: T } & Diagnostic extends infer U ? U : never
 ) => void | Promise<void>;
 
 export type DiagnosticEvent = 'diagnostic' | `diagnostic.${DiagnosticKind}`;

packages/core/data-transfer/src/file/providers/destination/index.ts

@@ -20,18 +20,18 @@ import { ProviderTransferError } from '../../../errors/providers';
 
 export interface ILocalFileDestinationProviderOptions {
   encryption: {
-    enabled: boolean;
-    key?: string;
+    enabled: boolean; // if the file should be encrypted
+    key?: string; // the key to use when encryption.enabled is true
   };
 
   compression: {
-    enabled: boolean;
+    enabled: boolean; // if the file should be compressed with gzip
   };
 
   file: {
-    path: string;
-    maxSize?: number;
-    maxSizeJsonl?: number;
+    path: string; // the filename to create
+    maxSize?: number; // the max size of a single backup file
+    maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file
   };
 }
 

packages/core/data-transfer/src/file/providers/source/index.ts

@@ -29,16 +29,16 @@ const METADATA_FILE_PATH = 'metadata.json';
  */
 export interface ILocalFileSourceProviderOptions {
   file: {
-    path: string;
+    path: string; // the file to load
   };
 
   encryption: {
-    enabled: boolean;
-    key?: string;
+    enabled: boolean; // if the file is encrypted (and should be decrypted)
+    key?: string; // the key to decrypt the file
   };
 
   compression: {
-    enabled: boolean;
+    enabled: boolean; // if the file is compressed (and should be decompressed)
   };
 }
 

packages/core/data-transfer/src/strapi/providers/local-destination/index.ts

@@ -18,10 +18,11 @@ export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge'];
 export const DEFAULT_CONFLICT_STRATEGY = 'restore';
 
 export interface ILocalStrapiDestinationProviderOptions {
-  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>;
-  autoDestroy?: boolean;
-  restore?: restore.IRestoreOptions;
-  strategy: 'restore' | 'merge';
+  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>; // return an initialized instance of Strapi
+
+  autoDestroy?: boolean; // shut down the instance returned by getStrapi() at the end of the transfer
+  restore?: restore.IRestoreOptions; // erase all data in strapi database before transfer
+  strategy: 'restore'; // conflict management strategy; only the restore strategy is available at this time
 }
 
 class LocalStrapiDestinationProvider implements IDestinationProvider {

packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts

@@ -3,16 +3,16 @@ import { ProviderTransferError } from '../../../../../errors/providers';
 import * as queries from '../../../../queries';
 
 export interface IRestoreOptions {
-  assets?: boolean;
+  assets?: boolean; // delete media library files before transfer
   configuration?: {
-    webhook?: boolean;
-    coreStore?: boolean;
+    webhook?: boolean; // delete webhooks before transfer
+    coreStore?: boolean; // delete core store before transfer
   };
   entities?: {
-    include?: string[];
-    exclude?: string[];
-    filters?: ((contentType: Schema.ContentType) => boolean)[];
-    params?: { [uid: string]: unknown };
+    include?: string[]; // only delete these stage entities before transfer
+    exclude?: string[]; // exclude these stage entities from deletion
+    filters?: ((contentType: Schema.ContentType) => boolean)[]; // custom filters to exclude a content type from deletion
+    params?: { [uid: string]: unknown }; // params object passed to deleteMany before transfer for custom deletions
   };
 }
 

packages/core/data-transfer/src/strapi/providers/local-source/index.ts

@@ -10,9 +10,9 @@ import * as utils from '../../../utils';
 import { assertValidStrapi } from '../../../utils/providers';
 
 export interface ILocalStrapiSourceProviderOptions {
-  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>;
+  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>; // return an initialized instance of Strapi
 
-  autoDestroy?: boolean;
+  autoDestroy?: boolean; // shut down the instance returned by getStrapi() at the end of the transfer
 }
 
 export const createLocalStrapiSourceProvider = (options: ILocalStrapiSourceProviderOptions) => {

packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts

@@ -7,21 +7,19 @@ import type { Schema, Utils } from '@strapi/strapi';
 import { createDispatcher, connectToWebsocket, trimTrailingSlash } from '../utils';
 
 import type { IDestinationProvider, IMetadata, ProviderType, IAsset } from '../../../../types';
-import type { Client, Server } from '../../../../types/remote/protocol';
+import type { Client, Server, Auth } from '../../../../types/remote/protocol';
 import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
 import { TRANSFER_PATH } from '../../remote/constants';
 import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers';
 
-interface ITransferTokenAuth {
-  type: 'token';
-  token: string;
-}
-
 export interface IRemoteStrapiDestinationProviderOptions
   extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
-  url: URL;
-  auth?: ITransferTokenAuth;
-  retryMessageOptions?: { retryMessageTimeout: number; retryMessageMaxRetries: number };
+  url: URL; // the url of the remote Strapi admin
+  auth?: Auth.ITransferTokenAuth;
+  retryMessageOptions?: {
+    retryMessageTimeout: number; // milliseconds to wait for a response from a message
+    retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
+  };
 }
 
 const jsonLength = (obj: object) => Buffer.byteLength(JSON.stringify(obj));

packages/core/data-transfer/src/strapi/providers/remote-source/index.ts

@@ -11,21 +11,19 @@ import type {
   ProviderType,
   TransferStage,
 } from '../../../../types';
-import { Client, Server } from '../../../../types/remote/protocol';
+import { Client, Server, Auth } from '../../../../types/remote/protocol';
 import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers';
 import { TRANSFER_PATH } from '../../remote/constants';
 import { ILocalStrapiSourceProviderOptions } from '../local-source';
 import { createDispatcher, connectToWebsocket, trimTrailingSlash } from '../utils';
 
-interface ITransferTokenAuth {
-  type: 'token';
-  token: string;
-}
-
 export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourceProviderOptions {
-  url: URL;
-  auth?: ITransferTokenAuth;
-  retryMessageOptions?: { retryMessageTimeout: number; retryMessageMaxRetries: number };
+  url: URL; // the url of the remote Strapi admin
+  auth?: Auth.ITransferTokenAuth;
+  retryMessageOptions?: {
+    retryMessageTimeout: number; // milliseconds to wait for a response from a message
+    retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
+  };
 }
 
 class RemoteStrapiSourceProvider implements ISourceProvider {

packages/core/data-transfer/types/providers.d.ts

@@ -12,17 +12,20 @@ export type ProviderType = 'source' | 'destination';
 
 export interface IProvider {
   type: ProviderType;
-  name: string;
-  results?: IProviderTransferResults;
+  name: string; // a unique name for this provider
+  results?: IProviderTransferResults; // optional object for tracking any data needed from outside the engine
 
+  /**
+   * bootstrap() is called during transfer engine bootstrap
+   * It is used for initialization operations such as making a database connection, opening a file, checking authorization, etc
+   */
   bootstrap?(): MaybePromise<void>;
-  close?(): MaybePromise<void>;
+  close?(): MaybePromise<void>; // called during transfer engine close
 
-  getMetadata(): MaybePromise<IMetadata | null>;
-  getSchemas?(): MaybePromise<Utils.StringRecord<Schema.Schema> | null>;
+  getMetadata(): MaybePromise<IMetadata | null>; // returns the transfer metadata to be used for version validation
+  getSchemas?(): MaybePromise<Utils.StringRecord<Schema.Schema> | null>; // returns the schemas for the schema validation
 
-  beforeTransfer?(): MaybePromise<void>;
-  validateOptions?(): MaybePromise<void>;
+  beforeTransfer?(): MaybePromise<void>; // called immediately before transfer stages are run
 }
 
 export interface ISourceProvider extends IProvider {
@@ -40,6 +43,7 @@ export interface IDestinationProvider extends IProvider {
 
   /**
    * Optional rollback implementation
+   * Called when an error is thrown during a transfer to allow rollback operations to be performed
    */
   rollback?<T extends Error = Error>(e: T): MaybePromise<void>;
 

packages/core/data-transfer/types/remote/protocol/auth.d.ts

@@ -0,0 +1,4 @@
+export interface ITransferTokenAuth {
+  type: 'token'; // the name of the auth strategy
+  token: string; // the transfer token
+}

packages/core/data-transfer/types/remote/protocol/index.d.ts

@@ -1,2 +1,3 @@
 export * as Client from './client';
 export * as Server from './server';
+export * as Auth from './auth';