Merge pull request #15175 from strapi/deits/import-links

This commit is contained in:
Ben Irvin 2022-12-16 15:15:20 +01:00 committed by GitHub
commit e7ec94dec7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 396 additions and 336 deletions

View File

@ -127,6 +127,21 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
throw new Error(`Invalid strategy supplied: "${strategy}"`);
}
async getLinksStream(): Promise<Writable> {
if (!this.strapi) {
throw new Error('Not able to stream links. Strapi instance not found');
}
const { strategy } = this.options;
const mapID = (uid: string, id: number): number | undefined => this.#entitiesMapper[uid]?.[id];
if (strategy === 'restore') {
return restore.createLinksWriteStream(mapID, this.strapi);
}
throw new Error(`Invalid strategy supplied: "${strategy}"`);
}
}
export const createLocalStrapiDestinationProvider = (

View File

@ -130,3 +130,4 @@ const useResults = (
export * from './entities';
export * from './configuration';
export * from './links';

View File

@ -0,0 +1,34 @@
import { Writable } from 'stream';
import { ILink } from '../../../../../types';
import { createLinkQuery } from '../../../shared/strapi/link';
export const createLinksWriteStream = (
mapID: (uid: string, id: number) => number | undefined,
strapi: Strapi.Strapi
) => {
return new Writable({
objectMode: true,
async write(link: ILink, _encoding, callback) {
const { left, right } = link;
const query = createLinkQuery(strapi);
// Map IDs if needed
left.ref = mapID(left.type, left.ref) ?? left.ref;
right.ref = mapID(right.type, right.ref) ?? right.ref;
try {
await query().insert(link);
} catch (e) {
if (e instanceof Error) {
return callback(e);
}
return callback(
new Error(`An error happened while trying to import a ${left.type} link. ${e}`)
);
}
callback(null);
},
});
};

View File

@ -1,10 +1,10 @@
import { Readable } from 'stream';
import { createLinksStream } from '../links';
import { getDeepPopulateQuery } from '../links/utils';
import { collect, getStrapiFactory } from '../../../__tests__/test-utils';
describe('Local Strapi Source Provider - Entities Streaming', () => {
// TODO: entityService needs to be replaced with a mocked wrapper of db.connection and provide real metadata
describe.skip('Local Strapi Source Provider - Entities Streaming', () => {
describe('Create Links Stream', () => {
test('should return an empty stream if no content types ', async () => {
const strapi = getStrapiFactory({
@ -326,99 +326,4 @@ describe('Local Strapi Source Provider - Entities Streaming', () => {
);
});
});
describe('Get Deep Populate Query', () => {
test('Should return an empty object if there is nothing to populate', () => {
const strapi = getStrapiFactory({
contentTypes: {
blog: {
uid: 'blog',
attributes: {
title: { type: 'string' },
age: { type: 'number' },
},
},
},
})();
const schema = strapi.contentTypes.blog;
const populate = getDeepPopulateQuery(schema, strapi);
expect(populate).toEqual({});
});
test("Should return an empty object if there are components or dynamic zones but they doesn't contain relations", () => {
const strapi = getStrapiFactory({
contentTypes: {
blog: {
uid: 'blog',
attributes: {
title: { type: 'string' },
age: { type: 'number' },
compo: { type: 'component', component: 'somecomponent' },
dz: { type: 'dynamiczone', components: ['somecomponent'] },
},
},
},
components: {
somecomponent: {
uid: 'somecomponent',
attributes: {
field: { type: 'string' },
},
},
},
})();
const schema = strapi.contentTypes.blog;
const populate = getDeepPopulateQuery(schema, strapi);
expect(populate).toEqual({});
});
test('Should return a nested populate object for first level of relations', () => {
const strapi = getStrapiFactory({
contentTypes: {
blog: {
uid: 'blog',
attributes: {
title: { type: 'string' },
age: { type: 'number' },
compo: { type: 'component', component: 'somecomponent' },
dz: { type: 'dynamiczone', components: ['somecomponent'] },
blog: { type: 'relation', target: 'blog', relation: 'oneToOne' },
},
},
},
components: {
somecomponent: {
uid: 'somecomponent',
attributes: {
field: { type: 'string' },
blog: { type: 'relation', target: 'blog', relation: 'oneToOne' },
},
},
},
})();
const schema = strapi.contentTypes.blog;
const populate = getDeepPopulateQuery(schema, strapi);
expect(populate).toMatchObject(
expect.objectContaining({
blog: {
fields: ['id'],
},
compo: {
fields: ['id'],
populate: { blog: { fields: ['id'] } },
},
dz: {
fields: ['id'],
populate: { blog: { fields: ['id'] } },
},
})
);
});
});
});

View File

@ -2,6 +2,8 @@ import { chain } from 'stream-chain';
import { Readable } from 'stream';
import { set } from 'lodash/fp';
import type { IConfiguration } from '../../../types';
/**
* Create a readable stream that export the Strapi app configuration
*/
@ -23,7 +25,7 @@ export const createConfigurationStream = (strapi: Strapi.Strapi): Readable => {
// Readable configuration stream
return Readable.from(
(async function* () {
(async function* configurationGenerator(): AsyncGenerator<IConfiguration> {
for (const stream of streams) {
for await (const item of stream) {
yield item;

View File

@ -2,6 +2,7 @@ import type { ContentTypeSchema } from '@strapi/strapi';
import { isObject, isArray, isEmpty, size } from 'lodash/fp';
import { Readable, PassThrough } from 'stream';
import { IEntity } from '../../../types';
/**
* Generate and consume content-types streams in order to stream each entity individually
@ -26,7 +27,10 @@ export const createEntitiesStream = (strapi: Strapi.Strapi): Readable => {
}
return Readable.from(
(async function* () {
(async function* entitiesGenerator(): AsyncGenerator<{
entity: IEntity;
contentType: ContentTypeSchema;
}> {
for await (const { stream, contentType } of contentTypeStreamGenerator()) {
for await (const entity of stream) {
yield { entity, contentType };

View File

@ -0,0 +1,26 @@
import { Readable } from 'stream';
import type { ILink } from '../../../types';
import { createLinkQuery } from '../shared/strapi/link';
/**
* Create a Readable which will stream all the links from a Strapi instance
*/
export const createLinksStream = (strapi: Strapi.Strapi): Readable => {
const uids = [...Object.keys(strapi.contentTypes), ...Object.keys(strapi.components)] as string[];
// Async generator stream that returns every link from a Strapi instance
return Readable.from(
(async function* linkGenerator(): AsyncGenerator<ILink> {
const query = createLinkQuery(strapi);
for (const uid of uids) {
const generator = query().generateAll(uid);
for await (const link of generator) {
yield link;
}
}
})()
);
};

View File

@ -1,45 +0,0 @@
import type { ContentTypeSchema } from '@strapi/strapi';
import { Readable } from 'stream';
import { castArray } from 'lodash/fp';
import { getDeepPopulateQuery, parseEntityLinks } from './utils';
/**
* Create a Duplex instance which will stream all the links from a Strapi instance
*/
export const createLinksStream = (strapi: Strapi.Strapi): Readable => {
const schemas: ContentTypeSchema[] = Object.values(strapi.contentTypes);
// Destroy the Duplex stream instance
const destroy = (): void => {
if (!stream.destroyed) {
stream.destroy();
}
};
// Async generator stream that returns every link from a Strapi instance
const stream = Readable.from(
(async function* () {
for (const schema of schemas) {
const populate = getDeepPopulateQuery(schema, strapi);
const query = { fields: ['id'], populate };
// TODO: Replace with the DB stream API
const results = await strapi.entityService.findMany(schema.uid, query);
for (const entity of castArray(results)) {
const links = parseEntityLinks(entity, populate, schema, strapi);
for (const link of links) {
yield link;
}
}
}
destroy();
})()
);
return stream;
};

View File

@ -1,190 +0,0 @@
import type { RelationsType } from '@strapi/strapi';
import { concat, set, isEmpty } from 'lodash/fp';
import type { ILink } from '../../../../types';
// TODO: Fix any typings when we'll have better Strapi types
/**
* Parse every links from an entity result (including nested components and dynamic zone levels)
*/
export const parseEntityLinks = (entity: any, populate: any, schema: any, strapi: any): any[] => {
if (!entity) {
return [];
}
if (Array.isArray(entity)) {
return entity
.map((item) => parseEntityLinks(item, populate, schema, strapi))
.reduce(concat, [])
.flat();
}
const { attributes } = schema;
const links = [];
for (const key of Object.keys(populate)) {
const attribute = attributes[key];
const value = entity[key];
const subPopulate = populate[key];
// Ignore nil values (relations, components or dynamic zones not set)
if (!value) {
continue;
}
// Components
// Recurse to find relations
if (attribute.type === 'component') {
const componentSchema = strapi.components[attribute.component];
const componentLinks = parseEntityLinks(value, subPopulate.populate, componentSchema, strapi);
links.push(...componentLinks);
}
// Dynamic Zones
// We need to extract links from each items in the DZ's components
if (attribute.type === 'dynamiczone') {
const dzLinks = value
.map(({ __component, ...item }: any) =>
parseEntityLinks(item, subPopulate.populate, strapi.components[__component], strapi)
)
.reduce((acc: any, rlinks: any) => acc.concat(...rlinks), []);
links.push(...dzLinks);
}
// Relations
// If it's a regular relation, extract the links but do not recurse further
if (attribute.type === 'relation') {
const relationLinks = parseRelationLinks({ entity, fieldName: key, value, schema });
links.push(...relationLinks);
}
}
return links;
};
/**
* Parse links contained in a relational attribute
*/
export const parseRelationLinks = ({ entity, schema, fieldName, value }: any): ILink[] => {
const attribute = schema.attributes[fieldName];
const { relation, target }: { relation: RelationsType; target: string } = attribute;
// Handle ToMany relations
if (Array.isArray(value)) {
return (
value
// Get links from value
.map((item) => parseRelationLinks({ entity, schema, fieldName, value: item }))
// Flatten the results, to make sure we're dealing with the right data structure
.flat()
// Update the pos with the relation index in the collection
.map((link, i) => set<ILink>('left.pos', i, link))
);
}
const isMorphRelation = relation.startsWith('morph');
const isCircularRelation = !isMorphRelation && target === schema.uid;
// eslint-disable-next-line no-nested-ternary
const kind: ILink['kind'] = isMorphRelation
? // Polymorphic relations
'relation.morph'
: isCircularRelation
? // Self referencing relations
'relation.circular'
: // Regular relations
'relation.basic';
const link = linkBuilder(kind, relation)
.left(schema.uid, entity.id, fieldName)
.right(target, value.id, attribute.inversedBy).value;
return link ? [link] : [];
};
/**
* Get a deep populate query for a given schema
* It will populate first level for relations and media as well as
* first-level relations for nested components and dynamic zones' components
*/
export const getDeepPopulateQuery = (schema: any, strapi: any) => {
const populate: { [key: string]: any } = {};
for (const [key, attribute] of Object.entries<any>(schema.attributes)) {
const setPopulateKey = (value: any) => {
populate[key] = value;
};
// Owning side of a relation
if (attribute.type === 'relation' && !attribute.mappedBy) {
setPopulateKey({ fields: ['id'] });
}
// Media
if (attribute.type === 'media') {
setPopulateKey({ fields: ['id'] });
}
// Dynamic zone (nested structure)
if (attribute.type === 'dynamiczone') {
const subPopulate: { [key: string]: any } = {};
for (const component of attribute.components) {
const componentSchema = strapi.components[component];
const componentPopulate = getDeepPopulateQuery(componentSchema, strapi);
// FIXME: Same problem as when trying to populate dynamic zones,
// we don't have a way to discriminate components queries (which
// can cause issue when components share same fields name)
Object.assign(subPopulate, componentPopulate);
}
if (!isEmpty(subPopulate)) {
setPopulateKey({ fields: ['id'], populate: subPopulate });
}
}
// Component (nested structure)
if (attribute.type === 'component') {
const componentSchema = strapi.components[attribute.component];
const componentPopulate = getDeepPopulateQuery(componentSchema, strapi);
if (!isEmpty(componentPopulate)) {
setPopulateKey({ fields: ['id'], populate: componentPopulate });
}
}
}
return populate;
};
/**
* Domain util to create a link
* TODO: Move that to the domain layer when we'll update it
*/
export const linkBuilder = <T extends ILink = ILink>(kind: T['kind'], relation: RelationsType) => {
const link: Partial<T> = {};
link.kind = kind;
link.relation = relation;
return {
left(type: string, ref: string | number, field: string, pos?: number) {
link.left = { type, ref, field, pos };
return this;
},
right(type: string, ref: string | number, field?: string) {
link.right = { type, ref, field };
return this;
},
get value() {
return link.left && link.right ? (link as ILink) : null;
},
};
};

View File

@ -1 +1,2 @@
export * as entity from './entity';
export * as link from './link';

View File

@ -0,0 +1,280 @@
import { clone, isNil } from 'lodash/fp';
import { ILink } from '../../../../types';
// TODO: Remove any types when we'll have types for DB metadata
export const createLinkQuery = (strapi: Strapi.Strapi) => {
const query = () => {
const { connection } = strapi.db;
async function* generateAllForAttribute(uid: string, fieldName: string): AsyncGenerator<ILink> {
const metadata = strapi.db.metadata.get(uid);
if (!metadata) {
throw new Error(`No metadata found for ${uid}`);
}
const attributes = filterValidRelationalAttributes(metadata.attributes);
if (!(fieldName in attributes)) {
throw new Error(`${fieldName} is not a valid relational attribute name`);
}
const attribute = attributes[fieldName];
const kind = getLinkKind(attribute, uid);
const { relation, target } = attribute;
// The relation is stored in the same table
// TODO: handle manyToOne joinColumn
if (attribute.joinColumn) {
const joinColumnName: string = attribute.joinColumn.name;
const qb = connection.queryBuilder().select('id', joinColumnName).from(metadata.tableName);
// TODO: stream the query to improve performances
const entries = await qb;
for (const entry of entries) {
const ref = entry[joinColumnName];
if (ref !== null) {
yield {
kind,
relation,
left: { type: uid, ref: entry.id, field: fieldName },
right: { type: target, ref },
};
}
}
}
// The relation uses a join table
if (attribute.joinTable) {
const {
name,
joinColumn,
inverseJoinColumn,
orderColumnName,
morphColumn,
inverseOrderColumnName,
} = attribute.joinTable;
const qb = connection.queryBuilder().from(name);
type Columns = {
left: { ref: string | null; order?: string };
right: { ref: string | null; order?: string; type?: string; field?: string };
};
const columns: Columns = {
left: { ref: null },
right: { ref: null },
};
const left: Partial<ILink['left']> = { type: uid, field: fieldName };
const right: Partial<ILink['right']> = {};
if (kind === 'relation.basic' || kind === 'relation.circular') {
right.type = attribute.target;
right.field = attribute.inversedBy;
columns.left.ref = joinColumn.name;
columns.right.ref = inverseJoinColumn.name;
if (orderColumnName) {
columns.left.order = orderColumnName as string;
}
if (inverseOrderColumnName) {
columns.right.order = inverseOrderColumnName as string;
}
}
if (kind === 'relation.morph') {
columns.left.ref = joinColumn.name;
columns.right.ref = morphColumn.idColumn.name;
columns.right.type = morphColumn.typeColumn.name;
columns.right.field = 'field';
columns.right.order = 'order';
}
const validColumns = [
// Left
columns.left.ref,
columns.left.order,
// Right
columns.right.ref,
columns.right.type,
columns.right.field,
columns.right.order,
].filter((column: string | null | undefined) => !isNil(column));
qb.select(validColumns);
// TODO: stream the query to improve performances
const entries = await qb;
for (const entry of entries) {
if (columns.left.ref) {
left.ref = entry[columns.left.ref];
}
if (columns.right.ref) {
right.ref = entry[columns.right.ref];
}
if (columns.left.order) {
left.pos = entry[columns.left.order as string];
}
if (columns.right.order) {
right.pos = entry[columns.right.order as string];
}
if (columns.right.type) {
right.type = entry[columns.right.type as string];
}
if (columns.right.field) {
right.field = entry[columns.right.field as string];
}
const link: ILink = {
kind,
relation,
left: clone(left as ILink['left']),
right: clone(right as ILink['right']),
};
yield link;
}
}
}
async function* generateAll(uid: string): AsyncGenerator<ILink> {
const metadata = strapi.db.metadata.get(uid);
if (!metadata) {
throw new Error(`No metadata found for ${uid}`);
}
const attributes = filterValidRelationalAttributes(metadata.attributes);
for (const fieldName of Object.keys(attributes)) {
for await (const link of generateAllForAttribute(uid, fieldName)) {
yield link;
}
}
}
const insert = async (link: ILink) => {
const { kind, left, right } = link;
const metadata = strapi.db.metadata.get(left.type);
const attribute = metadata.attributes[left.field];
const payload = {};
if (attribute.joinColumn) {
const joinColumnName = attribute.joinColumn.name;
await connection(metadata.tableName)
.where('id', left.ref)
.update({ [joinColumnName]: right.ref });
}
if (attribute.joinTable) {
const {
name,
joinColumn,
inverseJoinColumn,
orderColumnName,
inverseOrderColumnName,
morphColumn,
} = attribute.joinTable;
if (joinColumn) {
Object.assign(payload, { [joinColumn.name]: left.ref });
}
const assignInverseColumn = () => {
if (inverseJoinColumn) {
Object.assign(payload, {
[inverseJoinColumn.name]: right.ref,
});
}
};
const assignOrderColumns = () => {
if (orderColumnName) {
Object.assign(payload, { [orderColumnName]: left.pos ?? null });
}
if (inverseOrderColumnName) {
Object.assign(payload, { [inverseOrderColumnName]: right.pos ?? null });
}
};
const assignMorphColumns = () => {
const { idColumn, typeColumn } = morphColumn ?? {};
if (idColumn) {
Object.assign(payload, { [idColumn.name]: right.ref });
}
if (typeColumn) {
Object.assign(payload, { [typeColumn.name]: right.type });
}
Object.assign(payload, { order: right.pos ?? null, field: right.field ?? null });
};
if (kind === 'relation.basic' || kind === 'relation.circular') {
assignInverseColumn();
}
if (kind === 'relation.morph') {
assignMorphColumns();
}
assignOrderColumns();
await connection.insert(payload).into(name);
}
};
return { generateAll, generateAllForAttribute, insert };
};
return query;
};
const filterValidRelationalAttributes = (attributes: Record<string, any>) => {
const isOwner = (attribute: any) => {
return attribute.owner || (!attribute.mappedBy && !attribute.morphBy);
};
const isComponentLike = (attribute: any) => {
return attribute.component || attribute.components;
};
return Object.entries(attributes)
.filter(([, attribute]) => {
return attribute.type === 'relation' && isOwner(attribute) && !isComponentLike(attribute);
})
.reduce<Record<string, any>>((acc, [key, attribute]) => ({ ...acc, [key]: attribute }), {});
};
const getLinkKind = (attribute: any, uid: string): ILink['kind'] => {
if (attribute.relation.startsWith('morph')) {
return 'relation.morph';
}
if (attribute.target === uid) {
return 'relation.circular';
}
return 'relation.basic';
};

View File

@ -70,7 +70,7 @@ interface IDefaultLink {
/**
* Reference ID of the entity
*/
ref: number | string;
ref: number;
/**
* Field used to hold the link in the entity
*/
@ -94,11 +94,15 @@ interface IDefaultLink {
/**
* Reference ID of the entity
*/
ref: number | string;
ref: number;
/**
* Field used to hold the link in the entity
*/
field?: string;
/**
* If the link is part of a collection, keep its position here
*/
pos?: number;
};
}

View File

@ -1,3 +1,4 @@
import { Knex } from 'knex';
import { LifecycleProvider } from './lifecycles';
import { MigrationProvider } from './migrations';
import { SchemaProvider } from './schema';
@ -158,6 +159,8 @@ export interface Database {
migrations: MigrationProvider;
entityManager: EntityManager;
queryBuilder: any;
metadata: any;
connection: Knex;
query<T extends keyof AllTypes>(uid: T): QueryFromContentType<T>;
}

View File

@ -55,6 +55,16 @@ module.exports = async (opts) => {
strategy: 'restore', // for an export to file, strategy will always be 'restore'
versionMatching: 'ignore', // for an export to file, versionMatching will always be skipped
transforms: {
links: [
{
filter(link) {
return (
!DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) &&
!DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type)
);
},
},
],
entities: [
{
filter(entity) {

View File

@ -57,6 +57,16 @@ module.exports = async (opts) => {
versionMatching: opts.schemaComparison,
exclude: opts.exclude,
rules: {
links: [
{
filter(link) {
return (
!DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) &&
!DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type)
);
},
},
],
entities: [
{
filter: (entity) => !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type),