yujunjun/strapi
1
0
Fork 0
mirror of https://github.com/strapi/strapi.git synced 2026-01-03 18:53:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

migrate graphql upload to use streams + use fs-extra

Browse Source
This commit is contained in:
Pierre Noël 2022-01-05 17:36:21 +01:00
parent 186ebfd06e
commit 7cf2821451
12 changed files with 6112 additions and 959 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/cli/create-strapi-starter/package.json
View File

@ -43,7 +43,7 @@
"ci-info": "3.1.1",
"commander": "7.1.0",
"execa": "5.1.1",
"fs-extra": "9.1.0",
"fs-extra": "10.0.0",
"inquirer": "8.2.0",
"ora": "5.4.0"
},

2
packages/core/admin/package.json
View File

@ -72,7 +72,7 @@
"fast-deep-equal": "3.1.3",
"font-awesome": "^4.7.0",
"formik": "^2.2.6",
"fs-extra": "^9.1.0",
"fs-extra": "10.0.0",
"highlight.js": "^10.4.1",
"history": "^4.9.0",
"hoist-non-react-statics": "^3.3.0",

2
packages/core/content-type-builder/package.json
View File

@ -31,7 +31,7 @@
"@strapi/generators": "4.1.0",
"@strapi/helper-plugin": "4.1.0",
"@strapi/utils": "4.1.0",
"fs-extra": "^9.1.0",
"fs-extra": "10.0.0",
"lodash": "4.17.21",
"pluralize": "^8.0.0",
"react": "^17.0.2",

1
packages/core/upload/package.json
View File

@ -28,6 +28,7 @@
"@strapi/utils": "4.1.0",
"byte-size": "7.0.1",
"cropperjs": "1.5.11",
"fs-extra": "10.0.0",
"immer": "9.0.6",
"koa-range": "0.3.0",
"koa-static": "5.0.0",

6
packages/core/upload/server/controllers/admin-api.js
View File

@ -1,9 +1,9 @@
'use strict';
const fse = require('fs-extra');
const _ = require('lodash');
const { contentTypes: contentTypesUtils } = require('@strapi/utils');
const { ApplicationError, NotFoundError, ForbiddenError } = require('@strapi/utils').errors;
const rimraf = require('rimraf');
const { getService } = require('../utils');
const validateSettings = require('./validation/settings');
const validateUploadBody = require('./validation/upload');
@ -186,9 +186,9 @@ module.exports = {
await (id ? this.replaceFile : this.uploadFiles)(ctx);
} finally {
if (Array.isArray(files)) {
files.map(file => rimraf(file.path, () => {}));
await Promise.all(files.map(file => fse.remove(file.path)));
} else if (files.path) {
rimraf(files.path, () => {});
await fse.remove(files.path);
}
}
},

6
packages/core/upload/server/controllers/content-api.js
View File

@ -1,8 +1,8 @@
'use strict';
const fse = require('fs-extra');
const _ = require('lodash');
const utils = require('@strapi/utils');
const rimraf = require('rimraf');
const { getService } = require('../utils');
const validateSettings = require('./validation/settings');
const validateUploadBody = require('./validation/upload');
@ -138,9 +138,9 @@ module.exports = {
await (id ? this.replaceFile : this.uploadFiles)(ctx);
} finally {
if (Array.isArray(files)) {
files.map(file => rimraf(file.path, () => {}));
await Promise.all(files.map(file => fse.remove(file.path)));
} else if (files.path) {
rimraf(files.path, () => {});
await fse.remove(files.path);
}
}
},

80
packages/core/upload/server/graphql.js
View File

@ -1,7 +1,9 @@
'use strict';
const _ = require('lodash');
const { streamToBuffer } = require('./utils/file');
const path = require('path');
const os = require('os');
const fse = require('fs-extra');
const { getStreamSize } = require('./utils/file');
const UPLOAD_MUTATION_NAME = 'upload';
const MULTIPLE_UPLOAD_MUTATION_NAME = 'multipleUpload';
@ -40,27 +42,23 @@ module.exports = ({ strapi }) => {
* @param {object} metas
* @return {Promise<object>}
*/
const formatFile = async (upload, extraInfo, metas) => {
// ICI
const formatFile = async (upload, extraInfo, metas, { tmpFolderPath }) => {
const uploadService = getUploadService('upload');
const { filename, mimetype, createReadStream } = await upload;
const readBuffer = await streamToBuffer(createReadStream());
const { buffer, info } = await optimize(readBuffer);
const uploadService = getUploadService('upload');
const fileInfo = uploadService.formatFileInfo(
const currentFile = uploadService.formatFileInfo(
{
filename,
type: mimetype,
size: buffer.length,
size: await getStreamSize(createReadStream()),
},
extraInfo || {},
metas
);
currentFile.getStream = createReadStream;
return _.assign(fileInfo, info, { buffer });
const newFile = await optimize(currentFile, { tmpFolderPath });
return newFile;
};
/**
@ -99,12 +97,29 @@ module.exports = ({ strapi }) => {
},
async resolve(parent, args) {
const { file: upload, info, ...fields } = args;
// create temporary folder to store files for stream manipulation
const tmpFolderPath = await fse.mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
let sanitizedEntity;
const file = await formatFile(upload, info, fields);
const uploadedFile = await getUploadService('upload').uploadFileAndPersist(file);
try {
const { file: upload, info, ...fields } = args;
return toEntityResponse(uploadedFile, { args, resourceUID: fileTypeName });
const file = await formatFile(upload, info, fields, { tmpFolderPath });
const uploadedFile = await getUploadService('upload').uploadFileAndPersist(
file,
{},
{ tmpFolderPath }
);
sanitizedEntity = await toEntityResponse(uploadedFile, {
args,
resourceUID: fileTypeName,
});
} finally {
// delete temporary folder
await fse.remove(tmpFolderPath);
}
return sanitizedEntity;
},
});
@ -122,19 +137,32 @@ module.exports = ({ strapi }) => {
},
async resolve(parent, args) {
const { files: uploads, ...fields } = args;
// create temporary folder to store files for stream manipulation
const tmpFolderPath = await fse.mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
let sanitizedEntities = [];
const files = await Promise.all(uploads.map(upload => formatFile(upload, {}, fields)));
try {
const { files: uploads, ...fields } = args;
const uploadService = getUploadService('upload');
const files = await Promise.all(
uploads.map(upload => formatFile(upload, {}, fields, { tmpFolderPath }))
);
const uploadedFiles = await Promise.all(
files.map(file => uploadService.uploadFileAndPersist(file))
);
const uploadService = getUploadService('upload');
return uploadedFiles.map(file =>
toEntityResponse(file, { args, resourceUID: fileTypeName })
);
const uploadedFiles = await Promise.all(
files.map(file => uploadService.uploadFileAndPersist(file, {}, { tmpFolderPath }))
);
sanitizedEntities = uploadedFiles.map(file =>
toEntityResponse(file, { args, resourceUID: fileTypeName })
);
} finally {
// delete temporary folder
await fse.remove(tmpFolderPath);
}
return sanitizedEntities;
},
});

9
packages/core/upload/server/services/upload.js
View File

@ -6,12 +6,11 @@
* @description: A set of functions similar to controller's actions to avoid code duplication.
*/
const fs = require('fs');
const os = require('os');
const { mkdtemp } = require('fs').promises;
const path = require('path');
const crypto = require('crypto');
const rimraf = require('rimraf');
const fs = require('fs');
const fse = require('fs-extra');
const _ = require('lodash');
const {
sanitize,
@ -109,7 +108,7 @@ module.exports = ({ strapi }) => ({
async upload({ data, files }, { user } = {}) {
// create temporary folder to store files for stream manipulation
const tmpFolderPath = await mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
const tmpFolderPath = await fse.mkdtemp(path.join(os.tmpdir(), 'strapi-upload-'));
let uploadedFiles = [];
try {
@ -129,7 +128,7 @@ module.exports = ({ strapi }) => ({
);
} finally {
// delete temporary folder
rimraf(tmpFolderPath, () => {});
await fse.remove(tmpFolderPath);
}
return uploadedFiles;

10
packages/core/upload/server/utils/file.js
View File

@ -17,7 +17,17 @@ const streamToBuffer = stream =>
stream.on('error', reject);
});
const getStreamSize = stream =>
new Promise((resolve, reject) => {
let size = 0;
stream.on('data', chunk => (size += Buffer.byteLength(chunk)));
stream.on('close', () => resolve(size));
stream.on('error', reject);
stream.resume();
});
module.exports = {
streamToBuffer,
bytesToKbytes,
getStreamSize,
};

2
packages/generators/app/package.json
View File

@ -39,7 +39,7 @@
"@sentry/node": "6.3.0",
"chalk": "^4.1.1",
"execa": "^1.0.0",
"fs-extra": "^9.1.0",
"fs-extra": "10.0.0",
"inquirer": "8.2.0",
"lodash": "4.17.21",
"node-fetch": "^2.6.1",

2
packages/plugins/documentation/package.json
View File

@ -28,7 +28,7 @@
"@strapi/utils": "4.1.0",
"bcryptjs": "2.4.3",
"cheerio": "^1.0.0-rc.5",
"fs-extra": "^9.1.0",
"fs-extra": "10.0.0",
"koa-static": "^5.0.0",
"lodash": "4.17.21",
"path-to-regexp": "6.2.0",

6949
yarn.lock
View File

File diff suppressed because it is too large Load Diff