Merge pull request #5358 from strapi/media-lib/provider-settings

Media lib/provider settings
This commit is contained in:
Alexandre BODIN 2020-03-02 17:03:04 +01:00 committed by GitHub
commit 4c9e5b5185
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 273 additions and 313 deletions

View File

@ -272,44 +272,83 @@ In our second example, you can upload and attach multiple pictures to the restau
}
```
## Install providers
## Using a provider
By default Strapi provides a local file upload system. You might want to upload your files on AWS S3 or another provider.
By default Strapi provides a provider that upload files to a local directory. You might want to upload your files to another provider like AWS S3.
You can check all the available providers developed by the community on npmjs.org - [Providers list](https://www.npmjs.com/search?q=strapi-provider-upload-)
To install a new provider run:
```
$ npm install strapi-provider-upload-aws-s3@beta --save
$ npm install strapi-provider-upload-aws-s3 --save
```
::: tip
If the provider is not in the mono repo, you probably don't need `@beta` depending if the creator published it with this tag or not.
:::
or
Then, visit [http://localhost:1337/admin/plugins/upload/configurations/development](http://localhost:1337/admin/plugins/upload/configurations/development) on your web browser and configure the provider.
```
$ yarn add strapi-provider-upload-aws-s3
```
To enable the provider, create or edit the file at `./extensions/upload/config/settings.json`
```json
{
"provider": "aws-s3",
"providerOptions": {
"accessKeyId": "dev-key",
"secretAccessKey": "dev-secret",
"region": "aws-region",
"params": {
"Bucket": "my-bucket"
}
}
}
```
Make sure to read the provider's `README` to know what are the possible parameters.
## Create providers
If you want to create your own, make sure the name starts with `strapi-provider-upload-` (duplicating an existing one will be easier to create), modify the `auth` config object and customize the `upload` and `delete` functions.
You can create a Node.js module to implement a custom provider. Read the official documentation [here](https://docs.npmjs.com/creating-node-js-modules).
To use it you will have to publish it on **npm**.
To work with strapi, your provider name must match the pattern `strapi-provider-upload-{provider-name}`.
Your provider need to export the following interface:
```js
module.exports = {
init(providerOptions) {
// init your provider if necessary
return {
upload(file) {
// upload the file in the provider
},
delete(file) {
// delete the file in the provider
},
};
},
};
```
You can then publish it to make it available to the community.
### Create a local provider
If you want to create your own provider without publishing it on **npm** you can follow these steps:
- Create a `providers` folder in your application.
- Create your provider as explained in the documentation eg. `./providers/strapi-provider-upload-[...]/...`
- Then update your `package.json` to link your `strapi-provider-upload-[...]` dependency to the [local path](https://docs.npmjs.com/files/package.json#local-paths) of your new provider.
- Create a `./providers/strapi-provider-upload-{provider-name}` folder in your root application folder.
- Create your provider as explained in the [documentation](#create-providers) above.
- Then update your `package.json` to link your `strapi-provider-upload-{provider-name}` dependency to point to the [local path](https://docs.npmjs.com/files/package.json#local-paths) of your provider.
```json
{
...
"dependencies": {
...
"strapi-provider-upload-[...]": "file:providers/strapi-provider-upload-[...]",
"strapi-provider-upload-{provider-name}": "file:providers/strapi-provider-upload-{provider-name}"
...
}
}

View File

@ -0,0 +1,8 @@
module.exports = {
// provider: 'aws-s3',
// providerOptions: {
// cloud_name: '',
// api_key: '',
// api_secret: '',
// },
};

View File

@ -29,6 +29,8 @@
"strapi-plugin-users-permissions": "3.0.0-beta.18.7",
"strapi-provider-email-mailgun": "3.0.0-beta.18.7",
"strapi-provider-upload-aws-s3": "3.0.0-beta.18.7",
"strapi-provider-upload-cloudinary": "3.0.0-beta.18.7",
"strapi-provider-upload-rackspace": "3.0.0-beta.18.7",
"strapi-utils": "3.0.0-beta.18.7"
},
"strapi": {

View File

@ -0,0 +1,8 @@
{
"enabled": true,
"provider": "local",
"providerOptions": {
"sizeLimit": 1000
},
"providers": []
}

View File

@ -7,7 +7,6 @@
* This gives you an opportunity to set up your data model,
* run jobs, or perform some special logic.
*/
const _ = require('lodash');
module.exports = async () => {
// set plugin store
@ -17,20 +16,9 @@ module.exports = async () => {
key: 'settings',
});
Object.assign(strapi.plugins.upload.config, {
enabled: true,
provider: 'local',
sizeLimit: 1000000,
providers: [],
});
const installedProviders = Object.keys(strapi.config.info.dependencies)
.filter(d => d.includes('strapi-provider-upload-'))
.concat('strapi-provider-upload-local');
for (let installedProvider of _.uniq(installedProviders)) {
strapi.plugins.upload.config.providers.push(require(installedProvider));
}
strapi.plugins.upload.provider = createProvider(
strapi.plugins.upload.config || {}
);
// if provider config does not exist set one by default
const config = await configurator.get();
@ -45,3 +33,14 @@ module.exports = async () => {
});
}
};
const createProvider = ({ provider, providerOptions }) => {
try {
return require(`strapi-provider-upload-${provider}`).init(providerOptions);
} catch (err) {
strapi.log.error(err);
throw new Error(
`The provider package isn't installed. Please run \`npm install strapi-provider-upload-${provider}\``
);
}
};

View File

@ -36,9 +36,7 @@ module.exports = {
resolver: async (obj, { file: upload, ...fields }) => {
const file = await formatFile(upload, fields);
const config = strapi.plugins.upload.config;
const uploadedFiles = await strapi.plugins.upload.services.upload.upload([file], config);
const uploadedFiles = await strapi.plugins.upload.services.upload.upload([file]);
// Return response.
return uploadedFiles.length === 1 ? uploadedFiles[0] : uploadedFiles;
@ -50,9 +48,7 @@ module.exports = {
resolver: async (obj, { files: uploads, ...fields }) => {
const files = await Promise.all(uploads.map(upload => formatFile(upload, fields)));
const config = strapi.plugins.upload.config;
const uploadedFiles = await strapi.plugins.upload.services.upload.upload(files, config);
const uploadedFiles = await strapi.plugins.upload.services.upload.upload(files);
// Return response.
return uploadedFiles;
@ -79,7 +75,7 @@ const formatFile = async (upload, fields) => {
ext: path.extname(filename),
buffer,
mime: mimetype,
size: (buffer.length / 1000).toFixed(2),
size: Math.round((buffer.length / 1000) * 100) / 100,
};
const { refId, ref, source, field } = fields;

View File

@ -13,24 +13,15 @@ module.exports = {
const uploadService = strapi.plugins.upload.services.upload;
// Retrieve provider configuration.
const config = strapi.plugins.upload.config;
const { enabled } = strapi.plugins.upload.config;
// Verify if the file upload is enable.
if (config.enabled === false) {
return ctx.badRequest(
null,
[
{
messages: [
{
id: 'Upload.status.disabled',
message: 'File upload is disabled',
},
],
},
]
);
if (enabled === false) {
throw strapi.errors.badRequest(null, {
errors: [
{ id: 'Upload.status.disabled', message: 'File upload is disabled' },
],
});
}
// Extract optional relational data.
@ -38,31 +29,15 @@ module.exports = {
const { files = {} } = ctx.request.files || {};
if (_.isEmpty(files)) {
return ctx.badRequest(null, [
{
messages: [{ id: 'Upload.status.empty', message: 'Files are empty' }],
},
]);
throw strapi.errors.badRequest(null, {
errors: [{ id: 'Upload.status.empty', message: 'Files are empty' }],
});
}
// Transform stream files to buffer
const buffers = await uploadService.bufferize(files);
const enhancedFiles = buffers.map(file => {
if (file.size > config.sizeLimit) {
return ctx.badRequest(null, [
{
messages: [
{
id: 'Upload.status.sizeLimit',
message: `${file.name} file is bigger than limit size!`,
values: { file: file.name },
},
],
},
]);
}
// Add details to the file to be able to create the relationships.
if (refId && ref && field) {
Object.assign(file, {
@ -92,7 +67,7 @@ module.exports = {
return;
}
const uploadedFiles = await uploadService.upload(enhancedFiles, config);
const uploadedFiles = await uploadService.upload(enhancedFiles);
// Send 200 `ok`
ctx.send(uploadedFiles);
@ -151,7 +126,6 @@ module.exports = {
async destroy(ctx) {
const { id } = ctx.params;
const config = strapi.plugins.upload.config;
const file = await strapi.plugins['upload'].services.upload.fetch({ id });
@ -159,7 +133,7 @@ module.exports = {
return ctx.notFound('file.notFound');
}
await strapi.plugins['upload'].services.upload.remove(file, config);
await strapi.plugins['upload'].services.upload.remove(file);
ctx.send(file);
},

View File

@ -45,7 +45,7 @@ module.exports = {
ext: stream.name.split('.').length > 1 ? `.${_.last(stream.name.split('.'))}` : '',
buffer,
mime: stream.type,
size: (stream.size / 1000).toFixed(2),
size: Math.round((stream.size / 1000) * 100) / 100,
};
};
@ -53,35 +53,18 @@ module.exports = {
return Promise.all(files.map(stream => createBuffer(stream)));
},
async upload(files, config) {
// Get upload provider settings to configure the provider to use.
const provider = _.find(strapi.plugins.upload.config.providers, {
provider: config.provider,
});
if (!provider) {
throw new Error(
`The provider package isn't installed. Please run \`npm install strapi-provider-upload-${config.provider}\``
);
}
const actions = await provider.init(config);
async upload(files) {
const config = strapi.plugins.upload.config;
// upload a single file
const uploadFile = async file => {
await actions.upload(file);
await strapi.plugins.upload.provider.upload(file);
// Remove buffer to don't save it.
delete file.buffer;
file.provider = provider.provider;
file.provider = config.provider;
const res = await this.add(file);
// Remove temp file
if (file.tmpPath) {
fs.unlinkSync(file.tmpPath);
}
strapi.eventHub.emit('media.create', { media: res });
return res;
};
@ -108,19 +91,12 @@ module.exports = {
return strapi.query('file', 'upload').count(params);
},
async remove(file, config) {
// get upload provider settings to configure the provider to use
const provider = _.cloneDeep(
_.find(strapi.plugins.upload.config.providers, {
provider: config.provider,
})
);
_.assign(provider, config);
const actions = provider.init(config);
async remove(file) {
const config = strapi.plugins.upload.config;
// execute delete function of the provider
if (file.provider === provider.provider) {
await actions.delete(file);
if (file.provider === config.provider) {
await strapi.plugins.upload.provider.delete(file);
}
const media = await strapi.query('file', 'upload').findOne({
@ -133,9 +109,6 @@ module.exports = {
},
async uploadToEntity(params, files, source) {
// Retrieve provider settings from database.
const config = strapi.plugins.upload.config;
const model = strapi.getModel(params.model, source);
// Asynchronous upload.
@ -161,7 +134,7 @@ module.exports = {
});
// Make upload async.
return this.upload(enhancedFiles, config);
return this.upload(enhancedFiles);
})
);
},

View File

@ -5,13 +5,20 @@ describe('Upload plugin bootstrap function', () => {
const setStore = jest.fn(() => {});
global.strapi = {
log: {
error() {},
},
config: {
info: {
dependencies: {},
},
},
plugins: {
upload: { config: {} },
upload: {
config: {
provider: 'local',
},
},
},
store() {
return {

View File

@ -93,9 +93,6 @@ describe('Upload plugin end to end tests', () => {
});
expect(res.statusCode).toBe(400);
expect(res.body).toMatchObject({
message: [{ messages: [{ message: 'Files are empty' }] }],
});
});
});

View File

@ -1,9 +0,0 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.12.0
# ignores vulnerabilities until expiry date; change duration by modifying expiry date
ignore:
shelljs:
- '*':
reason: testing
expires: 2019-01-04T14:38:09.622Z
patch: {}

View File

@ -1,4 +1,26 @@
# strapi-provider-upload-s3
# strapi-provider-upload-aws-s3
## Configurations
Your configuration is passed down to the provider. (e.g: `new AWS.S3(config)`). You can see the complete list of options [here](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property)
**Example**
`./extensions/upload/config/settings.json`
```json
{
"provider": "aws-s3",
"providerOptions": {
"accessKeyId": "dev-key",
"secretAccessKey": "dev-secret",
"region": "aws-region",
"params": {
"Bucket": "my-bucket"
}
}
}
```
## Resources

View File

@ -9,67 +9,15 @@
const _ = require('lodash');
const AWS = require('aws-sdk');
const trimParam = str => (typeof str === 'string' ? str.trim() : undefined);
module.exports = {
provider: 'aws-s3',
name: 'Amazon Web Service S3',
auth: {
public: {
label: 'Access API Token',
type: 'text',
},
private: {
label: 'Secret Access Token',
type: 'text',
},
region: {
label: 'Region',
type: 'enum',
values: [
'us-east-1',
'us-east-2',
'us-west-1',
'us-west-2',
'ca-central-1',
'ap-south-1',
'ap-northeast-1',
'ap-northeast-2',
'ap-northeast-3',
'ap-southeast-1',
'ap-southeast-2',
'cn-north-1',
'cn-northwest-1',
'eu-central-1',
'eu-north-1',
'eu-west-1',
'eu-west-2',
'eu-west-3',
'sa-east-1',
],
},
bucket: {
label: 'Bucket',
type: 'text',
},
},
init: config => {
// configure AWS S3 bucket connection
AWS.config.update({
accessKeyId: trimParam(config.public),
secretAccessKey: trimParam(config.private),
region: config.region,
});
init(config) {
const S3 = new AWS.S3({
apiVersion: '2006-03-01',
params: {
Bucket: trimParam(config.bucket),
},
...config,
});
return {
upload: file => {
upload(file, customParams = {}) {
return new Promise((resolve, reject) => {
// upload file on S3 bucket
const path = file.path ? `${file.path}/` : '';
@ -79,6 +27,7 @@ module.exports = {
Body: Buffer.from(file.buffer, 'binary'),
ACL: 'public-read',
ContentType: file.mime,
...customParams,
},
(err, data) => {
if (err) {
@ -93,13 +42,14 @@ module.exports = {
);
});
},
delete: file => {
delete(file, customParams = {}) {
return new Promise((resolve, reject) => {
// delete file on S3 bucket
const path = file.path ? `${file.path}/` : '';
S3.deleteObject(
{
Key: `${path}${file.hash}${file.ext}`,
...customParams,
},
(err, data) => {
if (err) {

View File

@ -1,9 +0,0 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.12.0
# ignores vulnerabilities until expiry date; change duration by modifying expiry date
ignore:
shelljs:
- '*':
reason: testing
expires: 2019-01-04T14:38:11.663Z
patch: {}

View File

@ -1,9 +1,22 @@
# strapi-provider-upload-cloudinary
## ⏳ Installation
## Configurations
```bash
npm i --save strapi-provider-upload-cloudinary
Your configuration is passed down to the cloudinary configuration. (e.g: `cloudinary.config(config)`). You can see the complete list of options [here](https://cloudinary.com/documentation/cloudinary_sdks#configuration_parameters)
**Example**
`./extensions/upload/config/settings.json`
```json
{
"provider": "cloudinary",
"providerOptions": {
"cloud_name": "cloud-name",
"api_key": "api-key",
"api_secret": "api-secret"
}
}
```
## Resources

View File

@ -5,39 +5,18 @@
*/
// Public node modules.
/* eslint-disable prefer-template */
const cloudinary = require('cloudinary').v2;
const intoStream = require('into-stream');
module.exports = {
provider: 'cloudinary',
name: 'Cloudinary',
auth: {
cloud_name: {
label: 'Cloud name',
type: 'text',
},
api_key: {
label: 'API Key',
type: 'text',
},
api_secret: {
label: 'API Secret',
type: 'password',
},
},
init: config => {
cloudinary.config({
cloud_name: config.cloud_name,
api_key: config.api_key,
api_secret: config.api_secret,
});
init(config) {
cloudinary.config(config);
return {
upload(file) {
upload(file, customConfig = {}) {
return new Promise((resolve, reject) => {
const upload_stream = cloudinary.uploader.upload_stream(
{ resource_type: 'auto' },
{ resource_type: 'auto', ...customConfig },
(err, image) => {
if (err) {
return reject(err);
@ -50,16 +29,19 @@ module.exports = {
resolve();
}
);
intoStream(file.buffer).pipe(upload_stream);
});
},
async delete(file) {
async delete(file, customConfig = {}) {
try {
const { resource_type, public_id } = file.provider_metadata;
const response = await cloudinary.uploader.destroy(public_id, {
invalidate: true,
resource_type: resource_type || 'image',
...customConfig,
});
if (response.result !== 'ok') {
throw {
error: new Error(response.result),

View File

@ -1,9 +0,0 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.12.0
# ignores vulnerabilities until expiry date; change duration by modifying expiry date
ignore:
shelljs:
- '*':
reason: testing
expires: 2019-01-04T14:38:13.835Z
patch: {}

View File

@ -1,5 +1,24 @@
# strapi-provider-upload-local
## Configurations
Your configuration is passed down to the cloudinary configuration. (e.g: `cloudinary.config(config)`). You can see the complete list of options [here](https://cloudinary.com/documentation/cloudinary_sdks#configuration_parameters)
**Example**
`./extensions/upload/config/settings.json`
```json
{
"provider": "cloudinary",
"providerOptions": {
"sizeLimit": 100000
}
}
```
The `sizeLimit` parameter must be a number. Be aware that the unit is in KB.
## Resources
- [MIT License](LICENSE.md)

View File

@ -7,44 +7,68 @@
// Public node modules.
const fs = require('fs');
const path = require('path');
/* eslint-disable no-unused-vars */
module.exports = {
provider: 'local',
name: 'Local server',
init: (config) => {
init({ sizeLimit = 1000000 } = {}) {
const verifySize = file => {
if (file.size > sizeLimit) {
throw strapi.errors.badRequest('FileToBig', {
errors: [
{
id: 'Upload.status.sizeLimit',
message: `${file.name} file is bigger than limit size!`,
values: { file: file.name },
},
],
});
}
};
return {
upload: (file) => {
upload(file) {
verifySize(file);
return new Promise((resolve, reject) => {
// write file in public/assets folder
fs.writeFile(path.join(strapi.config.public.path, `/uploads/${file.hash}${file.ext}`), file.buffer, (err) => {
fs.writeFile(
path.join(
strapi.config.public.path,
`/uploads/${file.hash}${file.ext}`
),
file.buffer,
err => {
if (err) {
return reject(err);
}
file.url = `/uploads/${file.hash}${file.ext}`;
resolve();
}
);
});
},
delete(file) {
return new Promise((resolve, reject) => {
const filePath = path.join(
strapi.config.public.path,
`/uploads/${file.hash}${file.ext}`
);
if (!fs.existsSync(filePath)) {
return resolve("File doesn't exist");
}
// remove file from public/assets folder
fs.unlink(filePath, err => {
if (err) {
return reject(err);
}
file.url = `/uploads/${file.hash}${file.ext}`;
resolve();
});
});
},
delete: (file) => {
return new Promise((resolve, reject) => {
const filePath = path.join(strapi.config.public.path, `/uploads/${file.hash}${file.ext}`);
if (!fs.existsSync(filePath)) {
return resolve('File doesn\'t exist');
}
// remove file from public/assets folder
fs.unlink(filePath, (err) => {
if (err) {
return reject(err);
}
resolve();
});
});
}
};
}
},
};

View File

@ -1,9 +0,0 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.12.0
# ignores vulnerabilities until expiry date; change duration by modifying expiry date
ignore:
shelljs:
- '*':
reason: testing
expires: 2019-01-04T14:38:15.967Z
patch: {}

View File

@ -1,5 +1,24 @@
# strapi-provider-upload-rackspace
## Configurations
Your configuration is passed down to the client initialization. (e.g: `createClient(config)`). The implementation is based on the package `pkgcloud`. You can read the docs [here](https://github.com/pkgcloud/pkgcloud#storage).
**Example**
`./extensions/upload/config/settings.json`
```json
{
"provider": "rackspace",
"providerOptions": {
"username": "user-name",
"apiKey": "api-key",
"region": "IAD"
}
}
```
## Resources
- [MIT License](LICENSE.md)

View File

@ -7,43 +7,13 @@
// Public node modules.
const pkgcloud = require('pkgcloud');
const streamifier = require('streamifier');
/* eslint-disable no-unused-vars */
module.exports = {
provider: 'rackspace-cloudfiles',
name: 'Rackspace Cloud',
auth: {
username: {
label: 'Username',
type: 'text',
},
apiKey: {
label: 'API Key',
type: 'text',
},
container: {
label: 'Container Name',
type: 'text',
},
region: {
label: 'Region',
type: 'enum',
values: [
'DFW (Dallas-Fort Worth, TX, US)',
'HKG (Hong Kong, China)',
'IAD (Blacksburg, VA, US)',
'LON (London, England)',
'SYD (Sydney, Australia)',
],
},
},
init: config => {
init(config) {
const options = { container: config.container };
const client = pkgcloud.storage.createClient({
provider: 'rackspace',
username: config.username,
apiKey: config.apiKey,
region: config.region.replace(/(\s.*\))$/gi, ''),
...config,
});
const remoteURL = () =>
@ -54,21 +24,15 @@ module.exports = {
});
});
const byteSize = bytes => {
if (bytes === 0) return 0;
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10);
return i === 0 ? bytes : `${(bytes / 1024 ** i).toFixed(1)}`;
};
return {
upload: file => {
upload(file) {
const readStream = streamifier.createReadStream(file.buffer);
const writeStream = client.upload(
Object.assign({}, options, {
remote: file.name,
contentType: file.mime,
})
);
const writeStream = client.upload({
...options,
remote: file.name,
contentType: file.mime,
});
return new Promise((resolve, reject) => {
readStream.pipe(writeStream);
writeStream.on('error', error => error && reject(error));
@ -78,12 +42,8 @@ module.exports = {
resolve(
Object.assign(file, {
name: result.name,
hash: file.hash,
ext: file.ext,
mime: result.contentType,
size: file.size,
url: `${data.cdnSslUri}/${result.name}`,
provider: 'Rackspace Cloud',
})
)
)
@ -91,7 +51,7 @@ module.exports = {
});
});
},
delete: file => {
delete(file) {
return new Promise((resolve, reject) => {
client.removeFile(config.container, file.name, error => {
if (error) return reject(error);

View File

@ -13,5 +13,9 @@
"pkgcloud": "^2.0.0",
"streamifier": "^0.1.1"
},
"engines": {
"node": ">=10.0.0",
"npm": ">=6.0.0"
},
"gitHead": "c85658a19b8fef0f3164c19693a45db305dc07a9"
}