yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 18:07:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

improvement(upload-files): bring back changes from SaaS (#15181)

Browse Source 
Co-authored-by: Chris Collins <chriscollins3456@gmail.com>
This commit is contained in:
Purnima Garg 2025-11-04 21:17:01 +05:30 committed by GitHub
parent 76ecfa6e8e
commit 3d7d7a6eed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 69 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/files/GetPresignedUploadUrlResolverTest.java
View File

@ -38,7 +38,7 @@ public class GetPresignedUploadUrlResolverTest {
private static final String MOCKED_PRESIGNED_URL = "https://mocked.s3.url/test-key";
private static final Integer TEST_EXPIRATION_SECONDS = 3600; // Default from application.yaml
private static final String TEST_ASSET_PATH_PREFIX =
"product-assets"; // Default from application.yaml
"product_assets"; // Default from application.yaml
@Mock private S3Util mockS3Util;
@Mock private QueryContext mockQueryContext;

3
datahub-web-react/src/alchemy-components/components/Editor/extensions/fileDragDrop/FileDragDropExtension.tsx
View File

@ -74,6 +74,9 @@ class FileDragDropExtension extends NodeExtension<FileDragDropOptions> {
props: {
handleDOMEvents: {
drop: (view: EditorView, event: DragEvent) => {
if (!view.editable) {
return true; // prevents editor from handling the drop
}
const data = event.dataTransfer;
if (data && data.files && data.files.length > 0) {
// External file drop

18
datahub-web-react/src/alchemy-components/components/Editor/extensions/fileDragDrop/__tests__/fileUtils.test.ts
View File

@ -323,8 +323,22 @@ describe('fileUtils', () => {
expect(getFileIconFromExtension('PPTX')).toBe('FilePpt');
});
it('should return FileImage for image extensions', () => {
['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tiff'].forEach((ext) => {
it('should return FileJpg for image extensions', () => {
['jpg', 'jpeg'].forEach((ext) => {
expect(getFileIconFromExtension(ext)).toBe('FileJpg');
expect(getFileIconFromExtension(ext.toUpperCase())).toBe('FileJpg');
});
});
it('should return FilePng for png extensions', () => {
['png'].forEach((ext) => {
expect(getFileIconFromExtension(ext)).toBe('FilePng');
expect(getFileIconFromExtension(ext.toUpperCase())).toBe('FilePng');
});
});
it('should return FileImage for other image extensions', () => {
['gif', 'webp', 'bmp', 'tiff'].forEach((ext) => {
expect(getFileIconFromExtension(ext)).toBe('FileImage');
expect(getFileIconFromExtension(ext.toUpperCase())).toBe('FileImage');
});

59
datahub-web-react/src/alchemy-components/components/Editor/extensions/fileDragDrop/fileUtils.ts
View File

@ -156,6 +156,27 @@ export const getExtensionFromFileName = (fileName: string): string | undefined =
return fileName.slice(lastDotIndex + 1).toLowerCase();
};
/**
* Extract file type from URL
* @param url - the URL to extract type from
* @returns MIME type if detectable, empty string otherwise
*/
export const getFileTypeFromUrl = (url: string): string => {
const extension = getExtensionFromFileName(url);
if (!extension) return '';
return EXTENSION_TO_FILE_TYPE[extension] || '';
};
/**
* Extract file type from filename
* @param filename - the filename to extract type from
* @returns MIME type if detectable, empty string otherwise
*/
export const getFileTypeFromFilename = (filename: string): string => {
return getFileTypeFromUrl(filename);
};
/**
* Validate file before processing
*/
@ -167,6 +188,7 @@ export const validateFile = (
},
): { isValid: boolean; error?: string; displayError?: string; failureType?: FileUploadFailureType } => {
const { maxSize = MAX_FILE_SIZE_IN_BYTES, allowedTypes = SUPPORTED_FILE_TYPES } = options || {};
const fileType = file.type || getFileTypeFromFilename(file.name);
// Check file size
if (file.size > maxSize) {
@ -179,11 +201,11 @@ export const validateFile = (
}
// Check file type
if (!isFileTypeSupported(file.type, allowedTypes)) {
if (!isFileTypeSupported(fileType, allowedTypes)) {
const extension = getExtensionFromFileName(file.name);
return {
isValid: false,
error: `File type "${file.type}" is not allowed. Supported types: ${allowedTypes.join(', ')}`,
error: `File type "${fileType}" is not allowed. Supported types: ${allowedTypes.join(', ')}`,
displayError: `File type not supported${extension ? `: ${extension.toLocaleUpperCase()}` : ''}`,
failureType: FileUploadFailureType.FILE_TYPE,
};
@ -221,27 +243,6 @@ export const isFileUrl = (url: string): boolean => {
return url.includes('/openapi/v1/'); // Our internal file API
};
/**
* Extract file type from URL
* @param url - the URL to extract type from
* @returns MIME type if detectable, empty string otherwise
*/
export const getFileTypeFromUrl = (url: string): string => {
const extension = getExtensionFromFileName(url);
if (!extension) return '';
return EXTENSION_TO_FILE_TYPE[extension] || '';
};
/**
* Extract file type from filename
* @param filename - the filename to extract type from
* @returns MIME type if detectable, empty string otherwise
*/
export const getFileTypeFromFilename = (filename: string): string => {
return getFileTypeFromUrl(filename);
};
/**
* Get icon to show based on file extension
* @param extension - the extension of the file
@ -257,6 +258,8 @@ export const getFileIconFromExtension = (extension: string) => {
case 'txt':
case 'md':
case 'rtf':
case 'log':
case 'json':
return 'FileText';
case 'xls':
case 'xlsx':
@ -264,9 +267,13 @@ export const getFileIconFromExtension = (extension: string) => {
case 'ppt':
case 'pptx':
return 'FilePpt';
case 'svg':
return 'FileSvg';
case 'jpg':
case 'jpeg':
return 'FileJpg';
case 'png':
return 'FilePng';
case 'gif':
case 'webp':
case 'bmp':
@ -284,6 +291,12 @@ export const getFileIconFromExtension = (extension: string) => {
return 'FileZip';
case 'csv':
return 'FileCsv';
case 'html':
return 'FileHtml';
case 'py':
return 'FilePy';
case 'java':
return 'FileCode';
default:
return 'FileArrowDown';
}

5
datahub-web-react/src/alchemy-components/components/Notification/components/NotificationGlobalStyle.tsx
View File

@ -3,8 +3,13 @@ import { createGlobalStyle } from 'styled-components';
import { colors } from '@components/theme';
export const NotificationGlobalStyle = createGlobalStyle`
.ant-notification {
z-index: 1013; // one above antd modal (which is 1012)
}
.datahub-notification.ant-notification-notice {
padding: 8px;
border-radius: 8px;
}
.datahub-notification .ant-notification-notice-icon {

1
datahub-web-react/src/app/entityV2/summary/documentation/EditDescriptionModal.tsx
View File

@ -58,6 +58,7 @@ export default function EditDescriptionModal({
onCancel={closeModal}
width="80vw"
style={{ maxWidth: '1200px' }}
maskClosable={false}
buttons={[
{
text: 'Cancel',

2
datahub-web-react/src/app/shared/constants.ts
View File

@ -21,4 +21,4 @@ export const DEBOUNCE_SEARCH_MS = 300;
export const ANT_NOTIFICATION_Z_INDEX = 1010;
// S3 folder to store product assets
export const PRODUCT_ASSETS_FOLDER = 'product-assets';
export const PRODUCT_ASSETS_FOLDER = 'product_assets';

4
datahub-web-react/src/app/shared/hooks/__tests__/useCreateFile.test.ts
View File

@ -49,7 +49,7 @@ describe('useCreateFile', () => {
schemaField,
scenario,
sizeInBytes: mockFile.size,
storageKey: `product-assets/${fileId}`,
storageKey: `product_assets/${fileId}`,
contentHash: 'ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73', // Expected SHA-256 hash of 'content'
},
},
@ -132,7 +132,7 @@ describe('useCreateFile', () => {
schemaField: undefined,
scenario,
sizeInBytes: mockFile.size,
storageKey: `product-assets/${fileId}`,
storageKey: `product_assets/${fileId}`,
contentHash: 'ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73',
},
},

6
datahub-web-react/src/app/shared/hooks/__tests__/useFileUpload.test.tsx
View File

@ -107,7 +107,7 @@ describe('useFileUpload', () => {
await waitFor(async () => {
const url = await uploadPromise;
expect(url).toBe('http://example.com/openapi/v1/files/product-assets/file-123');
expect(url).toBe('http://example.com/openapi/v1/files/product_assets/file-123');
});
// Verify fetch was called with correct parameters
@ -285,7 +285,7 @@ describe('useFileUpload', () => {
await waitFor(async () => {
const url = await uploadPromise;
expect(url).toBe('http://example.com/openapi/v1/files/product-assets/file-456');
expect(url).toBe('http://example.com/openapi/v1/files/product_assets/file-456');
});
// Verify fetch was called with correct content type
@ -357,7 +357,7 @@ describe('useFileUpload', () => {
await waitFor(async () => {
const url = await uploadPromise;
expect(url).toBe('http://example.com/openapi/v1/files/product-assets/file-789');
expect(url).toBe('http://example.com/openapi/v1/files/product_assets/file-789');
});
expect(mockCreateFile).toHaveBeenCalledTimes(1);
expect(mockCreateFile).toHaveBeenCalledWith(mockFileId, mockFile);

2
metadata-service/configuration/src/main/resources/application.yaml
View File

@ -162,7 +162,7 @@ datahub:
roleArn: ${DATAHUB_ROLE_ARN:} # The AWS IAM role ARN to assume for S3 access
presignedUploadUrlExpirationSeconds: ${DATAHUB_PRESIGNED_UPLOAD_URL_EXPIRATION_SECONDS:3600} # 60 minutes
presignedDownloadUrlExpirationSeconds: ${DATAHUB_PRESIGNED_DOWNLOAD_URL_EXPIRATION_SECONDS:3600} # 60 minutes
assetPathPrefix: ${DATAHUB_S3_ASSET_PATH_PREFIX:product-assets}
assetPathPrefix: ${DATAHUB_S3_ASSET_PATH_PREFIX:product_assets}
entityService:
impl: ${ENTITY_SERVICE_IMPL:ebean}