mirror of
https://github.com/payloadcms/payload.git
synced 2026-09-14 20:07:19 +08:00
fix: avoid buffering large client uploads into memory (#17872)
Backport of https://github.com/payloadcms/payload/pull/17856 ## Summary Client uploads send a file straight from the browser to storage, bypassing the Payload server. The server still reads the file afterward, to save it locally or to read metadata such as width and height. It previously always downloaded the whole file into one in-memory buffer for that step, with no size limit. This is a real risk for the multi-gigabyte files that client uploads exist to support. The server now reads only the bytes each save operation actually needs, and streams any full-file read straight to disk instead of memory. ## Why Azure's client-upload support removes the old five-gigabyte upload ceiling, so a single upload can now be far larger than the server's available memory. Re-downloading that whole file into a buffer, just to check its size or resave it unchanged, does not scale with that change. ## How - A new content-requirement check decides how much of a file the server needs before it fetches anything: nothing, when the client-reported metadata is enough; a small byte range, when only the image dimensions are needed; or the full file, when local storage needs the real bytes, the file will be resized or reformatted, or a configured mime-type allow list needs to inspect the content. - A full-file read now streams straight to a temporary file on disk, replacing the single in-memory buffer. - An unmodified temporary file that only needs saving to disk-disabled storage is left untouched, instead of being read into memory and written back unchanged. - Fixed a check for the `disableLocalStorage` option. Collections that leave it unset, the most common case, could take the reduced-fetch path meant only for disabled local storage. That risked saving a truncated or empty file. - Fixed the byte-range probe's request clone. It broke native request properties that real storage handlers read, such as an abort signal. Every adapter in this repository that reads that property, including S3 and Azure, crashed against a client upload needing only its image dimensions. - Corrected the list of image formats treated as animated. It skipped multi-page TIFF files and wrongly flagged every AVIF file as animated, even single-frame ones. ## Testing Added tests for the new content-requirement decision and the streaming behavior. Added integration tests that complete a real client upload against the S3 and Azure storage adapters for a file needing only its dimensions, the exact path the request-clone fix above corrects. No integration test exercised that path before, which is how the bug shipped unnoticed. ## Alternatives considered Tried splitting a full-file read into repeated smaller range requests, on the idea that shorter-lived reads would free memory sooner. Measured memory during the change and found no improvement, so it was not kept. ## Related work Related to #17318 and #17319, which enabled Azure client uploads larger than 5 GB.
This commit is contained in:
Paul
@@ -11,6 +11,7 @@ import {
|
||||
headersWithCors,
|
||||
logError,
|
||||
mergeHeaders,
|
||||
unlinkClientUploadTempFile,
|
||||
} from 'payload'
|
||||
|
||||
const handleError = async ({
|
||||
@@ -112,49 +113,55 @@ export const POST =
|
||||
})
|
||||
}
|
||||
|
||||
await addDataAndFileToRequest(req)
|
||||
addLocalesToRequestFromData(req)
|
||||
try {
|
||||
await addDataAndFileToRequest(req)
|
||||
addLocalesToRequestFromData(req)
|
||||
|
||||
const { schema, validationRules } = await getGraphql(config)
|
||||
const { schema, validationRules } = await getGraphql(config)
|
||||
|
||||
const headers = {}
|
||||
const apiResponse = await createHandler({
|
||||
context: { headers, req },
|
||||
onOperation: async (request, args, result) => {
|
||||
const response =
|
||||
typeof payload.extensions === 'function'
|
||||
? await payload.extensions({
|
||||
args,
|
||||
req: request,
|
||||
result,
|
||||
})
|
||||
: result
|
||||
if (response.errors) {
|
||||
const errors = (await Promise.all(
|
||||
result.errors.map((error) => {
|
||||
return handleError({ err: error, payload, req })
|
||||
}),
|
||||
)) as GraphQLError[]
|
||||
// errors type should be FormattedGraphQLError[] but onOperation has a return type of ExecutionResult instead of FormattedExecutionResult
|
||||
return { ...response, errors }
|
||||
}
|
||||
return response
|
||||
},
|
||||
schema,
|
||||
validationRules: (_, args, defaultRules) => defaultRules.concat(validationRules(args)),
|
||||
})(originalRequest)
|
||||
const headers = {}
|
||||
const apiResponse = await createHandler({
|
||||
context: { headers, req },
|
||||
onOperation: async (request, args, result) => {
|
||||
const response =
|
||||
typeof payload.extensions === 'function'
|
||||
? await payload.extensions({
|
||||
args,
|
||||
req: request,
|
||||
result,
|
||||
})
|
||||
: result
|
||||
if (response.errors) {
|
||||
const errors = (await Promise.all(
|
||||
result.errors.map((error) => {
|
||||
return handleError({ err: error, payload, req })
|
||||
}),
|
||||
)) as GraphQLError[]
|
||||
// errors type should be FormattedGraphQLError[] but onOperation has a return type of ExecutionResult instead of FormattedExecutionResult
|
||||
return { ...response, errors }
|
||||
}
|
||||
return response
|
||||
},
|
||||
schema,
|
||||
validationRules: (_, args, defaultRules) => defaultRules.concat(validationRules(args)),
|
||||
})(originalRequest)
|
||||
|
||||
const resHeaders = headersWithCors({
|
||||
headers: new Headers(apiResponse.headers),
|
||||
req,
|
||||
})
|
||||
const resHeaders = headersWithCors({
|
||||
headers: new Headers(apiResponse.headers),
|
||||
req,
|
||||
})
|
||||
|
||||
for (const key in headers) {
|
||||
resHeaders.append(key, headers[key])
|
||||
for (const key in headers) {
|
||||
resHeaders.append(key, headers[key])
|
||||
}
|
||||
|
||||
return new Response(apiResponse.body, {
|
||||
headers: req.responseHeaders ? mergeHeaders(req.responseHeaders, resHeaders) : resHeaders,
|
||||
status: apiResponse.status,
|
||||
})
|
||||
} finally {
|
||||
// GraphQL parses the body itself instead of going through wrapInternalEndpoints, and an
|
||||
// operation that writes no document never reaches the operation-level cleanup.
|
||||
await unlinkClientUploadTempFile({ req })
|
||||
}
|
||||
|
||||
return new Response(apiResponse.body, {
|
||||
headers: req.responseHeaders ? mergeHeaders(req.responseHeaders, resHeaders) : resHeaders,
|
||||
status: apiResponse.status,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -453,6 +453,13 @@ export const createOperation = async <
|
||||
|
||||
return result
|
||||
} catch (error: unknown) {
|
||||
await unlinkTempFiles({
|
||||
collectionConfig: args.collection.config,
|
||||
config: args.req.payload.config,
|
||||
req: args.req,
|
||||
}).catch((unlinkError) => {
|
||||
args.req.payload.logger.error({ err: unlinkError, msg: 'Failed to remove temp file' })
|
||||
})
|
||||
await killTransaction(args.req)
|
||||
throw error
|
||||
}
|
||||
|
||||
@@ -316,12 +316,6 @@ export const updateOperation = async <
|
||||
return null
|
||||
})
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config,
|
||||
req,
|
||||
})
|
||||
|
||||
// Process sequentially when using single transaction mode to avoid shared state issues
|
||||
// Process in parallel when using one transaction for better performance
|
||||
let awaitedDocs: (DataFromCollectionSlug<TSlug> | null)[]
|
||||
@@ -334,6 +328,12 @@ export const updateOperation = async <
|
||||
awaitedDocs = await Promise.all(promises)
|
||||
}
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config,
|
||||
req,
|
||||
})
|
||||
|
||||
let result = {
|
||||
docs: awaitedDocs.filter(Boolean),
|
||||
errors,
|
||||
@@ -359,6 +359,13 @@ export const updateOperation = async <
|
||||
// @ts-expect-error - vestiges of when tsconfig was not strict. Feel free to improve
|
||||
return result
|
||||
} catch (error: unknown) {
|
||||
await unlinkTempFiles({
|
||||
collectionConfig: args.collection.config,
|
||||
config: args.req.payload.config,
|
||||
req: args.req,
|
||||
}).catch((unlinkError) => {
|
||||
args.req.payload.logger.error({ err: unlinkError, msg: 'Failed to remove temp file' })
|
||||
})
|
||||
await killTransaction(args.req)
|
||||
throw error
|
||||
}
|
||||
|
||||
@@ -263,6 +263,13 @@ export const updateByIDOperation = async <
|
||||
|
||||
return result
|
||||
} catch (error: unknown) {
|
||||
await unlinkTempFiles({
|
||||
collectionConfig: args.collection.config,
|
||||
config: args.req.payload.config,
|
||||
req: args.req,
|
||||
}).catch((unlinkError) => {
|
||||
args.req.payload.logger.error({ err: unlinkError, msg: 'Failed to remove temp file' })
|
||||
})
|
||||
await killTransaction(args.req)
|
||||
throw error
|
||||
}
|
||||
|
||||
@@ -1799,6 +1799,7 @@ export { getFileByPath } from './uploads/getFileByPath.js'
|
||||
export { _internal_safeFetchGlobal } from './uploads/safeFetch.js'
|
||||
|
||||
export type * from './uploads/types.js'
|
||||
export { unlinkClientUploadTempFile } from './uploads/unlinkClientUploadTempFile.js'
|
||||
export { addDataAndFileToRequest } from './utilities/addDataAndFileToRequest.js'
|
||||
export { addLocalesToRequestFromData, sanitizeLocales } from './utilities/addLocalesToRequest.js'
|
||||
export { canAccessAdmin } from './utilities/canAccessAdmin.js'
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { PayloadRequest } from '../types/index.js'
|
||||
import type { WithMetadata } from './optionallyAppendMetadata.js'
|
||||
import type { UploadEdits } from './types.js'
|
||||
|
||||
import { isAnimatedImage } from './isAnimatedImage.js'
|
||||
import { optionallyAppendMetadata } from './optionallyAppendMetadata.js'
|
||||
|
||||
const percentToPixel = (value: number, dimension: number) => {
|
||||
@@ -35,7 +36,7 @@ export async function cropImage({
|
||||
const { x, y } = cropData!
|
||||
const file = fileArg!
|
||||
|
||||
const fileIsAnimatedType = ['image/avif', 'image/gif', 'image/webp'].includes(file.mimetype)
|
||||
const fileIsAnimatedType = isAnimatedImage(file.mimetype)
|
||||
|
||||
const sharpOptions: SharpOptions = {}
|
||||
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import type { Collection } from '../collections/config/types.js'
|
||||
import type { SanitizedConfig } from '../config/types.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import fs from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { generateFileData } from './generateFileData.js'
|
||||
|
||||
// A minimal valid 1x1 transparent PNG, so `file-type` can detect `image/png` from it.
|
||||
const PNG_SIGNATURE = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg==',
|
||||
'base64',
|
||||
)
|
||||
|
||||
const createSharpMock = () => {
|
||||
const toBufferMock = vi.fn().mockResolvedValue({
|
||||
data: PNG_SIGNATURE,
|
||||
info: { height: 1, width: 1, size: PNG_SIGNATURE.length },
|
||||
})
|
||||
const metadataMock = vi.fn().mockResolvedValue({ height: 1, width: 1 })
|
||||
|
||||
const chain: any = {
|
||||
metadata: metadataMock,
|
||||
resize: vi.fn(() => chain),
|
||||
rotate: vi.fn(() => chain),
|
||||
toBuffer: toBufferMock,
|
||||
toFormat: vi.fn(() => chain),
|
||||
withMetadata: vi.fn(() => chain),
|
||||
}
|
||||
chain.trim = vi.fn(() => chain)
|
||||
|
||||
const sharp = vi.fn(() => chain)
|
||||
|
||||
return { sharp, toBufferMock }
|
||||
}
|
||||
|
||||
const createCollection = (uploadOverrides: Record<string, unknown> = {}): Collection =>
|
||||
({
|
||||
config: {
|
||||
slug: 'media',
|
||||
upload: {
|
||||
focalPoint: false,
|
||||
staticDir: os.tmpdir(),
|
||||
...uploadOverrides,
|
||||
},
|
||||
},
|
||||
}) as unknown as Collection
|
||||
|
||||
describe('generateFileData', () => {
|
||||
let tempFilePath: string
|
||||
|
||||
beforeEach(async () => {
|
||||
tempFilePath = path.join(os.tmpdir(), `generate-file-data-test-${randomUUID()}`)
|
||||
await fs.writeFile(tempFilePath, PNG_SIGNATURE)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tempFilePath, { force: true })
|
||||
})
|
||||
|
||||
const createReq = (sharp: unknown): PayloadRequest =>
|
||||
({
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'image/png',
|
||||
name: 'photo.png',
|
||||
size: PNG_SIGNATURE.length,
|
||||
tempFilePath,
|
||||
},
|
||||
payload: {
|
||||
config: { sharp },
|
||||
logger: { error: vi.fn() },
|
||||
},
|
||||
}) as unknown as PayloadRequest
|
||||
|
||||
describe('when local storage is disabled', () => {
|
||||
it('does not run full sharp processing on an image with no configured adjustments, even when it arrives via tempFilePath', async () => {
|
||||
const { sharp, toBufferMock } = createSharpMock()
|
||||
|
||||
await generateFileData({
|
||||
collection: createCollection({ disableLocalStorage: true }),
|
||||
config: {} as SanitizedConfig,
|
||||
data: {},
|
||||
operation: 'create',
|
||||
overwriteExistingFiles: true,
|
||||
req: createReq(sharp),
|
||||
})
|
||||
|
||||
expect(toBufferMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not save anything when no processing is needed', async () => {
|
||||
const { sharp } = createSharpMock()
|
||||
|
||||
const { files } = await generateFileData({
|
||||
collection: createCollection({ disableLocalStorage: true }),
|
||||
config: {} as SanitizedConfig,
|
||||
data: {},
|
||||
operation: 'create',
|
||||
overwriteExistingFiles: true,
|
||||
req: createReq(sharp),
|
||||
})
|
||||
|
||||
expect(files).toEqual([])
|
||||
})
|
||||
|
||||
it('still runs sharp processing when resize options are configured', async () => {
|
||||
const { sharp, toBufferMock } = createSharpMock()
|
||||
|
||||
await generateFileData({
|
||||
collection: createCollection({ disableLocalStorage: true, resizeOptions: { width: 100 } }),
|
||||
config: {} as SanitizedConfig,
|
||||
data: {},
|
||||
operation: 'create',
|
||||
overwriteExistingFiles: true,
|
||||
req: createReq(sharp),
|
||||
})
|
||||
|
||||
expect(toBufferMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when local storage is enabled (default)', () => {
|
||||
it('copies straight from the temp file instead of running sharp processing, when no adjustments are configured', async () => {
|
||||
const { sharp, toBufferMock } = createSharpMock()
|
||||
|
||||
const { files } = await generateFileData({
|
||||
collection: createCollection(),
|
||||
config: {} as SanitizedConfig,
|
||||
data: {},
|
||||
operation: 'create',
|
||||
overwriteExistingFiles: true,
|
||||
req: createReq(sharp),
|
||||
})
|
||||
|
||||
expect(toBufferMock).not.toHaveBeenCalled()
|
||||
expect(files).toEqual([{ path: `${os.tmpdir()}/photo.png`, sourcePath: tempFilePath }])
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { Collection } from '../collections/config/types.js'
|
||||
import type { SanitizedConfig } from '../config/types.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const mkdirMock = vi.fn().mockResolvedValue(undefined)
|
||||
const readFileMock = vi.fn().mockResolvedValue(Buffer.from('unused'))
|
||||
const writeFileMock = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
vi.mock('fs/promises', () => ({
|
||||
default: {
|
||||
mkdir: mkdirMock,
|
||||
readFile: readFileMock,
|
||||
writeFile: writeFileMock,
|
||||
},
|
||||
}))
|
||||
|
||||
const { generateFileData } = await import('./generateFileData.js')
|
||||
|
||||
const createCollection = (disableLocalStorage: boolean): Collection =>
|
||||
({
|
||||
config: {
|
||||
slug: 'media',
|
||||
upload: {
|
||||
disableLocalStorage,
|
||||
staticDir: '/tmp/media',
|
||||
},
|
||||
},
|
||||
}) as unknown as Collection
|
||||
|
||||
const createReq = (tempFilePath: string, size: number): PayloadRequest =>
|
||||
({
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'video/mp4',
|
||||
name: 'big-video.mp4',
|
||||
size,
|
||||
tempFilePath,
|
||||
},
|
||||
payload: {
|
||||
config: {},
|
||||
logger: { error: vi.fn() },
|
||||
},
|
||||
}) as unknown as PayloadRequest
|
||||
|
||||
describe('generateFileData - non-image temp file buffering', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
readFileMock.mockResolvedValue(Buffer.from('unused'))
|
||||
})
|
||||
|
||||
it('does not read or rewrite a temp file into memory when local storage is disabled', async () => {
|
||||
const req = createReq('/tmp/payload-upload-abc', 5_000_000_000)
|
||||
|
||||
const result = await generateFileData({
|
||||
collection: createCollection(true),
|
||||
config: {} as SanitizedConfig,
|
||||
data: {},
|
||||
operation: 'create',
|
||||
overwriteExistingFiles: true,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(readFileMock).not.toHaveBeenCalled()
|
||||
expect(writeFileMock).not.toHaveBeenCalled()
|
||||
expect(result.files).toEqual([])
|
||||
expect(result.data).toMatchObject({ filesize: 5_000_000_000, mimeType: 'video/mp4' })
|
||||
})
|
||||
|
||||
it('copies the temp file directly, without reading it into memory, when local storage is enabled', async () => {
|
||||
const req = createReq('/tmp/payload-upload-def', 5_000_000_000)
|
||||
|
||||
const result = await generateFileData({
|
||||
collection: createCollection(false),
|
||||
config: {} as SanitizedConfig,
|
||||
data: {},
|
||||
operation: 'create',
|
||||
overwriteExistingFiles: true,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(readFileMock).not.toHaveBeenCalled()
|
||||
expect(writeFileMock).not.toHaveBeenCalled()
|
||||
expect(result.files).toEqual([
|
||||
{ path: '/tmp/media/big-video.mp4', sourcePath: '/tmp/payload-upload-def' },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -19,6 +19,7 @@ import { getFileByPath } from './getFileByPath.js'
|
||||
import { getImageSize } from './getImageSize.js'
|
||||
import { getSafeFileName } from './getSafeFilename.js'
|
||||
import { createImageSizes } from './image-resizing/createImageSizes.js'
|
||||
import { isAnimatedImage } from './isAnimatedImage.js'
|
||||
import { isImage } from './isImage.js'
|
||||
import { optionallyAppendMetadata } from './optionallyAppendMetadata.js'
|
||||
type Args<T> = {
|
||||
@@ -66,6 +67,14 @@ const shouldReupload = (
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the document's file metadata and the list of files to write to disk.
|
||||
*
|
||||
* A file uploaded with `useTempFiles` enabled arrives as a temp file path instead of an
|
||||
* in-memory buffer, and that temp file can be far larger than server memory allows. To avoid
|
||||
* loading it into memory unnecessarily, this skips reading the temp file entirely when local
|
||||
* storage is disabled, and copies it straight to its destination when local storage is enabled.
|
||||
*/
|
||||
export const generateFileData = async <T>({
|
||||
collection: { config: collectionConfig },
|
||||
data,
|
||||
@@ -175,7 +184,7 @@ export const generateFileData = async <T>({
|
||||
let newData = incomingFileData as T
|
||||
const filesToSave: FileToSave[] = []
|
||||
const fileData: Partial<FileData> = {}
|
||||
const fileIsAnimatedType = ['image/avif', 'image/gif', 'image/webp'].includes(file.mimetype)
|
||||
const fileIsAnimatedType = isAnimatedImage(file.mimetype)
|
||||
const cropData =
|
||||
typeof uploadEdits === 'object' && 'crop' in uploadEdits ? uploadEdits.crop : undefined
|
||||
|
||||
@@ -187,11 +196,11 @@ export const generateFileData = async <T>({
|
||||
let fileBuffer!: { data: Buffer; info: OutputInfo }
|
||||
let ext
|
||||
let mime: string
|
||||
// Depends only on configured resize/format/trim options, not on whether the bytes are on
|
||||
// disk or in memory.
|
||||
const fileHasAdjustments =
|
||||
fileSupportsResize &&
|
||||
Boolean(
|
||||
resizeOptions || formatOptions || trimOptions || constructorOptions || file.tempFilePath,
|
||||
)
|
||||
Boolean(resizeOptions || formatOptions || trimOptions || constructorOptions)
|
||||
|
||||
const sharpOptions: SharpOptions = { ...constructorOptions }
|
||||
|
||||
@@ -351,31 +360,55 @@ export const generateFileData = async <T>({
|
||||
}
|
||||
} else {
|
||||
// For non-image files with useTempFiles, read the buffer from the temp file
|
||||
// since file.data is empty when using temp files
|
||||
let bufferToSave: Buffer
|
||||
if (fileBuffer?.data) {
|
||||
bufferToSave = fileBuffer.data
|
||||
} else if (file.tempFilePath) {
|
||||
bufferToSave = await fs.readFile(file.tempFilePath)
|
||||
// since file.data is empty when using temp files.
|
||||
//
|
||||
// When local storage is disabled, filesToSave is never written to disk (create/update
|
||||
// skip uploadFiles for it), and an unmodified temp file's bytes on disk are already
|
||||
// correct, so there's nothing to read into memory or write back out. Skipping this avoids
|
||||
// buffering the entire file just to discard or rewrite it unchanged - a tempFilePath can
|
||||
// point at a file far larger than server memory allows.
|
||||
const skipTempFileBuffer =
|
||||
disableLocalStorage && Boolean(file.tempFilePath) && !fileBuffer?.data
|
||||
|
||||
// When local storage is enabled and the temp file itself is unmodified, copy it straight
|
||||
// to its destination instead of reading it into memory first - a tempFilePath can point at
|
||||
// a file far larger than server memory allows.
|
||||
const shouldCopyFromTempFile =
|
||||
!fileBuffer?.data && Boolean(file.tempFilePath) && !disableLocalStorage
|
||||
|
||||
if (shouldCopyFromTempFile) {
|
||||
filesToSave.push({
|
||||
path: `${staticPath}/${fsSafeName}`,
|
||||
sourcePath: file.tempFilePath!,
|
||||
})
|
||||
} else {
|
||||
bufferToSave = file.data
|
||||
}
|
||||
|
||||
filesToSave.push({
|
||||
buffer: bufferToSave,
|
||||
path: `${staticPath}/${fsSafeName}`,
|
||||
})
|
||||
|
||||
// If using temp files and the image is being resized, write the file to the temp path
|
||||
if (fileBuffer?.data || bufferToSave.length > 0) {
|
||||
if (file.tempFilePath) {
|
||||
await fs.writeFile(file.tempFilePath, fileBuffer?.data || bufferToSave) // write fileBuffer to the temp path
|
||||
let bufferToSave: Buffer
|
||||
if (fileBuffer?.data) {
|
||||
bufferToSave = fileBuffer.data
|
||||
} else if (file.tempFilePath) {
|
||||
bufferToSave = skipTempFileBuffer ? Buffer.alloc(0) : await fs.readFile(file.tempFilePath)
|
||||
} else {
|
||||
// Assign the _possibly modified_ file to the request object
|
||||
req.file = {
|
||||
...file,
|
||||
data: fileBuffer?.data || bufferToSave,
|
||||
size: fileBuffer?.info.size,
|
||||
bufferToSave = file.data
|
||||
}
|
||||
|
||||
if (!skipTempFileBuffer) {
|
||||
filesToSave.push({
|
||||
buffer: bufferToSave,
|
||||
path: `${staticPath}/${fsSafeName}`,
|
||||
})
|
||||
|
||||
// If using temp files and the image is being resized, write the file to the temp path
|
||||
if (fileBuffer?.data || bufferToSave.length > 0) {
|
||||
if (file.tempFilePath) {
|
||||
await fs.writeFile(file.tempFilePath, fileBuffer?.data || bufferToSave) // write fileBuffer to the temp path
|
||||
} else {
|
||||
// Assign the _possibly modified_ file to the request object
|
||||
req.file = {
|
||||
...file,
|
||||
data: fileBuffer?.data || bufferToSave,
|
||||
size: fileBuffer?.info.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { SanitizedUploadConfig } from './types.js'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { getFileContentRequirement } from './getFileContentRequirement.js'
|
||||
|
||||
describe('getFileContentRequirement', () => {
|
||||
it.each([
|
||||
{ expected: 'full', mimeType: 'video/mp4', upload: { disableLocalStorage: false } },
|
||||
{
|
||||
expected: 'full',
|
||||
mimeType: 'video/mp4',
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
},
|
||||
{ expected: 'none', mimeType: 'video/mp4', upload: { disableLocalStorage: true } },
|
||||
{ expected: 'header', mimeType: 'image/png', upload: { disableLocalStorage: true } },
|
||||
{
|
||||
expected: 'full',
|
||||
mimeType: 'image/png',
|
||||
upload: { disableLocalStorage: true, resizeOptions: { width: 100 } },
|
||||
},
|
||||
{
|
||||
expected: 'full',
|
||||
mimeType: 'image/png',
|
||||
upload: { disableLocalStorage: true, imageSizes: [{ name: 'thumb', width: 100 }] },
|
||||
},
|
||||
{ expected: 'full', mimeType: 'image/gif', upload: { disableLocalStorage: true } },
|
||||
])('returns $expected for $mimeType and $upload', ({ expected, mimeType, upload }) => {
|
||||
expect(
|
||||
getFileContentRequirement({
|
||||
mimeType,
|
||||
uploadConfig: upload as SanitizedUploadConfig,
|
||||
}),
|
||||
).toBe(expected)
|
||||
})
|
||||
|
||||
it('requires full content when the request includes crop or size edits', () => {
|
||||
expect(
|
||||
getFileContentRequirement({
|
||||
hasSizeEdits: true,
|
||||
mimeType: 'image/png',
|
||||
uploadConfig: { disableLocalStorage: true } as SanitizedUploadConfig,
|
||||
}),
|
||||
).toBe('full')
|
||||
})
|
||||
|
||||
it('returns none for a restricted-type-allowed non-image when mimeTypes validation is bypassed', () => {
|
||||
expect(
|
||||
getFileContentRequirement({
|
||||
mimeType: 'video/mp4',
|
||||
uploadConfig: {
|
||||
allowRestrictedFileTypes: true,
|
||||
disableLocalStorage: true,
|
||||
mimeTypes: ['video/*'],
|
||||
} as SanitizedUploadConfig,
|
||||
}),
|
||||
).toBe('none')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { SanitizedUploadConfig } from './types.js'
|
||||
|
||||
import { canResizeImage } from './canResizeImage.js'
|
||||
import { isAnimatedImage } from './isAnimatedImage.js'
|
||||
import { isImage } from './isImage.js'
|
||||
|
||||
export const HEADER_PROBE_BYTE_LENGTH = 1024 * 1024
|
||||
|
||||
export type FileContentRequirement = 'full' | 'header' | 'none'
|
||||
|
||||
export function getFileContentRequirement({
|
||||
hasSizeEdits,
|
||||
mimeType,
|
||||
uploadConfig,
|
||||
}: {
|
||||
hasSizeEdits?: boolean
|
||||
mimeType: string
|
||||
uploadConfig: SanitizedUploadConfig
|
||||
}): FileContentRequirement {
|
||||
if (!uploadConfig.disableLocalStorage) {
|
||||
return 'full'
|
||||
}
|
||||
|
||||
const hasMimeTypeAllowList =
|
||||
!uploadConfig.allowRestrictedFileTypes &&
|
||||
Array.isArray(uploadConfig.mimeTypes) &&
|
||||
uploadConfig.mimeTypes.length > 0
|
||||
|
||||
if (hasMimeTypeAllowList) {
|
||||
return 'full'
|
||||
}
|
||||
|
||||
const isResizableImage = canResizeImage(mimeType)
|
||||
const hasConfiguredAdjustments = Boolean(
|
||||
uploadConfig.resizeOptions ||
|
||||
uploadConfig.formatOptions ||
|
||||
uploadConfig.trimOptions ||
|
||||
uploadConfig.constructorOptions ||
|
||||
(Array.isArray(uploadConfig.imageSizes) && uploadConfig.imageSizes.length > 0),
|
||||
)
|
||||
|
||||
if (hasSizeEdits || (isResizableImage && hasConfiguredAdjustments) || isAnimatedImage(mimeType)) {
|
||||
return 'full'
|
||||
}
|
||||
|
||||
if (isResizableImage || isImage(mimeType)) {
|
||||
return 'header'
|
||||
}
|
||||
|
||||
return 'none'
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
import type { SanitizedUploadConfig } from './types.js'
|
||||
|
||||
import fs from 'fs/promises'
|
||||
import os from 'os'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { HEADER_PROBE_BYTE_LENGTH } from './getFileContentRequirement.js'
|
||||
import { type ClientUploadData, getFileFromClientUpload } from './getFileFromClientUpload.js'
|
||||
import { getImageSize } from './getImageSize.js'
|
||||
|
||||
const MINIMAL_PNG = (() => {
|
||||
const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
||||
const ihdrData = Buffer.alloc(13)
|
||||
ihdrData.writeUInt32BE(400, 0) // width
|
||||
ihdrData.writeUInt32BE(300, 4) // height
|
||||
ihdrData[8] = 8 // bit depth
|
||||
ihdrData[9] = 2 // color type: RGB
|
||||
const ihdrChunk = Buffer.concat([
|
||||
Buffer.from([0, 0, 0, 13]),
|
||||
Buffer.from('IHDR'),
|
||||
ihdrData,
|
||||
Buffer.alloc(4), // crc (unchecked by dimension probing)
|
||||
])
|
||||
return Buffer.concat([signature, ihdrChunk])
|
||||
})()
|
||||
|
||||
const tempFilesToRemove: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllGlobals()
|
||||
for (const tempFilePath of tempFilesToRemove) {
|
||||
await fs.rm(tempFilePath, { force: true })
|
||||
}
|
||||
tempFilesToRemove.length = 0
|
||||
})
|
||||
|
||||
const createReq = ({
|
||||
handlers,
|
||||
upload,
|
||||
}: {
|
||||
handlers: NonNullable<SanitizedUploadConfig['handlers']>
|
||||
upload?: Partial<SanitizedUploadConfig>
|
||||
}): PayloadRequest => {
|
||||
const request = new Request('http://localhost/api/media')
|
||||
const req = request as unknown as PayloadRequest
|
||||
|
||||
req.query = {}
|
||||
req.payload = {
|
||||
collections: {
|
||||
media: {
|
||||
config: {
|
||||
upload: {
|
||||
disableLocalStorage: true,
|
||||
handlers,
|
||||
...upload,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
config: {
|
||||
upload: {},
|
||||
},
|
||||
logger: { error: vi.fn() },
|
||||
} as unknown as PayloadRequest['payload']
|
||||
|
||||
return req
|
||||
}
|
||||
|
||||
const videoFile = (overrides: Partial<ClientUploadData> = {}): ClientUploadData => ({
|
||||
clientUploadContext: { prefix: 'abc' },
|
||||
collectionSlug: 'media',
|
||||
filename: 'clip.mp4',
|
||||
mimeType: 'video/mp4',
|
||||
size: 10,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const imageFile = (overrides: Partial<ClientUploadData> = {}): ClientUploadData => ({
|
||||
clientUploadContext: { prefix: 'abc' },
|
||||
collectionSlug: 'media',
|
||||
filename: 'photo.png',
|
||||
mimeType: 'image/png',
|
||||
size: MINIMAL_PNG.length,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
describe('getFileFromClientUpload', () => {
|
||||
it('does not call a handler and returns empty data with an own clientUploadContext property when no content is required', async () => {
|
||||
const handler = vi.fn()
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = videoFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(handler).not.toHaveBeenCalled()
|
||||
expect(Object.prototype.hasOwnProperty.call(result, 'clientUploadContext')).toBe(true)
|
||||
expect(result.clientUploadContext).toEqual({ prefix: 'abc' })
|
||||
expect(result.data.length).toBe(0)
|
||||
expect(result.tempFilePath).toBeUndefined()
|
||||
expect(result.name).toBe('clip.mp4')
|
||||
expect(result.size).toBe(10)
|
||||
})
|
||||
|
||||
it('requests a bounded byte range and returns dimensions for a header-only image', async () => {
|
||||
const handler = vi.fn(async (handlerReq: PayloadRequest) => {
|
||||
expect(handlerReq.headers.get('Range')).toBe(`bytes=0-${HEADER_PROBE_BYTE_LENGTH - 1}`)
|
||||
return new Response(MINIMAL_PNG, {
|
||||
headers: { 'Content-Type': 'image/png' },
|
||||
status: 206,
|
||||
})
|
||||
})
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = imageFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(result.data.length).toBeLessThanOrEqual(HEADER_PROBE_BYTE_LENGTH)
|
||||
expect(result.tempFilePath).toBeUndefined()
|
||||
await expect(getImageSize({ file: result })).resolves.toEqual({ height: 300, width: 400 })
|
||||
})
|
||||
|
||||
it('caps the header read at the boundary and cancels the stream without pulling past it', async () => {
|
||||
let hasCancelled = false
|
||||
let hasReadPastBoundary = false
|
||||
const firstChunk = Buffer.concat([
|
||||
MINIMAL_PNG,
|
||||
Buffer.alloc(HEADER_PROBE_BYTE_LENGTH - MINIMAL_PNG.length),
|
||||
])
|
||||
const stream = new ReadableStream(
|
||||
{
|
||||
cancel() {
|
||||
hasCancelled = true
|
||||
},
|
||||
pull(controller) {
|
||||
hasReadPastBoundary = true
|
||||
controller.error(new Error('Read past header boundary'))
|
||||
},
|
||||
start(controller) {
|
||||
controller.enqueue(firstChunk)
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
)
|
||||
|
||||
const handler = vi.fn(
|
||||
async () => new Response(stream, { headers: { 'Content-Type': 'image/png' }, status: 206 }),
|
||||
)
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = imageFile({ size: HEADER_PROBE_BYTE_LENGTH })
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(result.data.length).toBe(HEADER_PROBE_BYTE_LENGTH)
|
||||
expect(hasCancelled).toBe(true)
|
||||
expect(hasReadPastBoundary).toBe(false)
|
||||
})
|
||||
|
||||
it('lets a handler read native Request properties like signal through the Range-scoped proxy', async () => {
|
||||
const handler = vi.fn(async (handlerReq: PayloadRequest) => {
|
||||
expect(() => handlerReq.signal).not.toThrow()
|
||||
expect(handlerReq.signal).toBeInstanceOf(AbortSignal)
|
||||
return new Response(MINIMAL_PNG, {
|
||||
headers: { 'Content-Type': 'image/png' },
|
||||
status: 206,
|
||||
})
|
||||
})
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = imageFile()
|
||||
|
||||
await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('falls back to a full fetch when the bounded header cannot be probed for dimensions', async () => {
|
||||
const handler = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(Buffer.from('not an image'), {
|
||||
headers: { 'Content-Type': 'image/png' },
|
||||
status: 206,
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(MINIMAL_PNG, { headers: { 'Content-Type': 'image/png' }, status: 200 }),
|
||||
)
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = imageFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(2)
|
||||
expect(result.tempFilePath).toBeDefined()
|
||||
expect(result.data.length).toBe(0)
|
||||
})
|
||||
|
||||
it('streams a full-content response to a temp file without buffering the whole body', async () => {
|
||||
const chunks = [
|
||||
Buffer.from('chunk-one-'),
|
||||
Buffer.from('chunk-two-'),
|
||||
Buffer.from('chunk-three'),
|
||||
]
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) {
|
||||
controller.enqueue(chunk)
|
||||
}
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
|
||||
const response = new Response(stream, {
|
||||
headers: { 'Content-Type': 'video/mp4' },
|
||||
status: 200,
|
||||
})
|
||||
const arrayBufferTripwire = vi.fn(async () => {
|
||||
throw new Error('Unexpected whole-body buffering')
|
||||
})
|
||||
Object.defineProperty(response, 'arrayBuffer', { value: arrayBufferTripwire })
|
||||
|
||||
const handler = vi.fn(async () => response)
|
||||
const req = createReq({
|
||||
handlers: [handler],
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
const file = videoFile({ size: 30 })
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(result.data).toEqual(Buffer.alloc(0))
|
||||
expect(arrayBufferTripwire).not.toHaveBeenCalled()
|
||||
const written = await fs.readFile(result.tempFilePath!)
|
||||
expect(written.toString()).toBe(chunks.map((chunk) => chunk.toString()).join(''))
|
||||
})
|
||||
|
||||
it('runs every handler and uses the last one that returns a response, matching v3', async () => {
|
||||
const firstHandler = vi.fn(async () => new Response(Buffer.from('first-response')))
|
||||
const secondHandler = vi.fn(async () => new Response(Buffer.from('second-response')))
|
||||
const req = createReq({
|
||||
handlers: [firstHandler, secondHandler],
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
const file = videoFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(firstHandler).toHaveBeenCalledTimes(1)
|
||||
expect(secondHandler).toHaveBeenCalledTimes(1)
|
||||
const written = await fs.readFile(result.tempFilePath!)
|
||||
expect(written.toString()).toBe('second-response')
|
||||
})
|
||||
|
||||
it('falls through to the next handler when the first returns nothing', async () => {
|
||||
const firstHandler = vi.fn(async () => undefined)
|
||||
const secondHandler = vi.fn(async () => new Response(Buffer.alloc(10), { status: 200 }))
|
||||
const req = createReq({
|
||||
handlers: [firstHandler, secondHandler],
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
const file = videoFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(firstHandler).toHaveBeenCalledTimes(1)
|
||||
expect(secondHandler).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('follows exactly one redirect from a handler response', async () => {
|
||||
const redirectTarget = 'http://storage.example.com/file.mp4'
|
||||
const handler = vi.fn(
|
||||
async () => new Response(null, { headers: { Location: redirectTarget }, status: 302 }),
|
||||
)
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response(Buffer.alloc(10), { headers: { 'Content-Type': 'video/mp4' }, status: 200 }),
|
||||
)
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
const req = createReq({
|
||||
handlers: [handler],
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
const file = videoFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
expect(fetchMock).toHaveBeenCalledWith(redirectTarget)
|
||||
})
|
||||
|
||||
it('forwards the Range header through a redirect during the bounded header probe', async () => {
|
||||
const redirectTarget = 'http://storage.example.com/photo.png'
|
||||
const handler = vi.fn(
|
||||
async () => new Response(null, { headers: { Location: redirectTarget }, status: 302 }),
|
||||
)
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response(MINIMAL_PNG, { headers: { 'Content-Type': 'image/png' }, status: 206 }),
|
||||
)
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = imageFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(result.tempFilePath).toBeUndefined()
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
expect(fetchMock).toHaveBeenCalledWith(redirectTarget, {
|
||||
headers: { Range: `bytes=0-${HEADER_PROBE_BYTE_LENGTH - 1}` },
|
||||
})
|
||||
})
|
||||
|
||||
it('removes the partial temp file when the stream fails before completion', async () => {
|
||||
const stream = new ReadableStream({
|
||||
pull(controller) {
|
||||
controller.error(new Error('stream broke'))
|
||||
},
|
||||
start(controller) {
|
||||
controller.enqueue(Buffer.from('partial-chunk'))
|
||||
},
|
||||
})
|
||||
const handler = vi.fn(
|
||||
async () => new Response(stream, { headers: { 'Content-Type': 'video/mp4' }, status: 200 }),
|
||||
)
|
||||
const req = createReq({
|
||||
handlers: [handler],
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
const file = videoFile()
|
||||
|
||||
const tempFileDir = os.tmpdir()
|
||||
const filesBefore = new Set(await fs.readdir(tempFileDir))
|
||||
|
||||
await expect(getFileFromClientUpload({ file, req })).rejects.toThrow('stream broke')
|
||||
|
||||
const filesAfter = await fs.readdir(tempFileDir)
|
||||
const leftoverTempFiles = filesAfter.filter(
|
||||
(entry) => !filesBefore.has(entry) && entry.startsWith('payload-client-upload-'),
|
||||
)
|
||||
expect(leftoverTempFiles).toEqual([])
|
||||
})
|
||||
|
||||
it('requires full content and skips the Range header when the request carries crop or size edits', async () => {
|
||||
const handler = vi.fn(async (handlerReq: PayloadRequest) => {
|
||||
expect(handlerReq.headers.get('Range')).toBeNull()
|
||||
return new Response(MINIMAL_PNG, { headers: { 'Content-Type': 'image/png' }, status: 200 })
|
||||
})
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
req.query = { uploadEdits: { heightInPixels: 100, widthInPixels: 100 } }
|
||||
const file = imageFile()
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1)
|
||||
expect(result.tempFilePath).toBeDefined()
|
||||
expect(result.data.length).toBe(0)
|
||||
})
|
||||
|
||||
it('treats a null response body as a legitimate zero-byte file', async () => {
|
||||
const handler = vi.fn(
|
||||
async () => new Response(null, { headers: { 'Content-Type': 'video/mp4' }, status: 200 }),
|
||||
)
|
||||
const req = createReq({
|
||||
handlers: [handler],
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
const file = videoFile({ size: 0 })
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
expect(result.tempFilePath).toBeDefined()
|
||||
const written = await fs.readFile(result.tempFilePath!)
|
||||
expect(written.length).toBe(0)
|
||||
})
|
||||
|
||||
it('streams the full file to disk when a non-image client upload carries no context', async () => {
|
||||
const handler = vi.fn(
|
||||
async () =>
|
||||
new Response(Buffer.from('full-video-bytes'), {
|
||||
headers: { 'Content-Type': 'video/mp4' },
|
||||
status: 200,
|
||||
}),
|
||||
)
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = videoFile({ clientUploadContext: undefined })
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1)
|
||||
expect(result.tempFilePath).toBeDefined()
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
const written = await fs.readFile(result.tempFilePath!)
|
||||
expect(written.toString()).toBe('full-video-bytes')
|
||||
})
|
||||
|
||||
it('streams the full file to disk when an image client upload carries no context', async () => {
|
||||
const handler = vi.fn(async (handlerReq: PayloadRequest) => {
|
||||
expect(handlerReq.headers.get('Range')).toBeNull()
|
||||
return new Response(MINIMAL_PNG, { headers: { 'Content-Type': 'image/png' }, status: 200 })
|
||||
})
|
||||
const req = createReq({ handlers: [handler], upload: { disableLocalStorage: true } })
|
||||
const file = imageFile({ clientUploadContext: undefined })
|
||||
|
||||
const result = await getFileFromClientUpload({ file, req })
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1)
|
||||
expect(result.tempFilePath).toBeDefined()
|
||||
tempFilesToRemove.push(result.tempFilePath!)
|
||||
|
||||
const written = await fs.readFile(result.tempFilePath!)
|
||||
expect(written).toEqual(MINIMAL_PNG)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,259 @@
|
||||
import { randomUUID } from 'crypto'
|
||||
import fs from 'fs'
|
||||
import { mkdir, rm, writeFile } from 'fs/promises'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { Readable } from 'stream'
|
||||
import { pipeline } from 'stream/promises'
|
||||
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
import type { SanitizedUploadConfig, UploadEdits } from './types.js'
|
||||
|
||||
import { APIError } from '../errors/APIError.js'
|
||||
import { isolateObjectProperty } from '../utilities/isolateObjectProperty.js'
|
||||
import { getFileContentRequirement, HEADER_PROBE_BYTE_LENGTH } from './getFileContentRequirement.js'
|
||||
import { getImageSize } from './getImageSize.js'
|
||||
|
||||
export type ClientUploadData = {
|
||||
clientUploadContext?: unknown
|
||||
collectionSlug: string
|
||||
filename: string
|
||||
mimeType: string
|
||||
size: number
|
||||
}
|
||||
|
||||
/**
|
||||
* `req.context` key `unlinkClientUploadTempFile` reads to find a materializer-created temp file
|
||||
* once `req.file` is gone - plugin-cloud-storage's afterChange hook clears `req.file` after
|
||||
* uploading generated image sizes, before cleanup runs, so tracking it only on `req.file`
|
||||
* would leak the temp file.
|
||||
*/
|
||||
export const CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY = 'payloadClientUploadTempFilePath'
|
||||
|
||||
export async function getFileFromClientUpload({
|
||||
file,
|
||||
req,
|
||||
}: {
|
||||
file: ClientUploadData
|
||||
req: PayloadRequest
|
||||
}): Promise<NonNullable<PayloadRequest['file']>> {
|
||||
const uploadConfig = req.payload.collections[file.collectionSlug]?.config.upload
|
||||
|
||||
if (!uploadConfig || !uploadConfig.handlers) {
|
||||
throw new APIError(`uploadConfig.handlers is not present for ${file.collectionSlug}`)
|
||||
}
|
||||
|
||||
const contentRequirement = file.clientUploadContext
|
||||
? getFileContentRequirement({
|
||||
hasSizeEdits: requestHasSizeEdits(req),
|
||||
mimeType: file.mimeType,
|
||||
uploadConfig,
|
||||
})
|
||||
: 'full'
|
||||
|
||||
if (contentRequirement === 'none') {
|
||||
return {
|
||||
name: file.filename,
|
||||
clientUploadContext: file.clientUploadContext,
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: file.mimeType,
|
||||
size: file.size,
|
||||
}
|
||||
}
|
||||
|
||||
if (contentRequirement === 'header') {
|
||||
const headerFile = await fetchHeaderOnly({ file, req, uploadConfig })
|
||||
if (headerFile) {
|
||||
return headerFile
|
||||
}
|
||||
}
|
||||
|
||||
const response = await fetchUploadResponse({ file, req, uploadConfig })
|
||||
const tempFilePath = await streamResponseToTempFile({ req, response })
|
||||
|
||||
if (!req.context) {
|
||||
req.context = {}
|
||||
}
|
||||
req.context[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY] = tempFilePath
|
||||
|
||||
return {
|
||||
name: file.filename,
|
||||
clientUploadContext: file.clientUploadContext,
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: response.headers.get('Content-Type') || file.mimeType,
|
||||
size: file.size,
|
||||
tempFilePath,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Only crop and explicit pixel edits require the original bytes - a focal-point-only
|
||||
* change re-derives from data already on the document, not the incoming upload.
|
||||
*/
|
||||
function requestHasSizeEdits(req: PayloadRequest): boolean {
|
||||
const uploadEdits =
|
||||
req.query?.uploadEdits && typeof req.query.uploadEdits === 'object'
|
||||
? (req.query.uploadEdits as UploadEdits)
|
||||
: undefined
|
||||
|
||||
return Boolean(uploadEdits?.crop || uploadEdits?.heightInPixels || uploadEdits?.widthInPixels)
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs every configured handler and returns the last one that responds, following a single
|
||||
* redirect if that response is one - matching the pre-existing v3 handler contract exactly
|
||||
* (v3 always ran every handler; only v4 stops at the first response, via an unrelated PR).
|
||||
*/
|
||||
async function fetchUploadResponse({
|
||||
file,
|
||||
req,
|
||||
uploadConfig,
|
||||
}: {
|
||||
file: ClientUploadData
|
||||
req: PayloadRequest
|
||||
uploadConfig: SanitizedUploadConfig
|
||||
}): Promise<Response> {
|
||||
let response: null | Response = null
|
||||
let error: unknown
|
||||
|
||||
for (const handler of uploadConfig.handlers!) {
|
||||
try {
|
||||
const result = await handler(req, {
|
||||
doc: null!,
|
||||
params: {
|
||||
clientUploadContext: file.clientUploadContext,
|
||||
collection: file.collectionSlug,
|
||||
filename: file.filename,
|
||||
},
|
||||
})
|
||||
|
||||
if (result) {
|
||||
response = result
|
||||
}
|
||||
// If we couldn't get the file from that handler, save the error and try other.
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
}
|
||||
|
||||
if (!response) {
|
||||
if (error) {
|
||||
req.payload.logger.error(error)
|
||||
}
|
||||
|
||||
throw new APIError('Expected response from the upload handler.')
|
||||
}
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
const redirectUrl = response.headers.get('Location')
|
||||
if (redirectUrl) {
|
||||
// Forward the Range header (if any) so a redirect from the header-probe path still
|
||||
// fetches only the bounded slice, instead of the whole object, from the redirect target.
|
||||
const rangeHeader = req.headers.get('Range')
|
||||
response = rangeHeader
|
||||
? await fetch(redirectUrl, { headers: { Range: rangeHeader } })
|
||||
: await fetch(redirectUrl)
|
||||
}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
/**
|
||||
* Requests a bounded byte range from the handlers and probes it for image dimensions, without
|
||||
* ever reading past `HEADER_PROBE_BYTE_LENGTH`. Returns `null` when the bounded bytes cannot be
|
||||
* probed, so the caller falls back to a full streamed fetch.
|
||||
*/
|
||||
async function fetchHeaderOnly({
|
||||
file,
|
||||
req,
|
||||
uploadConfig,
|
||||
}: {
|
||||
file: ClientUploadData
|
||||
req: PayloadRequest
|
||||
uploadConfig: SanitizedUploadConfig
|
||||
}): Promise<NonNullable<PayloadRequest['file']> | null> {
|
||||
const rangedHeaders = new Headers(req.headers)
|
||||
rangedHeaders.set('Range', `bytes=0-${HEADER_PROBE_BYTE_LENGTH - 1}`)
|
||||
|
||||
const scopedReq = isolateObjectProperty(req, 'headers')
|
||||
scopedReq.headers = rangedHeaders
|
||||
|
||||
const response = await fetchUploadResponse({ file, req: scopedReq, uploadConfig })
|
||||
|
||||
if (!response.body) {
|
||||
return null
|
||||
}
|
||||
|
||||
const reader = response.body.getReader()
|
||||
const chunks: Uint8Array[] = []
|
||||
let bytesRead = 0
|
||||
|
||||
try {
|
||||
while (bytesRead < HEADER_PROBE_BYTE_LENGTH) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done || !value) {
|
||||
break
|
||||
}
|
||||
|
||||
const remaining = HEADER_PROBE_BYTE_LENGTH - bytesRead
|
||||
const chunk = value.length > remaining ? value.slice(0, remaining) : value
|
||||
|
||||
chunks.push(chunk)
|
||||
bytesRead += chunk.length
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined)
|
||||
}
|
||||
|
||||
const probedFile = {
|
||||
name: file.filename,
|
||||
clientUploadContext: file.clientUploadContext,
|
||||
data: Buffer.concat(chunks),
|
||||
mimetype: response.headers.get('Content-Type') || file.mimeType,
|
||||
size: file.size,
|
||||
}
|
||||
|
||||
try {
|
||||
await getImageSize({ file: probedFile, sharp: req.payload.config.sharp })
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
return probedFile
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams the full response body straight to disk so a cloud object of unbounded size never
|
||||
* becomes a single in-memory Buffer.
|
||||
*/
|
||||
async function streamResponseToTempFile({
|
||||
req,
|
||||
response,
|
||||
}: {
|
||||
req: PayloadRequest
|
||||
response: Response
|
||||
}): Promise<string> {
|
||||
const tempFileDir = req.payload.config.upload?.tempFileDir || os.tmpdir()
|
||||
await mkdir(tempFileDir, { recursive: true })
|
||||
const tempFilePath = path.join(tempFileDir, `payload-client-upload-${randomUUID()}`)
|
||||
|
||||
// A null body is a legitimate zero-byte file (the Fetch API allows a Response to omit a
|
||||
// body entirely), not a failure - `response.arrayBuffer()` tolerated this the same way.
|
||||
if (!response.body) {
|
||||
await writeFile(tempFilePath, Buffer.alloc(0))
|
||||
return tempFilePath
|
||||
}
|
||||
|
||||
try {
|
||||
await pipeline(
|
||||
Readable.fromWeb(response.body as ReadableStream<Uint8Array>),
|
||||
fs.createWriteStream(tempFilePath),
|
||||
)
|
||||
} catch (error) {
|
||||
await rm(tempFilePath, { force: true })
|
||||
throw error
|
||||
}
|
||||
|
||||
return tempFilePath
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import type { WithMetadata } from '../optionallyAppendMetadata.js'
|
||||
import type { FileSize, FileSizes, FileToSave, FocalPoint, ProbedImageSize } from '../types.js'
|
||||
|
||||
import { fileExists } from '../fileExists.js'
|
||||
import { isAnimatedImage } from '../isAnimatedImage.js'
|
||||
import { optionallyAppendMetadata } from '../optionallyAppendMetadata.js'
|
||||
import { createImageSize } from './createImageSize.js'
|
||||
import { extractHeightFromImage } from './extractHeightFromImage.js'
|
||||
@@ -74,7 +75,7 @@ export async function createImageSizes({
|
||||
}
|
||||
|
||||
// Determine if the file is animated
|
||||
const fileIsAnimatedType = ['image/avif', 'image/gif', 'image/webp'].includes(file!.mimetype)
|
||||
const fileIsAnimatedType = isAnimatedImage(file!.mimetype)
|
||||
const sharpOptions: SharpOptions = {}
|
||||
|
||||
if (fileIsAnimatedType) {
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { isAnimatedImage } from './isAnimatedImage.js'
|
||||
|
||||
describe('isAnimatedImage', () => {
|
||||
it('returns true for the formats sharp documents as supporting its animated option', () => {
|
||||
expect(isAnimatedImage('image/gif')).toBe(true)
|
||||
expect(isAnimatedImage('image/webp')).toBe(true)
|
||||
expect(isAnimatedImage('image/tiff')).toBe(true)
|
||||
})
|
||||
|
||||
it('returns false for avif, since sharp does not read or write multi-frame avif sequences', () => {
|
||||
expect(isAnimatedImage('image/avif')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns false for static-only image formats', () => {
|
||||
expect(isAnimatedImage('image/png')).toBe(false)
|
||||
expect(isAnimatedImage('image/jpeg')).toBe(false)
|
||||
expect(isAnimatedImage('image/svg+xml')).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,15 @@
|
||||
// Matches the formats sharp's own `animated` input option documents as supported (reading every
|
||||
// frame/page instead of just the first): https://sharp.pixelplumbing.com/api-constructor - "Set to
|
||||
// true to read all frames/pages of an animated image (GIF, WebP, TIFF)". AVIF is deliberately
|
||||
// excluded even though the format itself supports animation: this sharp version's AVIF encoder
|
||||
// flattens `animated: true` frames into a single tall static image instead of a multi-page
|
||||
// sequence, so there is no working animated-AVIF path to account for here.
|
||||
const ANIMATED_IMAGE_MIME_TYPES = ['image/gif', 'image/tiff', 'image/webp']
|
||||
|
||||
/**
|
||||
* Whether sharp needs to read every frame/page of this mime type (rather than just the first) to
|
||||
* process it correctly - used to decide whether sharp should be asked to read all frames.
|
||||
*/
|
||||
export function isAnimatedImage(mimeType: string): boolean {
|
||||
return ANIMATED_IMAGE_MIME_TYPES.includes(mimeType)
|
||||
}
|
||||
@@ -347,16 +347,28 @@ export type File = {
|
||||
tempFilePath?: string
|
||||
}
|
||||
|
||||
export type FileToSave = {
|
||||
/**
|
||||
* The buffer of the file.
|
||||
*/
|
||||
buffer: Buffer
|
||||
/**
|
||||
* The path to save the file.
|
||||
*/
|
||||
path: string
|
||||
}
|
||||
export type FileToSave =
|
||||
| {
|
||||
/**
|
||||
* The buffer of the file.
|
||||
*/
|
||||
buffer: Buffer
|
||||
/**
|
||||
* The path to save the file.
|
||||
*/
|
||||
path: string
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* The path to save the file.
|
||||
*/
|
||||
path: string
|
||||
/**
|
||||
* An existing file on disk to copy to `path`, instead of `buffer` - avoids loading a file
|
||||
* that's already on disk (e.g. a temp file) fully into memory just to write it back out.
|
||||
*/
|
||||
sourcePath: string
|
||||
}
|
||||
|
||||
type Crop = {
|
||||
height: number
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import fs from 'fs/promises'
|
||||
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import { CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY } from './getFileFromClientUpload.js'
|
||||
|
||||
type Args = {
|
||||
/**
|
||||
* A path the caller has already removed, so the redundant unlink is skipped.
|
||||
*/
|
||||
alreadyUnlinkedPath?: string
|
||||
req: PayloadRequest
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the temp file a client upload materialized, if it is still on disk. Logs rather than
|
||||
* throws, so cleanup can never replace a response or mask the error that a caller is handling.
|
||||
*
|
||||
* The path is tracked on `req.context` rather than read back off `req.file` because the file
|
||||
* can outlive both: plugin-cloud-storage's afterChange hook clears `req.file` before cleanup
|
||||
* runs, and the request parser materializes the file for whatever `collectionSlug` the multipart
|
||||
* field names, which does not have to be a collection the endpoint writes a document to.
|
||||
*/
|
||||
export const unlinkClientUploadTempFile: (args: Args) => Promise<void> = async ({
|
||||
alreadyUnlinkedPath,
|
||||
req,
|
||||
}) => {
|
||||
const tempFilePath = req.context?.[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]
|
||||
|
||||
if (typeof tempFilePath !== 'string') {
|
||||
return
|
||||
}
|
||||
|
||||
delete req.context[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]
|
||||
|
||||
if (tempFilePath === alreadyUnlinkedPath) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.unlink(tempFilePath)
|
||||
} catch (error) {
|
||||
req.payload.logger.error({ err: error, msg: 'Failed to remove client upload temp file' })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
import type { SanitizedCollectionConfig } from '../collections/config/types.js'
|
||||
import type { SanitizedConfig } from '../config/types.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import fs from 'fs/promises'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY } from './getFileFromClientUpload.js'
|
||||
import { unlinkTempFiles } from './unlinkTempFiles.js'
|
||||
|
||||
const createTempFile = async (contents = 'temp-file-contents'): Promise<string> => {
|
||||
const tempFilePath = path.join(
|
||||
os.tmpdir(),
|
||||
`unlink-temp-files-spec-${Date.now()}-${Math.random()}`,
|
||||
)
|
||||
await fs.writeFile(tempFilePath, contents)
|
||||
return tempFilePath
|
||||
}
|
||||
|
||||
const fileExists = async (filePath: string): Promise<boolean> => {
|
||||
try {
|
||||
await fs.access(filePath)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const collectionConfig = {
|
||||
upload: { disableLocalStorage: true },
|
||||
} as unknown as SanitizedCollectionConfig
|
||||
|
||||
const nonUploadCollectionConfig = {} as unknown as SanitizedCollectionConfig
|
||||
|
||||
describe('unlinkTempFiles', () => {
|
||||
const tempFilesToRemove: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const tempFilePath of tempFilesToRemove) {
|
||||
await fs.rm(tempFilePath, { force: true })
|
||||
}
|
||||
tempFilesToRemove.length = 0
|
||||
})
|
||||
|
||||
it('removes a client-upload materialized temp file even when useTempFiles is false', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
const req = {
|
||||
file: {
|
||||
clientUploadContext: undefined,
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'video/mp4',
|
||||
name: 'clip.mp4',
|
||||
size: 10,
|
||||
tempFilePath,
|
||||
},
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config: { upload: { useTempFiles: false } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves a local-API temp file alone when useTempFiles is false', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
const req = {
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'video/mp4',
|
||||
name: 'clip.mp4',
|
||||
size: 10,
|
||||
tempFilePath,
|
||||
},
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config: { upload: { useTempFiles: false } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(true)
|
||||
})
|
||||
|
||||
it('removes a multipart temp file when useTempFiles is true', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
const req = {
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'video/mp4',
|
||||
name: 'clip.mp4',
|
||||
size: 10,
|
||||
tempFilePath,
|
||||
},
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config: { upload: { useTempFiles: true } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
})
|
||||
|
||||
it('removes a client-upload temp file tracked on req.context after req.file was cleared', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
// Mirrors plugin-cloud-storage's afterChange hook, which clears req.file after uploading
|
||||
// generated image sizes but leaves req.context untouched.
|
||||
const req = {
|
||||
context: { [CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]: tempFilePath },
|
||||
file: undefined,
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config: { upload: { useTempFiles: false } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
expect(req.context[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not attempt a second unlink when the context-tracked path matches req.file.tempFilePath', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
const req = {
|
||||
context: { [CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]: tempFilePath },
|
||||
file: {
|
||||
clientUploadContext: undefined,
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'video/mp4',
|
||||
name: 'clip.mp4',
|
||||
size: 10,
|
||||
tempFilePath,
|
||||
},
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await expect(
|
||||
unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config: { upload: { useTempFiles: false } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
}),
|
||||
).resolves.not.toThrow()
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
})
|
||||
|
||||
it('removes a client-upload temp file when the collection is not an upload collection', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
// A forged `collectionSlug` in the multipart file field materializes a temp file for a
|
||||
// collection that never accepts uploads, so nothing in the upload branch removes it.
|
||||
const req = {
|
||||
context: { [CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]: tempFilePath },
|
||||
file: {
|
||||
clientUploadContext: { prefix: '' },
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'video/mp4',
|
||||
name: 'clip.mp4',
|
||||
size: 10,
|
||||
tempFilePath,
|
||||
},
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await unlinkTempFiles({
|
||||
collectionConfig: nonUploadCollectionConfig,
|
||||
config: { upload: { useTempFiles: false } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
})
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
expect(req.context[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]).toBeUndefined()
|
||||
})
|
||||
|
||||
it('logs instead of throwing when a client-upload temp file can no longer be removed', async () => {
|
||||
const req = {
|
||||
context: {
|
||||
[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]: path.join(
|
||||
os.tmpdir(),
|
||||
'unlink-temp-files-spec-missing-file',
|
||||
),
|
||||
},
|
||||
payload: { logger: { error: vi.fn() } },
|
||||
} as unknown as PayloadRequest
|
||||
|
||||
await expect(
|
||||
unlinkTempFiles({
|
||||
collectionConfig,
|
||||
config: { upload: { useTempFiles: false } } as unknown as SanitizedConfig,
|
||||
req,
|
||||
}),
|
||||
).resolves.toBeUndefined()
|
||||
|
||||
expect(req.payload.logger.error).toHaveBeenCalledWith({
|
||||
err: expect.objectContaining({ code: 'ENOENT' }),
|
||||
msg: 'Failed to remove client upload temp file',
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -5,6 +5,7 @@ import type { SanitizedConfig } from '../config/types.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import { mapAsync } from '../utilities/mapAsync.js'
|
||||
import { unlinkClientUploadTempFile } from './unlinkClientUploadTempFile.js'
|
||||
|
||||
type Args = {
|
||||
collectionConfig: SanitizedCollectionConfig
|
||||
@@ -19,14 +20,22 @@ export const unlinkTempFiles: (args: Args) => Promise<void> = async ({
|
||||
config,
|
||||
req,
|
||||
}) => {
|
||||
if (config.upload?.useTempFiles && collectionConfig.upload) {
|
||||
const { file } = req
|
||||
const { file } = req
|
||||
const isClientUploadTempFile = Boolean(
|
||||
file?.tempFilePath && Object.prototype.hasOwnProperty.call(file, 'clientUploadContext'),
|
||||
)
|
||||
let unlinkedTempFilePath: string | undefined
|
||||
|
||||
if (collectionConfig.upload && (config.upload?.useTempFiles || isClientUploadTempFile)) {
|
||||
const fileArray = [{ file }]
|
||||
await mapAsync(fileArray, async ({ file }) => {
|
||||
// Still need this check because this will not be populated if using local API
|
||||
if (file?.tempFilePath) {
|
||||
await fs.unlink(file.tempFilePath)
|
||||
unlinkedTempFilePath = file.tempFilePath
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
await unlinkClientUploadTempFile({ alreadyUnlinkedPath: unlinkedTempFilePath, req })
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import type { Payload } from '../index.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const copyFileMock = vi.fn().mockResolvedValue(undefined)
|
||||
const writeFileMock = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
vi.mock('fs/promises', () => ({
|
||||
default: {
|
||||
copyFile: copyFileMock,
|
||||
writeFile: writeFileMock,
|
||||
},
|
||||
}))
|
||||
|
||||
const { uploadFiles } = await import('./uploadFiles.js')
|
||||
|
||||
describe('uploadFiles', () => {
|
||||
const payload = { logger: { error: vi.fn() } } as unknown as Payload
|
||||
const req = {} as unknown as PayloadRequest
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('writes a buffer entry to disk', async () => {
|
||||
const buffer = Buffer.from('hello')
|
||||
|
||||
await uploadFiles(payload, [{ buffer, path: '/tmp/media/hello.txt' }], req)
|
||||
|
||||
expect(writeFileMock).toHaveBeenCalledWith('/tmp/media/hello.txt', buffer)
|
||||
expect(copyFileMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('copies a sourcePath entry directly, without reading it into memory', async () => {
|
||||
await uploadFiles(
|
||||
payload,
|
||||
[{ path: '/tmp/media/video.mp4', sourcePath: '/tmp/payload-upload-abc' }],
|
||||
req,
|
||||
)
|
||||
|
||||
expect(copyFileMock).toHaveBeenCalledWith('/tmp/payload-upload-abc', '/tmp/media/video.mp4')
|
||||
expect(writeFileMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -1,3 +1,5 @@
|
||||
import fs from 'fs/promises'
|
||||
|
||||
import type { Payload } from '../index.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
import type { FileToSave } from './types.js'
|
||||
@@ -12,8 +14,12 @@ export const uploadFiles = async (
|
||||
): Promise<void> => {
|
||||
try {
|
||||
await Promise.all(
|
||||
files.map(async ({ buffer, path }) => {
|
||||
await saveBufferToFile(buffer, path)
|
||||
files.map(async (file) => {
|
||||
if ('sourcePath' in file) {
|
||||
await fs.copyFile(file.sourcePath, file.path)
|
||||
} else {
|
||||
await saveBufferToFile(file.buffer, file.path)
|
||||
}
|
||||
}),
|
||||
)
|
||||
} catch (err) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
import type { ClientUploadData } from '../uploads/getFileFromClientUpload.js'
|
||||
import type { SanitizedUploadConfig } from '../uploads/types.js'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import fs from 'fs/promises'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { addDataAndFileToRequest } from './addDataAndFileToRequest.js'
|
||||
|
||||
@@ -34,6 +37,50 @@ const createReqWithMultipartBody = (): MinimalReq => {
|
||||
}
|
||||
}
|
||||
|
||||
const createClientUploadReq = ({
|
||||
file,
|
||||
handler,
|
||||
upload,
|
||||
}: {
|
||||
file: ClientUploadData
|
||||
handler: NonNullable<SanitizedUploadConfig['handlers']>[number]
|
||||
upload?: Partial<SanitizedUploadConfig>
|
||||
}): MinimalReq => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', JSON.stringify(file))
|
||||
|
||||
const request = new Request('http://localhost/api/media', {
|
||||
body: formData,
|
||||
method: 'POST',
|
||||
})
|
||||
|
||||
return {
|
||||
body: request.body,
|
||||
headers: request.headers,
|
||||
method: request.method,
|
||||
payload: {
|
||||
collections: {
|
||||
media: {
|
||||
config: {
|
||||
upload: {
|
||||
disableLocalStorage: true,
|
||||
handlers: [handler],
|
||||
...upload,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
config: {
|
||||
bodyParser: {},
|
||||
upload: {},
|
||||
},
|
||||
logger: {
|
||||
error: () => {},
|
||||
},
|
||||
} as unknown as PayloadRequest['payload'],
|
||||
}
|
||||
}
|
||||
|
||||
describe('addDataAndFileToRequest', () => {
|
||||
it('should parse multipart form-data even when content-length is absent', async () => {
|
||||
const req = createReqWithMultipartBody()
|
||||
@@ -46,4 +93,85 @@ describe('addDataAndFileToRequest', () => {
|
||||
expect(req.file?.name).toBe('hello.txt')
|
||||
expect(req.file?.mimetype).toBe('text/plain')
|
||||
})
|
||||
|
||||
describe('client uploads', () => {
|
||||
const tempFilesToRemove: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const tempFilePath of tempFilesToRemove) {
|
||||
await fs.rm(tempFilePath, { force: true })
|
||||
}
|
||||
tempFilesToRemove.length = 0
|
||||
})
|
||||
|
||||
it('materializes client-upload metadata without buffering an unused cloud file', async () => {
|
||||
const handler = vi.fn(() => {
|
||||
throw new Error('No-content handler was invoked')
|
||||
})
|
||||
const req = createClientUploadReq({
|
||||
file: {
|
||||
clientUploadContext: { prefix: '' },
|
||||
collectionSlug: 'media',
|
||||
filename: 'large.mp4',
|
||||
mimeType: 'video/mp4',
|
||||
size: 5_000_000_000,
|
||||
},
|
||||
handler,
|
||||
upload: { disableLocalStorage: true },
|
||||
})
|
||||
|
||||
await addDataAndFileToRequest(req as PayloadRequest)
|
||||
|
||||
expect(handler).not.toHaveBeenCalled()
|
||||
expect(req.file).toMatchObject({
|
||||
clientUploadContext: { prefix: '' },
|
||||
mimetype: 'video/mp4',
|
||||
name: 'large.mp4',
|
||||
size: 5_000_000_000,
|
||||
})
|
||||
expect(req.file?.data.length).toBe(0)
|
||||
})
|
||||
|
||||
it('streams a full-content client upload to a temp file without buffering the whole body', async () => {
|
||||
const chunks = [Buffer.from('chunk-one-'), Buffer.from('chunk-two')]
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) {
|
||||
controller.enqueue(chunk)
|
||||
}
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
const response = new Response(stream, {
|
||||
headers: { 'Content-Type': 'video/mp4' },
|
||||
status: 200,
|
||||
})
|
||||
const arrayBufferTripwire = vi.fn(async () => {
|
||||
throw new Error('Unexpected whole-body buffering')
|
||||
})
|
||||
Object.defineProperty(response, 'arrayBuffer', { value: arrayBufferTripwire })
|
||||
|
||||
const handler = vi.fn(async () => response)
|
||||
const req = createClientUploadReq({
|
||||
file: {
|
||||
clientUploadContext: { prefix: '' },
|
||||
collectionSlug: 'media',
|
||||
filename: 'clip.mp4',
|
||||
mimeType: 'video/mp4',
|
||||
size: 20,
|
||||
},
|
||||
handler,
|
||||
upload: { disableLocalStorage: true, mimeTypes: ['video/*'] },
|
||||
})
|
||||
|
||||
await addDataAndFileToRequest(req as PayloadRequest)
|
||||
tempFilesToRemove.push(req.file!.tempFilePath!)
|
||||
|
||||
expect(arrayBufferTripwire).not.toHaveBeenCalled()
|
||||
expect(req.file?.data.length).toBe(0)
|
||||
expect(req.file?.tempFilePath).toBeDefined()
|
||||
const written = await fs.readFile(req.file!.tempFilePath!)
|
||||
expect(written.toString()).toBe(chunks.map((chunk) => chunk.toString()).join(''))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
import type { ClientUploadData } from '../uploads/getFileFromClientUpload.js'
|
||||
|
||||
import { APIError } from '../errors/APIError.js'
|
||||
import { processMultipartFormdata } from '../uploads/fetchAPI-multipart/index.js'
|
||||
import { getFileFromClientUpload } from '../uploads/getFileFromClientUpload.js'
|
||||
|
||||
type AddDataAndFileToRequest = (req: PayloadRequest) => Promise<void>
|
||||
|
||||
@@ -58,64 +60,18 @@ export const addDataAndFileToRequest: AddDataAndFileToRequest = async (req) => {
|
||||
}
|
||||
|
||||
if (!req.file && fields?.file && typeof fields?.file === 'string') {
|
||||
let clientUploadContext, collectionSlug, filename, mimeType, size
|
||||
let clientUploadFile: ClientUploadData
|
||||
|
||||
try {
|
||||
;({ clientUploadContext, collectionSlug, filename, mimeType, size } = JSON.parse(
|
||||
fields.file,
|
||||
))
|
||||
clientUploadFile = JSON.parse(fields.file) as ClientUploadData
|
||||
} catch {
|
||||
throw new APIError('A file name is required.', 400)
|
||||
}
|
||||
const uploadConfig = req.payload.collections[collectionSlug]!.config.upload
|
||||
|
||||
if (!uploadConfig.handlers) {
|
||||
throw new APIError('uploadConfig.handlers is not present for ' + collectionSlug)
|
||||
}
|
||||
|
||||
let response: null | Response = null
|
||||
let error: unknown
|
||||
|
||||
for (const handler of uploadConfig.handlers) {
|
||||
try {
|
||||
const result = await handler(req, {
|
||||
doc: null!,
|
||||
params: {
|
||||
clientUploadContext, // Pass additional specific to adapters context returned from UploadHandler, then staticHandler can use them.
|
||||
collection: collectionSlug,
|
||||
filename,
|
||||
},
|
||||
})
|
||||
if (result) {
|
||||
response = result
|
||||
}
|
||||
// If we couldn't get the file from that handler, save the error and try other.
|
||||
} catch (err) {
|
||||
error = err
|
||||
}
|
||||
}
|
||||
|
||||
if (!response) {
|
||||
if (error) {
|
||||
payload.logger.error(error)
|
||||
}
|
||||
|
||||
throw new APIError('Expected response from the upload handler.')
|
||||
}
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
const redirectUrl = response.headers.get('Location')
|
||||
if (redirectUrl) {
|
||||
response = await fetch(redirectUrl)
|
||||
}
|
||||
}
|
||||
|
||||
req.file = {
|
||||
name: filename,
|
||||
clientUploadContext,
|
||||
data: Buffer.from(await response.arrayBuffer()),
|
||||
mimetype: response.headers.get('Content-Type') || mimeType,
|
||||
size,
|
||||
}
|
||||
req.file = await getFileFromClientUpload({
|
||||
file: clientUploadFile,
|
||||
req,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
import type { Endpoint } from '../config/types.js'
|
||||
import type { PayloadRequest } from '../types/index.js'
|
||||
|
||||
import fs from 'fs/promises'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY } from '../uploads/getFileFromClientUpload.js'
|
||||
import { wrapInternalEndpoints } from './wrapInternalEndpoints.js'
|
||||
|
||||
const createTempFile = async (): Promise<string> => {
|
||||
const tempFilePath = path.join(
|
||||
os.tmpdir(),
|
||||
`wrap-internal-endpoints-spec-${Date.now()}-${Math.random()}`,
|
||||
)
|
||||
await fs.writeFile(tempFilePath, 'temp-file-contents')
|
||||
return tempFilePath
|
||||
}
|
||||
|
||||
const fileExists = async (filePath: string): Promise<boolean> => {
|
||||
try {
|
||||
await fs.access(filePath)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const createReq = (tempFilePath?: string): PayloadRequest =>
|
||||
({
|
||||
body: null,
|
||||
context: tempFilePath ? { [CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]: tempFilePath } : {},
|
||||
headers: new Headers(),
|
||||
method: 'POST',
|
||||
payload: {
|
||||
config: {},
|
||||
logger: { error: vi.fn() },
|
||||
},
|
||||
}) as unknown as PayloadRequest
|
||||
|
||||
const wrapEndpoint = (handler: Endpoint['handler']): Endpoint['handler'] => {
|
||||
const [endpoint] = wrapInternalEndpoints([{ handler, method: 'post', path: '/test' } as Endpoint])
|
||||
|
||||
return endpoint!.handler
|
||||
}
|
||||
|
||||
describe('wrapInternalEndpoints', () => {
|
||||
const tempFilesToRemove: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const tempFilePath of tempFilesToRemove) {
|
||||
await fs.rm(tempFilePath, { force: true })
|
||||
}
|
||||
tempFilesToRemove.length = 0
|
||||
})
|
||||
|
||||
it('removes a client-upload temp file no operation claimed', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
// An endpoint that runs no collection create or update - an auth endpoint, a global update,
|
||||
// or a custom endpoint - never reaches unlinkTempFiles.
|
||||
const handler = vi.fn(() => Promise.resolve(Response.json({})))
|
||||
const req = createReq(tempFilePath)
|
||||
|
||||
await wrapEndpoint(handler)(req)
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1)
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
expect(req.context[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]).toBeUndefined()
|
||||
})
|
||||
|
||||
it('removes a client-upload temp file when the handler throws', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
const handler = vi.fn(() => Promise.reject(new Error('handler failed')))
|
||||
const req = createReq(tempFilePath)
|
||||
|
||||
await expect(wrapEndpoint(handler)(req)).rejects.toThrow('handler failed')
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not throw when the operation already removed the temp file', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
const handler = vi.fn(async (req: PayloadRequest) => {
|
||||
delete req.context[CLIENT_UPLOAD_TEMP_FILE_PATH_CONTEXT_KEY]
|
||||
await fs.unlink(tempFilePath)
|
||||
return Response.json({})
|
||||
})
|
||||
const req = createReq(tempFilePath)
|
||||
|
||||
await expect(wrapEndpoint(handler)(req)).resolves.toBeInstanceOf(Response)
|
||||
})
|
||||
|
||||
it('leaves requests without a materialized temp file untouched', async () => {
|
||||
const handler = vi.fn(() => Promise.resolve(Response.json({})))
|
||||
const req = createReq()
|
||||
|
||||
await wrapEndpoint(handler)(req)
|
||||
|
||||
expect(handler).toHaveBeenCalledTimes(1)
|
||||
expect(req.payload.logger.error).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('removes the temp file only after the handler has finished with it', async () => {
|
||||
const tempFilePath = await createTempFile()
|
||||
tempFilesToRemove.push(tempFilePath)
|
||||
|
||||
let existsDuringHandler = false
|
||||
const handler = vi.fn(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10))
|
||||
existsDuringHandler = await fileExists(tempFilePath)
|
||||
return Response.json({})
|
||||
})
|
||||
const req = createReq(tempFilePath)
|
||||
|
||||
await wrapEndpoint(handler)(req)
|
||||
|
||||
expect(existsDuringHandler).toBe(true)
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
})
|
||||
|
||||
it('returns the handler response when the temp file can no longer be removed', async () => {
|
||||
const req = createReq(path.join(os.tmpdir(), `wrap-internal-endpoints-spec-missing-file`))
|
||||
const handler = vi.fn(() => Promise.resolve(Response.json({})))
|
||||
|
||||
await expect(wrapEndpoint(handler)(req)).resolves.toBeInstanceOf(Response)
|
||||
|
||||
expect(req.payload.logger.error).toHaveBeenCalledWith({
|
||||
err: expect.objectContaining({ code: 'ENOENT' }),
|
||||
msg: 'Failed to remove client upload temp file',
|
||||
})
|
||||
})
|
||||
|
||||
it('throws the handler error rather than a cleanup error', async () => {
|
||||
const req = createReq(path.join(os.tmpdir(), `wrap-internal-endpoints-spec-missing-file`))
|
||||
const handler = vi.fn(() => Promise.reject(new Error('handler failed')))
|
||||
|
||||
await expect(wrapEndpoint(handler)(req)).rejects.toThrow('handler failed')
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Endpoint } from '../config/types.js'
|
||||
|
||||
import { unlinkClientUploadTempFile } from '../uploads/unlinkClientUploadTempFile.js'
|
||||
import { addDataAndFileToRequest } from './addDataAndFileToRequest.js'
|
||||
import { addLocalesToRequestFromData } from './addLocalesToRequest.js'
|
||||
|
||||
@@ -9,9 +10,13 @@ export const wrapInternalEndpoints = (endpoints: Endpoint[]): Endpoint[] => {
|
||||
|
||||
if (['patch', 'post'].includes(endpoint.method)) {
|
||||
endpoint.handler = async (req) => {
|
||||
await addDataAndFileToRequest(req)
|
||||
addLocalesToRequestFromData(req)
|
||||
return handler(req)
|
||||
try {
|
||||
await addDataAndFileToRequest(req)
|
||||
addLocalesToRequestFromData(req)
|
||||
return await handler(req)
|
||||
} finally {
|
||||
await unlinkClientUploadTempFile({ req })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { CollectionConfig } from 'payload'
|
||||
|
||||
export const mediaHeaderOnlySlug = 'media-header-only'
|
||||
|
||||
const HEADER_PROBE_BYTE_LENGTH = 1024 * 1024
|
||||
|
||||
export const MediaHeaderOnly: CollectionConfig = {
|
||||
slug: mediaHeaderOnlySlug,
|
||||
fields: [],
|
||||
hooks: {
|
||||
beforeValidate: [
|
||||
({ data, req }) => {
|
||||
if (!req.file || req.file.tempFilePath || req.file.data.length > HEADER_PROBE_BYTE_LENGTH) {
|
||||
throw new Error('Header-only client upload exceeded its byte boundary')
|
||||
}
|
||||
return data
|
||||
},
|
||||
],
|
||||
},
|
||||
upload: { disableLocalStorage: true },
|
||||
versions: false,
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { CollectionConfig } from 'payload'
|
||||
|
||||
export const mediaHeaderOnlyWithSizesSlug = 'media-header-only-with-sizes'
|
||||
|
||||
export const MediaHeaderOnlyWithSizes: CollectionConfig = {
|
||||
slug: mediaHeaderOnlyWithSizesSlug,
|
||||
fields: [],
|
||||
hooks: {
|
||||
beforeValidate: [
|
||||
({ data, req }) => {
|
||||
// Skip the internal metadata-only update that plugin-cloud-storage issues after
|
||||
// uploading generated image sizes - it intentionally clears req.file first.
|
||||
if (req.context?.skipCloudStorage) {
|
||||
return data
|
||||
}
|
||||
if (!req.file || req.file.data.length !== 0 || !req.file.tempFilePath) {
|
||||
throw new Error('Full client upload was buffered instead of staged')
|
||||
}
|
||||
return data
|
||||
},
|
||||
],
|
||||
},
|
||||
upload: {
|
||||
disableLocalStorage: true,
|
||||
imageSizes: [{ name: 'thumbnail', height: 300, width: 400 }],
|
||||
},
|
||||
versions: false,
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { CollectionConfig } from 'payload'
|
||||
|
||||
export const mediaNoContentSlug = 'media-no-content'
|
||||
|
||||
export const MediaNoContent: CollectionConfig = {
|
||||
slug: mediaNoContentSlug,
|
||||
fields: [],
|
||||
hooks: {
|
||||
beforeValidate: [
|
||||
({ data, req }) => {
|
||||
if (!req.file || req.file.data.length !== 0 || req.file.tempFilePath) {
|
||||
throw new Error('No-content client upload was materialized')
|
||||
}
|
||||
return data
|
||||
},
|
||||
],
|
||||
},
|
||||
upload: { disableLocalStorage: true },
|
||||
versions: false,
|
||||
}
|
||||
@@ -9,6 +9,12 @@ import { Media } from '../collections/Media.js'
|
||||
import { MediaWithPrefix } from '../collections/MediaWithPrefix.js'
|
||||
import { Users } from '../collections/Users.js'
|
||||
import { mediaSlug, mediaWithPrefixSlug, prefix } from '../shared.js'
|
||||
import { MediaHeaderOnly, mediaHeaderOnlySlug } from './collections/MediaHeaderOnly.js'
|
||||
import {
|
||||
MediaHeaderOnlyWithSizes,
|
||||
mediaHeaderOnlyWithSizesSlug,
|
||||
} from './collections/MediaHeaderOnlyWithSizes.js'
|
||||
import { MediaNoContent, mediaNoContentSlug } from './collections/MediaNoContent.js'
|
||||
import { MediaWithDocPrefix, mediaWithDocPrefixSlug } from './collections/MediaWithDocPrefix.js'
|
||||
|
||||
const filename = fileURLToPath(import.meta.url)
|
||||
@@ -24,7 +30,15 @@ export default buildConfigWithDefaults({
|
||||
baseDir: path.resolve(dirname, '..'),
|
||||
},
|
||||
},
|
||||
collections: [Media, MediaWithPrefix, MediaWithDocPrefix, Users],
|
||||
collections: [
|
||||
Media,
|
||||
MediaWithPrefix,
|
||||
MediaWithDocPrefix,
|
||||
MediaNoContent,
|
||||
MediaHeaderOnly,
|
||||
MediaHeaderOnlyWithSizes,
|
||||
Users,
|
||||
],
|
||||
onInit: async (payload) => {
|
||||
await payload.create({
|
||||
collection: 'users',
|
||||
@@ -36,7 +50,15 @@ export default buildConfigWithDefaults({
|
||||
},
|
||||
plugins: [
|
||||
azureStorage({
|
||||
allowContainerCreate: process.env.AZURE_STORAGE_ALLOW_CONTAINER_CREATE === 'true',
|
||||
baseURL: process.env.AZURE_STORAGE_ACCOUNT_BASEURL!,
|
||||
clientUploads: {
|
||||
chunkLargeFiles: true,
|
||||
},
|
||||
collections: {
|
||||
[mediaHeaderOnlySlug]: true,
|
||||
[mediaHeaderOnlyWithSizesSlug]: true,
|
||||
[mediaNoContentSlug]: true,
|
||||
[mediaSlug]: true,
|
||||
[mediaWithPrefixSlug]: {
|
||||
prefix,
|
||||
@@ -47,11 +69,6 @@ export default buildConfigWithDefaults({
|
||||
prefix: 'docprefix-collection',
|
||||
},
|
||||
},
|
||||
allowContainerCreate: process.env.AZURE_STORAGE_ALLOW_CONTAINER_CREATE === 'true',
|
||||
baseURL: process.env.AZURE_STORAGE_ACCOUNT_BASEURL!,
|
||||
clientUploads: {
|
||||
chunkLargeFiles: true,
|
||||
},
|
||||
connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING!,
|
||||
containerName: process.env.AZURE_STORAGE_CONTAINER_NAME!,
|
||||
}),
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
import type { ContainerClient } from '@azure/storage-blob'
|
||||
import type { Payload } from 'payload'
|
||||
|
||||
import { BlobServiceClient } from '@azure/storage-blob'
|
||||
import { BlobServiceClient, BlockBlobClient } from '@azure/storage-blob'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import type { NextRESTClient } from '../../__helpers/shared/NextRESTClient.js'
|
||||
|
||||
import { initPayloadInt } from '../../__helpers/shared/initPayloadInt.js'
|
||||
import { mediaSlug } from '../shared.js'
|
||||
import { mediaHeaderOnlySlug } from './collections/MediaHeaderOnly.js'
|
||||
import { mediaHeaderOnlyWithSizesSlug } from './collections/MediaHeaderOnlyWithSizes.js'
|
||||
import { mediaNoContentSlug } from './collections/MediaNoContent.js'
|
||||
import { mediaWithDocPrefixSlug } from './collections/MediaWithDocPrefix.js'
|
||||
|
||||
const filename = fileURLToPath(import.meta.url)
|
||||
@@ -83,7 +86,7 @@ describe('@payloadcms/storage-azure clientUploads', () => {
|
||||
|
||||
expect(blobKey).toBe('duplicate-target-1.png')
|
||||
|
||||
await payload.delete({ collection: mediaSlug, id: seedDoc.id })
|
||||
await payload.delete({ id: seedDoc.id, collection: mediaSlug })
|
||||
})
|
||||
|
||||
it('preserves a user-defined prefix.defaultValue across the plugin', async () => {
|
||||
@@ -100,4 +103,123 @@ describe('@payloadcms/storage-azure clientUploads', () => {
|
||||
.getProperties()
|
||||
expect(props.contentLength).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
describe('content requirement retrieval paths', () => {
|
||||
const createdDocs: Array<{ collection: string; id: number | string }> = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const doc of createdDocs) {
|
||||
await payload.delete({ id: doc.id, collection: doc.collection })
|
||||
}
|
||||
createdDocs.length = 0
|
||||
})
|
||||
|
||||
const stageAzureClientUpload = async ({
|
||||
collectionSlug,
|
||||
file,
|
||||
filename,
|
||||
mimeType,
|
||||
}: {
|
||||
collectionSlug: string
|
||||
file: Buffer
|
||||
filename: string
|
||||
mimeType: string
|
||||
}) => {
|
||||
const signedResponse = await restClient.POST('/storage-azure-generate-signed-url', {
|
||||
body: JSON.stringify({ collectionSlug, filename, mimeType }),
|
||||
})
|
||||
expect(signedResponse.status).toBe(200)
|
||||
|
||||
const signed: { docPrefix: string; filename?: string; url: string } =
|
||||
await signedResponse.json()
|
||||
const storedFilename = signed.filename || filename
|
||||
|
||||
await new BlockBlobClient(signed.url).uploadData(file, {
|
||||
blobHTTPHeaders: { blobContentType: mimeType },
|
||||
})
|
||||
|
||||
const form = new FormData()
|
||||
form.append(
|
||||
'file',
|
||||
JSON.stringify({
|
||||
clientUploadContext: { prefix: signed.docPrefix },
|
||||
collectionSlug,
|
||||
filename: storedFilename,
|
||||
mimeType,
|
||||
size: file.length,
|
||||
}),
|
||||
)
|
||||
|
||||
return { collectionSlug, form }
|
||||
}
|
||||
|
||||
it('performs no server-side download for a non-image upload needing no bytes', async () => {
|
||||
const fileBuffer = await readFile(path.resolve(dirname, '../../uploads/audio.mp3'))
|
||||
const { collectionSlug, form } = await stageAzureClientUpload({
|
||||
collectionSlug: mediaNoContentSlug,
|
||||
file: fileBuffer,
|
||||
filename: 'no-content-tripwire.mp3',
|
||||
mimeType: 'audio/mpeg',
|
||||
})
|
||||
|
||||
const downloadSpy = vi.spyOn(BlockBlobClient.prototype, 'download')
|
||||
const res = await restClient.POST(`/${collectionSlug}`, { body: form })
|
||||
|
||||
expect(res.status).toBe(201)
|
||||
const { doc } = await res.json()
|
||||
createdDocs.push({ id: doc.id, collection: collectionSlug })
|
||||
|
||||
expect(doc.filesize).toBe(23_334)
|
||||
expect(doc.mimeType).toBe('audio/mpeg')
|
||||
expect(downloadSpy).not.toHaveBeenCalled()
|
||||
downloadSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('performs one bounded range download for an image needing only dimensions', async () => {
|
||||
const fileBuffer = await readFile(path.resolve(dirname, '../../uploads/2mb.jpg'))
|
||||
const { collectionSlug, form } = await stageAzureClientUpload({
|
||||
collectionSlug: mediaHeaderOnlySlug,
|
||||
file: fileBuffer,
|
||||
filename: 'header-only-tripwire.jpg',
|
||||
mimeType: 'image/jpeg',
|
||||
})
|
||||
|
||||
const downloadSpy = vi.spyOn(BlockBlobClient.prototype, 'download')
|
||||
const res = await restClient.POST(`/${collectionSlug}`, { body: form })
|
||||
|
||||
expect(res.status).toBe(201)
|
||||
const { doc } = await res.json()
|
||||
createdDocs.push({ id: doc.id, collection: collectionSlug })
|
||||
|
||||
expect(doc.width).toBe(9000)
|
||||
expect(doc.height).toBe(9000)
|
||||
expect(doc.filesize).toBe(2_215_474)
|
||||
expect(downloadSpy).toHaveBeenCalledTimes(1)
|
||||
expect(downloadSpy).toHaveBeenCalledWith(0, 1024 * 1024, expect.anything())
|
||||
downloadSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('performs one full streamed download for an image needing generated sizes', async () => {
|
||||
const fileBuffer = await readFile(path.resolve(dirname, '../../uploads/2mb.jpg'))
|
||||
const { collectionSlug, form } = await stageAzureClientUpload({
|
||||
collectionSlug: mediaHeaderOnlyWithSizesSlug,
|
||||
file: fileBuffer,
|
||||
filename: 'full-content-tripwire.jpg',
|
||||
mimeType: 'image/jpeg',
|
||||
})
|
||||
|
||||
const downloadSpy = vi.spyOn(BlockBlobClient.prototype, 'download')
|
||||
const res = await restClient.POST(`/${collectionSlug}`, { body: form })
|
||||
|
||||
expect(res.status).toBe(201)
|
||||
const { doc } = await res.json()
|
||||
createdDocs.push({ id: doc.id, collection: collectionSlug })
|
||||
|
||||
expect(doc.sizes?.thumbnail?.width).toBe(400)
|
||||
expect(doc.sizes?.thumbnail?.height).toBe(300)
|
||||
expect(downloadSpy).toHaveBeenCalledTimes(1)
|
||||
expect(downloadSpy).toHaveBeenCalledWith(0, undefined, expect.anything())
|
||||
downloadSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -70,6 +70,9 @@ export interface Config {
|
||||
media: Media;
|
||||
'media-with-prefix': MediaWithPrefix;
|
||||
'media-with-doc-prefix': MediaWithDocPrefix;
|
||||
'media-no-content': MediaNoContent;
|
||||
'media-header-only': MediaHeaderOnly;
|
||||
'media-header-only-with-sizes': MediaHeaderOnlyWithSize;
|
||||
users: User;
|
||||
'payload-kv': PayloadKv;
|
||||
'payload-locked-documents': PayloadLockedDocument;
|
||||
@@ -81,6 +84,9 @@ export interface Config {
|
||||
media: MediaSelect<false> | MediaSelect<true>;
|
||||
'media-with-prefix': MediaWithPrefixSelect<false> | MediaWithPrefixSelect<true>;
|
||||
'media-with-doc-prefix': MediaWithDocPrefixSelect<false> | MediaWithDocPrefixSelect<true>;
|
||||
'media-no-content': MediaNoContentSelect<false> | MediaNoContentSelect<true>;
|
||||
'media-header-only': MediaHeaderOnlySelect<false> | MediaHeaderOnlySelect<true>;
|
||||
'media-header-only-with-sizes': MediaHeaderOnlyWithSizesSelect<false> | MediaHeaderOnlyWithSizesSelect<true>;
|
||||
users: UsersSelect<false> | UsersSelect<true>;
|
||||
'payload-kv': PayloadKvSelect<false> | PayloadKvSelect<true>;
|
||||
'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
|
||||
@@ -196,6 +202,70 @@ export interface MediaWithDocPrefix {
|
||||
focalX?: number | null;
|
||||
focalY?: number | null;
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "media-no-content".
|
||||
*/
|
||||
export interface MediaNoContent {
|
||||
id: string;
|
||||
updatedAt: string;
|
||||
createdAt: string;
|
||||
url?: string | null;
|
||||
thumbnailURL?: string | null;
|
||||
filename?: string | null;
|
||||
mimeType?: string | null;
|
||||
filesize?: number | null;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
focalX?: number | null;
|
||||
focalY?: number | null;
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "media-header-only".
|
||||
*/
|
||||
export interface MediaHeaderOnly {
|
||||
id: string;
|
||||
updatedAt: string;
|
||||
createdAt: string;
|
||||
url?: string | null;
|
||||
thumbnailURL?: string | null;
|
||||
filename?: string | null;
|
||||
mimeType?: string | null;
|
||||
filesize?: number | null;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
focalX?: number | null;
|
||||
focalY?: number | null;
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "media-header-only-with-sizes".
|
||||
*/
|
||||
export interface MediaHeaderOnlyWithSize {
|
||||
id: string;
|
||||
updatedAt: string;
|
||||
createdAt: string;
|
||||
url?: string | null;
|
||||
thumbnailURL?: string | null;
|
||||
filename?: string | null;
|
||||
mimeType?: string | null;
|
||||
filesize?: number | null;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
focalX?: number | null;
|
||||
focalY?: number | null;
|
||||
sizes?: {
|
||||
thumbnail?: {
|
||||
url?: string | null;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
mimeType?: string | null;
|
||||
filesize?: number | null;
|
||||
filename?: string | null;
|
||||
};
|
||||
};
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "users".
|
||||
@@ -257,6 +327,18 @@ export interface PayloadLockedDocument {
|
||||
relationTo: 'media-with-doc-prefix';
|
||||
value: string | MediaWithDocPrefix;
|
||||
} | null)
|
||||
| ({
|
||||
relationTo: 'media-no-content';
|
||||
value: string | MediaNoContent;
|
||||
} | null)
|
||||
| ({
|
||||
relationTo: 'media-header-only';
|
||||
value: string | MediaHeaderOnly;
|
||||
} | null)
|
||||
| ({
|
||||
relationTo: 'media-header-only-with-sizes';
|
||||
value: string | MediaHeaderOnlyWithSize;
|
||||
} | null)
|
||||
| ({
|
||||
relationTo: 'users';
|
||||
value: string | User;
|
||||
@@ -381,6 +463,71 @@ export interface MediaWithDocPrefixSelect<T extends boolean = true> {
|
||||
focalX?: T;
|
||||
focalY?: T;
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "media-no-content_select".
|
||||
*/
|
||||
export interface MediaNoContentSelect<T extends boolean = true> {
|
||||
updatedAt?: T;
|
||||
createdAt?: T;
|
||||
url?: T;
|
||||
thumbnailURL?: T;
|
||||
filename?: T;
|
||||
mimeType?: T;
|
||||
filesize?: T;
|
||||
width?: T;
|
||||
height?: T;
|
||||
focalX?: T;
|
||||
focalY?: T;
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "media-header-only_select".
|
||||
*/
|
||||
export interface MediaHeaderOnlySelect<T extends boolean = true> {
|
||||
updatedAt?: T;
|
||||
createdAt?: T;
|
||||
url?: T;
|
||||
thumbnailURL?: T;
|
||||
filename?: T;
|
||||
mimeType?: T;
|
||||
filesize?: T;
|
||||
width?: T;
|
||||
height?: T;
|
||||
focalX?: T;
|
||||
focalY?: T;
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "media-header-only-with-sizes_select".
|
||||
*/
|
||||
export interface MediaHeaderOnlyWithSizesSelect<T extends boolean = true> {
|
||||
updatedAt?: T;
|
||||
createdAt?: T;
|
||||
url?: T;
|
||||
thumbnailURL?: T;
|
||||
filename?: T;
|
||||
mimeType?: T;
|
||||
filesize?: T;
|
||||
width?: T;
|
||||
height?: T;
|
||||
focalX?: T;
|
||||
focalY?: T;
|
||||
sizes?:
|
||||
| T
|
||||
| {
|
||||
thumbnail?:
|
||||
| T
|
||||
| {
|
||||
url?: T;
|
||||
width?: T;
|
||||
height?: T;
|
||||
mimeType?: T;
|
||||
filesize?: T;
|
||||
filename?: T;
|
||||
};
|
||||
};
|
||||
}
|
||||
/**
|
||||
* This interface was referenced by `Config`'s JSON-Schema
|
||||
* via the `definition` "users_select".
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
adminThumbnailSizeSlug,
|
||||
allowListMediaSlug,
|
||||
anyImagesSlug,
|
||||
bulkUploadsHookErrorSlug,
|
||||
draftReuploadMediaSlug,
|
||||
enlargeSlug,
|
||||
focalNoSizesSlug,
|
||||
@@ -2337,6 +2338,240 @@ describe('Collections - Uploads', () => {
|
||||
expect(response.status).toBe(403)
|
||||
})
|
||||
})
|
||||
|
||||
describe('temp file cleanup when an operation fails', () => {
|
||||
const createdIDs: (number | string)[] = []
|
||||
const createdTmpFiles: string[] = []
|
||||
let originalUploadConfig: typeof payload.config.upload
|
||||
|
||||
beforeAll(() => {
|
||||
originalUploadConfig = payload.config.upload
|
||||
payload.config.upload = { ...payload.config.upload, useTempFiles: true }
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
payload.config.upload = originalUploadConfig
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
for (const id of createdIDs) {
|
||||
await payload.delete({ collection: bulkUploadsHookErrorSlug as CollectionSlug, id })
|
||||
}
|
||||
createdIDs.length = 0
|
||||
|
||||
for (const tmpFile of createdTmpFiles) {
|
||||
await fs.promises.unlink(tmpFile).catch(() => undefined)
|
||||
}
|
||||
createdTmpFiles.length = 0
|
||||
})
|
||||
|
||||
const createTempFileCopy = async () => {
|
||||
const pngData = await fs.promises.readFile(path.resolve(dirname, './image.png'))
|
||||
const tmpFile = path.join(os.tmpdir(), `payload-test-${randomUUID()}.png`)
|
||||
createdTmpFiles.push(tmpFile)
|
||||
await fs.promises.writeFile(tmpFile, pngData)
|
||||
return { size: pngData.length, tmpFile }
|
||||
}
|
||||
|
||||
it('removes the temp file when a beforeChange hook throws during create', async () => {
|
||||
const { size, tmpFile } = await createTempFileCopy()
|
||||
|
||||
await expect(
|
||||
payload.create({
|
||||
collection: bulkUploadsHookErrorSlug as CollectionSlug,
|
||||
data: { shouldFail: true },
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'image/png',
|
||||
name: 'temp-cleanup-create.png',
|
||||
size,
|
||||
tempFilePath: tmpFile,
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow()
|
||||
|
||||
expect(await fileExists(tmpFile)).toBe(false)
|
||||
})
|
||||
|
||||
it('removes the temp file when a beforeChange hook throws during update', async () => {
|
||||
const initial = await createTempFileCopy()
|
||||
|
||||
const doc = await payload.create({
|
||||
collection: bulkUploadsHookErrorSlug as CollectionSlug,
|
||||
data: { shouldFail: false },
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'image/png',
|
||||
name: 'temp-cleanup-update-initial.png',
|
||||
size: initial.size,
|
||||
tempFilePath: initial.tmpFile,
|
||||
},
|
||||
})
|
||||
createdIDs.push(doc.id)
|
||||
|
||||
const { size, tmpFile } = await createTempFileCopy()
|
||||
|
||||
await expect(
|
||||
payload.update({
|
||||
collection: bulkUploadsHookErrorSlug as CollectionSlug,
|
||||
id: doc.id,
|
||||
data: { shouldFail: true },
|
||||
file: {
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'image/png',
|
||||
name: 'temp-cleanup-update.png',
|
||||
size,
|
||||
tempFilePath: tmpFile,
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow()
|
||||
|
||||
expect(await fileExists(tmpFile)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* A bulk update runs `generateFileData` once and hands the resulting `filesToUpload` to the
|
||||
* per-document promises, so the temp file it copies from has to outlive those writes.
|
||||
*/
|
||||
describe('temp file copy during a bulk update', () => {
|
||||
const createdIDs: (number | string)[] = []
|
||||
const tempFilesToClean: string[] = []
|
||||
let originalUploadConfig: typeof payload.config.upload
|
||||
|
||||
beforeAll(() => {
|
||||
originalUploadConfig = payload.config.upload
|
||||
payload.config.upload = { ...payload.config.upload, useTempFiles: true }
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
payload.config.upload = originalUploadConfig
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
for (const id of createdIDs) {
|
||||
await payload.delete({ id, collection: mediaSlug })
|
||||
}
|
||||
createdIDs.length = 0
|
||||
|
||||
for (const tempFilePath of tempFilesToClean) {
|
||||
await fs.promises.rm(tempFilePath, { force: true })
|
||||
}
|
||||
tempFilesToClean.length = 0
|
||||
})
|
||||
|
||||
it('copies the temp file before removing it', async () => {
|
||||
const alt = `bulk-temp-file-${randomUUID()}`
|
||||
|
||||
const existingDoc = await payload.create({
|
||||
collection: mediaSlug,
|
||||
data: { alt },
|
||||
file: {
|
||||
name: `bulk-temp-file-initial-${randomUUID()}.mp3`,
|
||||
data: Buffer.from('initial-audio-bytes'),
|
||||
mimetype: 'audio/mpeg',
|
||||
size: 19,
|
||||
},
|
||||
})
|
||||
|
||||
createdIDs.push(existingDoc.id)
|
||||
|
||||
const fileContents = Buffer.from(`bulk-audio-bytes-${randomUUID()}`)
|
||||
const tempFilePath = path.join(os.tmpdir(), `payload-test-bulk-temp-${randomUUID()}.mp3`)
|
||||
|
||||
await fs.promises.writeFile(tempFilePath, fileContents)
|
||||
tempFilesToClean.push(tempFilePath)
|
||||
|
||||
const result = await payload.update({
|
||||
collection: mediaSlug,
|
||||
data: { alt },
|
||||
file: {
|
||||
name: `bulk-temp-file-${randomUUID()}.mp3`,
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'audio/mpeg',
|
||||
size: fileContents.length,
|
||||
tempFilePath,
|
||||
},
|
||||
where: { alt: { equals: alt } },
|
||||
})
|
||||
|
||||
expect(result.errors).toEqual([])
|
||||
expect(result.docs).toHaveLength(1)
|
||||
|
||||
const savedFilePath = path.join(dirname, './media', result.docs[0]!.filename!)
|
||||
|
||||
expect(await fileExists(savedFilePath)).toBe(true)
|
||||
expect(await fs.promises.readFile(savedFilePath)).toEqual(fileContents)
|
||||
|
||||
expect(await fileExists(tempFilePath)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* When local storage is enabled and no image processing changes the bytes, generateFileData
|
||||
* copies straight from `file.tempFilePath` to its destination instead of reading the whole
|
||||
* file into memory (see generateFileData.ts). `mediaSlug` has no restrictions on non-image
|
||||
* mime types, so an audio file uploaded there skips all sharp processing and exercises that
|
||||
* copy against real disk I/O.
|
||||
*/
|
||||
describe('temp file copy to local storage', () => {
|
||||
const createdIDs: (number | string)[] = []
|
||||
const tempFilesToClean: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const id of createdIDs) {
|
||||
await payload.delete({ id, collection: mediaSlug })
|
||||
}
|
||||
createdIDs.length = 0
|
||||
|
||||
for (const tempFilePath of tempFilesToClean) {
|
||||
await fs.promises.rm(tempFilePath, { force: true })
|
||||
}
|
||||
tempFilesToClean.length = 0
|
||||
})
|
||||
|
||||
it('copies the temp file to its destination instead of reading it into memory', async () => {
|
||||
const fileContents = Buffer.from(`fake-audio-bytes-${randomUUID()}`)
|
||||
const tempFilePath = path.join(os.tmpdir(), `payload-test-temp-file-${randomUUID()}.mp3`)
|
||||
await fs.promises.writeFile(tempFilePath, fileContents)
|
||||
tempFilesToClean.push(tempFilePath)
|
||||
|
||||
// fs.promises is the same object `fs/promises` exports, so this observes the real calls
|
||||
// generateFileData.ts/uploadFiles.ts make - it doesn't replace their behavior.
|
||||
const copyFileSpy = vitest.spyOn(fs.promises, 'copyFile')
|
||||
const readFileSpy = vitest.spyOn(fs.promises, 'readFile')
|
||||
|
||||
const doc = await payload.create({
|
||||
collection: mediaSlug,
|
||||
data: {},
|
||||
file: {
|
||||
name: `temp-file-copy-${randomUUID()}.mp3`,
|
||||
data: Buffer.alloc(0),
|
||||
mimetype: 'audio/mpeg',
|
||||
size: fileContents.length,
|
||||
tempFilePath,
|
||||
},
|
||||
})
|
||||
|
||||
createdIDs.push(doc.id)
|
||||
|
||||
const savedFilePath = path.join(dirname, './media', doc.filename)
|
||||
|
||||
expect(copyFileSpy).toHaveBeenCalledWith(tempFilePath, savedFilePath)
|
||||
expect(readFileSpy).not.toHaveBeenCalledWith(tempFilePath)
|
||||
|
||||
copyFileSpy.mockRestore()
|
||||
readFileSpy.mockRestore()
|
||||
|
||||
expect(doc.filesize).toBe(fileContents.length)
|
||||
expect(await fileExists(savedFilePath)).toBe(true)
|
||||
expect(await fs.promises.readFile(savedFilePath)).toEqual(fileContents)
|
||||
|
||||
// Copied, not moved - the original temp file must be untouched.
|
||||
expect(await fileExists(tempFilePath)).toBe(true)
|
||||
expect(await fs.promises.readFile(tempFilePath)).toEqual(fileContents)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
async function fileExists(fileName: string): Promise<boolean> {
|
||||
|
||||
Reference in New Issue
Block a user