2024-09-16 13:02:08 -04:00
|
|
|
import fs from 'fs'
|
|
|
|
|
import path from 'node:path'
|
|
|
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
|
|
|
|
|
|
const filename = fileURLToPath(import.meta.url)
|
|
|
|
|
const dirname = path.dirname(filename)
|
|
|
|
|
|
2025-10-06 16:48:02 -04:00
|
|
|
const mongooseAdapterArgs = `
|
2024-11-12 14:42:25 -05:00
|
|
|
ensureIndexes: true,
|
chore: fix various e2e test setup issues (#12670)
I noticed a few issues when running e2e tests that will be resolved by
this PR:
- Most important: for some test suites (fields, fields-relationship,
versions, queues, lexical), the database was cleared and seeded
**twice** in between each test run. This is because the onInit function
was running the clear and seed script, when it should only have been
running the seed script. Clearing the database / the snapshot workflow
is being done by the reInit endpoint, which then calls onInit to seed
the actual data.
- The slowest part of `clearAndSeedEverything` is recreating indexes on
mongodb. This PR slightly improves performance here by:
- Skipping this process for the built-in `['payload-migrations',
'payload-preferences', 'payload-locked-documents']` collections
- Previously we were calling both `createIndexes` and `ensureIndexes`.
This was unnecessary - `ensureIndexes` is a deprecated alias of
`createIndexes`. This PR changes it to only call `createIndexes`
- Makes the reinit endpoint accept GET requests instead of POST requests
- this makes it easier to debug right in the browser
- Some typescript fixes
- Adds a `dev:memorydb` script to the package.json. For some reason,
`dev` is super unreliable on mongodb locally when running e2e tests - it
frequently fails during index creation. Using the memorydb fixes this
issue, with the bonus of more closely resembling the CI environment
- Previously, you were unable to run test suites using turbopack +
postgres. This fixes it, by explicitly installing `pg` as devDependency
in our monorepo
- Fixes jest open handles warning
2025-06-04 13:34:37 -07:00
|
|
|
// required for connect to detect that we are using a memory server
|
|
|
|
|
mongoMemoryServer: global._mongoMemoryServer,
|
2024-08-19 17:31:36 -04:00
|
|
|
url:
|
|
|
|
|
process.env.MONGODB_MEMORY_SERVER_URI ||
|
|
|
|
|
process.env.DATABASE_URI ||
|
|
|
|
|
'mongodb://127.0.0.1/payloadtests',
|
2025-10-06 16:48:02 -04:00
|
|
|
`
|
|
|
|
|
|
|
|
|
|
export const allDatabaseAdapters = {
|
|
|
|
|
mongodb: `
|
|
|
|
|
import { mongooseAdapter } from '@payloadcms/db-mongodb'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = mongooseAdapter({
|
|
|
|
|
${mongooseAdapterArgs}
|
|
|
|
|
})`,
|
|
|
|
|
cosmosdb: `
|
|
|
|
|
import { mongooseAdapter, compatibilityOptions } from '@payloadcms/db-mongodb'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = mongooseAdapter({
|
|
|
|
|
...compatibilityOptions.cosmosdb,
|
|
|
|
|
${mongooseAdapterArgs}
|
|
|
|
|
})`,
|
|
|
|
|
documentdb: `
|
|
|
|
|
import { mongooseAdapter, compatibilityOptions } from '@payloadcms/db-mongodb'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = mongooseAdapter({
|
|
|
|
|
...compatibilityOptions.documentdb,
|
|
|
|
|
${mongooseAdapterArgs}
|
2024-08-19 17:31:36 -04:00
|
|
|
})`,
|
fix(db-mongodb): improve compatibility with Firestore database (#12763)
### What?
Adds four more arguments to the `mongooseAdapter`:
```typescript
useJoinAggregations?: boolean /* The big one */
useAlternativeDropDatabase?: boolean
useBigIntForNumberIDs?: boolean
usePipelineInSortLookup?: boolean
```
Also export a new `compatabilityOptions` object from
`@payloadcms/db-mongodb` where each key is a mongo-compatible database
and the value is the recommended `mongooseAdapter` settings for
compatability.
### Why?
When using firestore and visiting
`/admin/collections/media/payload-folders`, we get:
```
MongoServerError: invalid field(s) in lookup: [let, pipeline], only lookup(from, localField, foreignField, as) is supported
```
Firestore doesn't support the full MongoDB aggregation API used by
Payload which gets used when building aggregations for populating join
fields.
There are several other compatability issues with Firestore:
- The invalid `pipeline` property is used in the `$lookup` aggregation
in `buildSortParams`
- Firestore only supports number IDs of type `Long`, but Mongoose
converts custom ID fields of type number to `Double`
- Firestore does not support the `dropDatabase` command
- Firestore does not support the `createIndex` command (not addressed in
this PR)
### How?
```typescript
useJoinAggregations?: boolean /* The big one */
```
When this is `false` we skip the `buildJoinAggregation()` pipeline and resolve the join fields through multiple queries. This can potentially be used with AWS DocumentDB and Azure Cosmos DB to support join fields, but I have not tested with either of these databases.
```typescript
useAlternativeDropDatabase?: boolean
```
When `true`, monkey-patch (replace) the `dropDatabase` function so that
it calls `collection.deleteMany({})` on every collection instead of
sending a single `dropDatabase` command to the database
```typescript
useBigIntForNumberIDs?: boolean
```
When `true`, use `mongoose.Schema.Types.BigInt` for custom ID fields of type `number` which converts to a firestore `Long` behind the scenes
```typescript
usePipelineInSortLookup?: boolean
```
When `false`, modify the sortAggregation pipeline in `buildSortParams()` so that we don't use the `pipeline` property in the `$lookup` aggregation. Results in slightly worse performance when sorting by relationship properties.
### Limitations
This PR does not add support for transactions or creating indexes in firestore.
### Fixes
Fixed a bug (and added a test) where you weren't able to sort by multiple properties on a relationship field.
### Future work
1. Firestore supports simple `$lookup` aggregations but other databases might not. Could add a `useSortAggregations` property which can be used to disable aggregations in sorting.
---------
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com>
2025-07-17 01:02:43 +05:45
|
|
|
firestore: `
|
2025-08-21 00:31:19 +08:00
|
|
|
import { mongooseAdapter, compatibilityOptions } from '@payloadcms/db-mongodb'
|
fix(db-mongodb): improve compatibility with Firestore database (#12763)
### What?
Adds four more arguments to the `mongooseAdapter`:
```typescript
useJoinAggregations?: boolean /* The big one */
useAlternativeDropDatabase?: boolean
useBigIntForNumberIDs?: boolean
usePipelineInSortLookup?: boolean
```
Also export a new `compatabilityOptions` object from
`@payloadcms/db-mongodb` where each key is a mongo-compatible database
and the value is the recommended `mongooseAdapter` settings for
compatability.
### Why?
When using firestore and visiting
`/admin/collections/media/payload-folders`, we get:
```
MongoServerError: invalid field(s) in lookup: [let, pipeline], only lookup(from, localField, foreignField, as) is supported
```
Firestore doesn't support the full MongoDB aggregation API used by
Payload which gets used when building aggregations for populating join
fields.
There are several other compatability issues with Firestore:
- The invalid `pipeline` property is used in the `$lookup` aggregation
in `buildSortParams`
- Firestore only supports number IDs of type `Long`, but Mongoose
converts custom ID fields of type number to `Double`
- Firestore does not support the `dropDatabase` command
- Firestore does not support the `createIndex` command (not addressed in
this PR)
### How?
```typescript
useJoinAggregations?: boolean /* The big one */
```
When this is `false` we skip the `buildJoinAggregation()` pipeline and resolve the join fields through multiple queries. This can potentially be used with AWS DocumentDB and Azure Cosmos DB to support join fields, but I have not tested with either of these databases.
```typescript
useAlternativeDropDatabase?: boolean
```
When `true`, monkey-patch (replace) the `dropDatabase` function so that
it calls `collection.deleteMany({})` on every collection instead of
sending a single `dropDatabase` command to the database
```typescript
useBigIntForNumberIDs?: boolean
```
When `true`, use `mongoose.Schema.Types.BigInt` for custom ID fields of type `number` which converts to a firestore `Long` behind the scenes
```typescript
usePipelineInSortLookup?: boolean
```
When `false`, modify the sortAggregation pipeline in `buildSortParams()` so that we don't use the `pipeline` property in the `$lookup` aggregation. Results in slightly worse performance when sorting by relationship properties.
### Limitations
This PR does not add support for transactions or creating indexes in firestore.
### Fixes
Fixed a bug (and added a test) where you weren't able to sort by multiple properties on a relationship field.
### Future work
1. Firestore supports simple `$lookup` aggregations but other databases might not. Could add a `useSortAggregations` property which can be used to disable aggregations in sorting.
---------
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com>
2025-07-17 01:02:43 +05:45
|
|
|
|
|
|
|
|
export const databaseAdapter = mongooseAdapter({
|
2025-08-21 00:31:19 +08:00
|
|
|
...compatibilityOptions.firestore,
|
2025-10-06 16:48:02 -04:00
|
|
|
${mongooseAdapterArgs}
|
fix(db-mongodb): improve compatibility with Firestore database (#12763)
### What?
Adds four more arguments to the `mongooseAdapter`:
```typescript
useJoinAggregations?: boolean /* The big one */
useAlternativeDropDatabase?: boolean
useBigIntForNumberIDs?: boolean
usePipelineInSortLookup?: boolean
```
Also export a new `compatabilityOptions` object from
`@payloadcms/db-mongodb` where each key is a mongo-compatible database
and the value is the recommended `mongooseAdapter` settings for
compatability.
### Why?
When using firestore and visiting
`/admin/collections/media/payload-folders`, we get:
```
MongoServerError: invalid field(s) in lookup: [let, pipeline], only lookup(from, localField, foreignField, as) is supported
```
Firestore doesn't support the full MongoDB aggregation API used by
Payload which gets used when building aggregations for populating join
fields.
There are several other compatability issues with Firestore:
- The invalid `pipeline` property is used in the `$lookup` aggregation
in `buildSortParams`
- Firestore only supports number IDs of type `Long`, but Mongoose
converts custom ID fields of type number to `Double`
- Firestore does not support the `dropDatabase` command
- Firestore does not support the `createIndex` command (not addressed in
this PR)
### How?
```typescript
useJoinAggregations?: boolean /* The big one */
```
When this is `false` we skip the `buildJoinAggregation()` pipeline and resolve the join fields through multiple queries. This can potentially be used with AWS DocumentDB and Azure Cosmos DB to support join fields, but I have not tested with either of these databases.
```typescript
useAlternativeDropDatabase?: boolean
```
When `true`, monkey-patch (replace) the `dropDatabase` function so that
it calls `collection.deleteMany({})` on every collection instead of
sending a single `dropDatabase` command to the database
```typescript
useBigIntForNumberIDs?: boolean
```
When `true`, use `mongoose.Schema.Types.BigInt` for custom ID fields of type `number` which converts to a firestore `Long` behind the scenes
```typescript
usePipelineInSortLookup?: boolean
```
When `false`, modify the sortAggregation pipeline in `buildSortParams()` so that we don't use the `pipeline` property in the `$lookup` aggregation. Results in slightly worse performance when sorting by relationship properties.
### Limitations
This PR does not add support for transactions or creating indexes in firestore.
### Fixes
Fixed a bug (and added a test) where you weren't able to sort by multiple properties on a relationship field.
### Future work
1. Firestore supports simple `$lookup` aggregations but other databases might not. Could add a `useSortAggregations` property which can be used to disable aggregations in sorting.
---------
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com>
2025-07-17 01:02:43 +05:45
|
|
|
// The following options prevent some tests from failing.
|
|
|
|
|
// More work needed to get tests succeeding without these options.
|
|
|
|
|
ensureIndexes: true,
|
|
|
|
|
disableIndexHints: false,
|
|
|
|
|
useAlternativeDropDatabase: false,
|
|
|
|
|
})`,
|
2024-08-19 17:31:36 -04:00
|
|
|
postgres: `
|
|
|
|
|
import { postgresAdapter } from '@payloadcms/db-postgres'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = postgresAdapter({
|
|
|
|
|
pool: {
|
|
|
|
|
connectionString: process.env.POSTGRES_URL || 'postgres://127.0.0.1:5432/payloadtests',
|
|
|
|
|
},
|
|
|
|
|
})`,
|
|
|
|
|
'postgres-custom-schema': `
|
|
|
|
|
import { postgresAdapter } from '@payloadcms/db-postgres'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = postgresAdapter({
|
|
|
|
|
pool: {
|
|
|
|
|
connectionString: process.env.POSTGRES_URL || 'postgres://127.0.0.1:5432/payloadtests',
|
|
|
|
|
},
|
|
|
|
|
schemaName: 'custom',
|
|
|
|
|
})`,
|
|
|
|
|
'postgres-uuid': `
|
|
|
|
|
import { postgresAdapter } from '@payloadcms/db-postgres'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = postgresAdapter({
|
|
|
|
|
idType: 'uuid',
|
|
|
|
|
pool: {
|
|
|
|
|
connectionString: process.env.POSTGRES_URL || 'postgres://127.0.0.1:5432/payloadtests',
|
|
|
|
|
},
|
|
|
|
|
})`,
|
2025-06-09 22:09:52 +03:00
|
|
|
'postgres-read-replica': `
|
|
|
|
|
import { postgresAdapter } from '@payloadcms/db-postgres'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = postgresAdapter({
|
|
|
|
|
pool: {
|
|
|
|
|
connectionString: process.env.POSTGRES_URL,
|
|
|
|
|
},
|
|
|
|
|
readReplicas: [process.env.POSTGRES_REPLICA_URL],
|
|
|
|
|
})
|
|
|
|
|
`,
|
|
|
|
|
'vercel-postgres-read-replica': `
|
|
|
|
|
import { vercelPostgresAdapter } from '@payloadcms/db-vercel-postgres'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = vercelPostgresAdapter({
|
|
|
|
|
pool: {
|
|
|
|
|
connectionString: process.env.POSTGRES_URL,
|
|
|
|
|
},
|
|
|
|
|
readReplicas: [process.env.POSTGRES_REPLICA_URL],
|
|
|
|
|
})
|
|
|
|
|
`,
|
2024-08-19 17:31:36 -04:00
|
|
|
sqlite: `
|
|
|
|
|
import { sqliteAdapter } from '@payloadcms/db-sqlite'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = sqliteAdapter({
|
|
|
|
|
client: {
|
|
|
|
|
url: process.env.SQLITE_URL || 'file:./payloadtests.db',
|
|
|
|
|
},
|
2024-12-20 22:13:28 +02:00
|
|
|
autoIncrement: true
|
2024-08-19 17:31:36 -04:00
|
|
|
})`,
|
2024-12-19 05:44:04 +02:00
|
|
|
'sqlite-uuid': `
|
|
|
|
|
import { sqliteAdapter } from '@payloadcms/db-sqlite'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = sqliteAdapter({
|
|
|
|
|
idType: 'uuid',
|
|
|
|
|
client: {
|
|
|
|
|
url: process.env.SQLITE_URL || 'file:./payloadtests.db',
|
|
|
|
|
},
|
|
|
|
|
})`,
|
2024-08-19 17:31:36 -04:00
|
|
|
supabase: `
|
|
|
|
|
import { postgresAdapter } from '@payloadcms/db-postgres'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = postgresAdapter({
|
|
|
|
|
pool: {
|
|
|
|
|
connectionString:
|
|
|
|
|
process.env.POSTGRES_URL || 'postgresql://postgres:postgres@127.0.0.1:54322/postgres',
|
|
|
|
|
},
|
|
|
|
|
})`,
|
2025-09-29 23:58:18 +03:00
|
|
|
d1: `
|
|
|
|
|
import { sqliteD1Adapter } from '@payloadcms/db-d1-sqlite'
|
|
|
|
|
|
|
|
|
|
export const databaseAdapter = sqliteD1Adapter({ binding: global.d1 })
|
|
|
|
|
`,
|
2024-08-19 17:31:36 -04:00
|
|
|
}
|
|
|
|
|
|
2024-09-16 13:02:08 -04:00
|
|
|
/**
|
|
|
|
|
* Write to databaseAdapter.ts
|
|
|
|
|
*/
|
|
|
|
|
export function generateDatabaseAdapter(dbAdapter) {
|
|
|
|
|
const databaseAdapter = allDatabaseAdapters[dbAdapter]
|
|
|
|
|
if (!databaseAdapter) {
|
|
|
|
|
throw new Error(`Unknown database adapter: ${dbAdapter}`)
|
|
|
|
|
}
|
|
|
|
|
fs.writeFileSync(
|
|
|
|
|
path.resolve(dirname, 'databaseAdapter.js'),
|
|
|
|
|
`
|
|
|
|
|
// DO NOT MODIFY. This file is automatically generated by the test suite.
|
|
|
|
|
|
|
|
|
|
${databaseAdapter}
|
|
|
|
|
`,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
console.log('Wrote', dbAdapter, 'db adapter')
|
|
|
|
|
return databaseAdapter
|
2024-08-19 17:31:36 -04:00
|
|
|
}
|