mirror of
https://github.com/civitai/civitai.git
synced 2026-09-20 22:08:18 +08:00
feat(image-feed): tag Meili requests with X-Search-Actor
Send X-Search-Actor on the metrics-search feed paths so the proxy can pin same-actor traffic to a single Meili replica (HRW) and so heavy or abusive callers are correlatable in upstream access logs. Logged-in users get `user:<id>`; anon callers get `anon:<sha256(ip|ua)[..16]>`. Untagged internal callers send no header and round-robin as before. Header is wired into both the abortable raw fetch path and the SDK client via a per-actor MeiliSearch factory. Actor is built upstream in the tRPC controller and the v1 REST handler; the service just forwards the opaque string. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,12 +2,14 @@ import type { TRPCError } from '@trpc/server';
|
||||
import { getHTTPStatusCodeFromError } from '@trpc/server/http';
|
||||
import dayjs from '~/shared/utils/dayjs';
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import requestIp from 'request-ip';
|
||||
import * as z from 'zod';
|
||||
import { getEdgeUrl } from '~/client-utils/cf-images-utils';
|
||||
import { isProd } from '~/env/other';
|
||||
import { constants } from '~/server/common/constants';
|
||||
import { ImageSort } from '~/server/common/enums';
|
||||
import { buildFliptContext, getFeatureFlags } from '~/server/services/feature-flags.service';
|
||||
import { buildSearchActor } from '~/server/meilisearch/client';
|
||||
import {
|
||||
getAllImages,
|
||||
getAllImagesIndex,
|
||||
@@ -136,6 +138,12 @@ export default PublicEndpoint(async function handler(req: NextApiRequest, res: N
|
||||
// those sorts are honored.
|
||||
const useLegacyMethod = data.imageId || (data.modelId && !data.modelVersionId);
|
||||
|
||||
const actor = buildSearchActor({
|
||||
userId: session?.user?.id,
|
||||
ip: requestIp.getClientIp(req),
|
||||
userAgent: req.headers['user-agent'],
|
||||
});
|
||||
|
||||
const { items, nextCursor } = useLegacyMethod
|
||||
? await getAllImages({
|
||||
...data,
|
||||
@@ -171,6 +179,7 @@ export default PublicEndpoint(async function handler(req: NextApiRequest, res: N
|
||||
disablePoi: true,
|
||||
headers: { src: '/api/v1/images' },
|
||||
dbTarget: features.datapacketRead ? 'datapacket' : 'read',
|
||||
actor,
|
||||
})
|
||||
: await getImagesFromFeedSearch({
|
||||
...data,
|
||||
@@ -187,6 +196,7 @@ export default PublicEndpoint(async function handler(req: NextApiRequest, res: N
|
||||
useCombinedNsfwLevel: !features.canViewNsfw,
|
||||
disableMinor: true,
|
||||
disablePoi: true,
|
||||
actor,
|
||||
});
|
||||
|
||||
const metadata: Metadata = {
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
updateImageNsfwLevel,
|
||||
updateImageReportStatusByReason,
|
||||
} from '~/server/services/image.service';
|
||||
import { buildSearchActor } from '~/server/meilisearch/client';
|
||||
import { getGallerySettingsByModelId } from '~/server/services/model.service';
|
||||
import { trackModActivity } from '~/server/services/moderator.service';
|
||||
import { createNotification } from '~/server/services/notification.service';
|
||||
@@ -323,6 +324,11 @@ export const getInfiniteImagesHandler = async ({
|
||||
// Forward pre-evaluated variant so getImagesFromSearch can skip a
|
||||
// duplicate Flipt evaluation. `null` means "skipBitdex" path.
|
||||
bitdexMode,
|
||||
actor: buildSearchActor({
|
||||
userId: user?.id,
|
||||
ip: ctx.ip,
|
||||
userAgent: ctx.req.headers['user-agent'],
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
return await getAllImages({
|
||||
@@ -437,6 +443,12 @@ export const getImagesAsPostsInfiniteHandler = async ({
|
||||
}
|
||||
}
|
||||
|
||||
const actor = buildSearchActor({
|
||||
userId: user?.id,
|
||||
ip: ctx.ip,
|
||||
userAgent: ctx.req.headers['user-agent'],
|
||||
});
|
||||
|
||||
while (true) {
|
||||
// TODO handle/remove all these (headers, include, ids)
|
||||
const { nextCursor, items } = await fetchFn({
|
||||
@@ -453,6 +465,7 @@ export const getImagesAsPostsInfiniteHandler = async ({
|
||||
// Forward pre-evaluated variant — getImagesFromSearch ignores it on the
|
||||
// DB path (getAllImages doesn't read it).
|
||||
bitdexMode,
|
||||
actor,
|
||||
});
|
||||
|
||||
// Merge images by postId
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from 'crypto';
|
||||
import type { EnqueuedTask, DocumentsQuery, ResourceResults } from 'meilisearch';
|
||||
import { MeiliSearch } from 'meilisearch';
|
||||
import { env } from '~/env/server';
|
||||
@@ -24,6 +25,38 @@ export const metricsSearchClient = shouldConnectToMetricsSearch
|
||||
})
|
||||
: null;
|
||||
|
||||
export const SEARCH_ACTOR_HEADER = 'X-Search-Actor';
|
||||
|
||||
export function buildSearchActor({
|
||||
userId,
|
||||
ip,
|
||||
userAgent,
|
||||
}: {
|
||||
userId?: number | null;
|
||||
ip?: string | null;
|
||||
userAgent?: string | null;
|
||||
}) {
|
||||
if (userId) return `user:${userId}`;
|
||||
const fp = createHash('sha256')
|
||||
.update(`${ip ?? ''}|${userAgent ?? ''}`)
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return `anon:${fp}`;
|
||||
}
|
||||
|
||||
// Returns a fresh MeiliSearch instance with the X-Search-Actor header pinned
|
||||
// for the lifetime of the call. The SDK applies requestConfig.headers to every
|
||||
// request, so we create one per logical caller rather than mutating a shared
|
||||
// instance. Construction is cheap — config-only, no socket pool.
|
||||
export function getMetricsSearchClient(actor: string) {
|
||||
if (!shouldConnectToMetricsSearch) return null;
|
||||
return new MeiliSearch({
|
||||
host: env.METRICS_SEARCH_HOST as string,
|
||||
apiKey: env.METRICS_SEARCH_API_KEY,
|
||||
requestConfig: { headers: { [SEARCH_ACTOR_HEADER]: actor } },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch documents via a raw HTTP call that honors an AbortSignal. The
|
||||
* meilisearch-js client we're on (<=0.34) doesn't expose signal on
|
||||
@@ -33,14 +66,15 @@ export const metricsSearchClient = shouldConnectToMetricsSearch
|
||||
export async function fetchDocumentsAbortable<T>(
|
||||
indexName: string,
|
||||
params: DocumentsQuery<T>,
|
||||
options: { host: string; apiKey?: string; signal?: AbortSignal }
|
||||
options: { host: string; apiKey?: string; signal?: AbortSignal; actor?: string }
|
||||
): Promise<ResourceResults<T[]>> {
|
||||
const { host, apiKey, signal } = options;
|
||||
const { host, apiKey, signal, actor } = options;
|
||||
const res = await fetch(`${host}/indexes/${indexName}/documents/fetch`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}),
|
||||
...(actor ? { [SEARCH_ACTOR_HEADER]: actor } : {}),
|
||||
},
|
||||
body: JSON.stringify(params),
|
||||
signal,
|
||||
|
||||
@@ -42,7 +42,11 @@ import {
|
||||
import { poolCounters } from '~/server/games/new-order/utils';
|
||||
import { logToAxiom, safeError } from '~/server/logging/client';
|
||||
import { withSpan } from '~/server/utils/otel-helpers';
|
||||
import { fetchDocumentsAbortable, metricsSearchClient } from '~/server/meilisearch/client';
|
||||
import {
|
||||
fetchDocumentsAbortable,
|
||||
getMetricsSearchClient,
|
||||
metricsSearchClient,
|
||||
} from '~/server/meilisearch/client';
|
||||
import { postMetrics } from '~/server/metrics';
|
||||
import { leakingContentCounter } from '~/server/prom/client';
|
||||
import { imageOnSiteSql, isImageMetaOnSite } from '~/server/utils/image-onsite';
|
||||
@@ -1086,6 +1090,9 @@ type GetAllImagesInput = GetInfiniteImagesOutput & {
|
||||
// Pre-evaluated BITDEX_IMAGE_SEARCH variant from the controller — pass-through
|
||||
// to avoid a second Flipt evaluation inside getImagesFromSearch.
|
||||
bitdexMode?: string | null;
|
||||
// Caller identity forwarded to Meili via X-Search-Actor for abuse/rate
|
||||
// correlation. Built upstream via buildSearchActor().
|
||||
actor?: string;
|
||||
};
|
||||
export type ImagesInfiniteModel = AsyncReturnType<typeof getAllImages>['items'][0];
|
||||
export const getAllImages = async (
|
||||
@@ -2172,6 +2179,7 @@ type ImageSearchInput = GetInfiniteImagesOutput & {
|
||||
// Pre-evaluated BITDEX_IMAGE_SEARCH variant from the caller (controller).
|
||||
// When provided, getImagesFromSearch skips its own Flipt evaluation.
|
||||
bitdexMode?: string | null;
|
||||
actor?: string;
|
||||
// Unhandled
|
||||
//prioritizedUserIds?: number[];
|
||||
//userIds?: number | number[];
|
||||
@@ -2942,6 +2950,8 @@ export async function getImagesFromSearchPreFilter(input: ImageSearchInput) {
|
||||
requestTotal.inc({ route }); // count every request up front
|
||||
|
||||
try {
|
||||
const actor = input.actor;
|
||||
|
||||
// Use the abortable raw fetch when a signal is available so client
|
||||
// disconnects (e.g. the feed's slow-fetch timeout) actually cancel the
|
||||
// underlying Meili request. The meilisearch-js client on 0.33/0.34 doesn't
|
||||
@@ -2952,8 +2962,9 @@ export async function getImagesFromSearchPreFilter(input: ImageSearchInput) {
|
||||
host: env.METRICS_SEARCH_HOST as string,
|
||||
apiKey: env.METRICS_SEARCH_API_KEY,
|
||||
signal: input.signal,
|
||||
actor,
|
||||
})
|
||||
: metricsSearchClient!
|
||||
: (actor ? getMetricsSearchClient(actor) : metricsSearchClient)!
|
||||
.index(METRICS_SEARCH_INDEX)
|
||||
.getDocuments<ImageMetricsSearchIndexRecord>(request)
|
||||
);
|
||||
@@ -3775,6 +3786,9 @@ export async function getImagesFromSearchPostFilter(input: ImageSearchInput) {
|
||||
sort: sorts,
|
||||
};
|
||||
|
||||
const actor = input.actor;
|
||||
const actorClient = actor ? getMetricsSearchClient(actor) : metricsSearchClient;
|
||||
|
||||
try {
|
||||
while (accumulatedHits.length < limit + 1 && iteration < MAX_ITERATIONS) {
|
||||
// Safety check for total processed results
|
||||
@@ -3796,9 +3810,10 @@ export async function getImagesFromSearchPostFilter(input: ImageSearchInput) {
|
||||
host: env.METRICS_SEARCH_HOST as string,
|
||||
apiKey: env.METRICS_SEARCH_API_KEY,
|
||||
signal: input.signal,
|
||||
actor,
|
||||
}
|
||||
)
|
||||
: await metricsSearchClient
|
||||
: await actorClient!
|
||||
.index(METRICS_SEARCH_INDEX)
|
||||
.getDocuments<ImageMetricsSearchIndexRecord>(request);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user