mirror of
https://github.com/civitai/civitai.git
synced 2026-09-20 22:08:18 +08:00
fix: Calibrate challenge judge scoring to reduce inflation (#2040)
* fix: Calibrate challenge judge scoring to reduce score inflation Addresses score inflation where Grok Fast consistently rates entries 8-10 instead of using the full 0-10 range. Three key changes: 1. Analysis-first JSON schema: Forces model to identify strengths/weaknesses BEFORE generating scores, anchoring scores to its critique 2. Lower temperature (1.0 → 0.4): More consistent, calibrated scoring 3. Scoring rubric with anchor points: Explicit meanings for each score level (5=average, 7=good, 9+=exceptional) Also adds opt-in two-pass multi-turn review mode for higher-quality judging (e.g. finals): first an objective persona-free analysis, then persona-scored review anchored by that analysis. Available via multiTurn flag in playground. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: Add JSON review template system for challenge judges Replace hardcoded multi-turn review logic with a flexible JSON message template system. Judges can now store a full message template (agent-workbench format) with {{variable}} placeholders, giving mods complete control over the LLM conversation shape including few-shot examples. - Add reviewTemplate field to ChallengeJudge schema + migration - Add template engine (parse, validate, resolve variables) - Rewrite generateReview() with template path + fallback path - Remove multi-turn review code and hardcoded scoring constants - Refactor CreateJudgeModal to use Form/InputText/InputTextArea - Add JsonInput for review template editing in playground panels - Remove userMessage override from all activity panels - Fix deprecated isLoading → isPending across playground - Support reviewTemplate override in playground without saving Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Inject response schema and user input in template path Ensures the template path in generateReview always includes the JSON response schema and user context (theme/creator/image), preventing score parsing crashes when templates omit these. Also replaces raw JsonInput with InputJson form wrapper, removes unused userMessageOverride plumbing, and extracts findLastIndex to shared array-helpers. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Fix schema injection fallback and add template engine tests Handle array content without text items in schema injection by pushing a new text item instead of silently skipping. Add unit tests for parseReviewTemplate and resolveTemplate. Rename vitest config to .mts to fix ESM compatibility. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Small lint fix * Add aestheticFlaws field to challenge entry reviews Store aesthetic flaws from AI judge reviews in the CollectionItem note JSON alongside score/summary, and display them in the judge playground ReviewImageActivity component. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Manuel Emilio Urena
committed by
GitHub
GitHub
parent
921f5ab2e1
commit
e6ede89f9c
@@ -0,0 +1,2 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "ChallengeJudge" ADD COLUMN "reviewTemplate" TEXT;
|
||||
@@ -3869,6 +3869,7 @@ model ChallengeJudge {
|
||||
collectionPrompt String? @db.Text
|
||||
contentPrompt String? @db.Text
|
||||
reviewPrompt String? @db.Text
|
||||
reviewTemplate String? @db.Text // JSON message template (agent-workbench format)
|
||||
winnerSelectionPrompt String? @db.Text
|
||||
active Boolean @default(true)
|
||||
|
||||
|
||||
@@ -1,38 +1,40 @@
|
||||
import {
|
||||
Button,
|
||||
Group,
|
||||
Loader,
|
||||
Modal,
|
||||
Select,
|
||||
Stack,
|
||||
TextInput,
|
||||
Textarea,
|
||||
} from '@mantine/core';
|
||||
import { Button, Group, Loader, Modal, Select, Stack } from '@mantine/core';
|
||||
import { useDebouncedValue } from '@mantine/hooks';
|
||||
import { IconPlus } from '@tabler/icons-react';
|
||||
import { useState } from 'react';
|
||||
import { useMemo, useState } from 'react';
|
||||
import * as z from 'zod';
|
||||
import { Form, InputJson, InputText, InputTextArea, useForm } from '~/libs/form';
|
||||
import { upsertJudgeSchema } from '~/server/schema/challenge.schema';
|
||||
import { showErrorNotification, showSuccessNotification } from '~/utils/notifications';
|
||||
import { trpc } from '~/utils/trpc';
|
||||
import { usePlaygroundStore } from './playground.store';
|
||||
import { TemplateVariableIndicators } from './TemplateVariableIndicators';
|
||||
|
||||
export function CreateJudgeModal({
|
||||
opened,
|
||||
onClose,
|
||||
}: {
|
||||
opened: boolean;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
// Form schema — derives from server schema, overrides userId to string for Select
|
||||
const schema = upsertJudgeSchema.omit({ id: true, userId: true, active: true }).extend({
|
||||
userId: z.string().nullish().default(null),
|
||||
});
|
||||
|
||||
const defaultValues: z.infer<typeof schema> = {
|
||||
name: '',
|
||||
userId: null,
|
||||
bio: null,
|
||||
sourceCollectionId: null,
|
||||
systemPrompt: null,
|
||||
collectionPrompt: null,
|
||||
contentPrompt: null,
|
||||
reviewPrompt: null,
|
||||
reviewTemplate: null,
|
||||
winnerSelectionPrompt: null,
|
||||
};
|
||||
|
||||
export function CreateJudgeModal({ opened, onClose }: { opened: boolean; onClose: () => void }) {
|
||||
const setSelectedJudgeId = usePlaygroundStore((s) => s.setSelectedJudgeId);
|
||||
|
||||
const [name, setName] = useState('');
|
||||
const [bio, setBio] = useState('');
|
||||
const [systemPrompt, setSystemPrompt] = useState('');
|
||||
const [contentPrompt, setContentPrompt] = useState('');
|
||||
const [reviewPrompt, setReviewPrompt] = useState('');
|
||||
const [winnerSelectionPrompt, setWinnerSelectionPrompt] = useState('');
|
||||
const [userSearch, setUserSearch] = useState('');
|
||||
const [selectedUserId, setSelectedUserId] = useState<string | null>(null);
|
||||
const form = useForm({ schema, defaultValues });
|
||||
|
||||
// User search state (external to form — drives the async Select)
|
||||
const [userSearch, setUserSearch] = useState('');
|
||||
const [debouncedSearch] = useDebouncedValue(userSearch, 300);
|
||||
|
||||
const { data: usersData, isLoading: usersLoading } = trpc.user.getAll.useQuery(
|
||||
@@ -40,10 +42,14 @@ export function CreateJudgeModal({
|
||||
{ enabled: debouncedSearch.length >= 2 }
|
||||
);
|
||||
|
||||
const userOptions = (usersData ?? []).map((u) => ({
|
||||
value: String(u.id),
|
||||
label: u.username ?? `User ${u.id}`,
|
||||
}));
|
||||
const userOptions = useMemo(
|
||||
() =>
|
||||
(usersData ?? []).map((u) => ({
|
||||
value: String(u.id),
|
||||
label: u.username ?? `User ${u.id}`,
|
||||
})),
|
||||
[usersData]
|
||||
);
|
||||
|
||||
const queryUtils = trpc.useUtils();
|
||||
const upsertMutation = trpc.challenge.upsertJudge.useMutation({
|
||||
@@ -59,110 +65,106 @@ export function CreateJudgeModal({
|
||||
});
|
||||
|
||||
const resetAndClose = () => {
|
||||
setName('');
|
||||
setBio('');
|
||||
setSystemPrompt('');
|
||||
setContentPrompt('');
|
||||
setReviewPrompt('');
|
||||
setWinnerSelectionPrompt('');
|
||||
form.reset(defaultValues);
|
||||
setUserSearch('');
|
||||
setSelectedUserId(null);
|
||||
onClose();
|
||||
};
|
||||
|
||||
const handleCreate = () => {
|
||||
const handleSubmit = (data: z.infer<typeof schema>) => {
|
||||
upsertMutation.mutate({
|
||||
name,
|
||||
userId: selectedUserId ? parseInt(selectedUserId, 10) : undefined,
|
||||
bio: bio || null,
|
||||
systemPrompt: systemPrompt || null,
|
||||
contentPrompt: contentPrompt || null,
|
||||
reviewPrompt: reviewPrompt || null,
|
||||
winnerSelectionPrompt: winnerSelectionPrompt || null,
|
||||
name: data.name,
|
||||
userId: data.userId ? parseInt(data.userId, 10) : undefined,
|
||||
bio: data.bio,
|
||||
systemPrompt: data.systemPrompt,
|
||||
contentPrompt: data.contentPrompt,
|
||||
reviewPrompt: data.reviewPrompt,
|
||||
reviewTemplate: data.reviewTemplate,
|
||||
winnerSelectionPrompt: data.winnerSelectionPrompt,
|
||||
});
|
||||
};
|
||||
|
||||
const nothingFoundMessage = usersLoading
|
||||
? 'Searching...'
|
||||
: debouncedSearch.length < 2
|
||||
? 'Type at least 2 characters'
|
||||
: 'No users found';
|
||||
|
||||
return (
|
||||
<Modal opened={opened} onClose={resetAndClose} title="Create Judge" size="lg">
|
||||
<Stack gap="sm">
|
||||
<Select
|
||||
label="User"
|
||||
description="Search for a user to associate with this judge"
|
||||
placeholder="Type a username..."
|
||||
data={userOptions}
|
||||
value={selectedUserId}
|
||||
onChange={setSelectedUserId}
|
||||
searchable
|
||||
searchValue={userSearch}
|
||||
onSearchChange={setUserSearch}
|
||||
nothingFoundMessage={
|
||||
usersLoading
|
||||
? 'Searching...'
|
||||
: debouncedSearch.length < 2
|
||||
? 'Type at least 2 characters'
|
||||
: 'No users found'
|
||||
}
|
||||
rightSection={usersLoading ? <Loader size="xs" /> : undefined}
|
||||
clearable
|
||||
/>
|
||||
<TextInput label="Name" required value={name} onChange={(e) => setName(e.currentTarget.value)} />
|
||||
<Textarea
|
||||
label="Bio"
|
||||
autosize
|
||||
minRows={2}
|
||||
maxRows={4}
|
||||
value={bio}
|
||||
onChange={(e) => setBio(e.currentTarget.value)}
|
||||
/>
|
||||
<Textarea
|
||||
label="System Prompt"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
value={systemPrompt}
|
||||
onChange={(e) => setSystemPrompt(e.currentTarget.value)}
|
||||
/>
|
||||
<Textarea
|
||||
label="Content Prompt"
|
||||
description="Used for challenge content generation"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
value={contentPrompt}
|
||||
onChange={(e) => setContentPrompt(e.currentTarget.value)}
|
||||
/>
|
||||
<Textarea
|
||||
label="Review Prompt"
|
||||
description="Used for image review scoring"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
value={reviewPrompt}
|
||||
onChange={(e) => setReviewPrompt(e.currentTarget.value)}
|
||||
/>
|
||||
<Textarea
|
||||
label="Winner Selection Prompt"
|
||||
description="Used for picking challenge winners"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
value={winnerSelectionPrompt}
|
||||
onChange={(e) => setWinnerSelectionPrompt(e.currentTarget.value)}
|
||||
/>
|
||||
<Group justify="flex-end" mt="sm">
|
||||
<Button variant="default" onClick={resetAndClose}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
leftSection={<IconPlus size={16} />}
|
||||
onClick={handleCreate}
|
||||
loading={upsertMutation.isLoading}
|
||||
disabled={!name}
|
||||
>
|
||||
Create Judge
|
||||
</Button>
|
||||
</Group>
|
||||
</Stack>
|
||||
<Form form={form} onSubmit={handleSubmit}>
|
||||
<Stack gap="sm">
|
||||
<Select
|
||||
label="User"
|
||||
description="Search for a user to associate with this judge"
|
||||
placeholder="Type a username..."
|
||||
data={userOptions}
|
||||
value={form.watch('userId')}
|
||||
onChange={(v) => form.setValue('userId', v)}
|
||||
searchable
|
||||
searchValue={userSearch}
|
||||
onSearchChange={setUserSearch}
|
||||
nothingFoundMessage={nothingFoundMessage}
|
||||
rightSection={usersLoading ? <Loader size="xs" /> : undefined}
|
||||
clearable
|
||||
/>
|
||||
<InputText name="name" label="Name" withAsterisk />
|
||||
<InputTextArea name="bio" label="Bio" autosize minRows={2} maxRows={4} />
|
||||
<InputTextArea
|
||||
name="systemPrompt"
|
||||
label="System Prompt"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
/>
|
||||
<InputTextArea
|
||||
name="contentPrompt"
|
||||
label="Content Prompt"
|
||||
description="Used for challenge content generation"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
/>
|
||||
<InputTextArea
|
||||
name="reviewPrompt"
|
||||
label="Review Prompt"
|
||||
description="Used for image review scoring"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
/>
|
||||
<InputJson
|
||||
name="reviewTemplate"
|
||||
label="Review Template (JSON)"
|
||||
description={<TemplateVariableIndicators value={form.watch('reviewTemplate') ?? ''} />}
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
formatOnBlur
|
||||
validationError="Invalid JSON"
|
||||
styles={{ input: { fontFamily: 'monospace', fontSize: '12px' } }}
|
||||
/>
|
||||
<InputTextArea
|
||||
name="winnerSelectionPrompt"
|
||||
label="Winner Selection Prompt"
|
||||
description="Used for picking challenge winners"
|
||||
autosize
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
/>
|
||||
<Group justify="flex-end" mt="sm">
|
||||
<Button variant="default" onClick={resetAndClose}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
type="submit"
|
||||
leftSection={<IconPlus size={16} />}
|
||||
loading={upsertMutation.isPending}
|
||||
>
|
||||
Create Judge
|
||||
</Button>
|
||||
</Group>
|
||||
</Stack>
|
||||
</Form>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ export function GenerateContentActivity() {
|
||||
const aiModel = usePlaygroundStore((s) => s.aiModel);
|
||||
const drafts = usePlaygroundStore((s) => s.drafts);
|
||||
const updateDraft = usePlaygroundStore((s) => s.updateDraft);
|
||||
const { modelVersionIds, userMessage } = usePlaygroundStore((s) => s.generateContentInputs);
|
||||
const { modelVersionIds } = usePlaygroundStore((s) => s.generateContentInputs);
|
||||
const updateInputs = usePlaygroundStore((s) => s.updateGenerateContentInputs);
|
||||
|
||||
const [result, setResult] = useState<GenerateResult | null>(null);
|
||||
@@ -57,7 +57,6 @@ export function GenerateContentActivity() {
|
||||
content: draft?.contentPrompt ?? undefined,
|
||||
}
|
||||
: undefined,
|
||||
userMessage: userMessage || undefined,
|
||||
aiModel: aiModel || undefined,
|
||||
});
|
||||
};
|
||||
@@ -90,19 +89,10 @@ export function GenerateContentActivity() {
|
||||
if (id != null) updateDraft(id, { contentPrompt: e.currentTarget.value || null });
|
||||
}}
|
||||
/>
|
||||
<Textarea
|
||||
label="User Message (override)"
|
||||
placeholder="Leave empty to use default (auto-generated from model info)"
|
||||
autosize
|
||||
minRows={2}
|
||||
maxRows={6}
|
||||
value={userMessage}
|
||||
onChange={(e) => updateInputs({ userMessage: e.currentTarget.value })}
|
||||
/>
|
||||
<Button
|
||||
leftSection={<IconPlayerPlay size={16} />}
|
||||
onClick={handleRun}
|
||||
loading={mutation.isLoading}
|
||||
loading={mutation.isPending}
|
||||
disabled={modelVersionIds.length === 0}
|
||||
>
|
||||
Generate Content
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
import { Button, Loader, ScrollArea, Stack, Text, TextInput, Textarea } from '@mantine/core';
|
||||
import {
|
||||
Button,
|
||||
JsonInput,
|
||||
Loader,
|
||||
ScrollArea,
|
||||
Stack,
|
||||
Text,
|
||||
TextInput,
|
||||
Textarea,
|
||||
} from '@mantine/core';
|
||||
import { IconDeviceFloppy } from '@tabler/icons-react';
|
||||
import { showErrorNotification, showSuccessNotification } from '~/utils/notifications';
|
||||
import { trpc } from '~/utils/trpc';
|
||||
import { ModelSelector } from './ModelSelector';
|
||||
import { usePlaygroundStore } from './playground.store';
|
||||
import { TemplateVariableIndicators } from './TemplateVariableIndicators';
|
||||
|
||||
export function JudgeSettingsPanel() {
|
||||
const selectedJudgeId = usePlaygroundStore((s) => s.selectedJudgeId);
|
||||
@@ -39,6 +49,7 @@ export function JudgeSettingsPanel() {
|
||||
const currentSystemPrompt = draft?.systemPrompt ?? judge?.systemPrompt ?? '';
|
||||
const currentContentPrompt = draft?.contentPrompt ?? judge?.contentPrompt ?? '';
|
||||
const currentReviewPrompt = draft?.reviewPrompt ?? judge?.reviewPrompt ?? '';
|
||||
const currentReviewTemplate = draft?.reviewTemplate ?? judge?.reviewTemplate ?? '';
|
||||
const currentWinnerPrompt = draft?.winnerSelectionPrompt ?? judge?.winnerSelectionPrompt ?? '';
|
||||
|
||||
const handleSave = () => {
|
||||
@@ -49,9 +60,10 @@ export function JudgeSettingsPanel() {
|
||||
name: currentName,
|
||||
bio: currentBio || null,
|
||||
systemPrompt: currentSystemPrompt || null,
|
||||
contentPrompt: draft?.contentPrompt ?? judge?.contentPrompt ?? null,
|
||||
reviewPrompt: draft?.reviewPrompt ?? judge?.reviewPrompt ?? null,
|
||||
winnerSelectionPrompt: draft?.winnerSelectionPrompt ?? judge?.winnerSelectionPrompt ?? null,
|
||||
contentPrompt: currentContentPrompt || null,
|
||||
reviewPrompt: currentReviewPrompt || null,
|
||||
reviewTemplate: currentReviewTemplate || null,
|
||||
winnerSelectionPrompt: currentWinnerPrompt || null,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -134,6 +146,21 @@ export function JudgeSettingsPanel() {
|
||||
updateDraft(selectedJudgeId, { reviewPrompt: e.currentTarget.value || null });
|
||||
}}
|
||||
/>
|
||||
<JsonInput
|
||||
label="Review Template (JSON)"
|
||||
description={<TemplateVariableIndicators value={currentReviewTemplate ?? ''} />}
|
||||
autosize
|
||||
minRows={4}
|
||||
maxRows={14}
|
||||
formatOnBlur
|
||||
validationError="Invalid JSON"
|
||||
styles={{ input: { fontFamily: 'monospace', fontSize: '12px' } }}
|
||||
value={currentReviewTemplate ?? ''}
|
||||
onChange={(value) => {
|
||||
if (selectedJudgeId != null)
|
||||
updateDraft(selectedJudgeId, { reviewTemplate: value || null });
|
||||
}}
|
||||
/>
|
||||
<Textarea
|
||||
label="Winner Selection Prompt"
|
||||
description="Used for picking challenge winners"
|
||||
@@ -155,7 +182,7 @@ export function JudgeSettingsPanel() {
|
||||
leftSection={<IconDeviceFloppy size={16} />}
|
||||
m="sm"
|
||||
onClick={handleSave}
|
||||
loading={upsertMutation.isLoading}
|
||||
loading={upsertMutation.isPending}
|
||||
disabled={!currentName}
|
||||
>
|
||||
Save Judge
|
||||
|
||||
@@ -38,7 +38,7 @@ export function PickWinnersActivity() {
|
||||
const aiModel = usePlaygroundStore((s) => s.aiModel);
|
||||
const drafts = usePlaygroundStore((s) => s.drafts);
|
||||
const updateDraft = usePlaygroundStore((s) => s.updateDraft);
|
||||
const { challengeId, userMessage } = usePlaygroundStore((s) => s.pickWinnersInputs);
|
||||
const { challengeId } = usePlaygroundStore((s) => s.pickWinnersInputs);
|
||||
const updateInputs = usePlaygroundStore((s) => s.updatePickWinnersInputs);
|
||||
|
||||
const [result, setResult] = useState<PickWinnersResult | null>(null);
|
||||
@@ -81,7 +81,6 @@ export function PickWinnersActivity() {
|
||||
winner: draft?.winnerSelectionPrompt ?? undefined,
|
||||
}
|
||||
: undefined,
|
||||
userMessage: userMessage || undefined,
|
||||
aiModel: aiModel || undefined,
|
||||
});
|
||||
};
|
||||
@@ -111,19 +110,10 @@ export function PickWinnersActivity() {
|
||||
if (id != null) updateDraft(id, { winnerSelectionPrompt: e.currentTarget.value || null });
|
||||
}}
|
||||
/>
|
||||
<Textarea
|
||||
label="User Message (override)"
|
||||
placeholder="Leave empty to use default (Theme + Entries JSON)"
|
||||
autosize
|
||||
minRows={2}
|
||||
maxRows={6}
|
||||
value={userMessage}
|
||||
onChange={(e) => updateInputs({ userMessage: e.currentTarget.value })}
|
||||
/>
|
||||
<Button
|
||||
leftSection={<IconPlayerPlay size={16} />}
|
||||
onClick={handleRun}
|
||||
loading={mutation.isLoading}
|
||||
loading={mutation.isPending}
|
||||
disabled={!challengeId}
|
||||
>
|
||||
Pick Winners
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
Button,
|
||||
Card,
|
||||
Group,
|
||||
JsonInput,
|
||||
Paper,
|
||||
Progress,
|
||||
ScrollArea,
|
||||
@@ -15,12 +16,14 @@ import { useState } from 'react';
|
||||
import { showErrorNotification } from '~/utils/notifications';
|
||||
import { trpc } from '~/utils/trpc';
|
||||
import { usePlaygroundStore } from './playground.store';
|
||||
import { TemplateVariableIndicators } from './TemplateVariableIndicators';
|
||||
|
||||
type ReviewResult = {
|
||||
score: { theme: number; wittiness: number; humor: number; aesthetic: number };
|
||||
reaction: string;
|
||||
comment: string;
|
||||
summary: string;
|
||||
aestheticFlaws?: string[];
|
||||
};
|
||||
|
||||
const SCORE_COLORS: Record<string, string> = {
|
||||
@@ -55,9 +58,7 @@ export function ReviewImageActivity() {
|
||||
const aiModel = usePlaygroundStore((s) => s.aiModel);
|
||||
const drafts = usePlaygroundStore((s) => s.drafts);
|
||||
const updateDraft = usePlaygroundStore((s) => s.updateDraft);
|
||||
const { imageInput, theme, creator, userMessage } = usePlaygroundStore(
|
||||
(s) => s.reviewImageInputs
|
||||
);
|
||||
const { imageInput, theme, creator } = usePlaygroundStore((s) => s.reviewImageInputs);
|
||||
const updateInputs = usePlaygroundStore((s) => s.updateReviewImageInputs);
|
||||
|
||||
const [result, setResult] = useState<ReviewResult | null>(null);
|
||||
@@ -65,6 +66,7 @@ export function ReviewImageActivity() {
|
||||
const draft =
|
||||
selectedJudgeId != null && selectedJudgeId > 0 ? drafts[selectedJudgeId] : undefined;
|
||||
const reviewPrompt = draft?.reviewPrompt ?? '';
|
||||
const reviewTemplate = draft?.reviewTemplate ?? '';
|
||||
|
||||
const parsedImageId = parseImageInput(imageInput);
|
||||
|
||||
@@ -89,7 +91,7 @@ export function ReviewImageActivity() {
|
||||
review: draft?.reviewPrompt ?? undefined,
|
||||
}
|
||||
: undefined,
|
||||
userMessage: userMessage || undefined,
|
||||
reviewTemplate: draft?.reviewTemplate ?? undefined,
|
||||
aiModel: aiModel || undefined,
|
||||
});
|
||||
};
|
||||
@@ -132,19 +134,26 @@ export function ReviewImageActivity() {
|
||||
if (id != null) updateDraft(id, { reviewPrompt: e.currentTarget.value || null });
|
||||
}}
|
||||
/>
|
||||
<Textarea
|
||||
label="User Message (override)"
|
||||
placeholder="Leave empty to use default (Theme + Creator)"
|
||||
<JsonInput
|
||||
label="Review Template (override)"
|
||||
placeholder="Leave empty to use judge's default"
|
||||
description={<TemplateVariableIndicators value={reviewTemplate} />}
|
||||
autosize
|
||||
minRows={2}
|
||||
maxRows={6}
|
||||
value={userMessage}
|
||||
onChange={(e) => updateInputs({ userMessage: e.currentTarget.value })}
|
||||
minRows={3}
|
||||
maxRows={8}
|
||||
formatOnBlur
|
||||
validationError="Invalid JSON"
|
||||
styles={{ input: { fontFamily: 'monospace', fontSize: '12px' } }}
|
||||
value={reviewTemplate}
|
||||
onChange={(value) => {
|
||||
const id = selectedJudgeId != null && selectedJudgeId > 0 ? selectedJudgeId : null;
|
||||
if (id != null) updateDraft(id, { reviewTemplate: value || null });
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
leftSection={<IconPlayerPlay size={16} />}
|
||||
onClick={handleRun}
|
||||
loading={mutation.isLoading}
|
||||
loading={mutation.isPending}
|
||||
disabled={!parsedImageId || !theme}
|
||||
>
|
||||
Review Image
|
||||
@@ -157,7 +166,7 @@ export function ReviewImageActivity() {
|
||||
Scores
|
||||
</Text>
|
||||
<Paper withBorder p="sm">
|
||||
{Object.entries(result.score).map(([key, value]) => (
|
||||
{Object.entries(result.score ?? {}).map(([key, value]) => (
|
||||
<div key={key}>
|
||||
<Group justify="space-between" mb={4}>
|
||||
<Text size="xs" tt="capitalize">
|
||||
@@ -183,6 +192,22 @@ export function ReviewImageActivity() {
|
||||
<Paper withBorder p="sm">
|
||||
<Text size="sm">{result.comment}</Text>
|
||||
</Paper>
|
||||
{result.aestheticFlaws && result.aestheticFlaws.length > 0 && (
|
||||
<>
|
||||
<Text size="sm" fw={600} mb={4}>
|
||||
Aesthetic Flaws
|
||||
</Text>
|
||||
<Paper withBorder p="sm">
|
||||
<Stack gap={4}>
|
||||
{result.aestheticFlaws.map((flaw, i) => (
|
||||
<Text key={i} size="sm">
|
||||
• {flaw}
|
||||
</Text>
|
||||
))}
|
||||
</Stack>
|
||||
</Paper>
|
||||
</>
|
||||
)}
|
||||
</Stack>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { Badge, CopyButton, Group } from '@mantine/core';
|
||||
|
||||
const TEMPLATE_VARIABLES = [
|
||||
{ name: 'systemPrompt', description: 'The judge system prompt' },
|
||||
{ name: 'reviewPrompt', description: 'The judge review prompt' },
|
||||
{ name: 'theme', description: 'The challenge theme' },
|
||||
] as const;
|
||||
|
||||
export function TemplateVariableIndicators({ value }: { value: string }) {
|
||||
return (
|
||||
<Group gap={6}>
|
||||
{TEMPLATE_VARIABLES.map((v) => {
|
||||
const variable = `{{${v.name}}}`;
|
||||
const isUsed = value.includes(variable);
|
||||
return (
|
||||
<CopyButton key={v.name} value={variable}>
|
||||
{({ copied, copy }) => (
|
||||
<Badge
|
||||
variant="light"
|
||||
color={copied ? 'teal' : isUsed ? 'green' : 'gray'}
|
||||
size="xs"
|
||||
title={copied ? 'Copied!' : `${v.description} — click to copy`}
|
||||
style={{ cursor: 'pointer', textTransform: 'none' }}
|
||||
onClick={copy}
|
||||
>
|
||||
{copied ? 'Copied!' : variable}
|
||||
</Badge>
|
||||
)}
|
||||
</CopyButton>
|
||||
);
|
||||
})}
|
||||
</Group>
|
||||
);
|
||||
}
|
||||
@@ -10,24 +10,22 @@ export type JudgeDraft = {
|
||||
systemPrompt?: string | null;
|
||||
contentPrompt?: string | null;
|
||||
reviewPrompt?: string | null;
|
||||
reviewTemplate?: string | null;
|
||||
winnerSelectionPrompt?: string | null;
|
||||
};
|
||||
|
||||
type GenerateContentInputs = {
|
||||
modelVersionIds: number[];
|
||||
userMessage: string;
|
||||
};
|
||||
|
||||
type ReviewImageInputs = {
|
||||
imageInput: string;
|
||||
theme: string;
|
||||
creator: string;
|
||||
userMessage: string;
|
||||
};
|
||||
|
||||
type PickWinnersInputs = {
|
||||
challengeId: string | null;
|
||||
userMessage: string;
|
||||
};
|
||||
|
||||
type PlaygroundState = {
|
||||
@@ -61,9 +59,9 @@ export const usePlaygroundStore = create<PlaygroundState & PlaygroundActions>()(
|
||||
aiModel: 'x-ai/grok-4.1-fast',
|
||||
customModelId: '',
|
||||
drafts: {},
|
||||
generateContentInputs: { modelVersionIds: [], userMessage: '' },
|
||||
reviewImageInputs: { imageInput: '', theme: '', creator: '', userMessage: '' },
|
||||
pickWinnersInputs: { challengeId: null, userMessage: '' },
|
||||
generateContentInputs: { modelVersionIds: [] },
|
||||
reviewImageInputs: { imageInput: '', theme: '', creator: '' },
|
||||
pickWinnersInputs: { challengeId: null },
|
||||
|
||||
setSelectedJudgeId: (id) =>
|
||||
set((state) => {
|
||||
|
||||
@@ -106,6 +106,7 @@ export default WebhookEndpoint(async function (req: NextApiRequest, res: NextApi
|
||||
score: review.score,
|
||||
summary: review.summary,
|
||||
judgeId: judgingConfig.judgeId,
|
||||
...(review.aestheticFlaws?.length && { aestheticFlaws: review.aestheticFlaws }),
|
||||
});
|
||||
await dbWrite.$executeRaw`
|
||||
UPDATE "CollectionItem"
|
||||
|
||||
@@ -33,6 +33,7 @@ const judgingConfigSchema = z.object({
|
||||
userId: z.number(),
|
||||
sourceCollectionId: z.number().nullable(),
|
||||
prompts: challengePromptsSchema,
|
||||
reviewTemplate: z.string().nullable().default(null),
|
||||
});
|
||||
|
||||
const challengeConfigSchema = z.object({
|
||||
@@ -160,6 +161,7 @@ async function fetchJudgingConfigFromDb(judgeId: number): Promise<JudgingConfig
|
||||
collectionPrompt: true,
|
||||
contentPrompt: true,
|
||||
reviewPrompt: true,
|
||||
reviewTemplate: true,
|
||||
winnerSelectionPrompt: true,
|
||||
},
|
||||
});
|
||||
@@ -178,6 +180,7 @@ async function fetchJudgingConfigFromDb(judgeId: number): Promise<JudgingConfig
|
||||
review: judge.reviewPrompt ?? '',
|
||||
winner: judge.winnerSelectionPrompt ?? '',
|
||||
},
|
||||
reviewTemplate: judge.reviewTemplate ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -229,6 +232,7 @@ export type JudgingConfig = {
|
||||
userId: number;
|
||||
sourceCollectionId: number | null; // Collection to pick model resources from
|
||||
prompts: ChallengePrompts;
|
||||
reviewTemplate: string | null; // JSON message template (agent-workbench format)
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -253,6 +257,7 @@ export async function getJudgingConfig(
|
||||
collectionPrompt: true,
|
||||
contentPrompt: true,
|
||||
reviewPrompt: true,
|
||||
reviewTemplate: true,
|
||||
winnerSelectionPrompt: true,
|
||||
},
|
||||
});
|
||||
@@ -271,6 +276,7 @@ export async function getJudgingConfig(
|
||||
review: judgingPromptOverride ?? judge.reviewPrompt ?? '',
|
||||
winner: judge.winnerSelectionPrompt ?? '',
|
||||
},
|
||||
reviewTemplate: judge.reviewTemplate ?? null,
|
||||
};
|
||||
}
|
||||
type ChallengeTypeRow = {
|
||||
|
||||
@@ -5,9 +5,16 @@ import type {
|
||||
Score,
|
||||
} from '~/server/games/daily-challenge/daily-challenge.utils';
|
||||
import { openrouter, AI_MODELS, type AIModel } from '~/server/services/ai/openrouter';
|
||||
import type { SimpleMessage } from '~/server/services/ai/openrouter';
|
||||
import type { ReviewReactions } from '~/shared/utils/prisma/enums';
|
||||
import { findLastIndex } from '~/utils/array-helpers';
|
||||
import { markdownToHtml } from '~/utils/markdown-helpers';
|
||||
import { stripLeadingWhitespace } from '~/utils/string-helpers';
|
||||
import {
|
||||
parseReviewTemplate,
|
||||
resolveTemplate,
|
||||
type ReviewTemplateVariables,
|
||||
} from './template-engine';
|
||||
|
||||
type GenerateCollectionDetailsInput = {
|
||||
resource: {
|
||||
@@ -79,7 +86,6 @@ type GenerateArticleInput = {
|
||||
allowedNsfwLevel: number;
|
||||
config: JudgingConfig;
|
||||
model?: AIModel;
|
||||
userMessageOverride?: string;
|
||||
};
|
||||
type GeneratedArticle = {
|
||||
title: string;
|
||||
@@ -87,18 +93,10 @@ type GeneratedArticle = {
|
||||
invitation: string;
|
||||
theme: string;
|
||||
};
|
||||
export async function generateArticle({
|
||||
resource,
|
||||
image,
|
||||
config,
|
||||
model,
|
||||
userMessageOverride,
|
||||
}: GenerateArticleInput) {
|
||||
export async function generateArticle({ resource, image, config, model }: GenerateArticleInput) {
|
||||
if (!openrouter) throw new Error('OpenRouter not connected');
|
||||
|
||||
const userText =
|
||||
userMessageOverride ??
|
||||
`Resource title: ${resource.title}\nResource link: https://civitai.com/models/${resource.modelId}\nCreator: ${resource.creator}\nCreator link: https://civitai.com/user/${resource.creator}`;
|
||||
const userText = `Resource title: ${resource.title}\nResource link: https://civitai.com/models/${resource.modelId}\nCreator: ${resource.creator}\nCreator link: https://civitai.com/user/${resource.creator}`;
|
||||
|
||||
const result = await openrouter.getJsonCompletion<GeneratedArticle>({
|
||||
retries: 3,
|
||||
@@ -151,57 +149,125 @@ type GenerateReviewInput = {
|
||||
imageUrl: string;
|
||||
config: JudgingConfig;
|
||||
model?: AIModel;
|
||||
userMessageOverride?: string;
|
||||
};
|
||||
type GeneratedReview = {
|
||||
score: Score;
|
||||
reaction: ReviewReactions;
|
||||
comment: string;
|
||||
summary: string;
|
||||
aestheticFlaws?: string[];
|
||||
};
|
||||
export async function generateReview(input: GenerateReviewInput) {
|
||||
|
||||
const RESPONSE_SCHEMA = `{
|
||||
"score": {
|
||||
"theme": number, // 0-10
|
||||
"wittiness": number, // 0-10
|
||||
"humor": number, // 0-10
|
||||
"aesthetic": number // 0-10
|
||||
},
|
||||
"reaction": "Laugh" | "Heart" | "Like" | "Cry",
|
||||
"comment": "your review comment (2-3 sentences)",
|
||||
"summary": "concise factual summary of the image"
|
||||
"aestheticFlaws": ["string describing flaw 1","string describing flaw 2",...] // optional array of strings describing specific aesthetic flaws in the image
|
||||
}`;
|
||||
|
||||
export async function generateReview(input: GenerateReviewInput): Promise<GeneratedReview> {
|
||||
if (!openrouter) throw new Error('OpenRouter not connected');
|
||||
|
||||
const userText = input.userMessageOverride ?? `Theme: ${input.theme}\nCreator: ${input.creator}`;
|
||||
let messages: SimpleMessage[];
|
||||
if (input.config.reviewTemplate) {
|
||||
try {
|
||||
messages = buildMessagesFromTemplate(input);
|
||||
} catch (e) {
|
||||
console.warn('[generateReview] Invalid reviewTemplate, falling back to default prompts:', e);
|
||||
messages = buildFallbackMessages(input);
|
||||
}
|
||||
} else {
|
||||
messages = buildFallbackMessages(input);
|
||||
}
|
||||
|
||||
const result = await openrouter.getJsonCompletion<GeneratedReview>({
|
||||
retries: 3,
|
||||
model: input.model ?? AI_MODELS.GROK,
|
||||
messages: [
|
||||
prepareSystemMessage(
|
||||
input.config,
|
||||
'review',
|
||||
`{
|
||||
"score": {
|
||||
"theme": number, // 0-10 how well it adheres to the theme
|
||||
"wittiness": number, // 0-10 how witty it is
|
||||
"humor": number, // 0-10 how funny it is
|
||||
"aesthetic": number // 0-10 how aesthetically pleasing it is
|
||||
},
|
||||
"reaction": "a single emoji reaction", // options are "Laugh", "Heart", "Like", "Cry"
|
||||
"comment": "the content of the comment",
|
||||
"summary": "concise summary of the content of the image"
|
||||
}`
|
||||
),
|
||||
{
|
||||
role: 'user' as const,
|
||||
content: [
|
||||
{
|
||||
type: 'text' as const,
|
||||
text: userText,
|
||||
},
|
||||
{
|
||||
type: 'image_url' as const,
|
||||
image_url: {
|
||||
url: input.imageUrl,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
messages,
|
||||
});
|
||||
|
||||
return {
|
||||
score: result.score,
|
||||
reaction: result.reaction,
|
||||
comment: result.comment,
|
||||
summary: result.summary,
|
||||
aestheticFlaws: result.aestheticFlaws,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build messages from a JSON review template with variable substitution.
|
||||
*/
|
||||
function buildMessagesFromTemplate(input: GenerateReviewInput): SimpleMessage[] {
|
||||
const template = parseReviewTemplate(input.config.reviewTemplate!);
|
||||
|
||||
const variables: ReviewTemplateVariables = {
|
||||
systemPrompt: input.config.prompts.systemMessage,
|
||||
reviewPrompt: input.config.prompts.review,
|
||||
theme: input.theme,
|
||||
};
|
||||
|
||||
const messages = resolveTemplate(template, variables);
|
||||
|
||||
// Inject response schema into the last system message
|
||||
const schemaInstruction = `\n\nReply with json\n\n${stripLeadingWhitespace(RESPONSE_SCHEMA)}`;
|
||||
const lastSystemIdx = findLastIndex(messages, (m) => m.role === 'system');
|
||||
if (lastSystemIdx >= 0) {
|
||||
const msg = messages[lastSystemIdx];
|
||||
if (typeof msg.content === 'string') {
|
||||
messages[lastSystemIdx] = { ...msg, content: msg.content + schemaInstruction };
|
||||
} else if (Array.isArray(msg.content)) {
|
||||
const lastTextIdx = findLastIndex(msg.content, (item) => item.type === 'text');
|
||||
if (lastTextIdx >= 0) {
|
||||
const items = [...msg.content];
|
||||
const textItem = items[lastTextIdx] as { type: 'text'; text: string };
|
||||
items[lastTextIdx] = { type: 'text', text: textItem.text + schemaInstruction };
|
||||
messages[lastSystemIdx] = { ...msg, content: items };
|
||||
} else {
|
||||
const items = [...msg.content];
|
||||
items.push({ type: 'text', text: schemaInstruction.trimStart() });
|
||||
messages[lastSystemIdx] = { ...msg, content: items };
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No system message in template — prepend one
|
||||
messages.unshift({ role: 'system', content: schemaInstruction.trimStart() });
|
||||
}
|
||||
|
||||
// Append user message with theme, creator, and image
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: `Theme: ${input.theme}\nCreator: ${input.creator}` },
|
||||
{ type: 'image_url', image_url: { url: input.imageUrl } },
|
||||
],
|
||||
});
|
||||
|
||||
return result;
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build simple 2-message array from systemPrompt + reviewPrompt fields (fallback path).
|
||||
*/
|
||||
function buildFallbackMessages(input: GenerateReviewInput): SimpleMessage[] {
|
||||
const userText = `Theme: ${input.theme}\nCreator: ${input.creator}`;
|
||||
|
||||
return [
|
||||
prepareSystemMessage(input.config, 'review', RESPONSE_SCHEMA),
|
||||
{
|
||||
role: 'user' as const,
|
||||
content: [
|
||||
{ type: 'text' as const, text: userText },
|
||||
{ type: 'image_url' as const, image_url: { url: input.imageUrl } },
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
type GenerateWinnersInput = {
|
||||
@@ -214,7 +280,6 @@ type GenerateWinnersInput = {
|
||||
theme: string;
|
||||
config: JudgingConfig;
|
||||
model?: AIModel;
|
||||
userMessageOverride?: string;
|
||||
};
|
||||
type GeneratedWinners = {
|
||||
winners: Array<{
|
||||
@@ -228,13 +293,11 @@ type GeneratedWinners = {
|
||||
export async function generateWinners(input: GenerateWinnersInput) {
|
||||
if (!openrouter) throw new Error('OpenRouter not connected');
|
||||
|
||||
const userText =
|
||||
input.userMessageOverride ??
|
||||
`Theme: ${input.theme}\nEntries:\n\`\`\`json \n${JSON.stringify(
|
||||
input.entries,
|
||||
null,
|
||||
2
|
||||
)}\n\`\`\``;
|
||||
const userText = `Theme: ${input.theme}\nEntries:\n\`\`\`json \n${JSON.stringify(
|
||||
input.entries,
|
||||
null,
|
||||
2
|
||||
)}\n\`\`\``;
|
||||
|
||||
const result = await openrouter.getJsonCompletion<GeneratedWinners>({
|
||||
retries: 3,
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { parseReviewTemplate, resolveTemplate } from './template-engine';
|
||||
import type { ReviewTemplate } from './template-engine';
|
||||
|
||||
describe('parseReviewTemplate', () => {
|
||||
it('parses a valid template', () => {
|
||||
const json = JSON.stringify({
|
||||
messages: [
|
||||
{ role: 'system', content: 'You are a judge.' },
|
||||
{ role: 'user', content: [{ type: 'text', text: 'Review this' }] },
|
||||
],
|
||||
});
|
||||
|
||||
const result = parseReviewTemplate(json);
|
||||
expect(result.messages).toHaveLength(2);
|
||||
expect(result.messages[0].role).toBe('system');
|
||||
expect(result.messages[0].content).toBe('You are a judge.');
|
||||
});
|
||||
|
||||
it('throws on invalid JSON', () => {
|
||||
expect(() => parseReviewTemplate('not json')).toThrow();
|
||||
});
|
||||
|
||||
it('throws on schema mismatch — missing messages', () => {
|
||||
expect(() => parseReviewTemplate(JSON.stringify({ prompts: [] }))).toThrow();
|
||||
});
|
||||
|
||||
it('throws on schema mismatch — wrong role', () => {
|
||||
const json = JSON.stringify({
|
||||
messages: [{ role: 'moderator', content: 'hello' }],
|
||||
});
|
||||
expect(() => parseReviewTemplate(json)).toThrow();
|
||||
});
|
||||
|
||||
it('throws on empty messages array', () => {
|
||||
const json = JSON.stringify({ messages: [] });
|
||||
expect(() => parseReviewTemplate(json)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveTemplate', () => {
|
||||
const variables = {
|
||||
systemPrompt: 'You are a challenge judge.',
|
||||
reviewPrompt: 'Rate this image.',
|
||||
theme: 'sunset landscape',
|
||||
};
|
||||
|
||||
it('replaces variables in string content', () => {
|
||||
const template: ReviewTemplate = {
|
||||
messages: [
|
||||
{ role: 'system', content: '{{systemPrompt}}' },
|
||||
{ role: 'user', content: '{{reviewPrompt}} Theme: {{theme}}' },
|
||||
],
|
||||
};
|
||||
|
||||
const result = resolveTemplate(template, variables);
|
||||
expect(result[0].content).toBe('You are a challenge judge.');
|
||||
expect(result[1].content).toBe('Rate this image. Theme: sunset landscape');
|
||||
});
|
||||
|
||||
it('replaces variables in array content text items', () => {
|
||||
const template: ReviewTemplate = {
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: [{ type: 'text', text: 'Prompt: {{systemPrompt}}' }],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = resolveTemplate(template, variables);
|
||||
const content = result[0].content as Array<{ type: 'text'; text: string }>;
|
||||
expect(content[0].text).toBe('Prompt: You are a challenge judge.');
|
||||
});
|
||||
|
||||
it('replaces variables in array content image_url items', () => {
|
||||
const template: ReviewTemplate = {
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'image_url', image_url: { url: 'https://example.com/{{theme}}.png' } }],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = resolveTemplate(template, variables);
|
||||
const content = result[0].content as Array<{
|
||||
type: 'image_url';
|
||||
image_url: { url: string };
|
||||
}>;
|
||||
expect(content[0].image_url.url).toBe('https://example.com/sunset landscape.png');
|
||||
});
|
||||
|
||||
it('leaves unrecognized variables as-is', () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => null);
|
||||
|
||||
const template: ReviewTemplate = {
|
||||
messages: [{ role: 'system', content: '{{unknownVar}} and {{systemPrompt}}' }],
|
||||
};
|
||||
|
||||
const result = resolveTemplate(template, variables);
|
||||
expect(result[0].content).toBe('{{unknownVar}} and You are a challenge judge.');
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('{{unknownVar}}'));
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('replaces multiple variables in the same string', () => {
|
||||
const template: ReviewTemplate = {
|
||||
messages: [{ role: 'user', content: '{{systemPrompt}} | {{reviewPrompt}} | {{theme}}' }],
|
||||
};
|
||||
|
||||
const result = resolveTemplate(template, variables);
|
||||
expect(result[0].content).toBe(
|
||||
'You are a challenge judge. | Rate this image. | sunset landscape'
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import * as z from 'zod';
|
||||
import type { SimpleMessage } from '~/server/services/ai/openrouter';
|
||||
|
||||
// Zod schema for validating review template JSON
|
||||
const contentItemSchema = z.discriminatedUnion('type', [
|
||||
z.object({ type: z.literal('text'), text: z.string() }),
|
||||
z.object({
|
||||
type: z.literal('image_url'),
|
||||
image_url: z.object({ url: z.string() }),
|
||||
}),
|
||||
]);
|
||||
|
||||
export const reviewTemplateSchema = z.object({
|
||||
messages: z
|
||||
.array(
|
||||
z.object({
|
||||
role: z.enum(['system', 'user', 'assistant']),
|
||||
content: z.union([z.string(), z.array(contentItemSchema)]),
|
||||
})
|
||||
)
|
||||
.min(1),
|
||||
});
|
||||
|
||||
export type ReviewTemplate = z.infer<typeof reviewTemplateSchema>;
|
||||
|
||||
export type ReviewTemplateVariables = {
|
||||
systemPrompt: string;
|
||||
reviewPrompt: string;
|
||||
theme: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse and validate a JSON review template string.
|
||||
* Throws on invalid JSON or schema mismatch.
|
||||
*/
|
||||
export function parseReviewTemplate(json: string): ReviewTemplate {
|
||||
const parsed = JSON.parse(json);
|
||||
return reviewTemplateSchema.parse(parsed);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep-walk a template's message tree, replacing {{var}} placeholders
|
||||
* with values from the variables map. Returns resolved SimpleMessage[].
|
||||
*/
|
||||
export function resolveTemplate(
|
||||
template: ReviewTemplate,
|
||||
variables: ReviewTemplateVariables
|
||||
): SimpleMessage[] {
|
||||
const vars = variables as Record<string, string>;
|
||||
|
||||
function replaceVars(text: string): string {
|
||||
return text.replace(/\{\{(\w+)\}\}/g, (match, key: string) => {
|
||||
if (key in vars) return vars[key];
|
||||
console.warn(`[template-engine] Unrecognized template variable: ${match}`);
|
||||
return match;
|
||||
});
|
||||
}
|
||||
|
||||
return template.messages.map((msg) => {
|
||||
if (typeof msg.content === 'string') {
|
||||
return { role: msg.role, content: replaceVars(msg.content) };
|
||||
}
|
||||
|
||||
return {
|
||||
role: msg.role,
|
||||
content: msg.content.map((item) => {
|
||||
if (item.type === 'text') {
|
||||
return { type: 'text' as const, text: replaceVars(item.text) };
|
||||
}
|
||||
return {
|
||||
type: 'image_url' as const,
|
||||
image_url: { url: replaceVars(item.image_url.url) },
|
||||
};
|
||||
}),
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -886,6 +886,7 @@ async function reviewEntriesForChallenge(currentChallenge: DailyChallengeDetails
|
||||
score: review.score,
|
||||
summary: review.summary,
|
||||
judgeId: judgingConfig.judgeId,
|
||||
...(review.aestheticFlaws?.length && { aestheticFlaws: review.aestheticFlaws }),
|
||||
});
|
||||
await dbWrite.$executeRaw`
|
||||
UPDATE "CollectionItem"
|
||||
|
||||
@@ -442,6 +442,7 @@ export const upsertJudgeSchema = z.object({
|
||||
collectionPrompt: z.string().optional().nullable(),
|
||||
contentPrompt: z.string().optional().nullable(),
|
||||
reviewPrompt: z.string().optional().nullable(),
|
||||
reviewTemplate: z.string().optional().nullable(),
|
||||
winnerSelectionPrompt: z.string().optional().nullable(),
|
||||
active: z.boolean().optional(),
|
||||
});
|
||||
@@ -457,7 +458,6 @@ export const playgroundGenerateContentSchema = z.object({
|
||||
content: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
userMessage: z.string().optional(),
|
||||
aiModel: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
@@ -474,7 +474,7 @@ export const playgroundReviewImageSchema = z.object({
|
||||
review: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
userMessage: z.string().optional(),
|
||||
reviewTemplate: z.string().optional(),
|
||||
aiModel: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
@@ -489,6 +489,5 @@ export const playgroundPickWinnersSchema = z.object({
|
||||
winner: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
userMessage: z.string().optional(),
|
||||
aiModel: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
@@ -60,6 +60,7 @@ import {
|
||||
generateReview,
|
||||
generateWinners,
|
||||
} from '~/server/games/daily-challenge/generative-content';
|
||||
import { reviewTemplateSchema } from '~/server/games/daily-challenge/template-engine';
|
||||
import { getCoverOfModel, getJudgedEntries } from '~/server/jobs/daily-challenge-processing';
|
||||
import { collectionsSearchIndex } from '~/server/search-index';
|
||||
import { SearchIndexUpdateQueueAction } from '~/server/common/enums';
|
||||
@@ -1905,6 +1906,7 @@ export async function getJudgeById(id: number) {
|
||||
collectionPrompt: true,
|
||||
contentPrompt: true,
|
||||
reviewPrompt: true,
|
||||
reviewTemplate: true,
|
||||
winnerSelectionPrompt: true,
|
||||
},
|
||||
});
|
||||
@@ -1919,6 +1921,19 @@ export async function getJudgeById(id: number) {
|
||||
export async function upsertJudge(input: UpsertJudgeInput & { userId: number }) {
|
||||
const { id, userId, ...data } = input;
|
||||
|
||||
// Validate reviewTemplate JSON if provided
|
||||
if (data.reviewTemplate) {
|
||||
try {
|
||||
const parsed = JSON.parse(data.reviewTemplate);
|
||||
reviewTemplateSchema.parse(parsed);
|
||||
} catch (e) {
|
||||
throw new TRPCError({
|
||||
code: 'BAD_REQUEST',
|
||||
message: `Invalid review template: ${e instanceof Error ? e.message : 'Invalid JSON'}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const judge = await dbWrite.challengeJudge.upsert({
|
||||
where: { id: id ?? -1 },
|
||||
create: {
|
||||
@@ -1930,6 +1945,7 @@ export async function upsertJudge(input: UpsertJudgeInput & { userId: number })
|
||||
collectionPrompt: data.collectionPrompt ?? null,
|
||||
contentPrompt: data.contentPrompt ?? null,
|
||||
reviewPrompt: data.reviewPrompt ?? null,
|
||||
reviewTemplate: data.reviewTemplate ?? null,
|
||||
winnerSelectionPrompt: data.winnerSelectionPrompt ?? null,
|
||||
active: data.active ?? true,
|
||||
},
|
||||
@@ -1943,6 +1959,7 @@ export async function upsertJudge(input: UpsertJudgeInput & { userId: number })
|
||||
...(data.collectionPrompt !== undefined && { collectionPrompt: data.collectionPrompt }),
|
||||
...(data.contentPrompt !== undefined && { contentPrompt: data.contentPrompt }),
|
||||
...(data.reviewPrompt !== undefined && { reviewPrompt: data.reviewPrompt }),
|
||||
...(data.reviewTemplate !== undefined && { reviewTemplate: data.reviewTemplate }),
|
||||
...(data.winnerSelectionPrompt !== undefined && {
|
||||
winnerSelectionPrompt: data.winnerSelectionPrompt,
|
||||
}),
|
||||
@@ -2018,7 +2035,6 @@ export async function playgroundGenerateContent(input: PlaygroundGenerateContent
|
||||
allowedNsfwLevel: 1,
|
||||
config: judgingConfig,
|
||||
model: (input.aiModel || undefined) as AIModel | undefined,
|
||||
userMessageOverride: input.userMessage,
|
||||
});
|
||||
|
||||
return result;
|
||||
@@ -2034,6 +2050,11 @@ export async function playgroundReviewImage(input: PlaygroundReviewImageInput) {
|
||||
|
||||
judgingConfig = applyPromptOverrides(judgingConfig, input.promptOverrides);
|
||||
|
||||
// Apply reviewTemplate override from playground draft
|
||||
if (input.reviewTemplate != null) {
|
||||
judgingConfig = { ...judgingConfig, reviewTemplate: input.reviewTemplate || null };
|
||||
}
|
||||
|
||||
// Resolve imageId to an image URL
|
||||
const image = await dbRead.image.findUnique({
|
||||
where: { id: input.imageId },
|
||||
@@ -2041,7 +2062,7 @@ export async function playgroundReviewImage(input: PlaygroundReviewImageInput) {
|
||||
});
|
||||
if (!image) throw new TRPCError({ code: 'NOT_FOUND', message: 'Image not found' });
|
||||
|
||||
const imageUrl = getEdgeUrl(image.url, { width: 1200, name: 'image' });
|
||||
const imageUrl = getEdgeUrl(image.url, { width: 1200, name: 'image', optimized: true });
|
||||
|
||||
const result = await generateReview({
|
||||
theme: input.theme,
|
||||
@@ -2049,7 +2070,6 @@ export async function playgroundReviewImage(input: PlaygroundReviewImageInput) {
|
||||
imageUrl,
|
||||
config: judgingConfig,
|
||||
model: (input.aiModel || undefined) as AIModel | undefined,
|
||||
userMessageOverride: input.userMessage,
|
||||
});
|
||||
|
||||
return result;
|
||||
@@ -2087,7 +2107,6 @@ export async function playgroundPickWinners(input: PlaygroundPickWinnersInput) {
|
||||
theme: challenge.theme ?? 'Unknown',
|
||||
config: judgingConfig,
|
||||
model: (input.aiModel || undefined) as AIModel | undefined,
|
||||
userMessageOverride: input.userMessage,
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
@@ -2899,6 +2899,7 @@ export interface ChallengeJudge {
|
||||
collectionPrompt: string | null;
|
||||
contentPrompt: string | null;
|
||||
reviewPrompt: string | null;
|
||||
reviewTemplate: string | null;
|
||||
winnerSelectionPrompt: string | null;
|
||||
active: boolean;
|
||||
createdAt: Date;
|
||||
|
||||
@@ -96,3 +96,10 @@ export function insertSorted(arr: number[], toInsert: number, order: 'asc' | 'de
|
||||
// Insert at the correct position
|
||||
arr.splice(left, 0, toInsert);
|
||||
}
|
||||
|
||||
export function findLastIndex<T>(arr: T[], predicate: (item: T) => boolean): number {
|
||||
for (let i = arr.length - 1; i >= 0; i--) {
|
||||
if (predicate(arr[i])) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user