mirror of
https://github.com/civitai/civitai.git
synced 2026-09-20 22:08:18 +08:00
Merge branch 'main' into feature/opinionated-server
This commit is contained in:
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"presets": ["next/babel"],
|
||||
"plugins": [
|
||||
"superjson-next"
|
||||
]
|
||||
}
|
||||
+2
-1
@@ -38,4 +38,5 @@ JOB_TOKEN=
|
||||
WEBHOOK_TOKEN=
|
||||
|
||||
# Security
|
||||
SCANNING_ENDPOINT=
|
||||
SCANNING_ENDPOINT=http://scanner.civitai.com/enqueue
|
||||
SCANNING_TOKEN=canihaztoken
|
||||
Generated
+11
@@ -46,6 +46,7 @@
|
||||
"cron-parser": "^4.6.0",
|
||||
"dayjs": "^1.11.6",
|
||||
"embla-carousel-react": "^7.0.3",
|
||||
"exifr": "^7.1.3",
|
||||
"gray-matter": "^4.0.3",
|
||||
"immer": "^9.0.15",
|
||||
"lodash": "^4.17.21",
|
||||
@@ -5238,6 +5239,11 @@
|
||||
"node": ">=0.8.x"
|
||||
}
|
||||
},
|
||||
"node_modules/exifr": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/exifr/-/exifr-7.1.3.tgz",
|
||||
"integrity": "sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw=="
|
||||
},
|
||||
"node_modules/expand-template": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
|
||||
@@ -13116,6 +13122,11 @@
|
||||
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
|
||||
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="
|
||||
},
|
||||
"exifr": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/exifr/-/exifr-7.1.3.tgz",
|
||||
"integrity": "sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw=="
|
||||
},
|
||||
"expand-template": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
|
||||
|
||||
@@ -61,6 +61,7 @@
|
||||
"cron-parser": "^4.6.0",
|
||||
"dayjs": "^1.11.6",
|
||||
"embla-carousel-react": "^7.0.3",
|
||||
"exifr": "^7.1.3",
|
||||
"gray-matter": "^4.0.3",
|
||||
"immer": "^9.0.15",
|
||||
"lodash": "^4.17.21",
|
||||
|
||||
@@ -3,23 +3,26 @@ import { IconUpload, IconCircleCheck, IconBan } from '@tabler/icons';
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useS3Upload } from '~/hooks/useS3Upload';
|
||||
import useIsClient from '~/hooks/useIsClient';
|
||||
import { UploadType, UploadTypeUnion } from '~/server/common/enums';
|
||||
import { formatBytes, formatSeconds } from '~/utils/number-helpers';
|
||||
import { getFileExtension } from '~/utils/string-helpers';
|
||||
import { toStringList } from '~/utils/array-helpers';
|
||||
import { useDidUpdate } from '@mantine/hooks';
|
||||
import { FileProps } from '~/server/common/validation/model';
|
||||
import isEqual from 'lodash/isEqual';
|
||||
import { ModelFileType } from '@prisma/client';
|
||||
import { bytesToKB } from '~/utils/number-helpers';
|
||||
|
||||
//TODO File Safety: Limit to the specific file extensions we want to allow
|
||||
export function FileInputUpload({
|
||||
uploadType = 'default',
|
||||
uploadType = 'Model',
|
||||
onChange,
|
||||
onLoading,
|
||||
value,
|
||||
// fileName = decodeURIComponent(value?.split('/').pop() ?? ''),
|
||||
error,
|
||||
fileName = value?.name,
|
||||
...props
|
||||
}: Props) {
|
||||
const isClient = useIsClient();
|
||||
const [state, setState] = useState<string | null>(value ?? null);
|
||||
const [state, setState] = useState<FileProps | undefined>(value);
|
||||
const { files, uploadToS3, resetFiles } = useS3Upload();
|
||||
const { file, progress, speed, timeRemaining, status, abort } = files[0] ?? {
|
||||
file: null,
|
||||
@@ -31,10 +34,9 @@ export function FileInputUpload({
|
||||
|
||||
const [fileTypeError, setFileTypeError] = useState('');
|
||||
|
||||
const fileName = decodeURIComponent(state?.split('/').pop() ?? '');
|
||||
|
||||
useDidUpdate(() => {
|
||||
if (value !== state) setState(value ?? null);
|
||||
const shouldUpdate = !isEqual(value, state);
|
||||
if (shouldUpdate) setState(value);
|
||||
}, [value]);
|
||||
|
||||
const handleOnChange: FileInputProps['onChange'] = async (file) => {
|
||||
@@ -51,18 +53,27 @@ export function FileInputUpload({
|
||||
acceptTypes.includes(fileExt) // Check with file extension
|
||||
) {
|
||||
onLoading?.(true);
|
||||
const uploaded = await uploadToS3(file, uploadType);
|
||||
const uploaded = await uploadToS3(
|
||||
file,
|
||||
uploadType === 'Model' ? 'model' : 'training-images'
|
||||
);
|
||||
url = uploaded.url;
|
||||
onLoading?.(false);
|
||||
setState(url);
|
||||
onChange?.(url, file);
|
||||
const value: FileProps = {
|
||||
sizeKB: file.size ? bytesToKB(file.size) : 0,
|
||||
type: uploadType,
|
||||
url,
|
||||
name: file.name,
|
||||
};
|
||||
setState(value);
|
||||
onChange?.(value);
|
||||
} else {
|
||||
setFileTypeError(`This input only accepts ${toStringList(acceptTypes)} files`);
|
||||
setState(null);
|
||||
setState(undefined);
|
||||
}
|
||||
} else {
|
||||
resetFiles();
|
||||
setState(null);
|
||||
setState(undefined);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -75,8 +86,8 @@ export function FileInputUpload({
|
||||
return (
|
||||
<Stack>
|
||||
<FileInput
|
||||
error={fileTypeError}
|
||||
{...props}
|
||||
error={error ?? fileTypeError}
|
||||
icon={<IconUpload size={16} />}
|
||||
onChange={handleOnChange}
|
||||
value={file ?? localFile}
|
||||
@@ -127,9 +138,9 @@ export function FileInputUpload({
|
||||
}
|
||||
|
||||
type Props = Omit<FileInputProps, 'icon' | 'onChange' | 'value'> & {
|
||||
value?: string;
|
||||
onChange?: (url: string | null, file: File | null) => void;
|
||||
value?: FileProps;
|
||||
onChange?: (value?: FileProps) => void;
|
||||
onLoading?: (loading: boolean) => void;
|
||||
uploadType?: UploadType | UploadTypeUnion;
|
||||
uploadType?: ModelFileType;
|
||||
fileName?: string;
|
||||
};
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
import { Stack, Text, Code, Popover, PopoverProps, Title } from '@mantine/core';
|
||||
import { ImageMetaProps } from '~/server/schema/image.schema';
|
||||
import { Stack, Text, Code, Popover, PopoverProps, Group, SimpleGrid, Button } from '@mantine/core';
|
||||
import { useClipboard } from '@mantine/hooks';
|
||||
import { IconCheck, IconCopy } from '@tabler/icons';
|
||||
import { useMemo } from 'react';
|
||||
import { encodeMetadata } from '~/utils/image-metadata';
|
||||
|
||||
type Props = {
|
||||
meta: ImageMetaProps;
|
||||
};
|
||||
type MetaDisplay = {
|
||||
label: string;
|
||||
value: string;
|
||||
};
|
||||
|
||||
const labelDictionary: Record<keyof ImageMetaProps, string> = {
|
||||
prompt: 'Prompt',
|
||||
@@ -15,16 +23,55 @@ const labelDictionary: Record<keyof ImageMetaProps, string> = {
|
||||
};
|
||||
|
||||
export function ImageMeta({ meta }: Props) {
|
||||
const keys = Object.keys(labelDictionary) as Array<keyof ImageMetaProps>;
|
||||
const { copied, copy } = useClipboard();
|
||||
// TODO only show keys in our meta list
|
||||
const metas = useMemo(() => {
|
||||
const long: MetaDisplay[] = [];
|
||||
const short: MetaDisplay[] = [];
|
||||
for (const key of Object.keys(labelDictionary)) {
|
||||
const value = meta[key]?.toString();
|
||||
if (!value) continue;
|
||||
(value.length > 15 ? long : short).push({
|
||||
label: labelDictionary[key],
|
||||
value,
|
||||
});
|
||||
}
|
||||
return { long, short };
|
||||
}, [meta]);
|
||||
|
||||
return (
|
||||
<Stack spacing="xs">
|
||||
{keys
|
||||
.filter((key) => !!meta[key])
|
||||
.map((key) => (
|
||||
<Text key={key}>
|
||||
{labelDictionary[key]}: <Code>{meta[key]}</Code>
|
||||
{metas.long.map(({ label, value }) => (
|
||||
<Stack key={label} spacing={0}>
|
||||
<Text size="sm" weight={500}>
|
||||
{label}
|
||||
</Text>
|
||||
<Code block sx={{ whiteSpace: 'normal' }}>
|
||||
{value}
|
||||
</Code>
|
||||
</Stack>
|
||||
))}
|
||||
<SimpleGrid cols={2} verticalSpacing="xs">
|
||||
{metas.short.map(({ label, value }) => (
|
||||
<Group key={label} spacing={0}>
|
||||
<Text size="sm" mr="xs" weight={500}>
|
||||
{label}
|
||||
</Text>
|
||||
<Code sx={{ flex: '1', textAlign: 'right' }}>{value}</Code>
|
||||
</Group>
|
||||
))}
|
||||
</SimpleGrid>
|
||||
<Button
|
||||
size="xs"
|
||||
color={copied ? 'teal' : 'blue'}
|
||||
variant="light"
|
||||
leftIcon={copied ? <IconCheck size={16} /> : <IconCopy size={16} />}
|
||||
onClick={() => {
|
||||
copy(encodeMetadata(meta));
|
||||
}}
|
||||
>
|
||||
{copied ? 'Copied' : 'Copy Generation Data'}
|
||||
</Button>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
@@ -35,20 +82,10 @@ export function ImageMetaPopover({
|
||||
...popoverProps
|
||||
}: Props & { children: React.ReactElement } & PopoverProps) {
|
||||
return (
|
||||
<Popover
|
||||
width={350}
|
||||
shadow="md"
|
||||
position="bottom-start"
|
||||
withArrow
|
||||
withinPortal
|
||||
{...popoverProps}
|
||||
>
|
||||
<Popover width={350} shadow="md" position="top-end" withArrow withinPortal {...popoverProps}>
|
||||
<Popover.Target>{children}</Popover.Target>
|
||||
<Popover.Dropdown>
|
||||
<Stack>
|
||||
<Title order={4}>Metadata</Title>
|
||||
<ImageMeta meta={meta} />
|
||||
</Stack>
|
||||
<ImageMeta meta={meta} />
|
||||
</Popover.Dropdown>
|
||||
</Popover>
|
||||
);
|
||||
|
||||
@@ -36,7 +36,7 @@ export function ImagePreview({
|
||||
};
|
||||
|
||||
return (
|
||||
<Paper style={{ overflow: 'hidden', position: 'relative', ...style }} {...props}>
|
||||
<Paper radius={0} style={{ overflow: 'hidden', position: 'relative', ...style }} {...props}>
|
||||
<AspectRatio ratio={aspectRatio ?? (width ?? 16) / (height ?? 9)}>
|
||||
{nsfw ? (
|
||||
<MediaHash hash={hash} width={width} height={height} />
|
||||
@@ -53,10 +53,11 @@ export function ImagePreview({
|
||||
{!nsfw && withMeta && meta && (
|
||||
<ImageMetaPopover meta={meta as ImageMetaProps}>
|
||||
<ActionIcon
|
||||
style={{ position: 'absolute', top: '5px', left: '5px', zIndex: 100 }}
|
||||
variant="transparent"
|
||||
style={{ position: 'absolute', bottom: '5px', right: '5px', zIndex: 100 }}
|
||||
size="lg"
|
||||
>
|
||||
<IconInfoCircle />
|
||||
<IconInfoCircle color="white" />
|
||||
</ActionIcon>
|
||||
</ImageMetaPopover>
|
||||
)}
|
||||
|
||||
@@ -24,21 +24,19 @@ import {
|
||||
Popover,
|
||||
Textarea,
|
||||
NumberInput,
|
||||
ScrollArea,
|
||||
Divider,
|
||||
Grid,
|
||||
Select,
|
||||
} from '@mantine/core';
|
||||
import { FileWithPath, Dropzone, IMAGE_MIME_TYPE } from '@mantine/dropzone';
|
||||
import { useDidUpdate, useListState } from '@mantine/hooks';
|
||||
import { IconPencil, IconTrash } from '@tabler/icons';
|
||||
import { cloneElement, useEffect, useState } from 'react';
|
||||
import { cloneElement, useState } from 'react';
|
||||
import { blurHashImage, loadImage } from '../../utils/blurhash';
|
||||
import { ImageUploadPreview } from '~/components/ImageUpload/ImageUploadPreview';
|
||||
import { useCFImageUpload } from '~/hooks/useCFImageUpload';
|
||||
import useIsClient from '~/hooks/useIsClient';
|
||||
import { ImageMetaProps } from '~/server/validators/image/schemas';
|
||||
import isEqual from 'lodash/isEqual';
|
||||
import { ImageMetaProps } from '~/server/schema/image.schema';
|
||||
import { getMetadata } from '~/utils/image-metadata';
|
||||
|
||||
type Props = Omit<InputWrapperProps, 'children' | 'onChange'> & {
|
||||
hasPrimaryImage?: boolean;
|
||||
@@ -68,15 +66,12 @@ export function ImageUpload({
|
||||
const [files, filesHandlers] = useListState<CustomFile>(value);
|
||||
const [activeId, setActiveId] = useState<UniqueIdentifier>();
|
||||
|
||||
useEffect(() => {
|
||||
// clear any remaining urls when unmounting
|
||||
return () => files.forEach((file) => URL.revokeObjectURL(file.url));
|
||||
}, [files]);
|
||||
|
||||
useDidUpdate(() => {
|
||||
const shouldReset = !isEqual(value, files);
|
||||
if (shouldReset) filesHandlers.setState(value);
|
||||
}, [value]);
|
||||
// Disabled this because it seemed to cause state loop...
|
||||
// useDidUpdate(() => {
|
||||
// const shouldReset = !isEqual(value, files);
|
||||
// console.log('did update');
|
||||
// if (shouldReset) filesHandlers.setState(value);
|
||||
// }, [value]);
|
||||
|
||||
useDidUpdate(() => {
|
||||
if (files) onChange?.(files);
|
||||
@@ -87,12 +82,15 @@ export function ImageUpload({
|
||||
const toUpload = await Promise.all(
|
||||
droppedFiles.map(async (file) => {
|
||||
const src = URL.createObjectURL(file);
|
||||
const meta = await getMetadata(file);
|
||||
const img = await loadImage(src);
|
||||
const hashResult = blurHashImage(img);
|
||||
return {
|
||||
name: file.name,
|
||||
url: src,
|
||||
previewUrl: src,
|
||||
file,
|
||||
meta,
|
||||
...hashResult,
|
||||
};
|
||||
})
|
||||
@@ -100,17 +98,23 @@ export function ImageUpload({
|
||||
|
||||
filesHandlers.setState((current) => [...current, ...toUpload]);
|
||||
|
||||
await Promise.all(
|
||||
toUpload.map(async (image) => {
|
||||
const { id } = await uploadToCF(image.file);
|
||||
filesHandlers.setState((state) => {
|
||||
const index = state.findIndex((item) => item.file === image.file);
|
||||
if (index === -1) return state;
|
||||
const cloned = [...state];
|
||||
cloned[index] = { ...cloned[index], url: id, file: undefined };
|
||||
return cloned;
|
||||
});
|
||||
URL.revokeObjectURL(image.url);
|
||||
const uploads = await Promise.all(
|
||||
toUpload.map(async ({ url, file, previewUrl }) => {
|
||||
const { id } = await uploadToCF(file);
|
||||
return { url, file, id, previewUrl };
|
||||
})
|
||||
);
|
||||
|
||||
filesHandlers.setState((states) =>
|
||||
states.map((state) => {
|
||||
const matchingUpload = uploads.find((x) => x.file == state.file);
|
||||
if (!matchingUpload) return state;
|
||||
return {
|
||||
...state,
|
||||
url: matchingUpload.id,
|
||||
onLoad: () => URL.revokeObjectURL(matchingUpload.previewUrl),
|
||||
file: null,
|
||||
};
|
||||
})
|
||||
);
|
||||
};
|
||||
@@ -190,7 +194,9 @@ export function ImageUpload({
|
||||
<ActionIcon
|
||||
variant="outline"
|
||||
color={
|
||||
image.meta && Object.keys(image.meta).length ? 'green' : undefined
|
||||
image.meta && Object.keys(image.meta).length
|
||||
? 'primary'
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
<IconPencil />
|
||||
@@ -285,14 +291,13 @@ function ImageMetaPopover({
|
||||
};
|
||||
|
||||
const handleSubmit = () => {
|
||||
const meta: ImageMetaProps = {};
|
||||
if (prompt) meta.prompt = prompt;
|
||||
if (negativePrompt) meta.negativePrompt = negativePrompt;
|
||||
if (cfgScale) meta.cfgScale = cfgScale;
|
||||
if (steps) meta.steps = steps;
|
||||
if (sampler) meta.sampler = sampler;
|
||||
if (seed) meta.seed = seed;
|
||||
onSubmit?.(Object.keys(meta).length ? meta : null);
|
||||
const newMeta = { ...meta, prompt, negativePrompt, cfgScale, steps, sampler, seed };
|
||||
const keys = Object.keys(newMeta) as Array<keyof typeof newMeta>;
|
||||
const toSubmit = keys.reduce<ImageMetaProps>((acc, key) => {
|
||||
if (newMeta[key]) return { ...acc, [key]: newMeta[key] };
|
||||
return acc;
|
||||
}, {});
|
||||
onSubmit?.(Object.keys(toSubmit).length ? toSubmit : null);
|
||||
setOpened(false);
|
||||
};
|
||||
|
||||
@@ -300,71 +305,68 @@ function ImageMetaPopover({
|
||||
<Popover opened={opened} onClose={handleClose} withArrow withinPortal width={400}>
|
||||
<Popover.Target>{cloneElement(children, { onClick: handleClose })}</Popover.Target>
|
||||
<Popover.Dropdown>
|
||||
<Stack spacing="sm">
|
||||
<Title order={4}>Image Meta</Title>
|
||||
<Grid>
|
||||
<Grid.Col span={12}>
|
||||
<Textarea
|
||||
value={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value)}
|
||||
label="Prompt"
|
||||
autosize
|
||||
maxRows={3}
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={12}>
|
||||
<Textarea
|
||||
value={negativePrompt}
|
||||
onChange={(e) => setNegativePrompt(e.target.value)}
|
||||
label="Negative prompt"
|
||||
autosize
|
||||
maxRows={3}
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<NumberInput
|
||||
value={cfgScale}
|
||||
onChange={(number) => setCfgScale(number)}
|
||||
label="Guidance scale"
|
||||
min={0}
|
||||
max={30}
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<NumberInput value={steps} onChange={(value) => setSteps(value)} label="Steps" />
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<Select
|
||||
clearable
|
||||
searchable
|
||||
data={[
|
||||
'Euler a',
|
||||
'Euler',
|
||||
'LMS',
|
||||
'Heun',
|
||||
'DPM2',
|
||||
'DPM2 a',
|
||||
'DPM fast',
|
||||
'DPM adaptive',
|
||||
'LMS Karras',
|
||||
'DPM2 Karras',
|
||||
'DPM2 a Karras',
|
||||
'DDIM',
|
||||
'PLMS',
|
||||
]}
|
||||
value={sampler}
|
||||
onChange={(value) => setSampler(value ?? undefined)}
|
||||
label="Sampler"
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<NumberInput value={seed} onChange={(value) => setSeed(value)} label="Seed" />
|
||||
</Grid.Col>
|
||||
</Grid>
|
||||
</Stack>
|
||||
<Divider pb="sm" />
|
||||
<Button fullWidth onClick={() => handleSubmit()}>
|
||||
Submit
|
||||
<Title order={4}>Generation details</Title>
|
||||
<Grid gutter="xs">
|
||||
<Grid.Col span={12}>
|
||||
<Textarea
|
||||
value={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value)}
|
||||
label="Prompt"
|
||||
autosize
|
||||
maxRows={3}
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={12}>
|
||||
<Textarea
|
||||
value={negativePrompt}
|
||||
onChange={(e) => setNegativePrompt(e.target.value)}
|
||||
label="Negative prompt"
|
||||
autosize
|
||||
maxRows={3}
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<NumberInput
|
||||
value={cfgScale}
|
||||
onChange={(number) => setCfgScale(number)}
|
||||
label="Guidance scale"
|
||||
min={0}
|
||||
max={30}
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<NumberInput value={steps} onChange={(value) => setSteps(value)} label="Steps" />
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<Select
|
||||
clearable
|
||||
searchable
|
||||
data={[
|
||||
'Euler a',
|
||||
'Euler',
|
||||
'LMS',
|
||||
'Heun',
|
||||
'DPM2',
|
||||
'DPM2 a',
|
||||
'DPM fast',
|
||||
'DPM adaptive',
|
||||
'LMS Karras',
|
||||
'DPM2 Karras',
|
||||
'DPM2 a Karras',
|
||||
'DDIM',
|
||||
'PLMS',
|
||||
]}
|
||||
value={sampler}
|
||||
onChange={(value) => setSampler(value ?? undefined)}
|
||||
label="Sampler"
|
||||
/>
|
||||
</Grid.Col>
|
||||
<Grid.Col span={6}>
|
||||
<NumberInput value={seed} onChange={(value) => setSeed(value)} label="Seed" />
|
||||
</Grid.Col>
|
||||
</Grid>
|
||||
<Button mt="xs" fullWidth onClick={() => handleSubmit()}>
|
||||
Save
|
||||
</Button>
|
||||
</Popover.Dropdown>
|
||||
</Popover>
|
||||
@@ -400,6 +402,7 @@ const useStyles = createStyles((theme, _params, getRef) => ({
|
||||
ref: getRef('actionsGroup'),
|
||||
position: 'absolute',
|
||||
background: theme.fn.rgba(theme.colors.dark[9], 0.6),
|
||||
borderBottomLeftRadius: theme.radius.sm,
|
||||
top: 0,
|
||||
right: 0,
|
||||
},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { forwardRef, CSSProperties } from 'react';
|
||||
import { ActionIcon, Center, createStyles, Paper } from '@mantine/core';
|
||||
import { EdgeImage } from '~/components/EdgeImage/EdgeImage';
|
||||
import { forwardRef, CSSProperties, useState } from 'react';
|
||||
import { Center, createStyles, Paper } from '@mantine/core';
|
||||
import { EdgeImage, EdgeImageProps } from '~/components/EdgeImage/EdgeImage';
|
||||
import { useSortable } from '@dnd-kit/sortable';
|
||||
import { CSS } from '@dnd-kit/utilities';
|
||||
import { UniqueIdentifier } from '@dnd-kit/core';
|
||||
@@ -16,9 +16,9 @@ type Props = {
|
||||
|
||||
export const ImageUploadPreview = forwardRef<HTMLDivElement, Props>(
|
||||
({ image, children, isPrimary, disabled, id, ...props }, ref) => {
|
||||
const url = image?.url.startsWith('http') ? `${image.url}/preview` : image?.url;
|
||||
const { classes } = useStyles({ url, isPrimary });
|
||||
const { classes } = useStyles({ isPrimary });
|
||||
const { classes: imageClasses } = useImageStyles();
|
||||
const [ready, setReady] = useState(false);
|
||||
|
||||
const sortable = useSortable({ id });
|
||||
|
||||
@@ -32,13 +32,23 @@ export const ImageUploadPreview = forwardRef<HTMLDivElement, Props>(
|
||||
|
||||
if (!image) return null;
|
||||
return (
|
||||
<div
|
||||
<Paper
|
||||
ref={setNodeRef}
|
||||
className={classes.root}
|
||||
{...props}
|
||||
radius="sm"
|
||||
style={{ ...style, ...props.style }}
|
||||
>
|
||||
<EdgeImage className={imageClasses.root} src={image?.url} height={isPrimary ? 410 : 200} />
|
||||
{!ready && image.previewUrl && <StyledEdgeImage src={image.previewUrl} />}
|
||||
{image.url && image.url != image.previewUrl && (
|
||||
<StyledEdgeImage
|
||||
src={image.url}
|
||||
onLoad={() => {
|
||||
image.onLoad?.();
|
||||
setReady(true);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
<Center className={classes.draggable} {...listeners} {...attributes}>
|
||||
<Paper className={classes.draggableIcon} p="xl" radius={100}>
|
||||
@@ -51,23 +61,37 @@ export const ImageUploadPreview = forwardRef<HTMLDivElement, Props>(
|
||||
</Paper>
|
||||
</Center>
|
||||
{children}
|
||||
</div>
|
||||
</Paper>
|
||||
);
|
||||
}
|
||||
);
|
||||
ImageUploadPreview.displayName = 'ImagePreview';
|
||||
|
||||
const StyledEdgeImage = (props: EdgeImageProps) => (
|
||||
<EdgeImage
|
||||
{...props}
|
||||
height={410}
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
objectFit: 'cover',
|
||||
objectPosition: '50% 50%',
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
const useStyles = createStyles(
|
||||
(
|
||||
theme,
|
||||
{
|
||||
// index,
|
||||
url,
|
||||
faded,
|
||||
isPrimary,
|
||||
}: {
|
||||
// index: number;
|
||||
url?: string;
|
||||
faded?: boolean;
|
||||
isPrimary?: boolean;
|
||||
}
|
||||
@@ -79,7 +103,6 @@ const useStyles = createStyles(
|
||||
height: isPrimary ? 410 : 200,
|
||||
gridRowStart: isPrimary ? 'span 2' : undefined,
|
||||
gridColumnStart: isPrimary ? 'span 2' : undefined,
|
||||
backgroundImage: `url("${url}")`,
|
||||
backgroundSize: 'cover',
|
||||
backgroundPosition: 'center',
|
||||
backgroundColor: 'grey',
|
||||
|
||||
@@ -2,22 +2,18 @@ import { Carousel, Embla } from '@mantine/carousel';
|
||||
import {
|
||||
Box,
|
||||
CloseButton,
|
||||
Text,
|
||||
Code,
|
||||
Stack,
|
||||
Paper,
|
||||
Title,
|
||||
Group,
|
||||
ActionIcon,
|
||||
createStyles,
|
||||
Popover,
|
||||
MantineProvider,
|
||||
} from '@mantine/core';
|
||||
import { useHotkeys } from '@mantine/hooks';
|
||||
import { ContextModalProps } from '@mantine/modals';
|
||||
import { IconInfoCircle, IconMinus, IconPlus, IconX } from '@tabler/icons';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { IconInfoCircle, IconMinus } from '@tabler/icons';
|
||||
import { useRef, useState } from 'react';
|
||||
import { EdgeImage } from '~/components/EdgeImage/EdgeImage';
|
||||
import { ImageMeta, ImageMetaPopover } from '~/components/ImageMeta/ImageMeta';
|
||||
import { ImageMeta } from '~/components/ImageMeta/ImageMeta';
|
||||
import { ImageMetaProps } from '~/server/schema/image.schema';
|
||||
import { ImageModel } from '~/server/validators/image/selectors';
|
||||
|
||||
@@ -32,35 +28,20 @@ export default function LightboxImageCarousel({
|
||||
innerProps,
|
||||
}: ContextModalProps<Props>) {
|
||||
const { initialSlide, images = [] } = innerProps;
|
||||
const [embla, setEmbla] = useState<Embla | null>(null);
|
||||
const [show, setShow] = useState(true);
|
||||
const [show, setShow] = useState(false);
|
||||
const [index, setIndex] = useState(initialSlide ?? 0);
|
||||
|
||||
const emblaRef = useRef<Embla | null>(null);
|
||||
|
||||
const { classes, cx } = useStyles();
|
||||
|
||||
const handlePrev = () => {
|
||||
setIndex((prev) => {
|
||||
const index = prev - 1;
|
||||
return index === -1 ? images.length - 1 : index;
|
||||
});
|
||||
};
|
||||
|
||||
const handleNext = () => {
|
||||
setIndex((prev) => {
|
||||
const index = prev + 1;
|
||||
return index === images.length ? 0 : index;
|
||||
});
|
||||
};
|
||||
|
||||
useHotkeys([
|
||||
['ArrowLeft', () => embla?.scrollPrev()],
|
||||
['ArrowRight', () => embla?.scrollNext()],
|
||||
['ArrowLeft', () => emblaRef.current?.scrollPrev()],
|
||||
['ArrowRight', () => emblaRef.current?.scrollNext()],
|
||||
]);
|
||||
|
||||
useEffect(() => console.log({ show }), [show]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<MantineProvider theme={{ colorScheme: 'dark' }}>
|
||||
<CloseButton
|
||||
style={{ position: 'absolute', top: 15, right: 15, zIndex: 100 }}
|
||||
size="lg"
|
||||
@@ -84,9 +65,11 @@ export default function LightboxImageCarousel({
|
||||
initialSlide={initialSlide}
|
||||
withIndicators
|
||||
loop
|
||||
onPreviousSlide={handlePrev}
|
||||
onNextSlide={handleNext}
|
||||
getEmblaApi={setEmbla}
|
||||
onSlideChange={(index) => setIndex(index)}
|
||||
withKeyboardEvents={false}
|
||||
getEmblaApi={(embla) => {
|
||||
emblaRef.current = embla;
|
||||
}}
|
||||
styles={{
|
||||
control: {
|
||||
zIndex: 100,
|
||||
@@ -118,45 +101,12 @@ export default function LightboxImageCarousel({
|
||||
width={image.width ?? 1200}
|
||||
/>
|
||||
</div>
|
||||
{/* {image.meta && (
|
||||
<Box
|
||||
style={{
|
||||
position: 'absolute',
|
||||
left: 0,
|
||||
top: 0,
|
||||
width: '350px',
|
||||
maxWidth: '100%',
|
||||
}}
|
||||
sx={(theme) => ({
|
||||
background: theme.fn.rgba(theme.black, 0.65),
|
||||
})}
|
||||
p="md"
|
||||
>
|
||||
<Stack>
|
||||
<Title order={4}>Metadata</Title>
|
||||
<ImageMeta meta={image.meta as ImageMetaProps} />
|
||||
</Stack>
|
||||
</Box>
|
||||
)} */}
|
||||
</Carousel.Slide>
|
||||
))}
|
||||
</Carousel>
|
||||
{/* {images[index]?.meta && (
|
||||
<ImageMetaPopover meta={images[index].meta as ImageMetaProps}>
|
||||
<ActionIcon
|
||||
style={{ position: 'absolute', top: 15, left: 15, zIndex: 100 }}
|
||||
size="lg"
|
||||
variant="default"
|
||||
>
|
||||
<IconInfoCircle />
|
||||
</ActionIcon>
|
||||
</ImageMetaPopover>
|
||||
)} */}
|
||||
{images[index]?.meta && (
|
||||
<Paper className={cx(classes.meta, { [classes.metaActive]: show })} p="md" withBorder>
|
||||
<Stack>
|
||||
<Title order={4}>Metadata</Title>
|
||||
|
||||
<ActionIcon
|
||||
onClick={() => setShow((v) => !v)}
|
||||
className={cx(classes.metaButton, { [classes.metaActive]: show })}
|
||||
@@ -170,7 +120,7 @@ export default function LightboxImageCarousel({
|
||||
</Paper>
|
||||
)}
|
||||
</Box>
|
||||
</>
|
||||
</MantineProvider>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ import {
|
||||
import { modelSchema } from '~/server/common/validation/model';
|
||||
import { ImageMetaProps } from '~/server/schema/image.schema';
|
||||
import { ModelById } from '~/types/router';
|
||||
import { bytesToKB } from '~/utils/number-helpers';
|
||||
import { splitUppercase } from '~/utils/string-helpers';
|
||||
import { trpc } from '~/utils/trpc';
|
||||
import { isDefined } from '~/utils/type-guards';
|
||||
@@ -76,7 +75,7 @@ export function ModelForm({ model }: Props) {
|
||||
modelVersions: model?.modelVersions.map(({ trainedWords, images, ...version }) => ({
|
||||
...version,
|
||||
trainedWords: trainedWords ?? [],
|
||||
// Casting image.meta to hotfix type issue with generated prisma schema
|
||||
// HOTFIX: Casting image.meta type issue with generated prisma schema
|
||||
images: images.map(({ image }) => ({ ...image, meta: image.meta as ImageMetaProps })) ?? [],
|
||||
})) ?? [defaultModelVersion],
|
||||
};
|
||||
@@ -132,12 +131,8 @@ export function ModelForm({ model }: Props) {
|
||||
},
|
||||
};
|
||||
|
||||
const data = {
|
||||
const data: CreateModelProps | UpdateModelProps = {
|
||||
...values,
|
||||
modelVersions: values.modelVersions.map(({ trainingDataFile, ...version }) => ({
|
||||
...version,
|
||||
trainingDataFile: trainingDataFile?.url ? null : trainingDataFile,
|
||||
})),
|
||||
tagsOnModels: values.tagsOnModels?.map((name) => {
|
||||
const match = tags.find((x) => x.name === name);
|
||||
return match ?? { name };
|
||||
@@ -254,54 +249,26 @@ export function ModelForm({ model }: Props) {
|
||||
</Grid.Col>
|
||||
<Grid.Col span={12}>
|
||||
<InputFileUpload
|
||||
name={`modelVersions.${index}.modelFile.url`}
|
||||
name={`modelVersions.${index}.modelFile`}
|
||||
label="Model File"
|
||||
placeholder="Pick your model"
|
||||
uploadType="model"
|
||||
uploadType="Model"
|
||||
accept=".ckpt,.pt"
|
||||
onLoading={setUploading}
|
||||
onChange={(url, file) => {
|
||||
if (file) {
|
||||
form.setValue(
|
||||
`modelVersions.${index}.modelFile.sizeKB`,
|
||||
file.size ? bytesToKB(file.size) : 0
|
||||
);
|
||||
form.setValue(
|
||||
`modelVersions.${index}.modelFile.name`,
|
||||
file.name
|
||||
);
|
||||
}
|
||||
}}
|
||||
withAsterisk
|
||||
/>
|
||||
</Grid.Col>
|
||||
{/* <Grid.Col span={12}>
|
||||
<Grid.Col span={12}>
|
||||
<InputFileUpload
|
||||
name={`modelVersions.${index}.trainingDataFile.url`}
|
||||
name={`modelVersions.${index}.trainingDataFile`}
|
||||
label="Training Data"
|
||||
placeholder="Pick your training data"
|
||||
description="The data you used to train your model (as .zip archive)"
|
||||
uploadType="training-images"
|
||||
uploadType="TrainingData"
|
||||
accept=".zip"
|
||||
onLoading={setUploading}
|
||||
onChange={(url, file) => {
|
||||
if (file) {
|
||||
form.setValue(
|
||||
`modelVersions.${index}.trainingDataFile.type`,
|
||||
ModelFileType.TrainingData
|
||||
);
|
||||
form.setValue(
|
||||
`modelVersions.${index}.trainingDataFile.sizeKB`,
|
||||
file.size ? bytesToKB(file.size) : 0
|
||||
);
|
||||
form.setValue(
|
||||
`modelVersions.${index}.trainingDataFile.name`,
|
||||
file.name
|
||||
);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Grid.Col> */}
|
||||
</Grid.Col>
|
||||
<Grid.Col span={12}>
|
||||
<InputImageUpload
|
||||
name={`modelVersions.${index}.images`}
|
||||
|
||||
@@ -76,10 +76,10 @@ function ReactionBadge({ reaction, reactions }: ReactionBadgeProps) {
|
||||
const { onEmojiClick, user, disabled } = useReactionPickerContext();
|
||||
const tooltip = toStringList(
|
||||
reactions.map((reaction) =>
|
||||
reaction.user.name === user?.name ? 'You' : reaction.user.name ?? '<deleted user>'
|
||||
reaction.user.username === user?.username ? 'You' : reaction.user.username ?? '<deleted user>'
|
||||
)
|
||||
);
|
||||
const reacted = reactions.findIndex((reaction) => reaction.user.name === user?.name) > -1;
|
||||
const reacted = reactions.findIndex((reaction) => reaction.user.username === user?.username) > -1;
|
||||
const canClick = user && !disabled;
|
||||
|
||||
return (
|
||||
|
||||
Vendored
+1
@@ -36,6 +36,7 @@ export const serverSchema = z.object({
|
||||
JOB_TOKEN: z.string(),
|
||||
WEBHOOK_TOKEN: z.string(),
|
||||
SCANNING_ENDPOINT: z.string(),
|
||||
SCANNING_TOKEN: z.string(),
|
||||
});
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { getGetUrl } from '~/utils/s3-utils';
|
||||
import { ModelFileType, UserActivityType } from '@prisma/client';
|
||||
import { ModelFileType, ModelType, UserActivityType } from '@prisma/client';
|
||||
import { getServerAuthSession } from '~/server/common/get-server-auth-session';
|
||||
import { prisma } from '~/server/db/client';
|
||||
import { filenamize } from '~/utils/string-helpers';
|
||||
@@ -14,8 +14,9 @@ export default async function downloadModel(req: NextApiRequest, res: NextApiRes
|
||||
const modelVersion = await prisma.modelVersion.findFirst({
|
||||
where: { id: parseInt(modelVersionId) },
|
||||
select: {
|
||||
model: { select: { id: true, name: true } },
|
||||
model: { select: { id: true, name: true, type: true } },
|
||||
name: true,
|
||||
trainedWords: true,
|
||||
files: { where: { type: ModelFileType.Model }, select: { url: true, name: true } },
|
||||
},
|
||||
});
|
||||
@@ -46,7 +47,12 @@ export default async function downloadModel(req: NextApiRequest, res: NextApiRes
|
||||
|
||||
const [modelFile] = modelVersion.files;
|
||||
const ext = modelFile.name.split('.').pop();
|
||||
const fileName = `${filenamize(modelVersion.model.name)}_${filenamize(modelVersion.name)}.${ext}`;
|
||||
let fileName = modelFile.name;
|
||||
if (modelVersion.model.type === ModelType.TextualInversion) {
|
||||
const trainedWord = modelVersion.trainedWords[0] ?? modelVersion.model.name;
|
||||
fileName = `${trainedWord}.pt`;
|
||||
} else
|
||||
fileName = `${filenamize(modelVersion.model.name)}_${filenamize(modelVersion.name)}.${ext}`;
|
||||
const { url } = await getGetUrl(modelFile.url, { fileName });
|
||||
|
||||
res.redirect(url);
|
||||
|
||||
@@ -1,27 +1,36 @@
|
||||
import { WebhookEndpoint } from '~/server/common/endpoint-helpers';
|
||||
import { scanFilesJob } from '~/server/jobs/scan-files';
|
||||
import { updateMetricsJob } from '~/server/jobs/update-metrics';
|
||||
import { processImportsJob } from '~/server/jobs/process-imports';
|
||||
import cronParser from 'cron-parser';
|
||||
import dayjs from 'dayjs';
|
||||
import { z } from 'zod';
|
||||
|
||||
const jobs = [scanFilesJob, updateMetricsJob];
|
||||
const jobs = [scanFilesJob, updateMetricsJob, processImportsJob];
|
||||
|
||||
export default WebhookEndpoint(async (req, res) => {
|
||||
const { run: runJob } = querySchema.parse(req.query);
|
||||
const ran = [];
|
||||
const toRun = [];
|
||||
const afterResponse = [];
|
||||
|
||||
const now = new Date();
|
||||
for (const { name, cron, run } of jobs) {
|
||||
for (const { name, cron, run, options } of jobs) {
|
||||
if (runJob) {
|
||||
if (runJob !== name) continue;
|
||||
} else if (!isCronMatch(cron, now)) continue;
|
||||
|
||||
await run();
|
||||
ran.push(name);
|
||||
if (options.shouldWait) {
|
||||
await run();
|
||||
ran.push(name);
|
||||
} else {
|
||||
afterResponse.push(run);
|
||||
toRun.push(name);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(200).json({ ok: true, ran });
|
||||
res.status(200).json({ ok: true, ran, toRun });
|
||||
await Promise.all(afterResponse.map((run) => run()));
|
||||
});
|
||||
|
||||
// https://github.com/harrisiirak/cron-parser/issues/153#issuecomment-590099607
|
||||
|
||||
@@ -73,12 +73,14 @@ export default WebhookEndpoint(async (req, res) => {
|
||||
});
|
||||
|
||||
enum ScanExitCode {
|
||||
Pending = -1,
|
||||
Success = 0,
|
||||
Danger = 1,
|
||||
Error = 2,
|
||||
}
|
||||
|
||||
const resultCodeMap = {
|
||||
[ScanExitCode.Pending]: ScanResultCode.Pending,
|
||||
[ScanExitCode.Success]: ScanResultCode.Success,
|
||||
[ScanExitCode.Danger]: ScanResultCode.Danger,
|
||||
[ScanExitCode.Error]: ScanResultCode.Error,
|
||||
@@ -88,9 +90,9 @@ type ScanResult = {
|
||||
url: string;
|
||||
fileExists: number;
|
||||
picklescanExitCode: ScanExitCode;
|
||||
picklescanOutput: string;
|
||||
picklescanGlobalImports: string[];
|
||||
picklescanDangerousImports: string[];
|
||||
picklescanOutput?: string;
|
||||
picklescanGlobalImports?: string[];
|
||||
picklescanDangerousImports?: string[];
|
||||
clamscanExitCode: ScanExitCode;
|
||||
clamscanOutput: string;
|
||||
};
|
||||
@@ -109,11 +111,17 @@ function processImport(importStr: string) {
|
||||
const specialImports: string[] = ['pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint'];
|
||||
|
||||
function examinePickleScanMessage({
|
||||
picklescanExitCode,
|
||||
picklescanDangerousImports,
|
||||
picklescanGlobalImports,
|
||||
}: ScanResult) {
|
||||
const importCount = picklescanDangerousImports.length + picklescanGlobalImports.length;
|
||||
if (importCount === 0)
|
||||
if (picklescanExitCode === ScanExitCode.Pending) return {};
|
||||
picklescanDangerousImports ??= [];
|
||||
picklescanGlobalImports ??= [];
|
||||
|
||||
const importCount =
|
||||
(picklescanDangerousImports?.length ?? 0) + (picklescanGlobalImports?.length ?? 0);
|
||||
if (importCount === 0 || (!picklescanDangerousImports && !picklescanGlobalImports))
|
||||
return {
|
||||
pickleScanMessage: 'No Pickle imports',
|
||||
hasDanger: false,
|
||||
|
||||
@@ -34,15 +34,14 @@ import {
|
||||
IconExclamationMark,
|
||||
IconFilter,
|
||||
IconFlag,
|
||||
IconInfoCircle,
|
||||
IconLicense,
|
||||
IconPlus,
|
||||
IconTrash,
|
||||
} from '@tabler/icons';
|
||||
import { createProxySSGHelpers } from '@trpc/react-query/ssg';
|
||||
import startCase from 'lodash/startCase';
|
||||
import { GetServerSideProps, InferGetServerSidePropsType, NextPage } from 'next';
|
||||
import { getSession, useSession } from 'next-auth/react';
|
||||
import { GetServerSideProps } from 'next';
|
||||
import { useSession } from 'next-auth/react';
|
||||
import Link from 'next/link';
|
||||
import { useRouter } from 'next/router';
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
@@ -78,10 +77,7 @@ import { isNumber } from '~/utils/type-guards';
|
||||
import { LoginRedirect } from '~/components/LoginRedirect/LoginRedirect';
|
||||
import { VerifiedShield } from '~/components/VerifiedShield/VerifiedShield';
|
||||
import { getEdgeUrl } from '~/components/EdgeImage/EdgeImage';
|
||||
import { getModelHandler } from '~/server/controllers/model.controller';
|
||||
import { getServerAuthSession } from '~/server/common/get-server-auth-session';
|
||||
import { prisma } from '~/server/db/client';
|
||||
import { unstable_getServerSession } from 'next-auth';
|
||||
|
||||
type PageProps = {
|
||||
id: number;
|
||||
@@ -428,7 +424,7 @@ export default function ModelDetail(props: PageProps) {
|
||||
</Menu.Item>
|
||||
</>
|
||||
) : null}
|
||||
{session && published ? (
|
||||
{session && isOwner && published ? (
|
||||
<Menu.Item
|
||||
icon={<IconBan size={14} stroke={1.5} />}
|
||||
color="yellow"
|
||||
|
||||
@@ -33,6 +33,8 @@ export const fileSchema = z.object({
|
||||
type: z.nativeEnum(ModelFileType),
|
||||
});
|
||||
|
||||
export type FileProps = z.infer<typeof fileSchema>;
|
||||
|
||||
export const modelVersionSchema = z.object({
|
||||
id: z.number().optional(),
|
||||
name: z.string().min(1, 'Name cannot be empty.'),
|
||||
@@ -40,7 +42,7 @@ export const modelVersionSchema = z.object({
|
||||
steps: z.number().nullish(),
|
||||
epochs: z.number().nullish(),
|
||||
modelFile: fileSchema,
|
||||
trainingDataFile: fileSchema.optional(),
|
||||
trainingDataFile: fileSchema.nullish(),
|
||||
images: z
|
||||
.array(imageSchema)
|
||||
.min(1, 'At least one example image must be uploaded')
|
||||
|
||||
@@ -2,7 +2,6 @@ import { ImportStatus, ModelFileType, ModelType, Prisma } from '@prisma/client';
|
||||
import { createImporter } from '~/server/importers/importer';
|
||||
import { prisma } from '~/server/db/client';
|
||||
import { uploadViaUrl } from '~/utils/cf-images-utils';
|
||||
import { getEdgeUrl } from '~/components/EdgeImage/EdgeImage';
|
||||
import { markdownToHtml } from '~/utils/markdown-helpers';
|
||||
import { bytesToKB } from '~/utils/number-helpers';
|
||||
import { imageToBlurhash } from '~/utils/image-utils';
|
||||
@@ -105,7 +104,7 @@ export async function importModelFromHuggingFace(
|
||||
|
||||
// Upload images
|
||||
const imageFiles = files.filter((f) => isImage(f.name));
|
||||
if (!imageFiles.length)
|
||||
if (imageFiles.length === 0)
|
||||
// if no images, use the default
|
||||
imageFiles.push({
|
||||
name: 'default.png',
|
||||
@@ -115,11 +114,11 @@ export async function importModelFromHuggingFace(
|
||||
// Process images
|
||||
for (const { name, url } of imageFiles) {
|
||||
try {
|
||||
const { hash, height, width } = await imageToBlurhash(url);
|
||||
const { id } = await uploadViaUrl(url, {
|
||||
userId: 1,
|
||||
source: 'huggingface',
|
||||
});
|
||||
const { hash, height, width } = await getImageProps(id);
|
||||
imagesToCreate.push({ name, url: id, userId: 1, hash, height, width });
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
@@ -157,6 +156,7 @@ export async function importModelFromHuggingFace(
|
||||
data.images = {
|
||||
create: images.map((image, index) => ({ imageId: image.id, index })),
|
||||
};
|
||||
console.log(data.images);
|
||||
await tx.modelVersion.create({ data });
|
||||
}
|
||||
},
|
||||
@@ -198,11 +198,6 @@ function fileToModelType(filename: string, sizeKB: number) {
|
||||
return ModelType.Checkpoint;
|
||||
}
|
||||
|
||||
async function getImageProps(id: string) {
|
||||
const url = getEdgeUrl(id, { width: 512 });
|
||||
return await imageToBlurhash(url);
|
||||
}
|
||||
|
||||
type HuggingFaceModel = {
|
||||
id: string;
|
||||
author: string;
|
||||
|
||||
+15
-1
@@ -2,12 +2,26 @@ export type Job = {
|
||||
name: string;
|
||||
run: () => Promise<void>;
|
||||
cron: string;
|
||||
options: JobOptions;
|
||||
};
|
||||
|
||||
export function createJob(name: string, cron: string, fn: () => Promise<void>) {
|
||||
export type JobOptions = {
|
||||
shouldWait?: boolean;
|
||||
};
|
||||
|
||||
export function createJob(
|
||||
name: string,
|
||||
cron: string,
|
||||
fn: () => Promise<void>,
|
||||
options: JobOptions = {}
|
||||
) {
|
||||
return {
|
||||
name,
|
||||
cron,
|
||||
run: fn,
|
||||
options: {
|
||||
shouldWait: true,
|
||||
...options,
|
||||
},
|
||||
} as Job;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { createJob } from './job';
|
||||
import { prisma } from '~/server/db/client';
|
||||
import { ImportStatus } from '@prisma/client';
|
||||
import dayjs from 'dayjs';
|
||||
import { chunk } from 'lodash';
|
||||
import { processImport } from '~/server/importers/importRouter';
|
||||
|
||||
export const processImportsJob = createJob(
|
||||
'process-imports',
|
||||
'1 */1 * * *',
|
||||
async () => {
|
||||
// Get pending import jobs that are older than 30 minutes
|
||||
const importJobs = await prisma.import.findMany({
|
||||
where: {
|
||||
status: ImportStatus.Pending,
|
||||
createdAt: { lt: dayjs().add(-30, 'minutes').toDate() },
|
||||
},
|
||||
});
|
||||
|
||||
// Process the pending jobs
|
||||
for (const batch of chunk(importJobs, 10)) {
|
||||
try {
|
||||
await Promise.all(batch.map((job) => processImport(job)));
|
||||
} catch (e) {} // We handle this inside the processImport...
|
||||
}
|
||||
},
|
||||
{
|
||||
shouldWait: false,
|
||||
}
|
||||
);
|
||||
@@ -50,6 +50,7 @@ async function requestFileScan(
|
||||
env.SCANNING_ENDPOINT +
|
||||
'?' +
|
||||
new URLSearchParams({
|
||||
token: env.SCANNING_TOKEN,
|
||||
fileUrl,
|
||||
callbackUrl,
|
||||
});
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
const stringToNumber = z.preprocess((value) => Number(value), z.number());
|
||||
|
||||
export const imageMetaSchema = z
|
||||
.object({
|
||||
prompt: z.string(),
|
||||
negativePrompt: z.string(),
|
||||
cfgScale: z.preprocess((value) => Number(value), z.number()),
|
||||
steps: z.preprocess((value) => Number(value), z.number()),
|
||||
cfgScale: stringToNumber,
|
||||
steps: stringToNumber,
|
||||
sampler: z.string(),
|
||||
seed: z.preprocess((value) => Number(value), z.number()),
|
||||
seed: stringToNumber,
|
||||
})
|
||||
.partial()
|
||||
.passthrough();
|
||||
|
||||
+137
-127
@@ -51,11 +51,15 @@ const isOwnerOrModerator = middleware(async ({ ctx, next, input = {} }) => {
|
||||
if (!ctx.user) throw new TRPCError({ code: 'UNAUTHORIZED' });
|
||||
|
||||
const { id } = input as { id: number };
|
||||
|
||||
const userId = ctx.user.id;
|
||||
const isModerator = ctx.user?.isModerator;
|
||||
const ownerId = (await prisma.model.findUnique({ where: { id } }))?.userId ?? 0;
|
||||
if (!isModerator) {
|
||||
if (ownerId !== userId) throw handleAuthorizationError();
|
||||
let ownerId = userId;
|
||||
if (id) {
|
||||
const isModerator = ctx?.user?.isModerator;
|
||||
ownerId = (await prisma.model.findUnique({ where: { id } }))?.userId ?? 0;
|
||||
if (!isModerator) {
|
||||
if (ownerId !== userId) throw handleAuthorizationError();
|
||||
}
|
||||
}
|
||||
|
||||
return next({
|
||||
@@ -311,141 +315,147 @@ export const modelRouter = router({
|
||||
.filter((version) => !versionIds.includes(version.id))
|
||||
.map(({ id }) => id);
|
||||
|
||||
const model = await prisma.$transaction(async (tx) => {
|
||||
const imagesToUpdate = modelVersions.flatMap((x) => x.images).filter((x) => !!x.id);
|
||||
await Promise.all(
|
||||
imagesToUpdate.map(async (image) =>
|
||||
tx.image.update({
|
||||
where: { id: image.id },
|
||||
data: {
|
||||
...image,
|
||||
meta: (image.meta as Prisma.JsonObject) ?? Prisma.JsonNull,
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
const model = await prisma.$transaction(
|
||||
async (tx) => {
|
||||
const imagesToUpdate = modelVersions.flatMap((x) => x.images).filter((x) => !!x.id);
|
||||
await Promise.all(
|
||||
imagesToUpdate.map(async (image) =>
|
||||
tx.image.update({
|
||||
where: { id: image.id },
|
||||
data: {
|
||||
...image,
|
||||
meta: (image.meta as Prisma.JsonObject) ?? Prisma.JsonNull,
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
// TODO Model Status: Allow them to save as draft and publish/unpublish
|
||||
return await prisma.model.update({
|
||||
where: { id },
|
||||
data: {
|
||||
...data,
|
||||
status: data.status,
|
||||
modelVersions: {
|
||||
deleteMany:
|
||||
versionsToDelete.length > 0 ? { id: { in: versionsToDelete } } : undefined,
|
||||
upsert: modelVersions.map(
|
||||
({ id = -1, images, modelFile, trainingDataFile, ...version }) => {
|
||||
const imagesWithIndex = images.map((image, index) => ({
|
||||
index,
|
||||
userId: ownerId,
|
||||
...image,
|
||||
meta: (image.meta as Prisma.JsonObject) ?? Prisma.JsonNull,
|
||||
}));
|
||||
const existingVersion = currentVersions.find((x) => x.id === id);
|
||||
// TODO Model Status: Allow them to save as draft and publish/unpublish
|
||||
return await tx.model.update({
|
||||
where: { id },
|
||||
data: {
|
||||
...data,
|
||||
status: data.status,
|
||||
modelVersions: {
|
||||
deleteMany:
|
||||
versionsToDelete.length > 0 ? { id: { in: versionsToDelete } } : undefined,
|
||||
upsert: modelVersions.map(
|
||||
({ id = -1, images, modelFile, trainingDataFile, ...version }) => {
|
||||
const imagesWithIndex = images.map((image, index) => ({
|
||||
index,
|
||||
userId: ownerId,
|
||||
...image,
|
||||
meta: (image.meta as Prisma.JsonObject) ?? Prisma.JsonNull,
|
||||
}));
|
||||
const existingVersion = currentVersions.find((x) => x.id === id);
|
||||
|
||||
// Determine what files to create/update
|
||||
const existingFileUrls: Record<string, string> = {};
|
||||
for (const existingFile of existingVersion?.files ?? [])
|
||||
existingFileUrls[existingFile.type] = existingFile.url;
|
||||
// Determine what files to create/update
|
||||
const existingFileUrls: Record<string, string> = {};
|
||||
for (const existingFile of existingVersion?.files ?? [])
|
||||
existingFileUrls[existingFile.type] = existingFile.url;
|
||||
|
||||
const files = prepareFiles(modelFile, trainingDataFile) as typeof modelFile[];
|
||||
const filesToCreate: typeof modelFile[] = [];
|
||||
const filesToUpdate: typeof modelFile[] = [];
|
||||
for (const file of files) {
|
||||
if (!file.type) continue;
|
||||
const existingUrl = existingFileUrls[file.type];
|
||||
if (!existingUrl) filesToCreate.push(file);
|
||||
else if (existingUrl !== file.url) filesToUpdate.push(file);
|
||||
}
|
||||
const files = prepareFiles(modelFile, trainingDataFile) as typeof modelFile[];
|
||||
const filesToCreate: typeof modelFile[] = [];
|
||||
const filesToUpdate: typeof modelFile[] = [];
|
||||
for (const file of files) {
|
||||
if (!file.type) continue;
|
||||
const existingUrl = existingFileUrls[file.type];
|
||||
if (!existingUrl) filesToCreate.push(file);
|
||||
else if (existingUrl !== file.url) filesToUpdate.push(file);
|
||||
}
|
||||
|
||||
// Determine what images to create/update
|
||||
const imagesToUpdate = imagesWithIndex.filter((x) => !!x.id);
|
||||
const imagesToCreate = imagesWithIndex.filter((x) => !x.id);
|
||||
// Determine what images to create/update
|
||||
const imagesToUpdate = imagesWithIndex.filter((x) => !!x.id);
|
||||
const imagesToCreate = imagesWithIndex.filter((x) => !x.id);
|
||||
|
||||
// TODO Model Status: Allow them to save as draft and publish/unpublish
|
||||
return {
|
||||
where: { id },
|
||||
create: {
|
||||
...version,
|
||||
status: data.status,
|
||||
files: {
|
||||
create: filesToCreate.map(({ name, type, url, sizeKB }) => ({
|
||||
name,
|
||||
type,
|
||||
url,
|
||||
sizeKB,
|
||||
...unscannedFile,
|
||||
})),
|
||||
},
|
||||
images: {
|
||||
create: imagesWithIndex.map(({ index, ...image }) => ({
|
||||
index,
|
||||
image: { create: image },
|
||||
})),
|
||||
},
|
||||
},
|
||||
update: {
|
||||
...version,
|
||||
epochs: version.epochs ?? null,
|
||||
steps: version.steps ?? null,
|
||||
status: data.status,
|
||||
files: {
|
||||
create: filesToCreate.map(({ name, type, url, sizeKB }) => ({
|
||||
name,
|
||||
type,
|
||||
url,
|
||||
sizeKB,
|
||||
...unscannedFile,
|
||||
})),
|
||||
update: filesToUpdate.map(({ type, url, name, sizeKB }) => ({
|
||||
where: { modelVersionId_type: { modelVersionId: id, type } },
|
||||
data: {
|
||||
url,
|
||||
// TODO Model Status: Allow them to save as draft and publish/unpublish
|
||||
return {
|
||||
where: { id },
|
||||
create: {
|
||||
...version,
|
||||
status: data.status,
|
||||
files: {
|
||||
create: filesToCreate.map(({ name, type, url, sizeKB }) => ({
|
||||
name,
|
||||
type,
|
||||
url,
|
||||
sizeKB,
|
||||
...unscannedFile,
|
||||
},
|
||||
})),
|
||||
},
|
||||
images: {
|
||||
deleteMany: {
|
||||
NOT: images.map((image) => ({ imageId: image.id })),
|
||||
})),
|
||||
},
|
||||
create: imagesToCreate.map(({ index, ...image }) => ({
|
||||
index,
|
||||
image: { create: image },
|
||||
})),
|
||||
update: imagesToUpdate.map(({ index, ...image }) => ({
|
||||
where: {
|
||||
imageId_modelVersionId: {
|
||||
imageId: image.id as number,
|
||||
modelVersionId: id,
|
||||
},
|
||||
},
|
||||
data: {
|
||||
images: {
|
||||
create: imagesWithIndex.map(({ index, ...image }) => ({
|
||||
index,
|
||||
},
|
||||
})),
|
||||
image: { create: image },
|
||||
})),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
),
|
||||
update: {
|
||||
...version,
|
||||
epochs: version.epochs ?? null,
|
||||
steps: version.steps ?? null,
|
||||
status: data.status,
|
||||
files: {
|
||||
create: filesToCreate.map(({ name, type, url, sizeKB }) => ({
|
||||
name,
|
||||
type,
|
||||
url,
|
||||
sizeKB,
|
||||
...unscannedFile,
|
||||
})),
|
||||
update: filesToUpdate.map(({ type, url, name, sizeKB }) => ({
|
||||
where: { modelVersionId_type: { modelVersionId: id, type } },
|
||||
data: {
|
||||
url,
|
||||
name,
|
||||
sizeKB,
|
||||
...unscannedFile,
|
||||
},
|
||||
})),
|
||||
},
|
||||
images: {
|
||||
deleteMany: {
|
||||
NOT: images.map((image) => ({ imageId: image.id })),
|
||||
},
|
||||
create: imagesToCreate.map(({ index, ...image }) => ({
|
||||
index,
|
||||
image: { create: image },
|
||||
})),
|
||||
update: imagesToUpdate.map(({ index, ...image }) => ({
|
||||
where: {
|
||||
imageId_modelVersionId: {
|
||||
imageId: image.id as number,
|
||||
modelVersionId: id,
|
||||
},
|
||||
},
|
||||
data: {
|
||||
index,
|
||||
},
|
||||
})),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
),
|
||||
},
|
||||
tagsOnModels: {
|
||||
deleteMany: {},
|
||||
connectOrCreate: tagsToUpdate.map((tag) => ({
|
||||
where: { modelId_tagId: { modelId: id, tagId: tag.id as number } },
|
||||
create: { tagId: tag.id as number },
|
||||
})),
|
||||
create: tagsToCreate.map((tag) => ({
|
||||
tag: { create: { name: tag.name.toLowerCase() } },
|
||||
})),
|
||||
},
|
||||
},
|
||||
tagsOnModels: {
|
||||
deleteMany: {},
|
||||
connectOrCreate: tagsToUpdate.map((tag) => ({
|
||||
where: { modelId_tagId: { modelId: id, tagId: tag.id as number } },
|
||||
create: { tagId: tag.id as number },
|
||||
})),
|
||||
create: tagsToCreate.map((tag) => ({
|
||||
tag: { create: { name: tag.name.toLowerCase() } },
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
{
|
||||
maxWait: 5000,
|
||||
timeout: 10000,
|
||||
}
|
||||
);
|
||||
|
||||
if (!model) {
|
||||
return handleDbError({
|
||||
|
||||
@@ -14,10 +14,13 @@ const isOwnerOrModerator = middleware(async ({ ctx, next, input }) => {
|
||||
|
||||
const { id } = input as { id: number };
|
||||
const userId = ctx.user.id;
|
||||
const isModerator = ctx.user?.isModerator;
|
||||
const ownerId = (await prisma.review.findUnique({ where: { id } }))?.userId ?? 0;
|
||||
if (!isModerator && ownerId) {
|
||||
if (ownerId !== userId) throw handleAuthorizationError();
|
||||
let ownerId: number = userId;
|
||||
if (id) {
|
||||
const isModerator = ctx?.user?.isModerator;
|
||||
ownerId = (await prisma.review.findUnique({ where: { id } }))?.userId ?? 0;
|
||||
if (!isModerator && ownerId) {
|
||||
if (ownerId !== userId) throw handleAuthorizationError();
|
||||
}
|
||||
}
|
||||
|
||||
return next({
|
||||
|
||||
Vendored
+2
@@ -16,6 +16,8 @@ declare global {
|
||||
type CustomFile = {
|
||||
id?: number;
|
||||
url: string;
|
||||
previewUrl?: string;
|
||||
onLoad?: () => void;
|
||||
name?: string;
|
||||
meta?: Record<string, unknown> | null;
|
||||
file?: FileWithPath;
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { FileWithPath } from '@mantine/dropzone';
|
||||
import exifr from 'exifr';
|
||||
import { ImageMetaProps, imageMetaSchema } from '~/server/schema/image.schema';
|
||||
|
||||
export async function getMetadata(file: FileWithPath) {
|
||||
const exif = await exifr.parse(file, {
|
||||
userComment: true,
|
||||
});
|
||||
let generationDetails = null;
|
||||
if (exif?.userComment) {
|
||||
const p = document.createElement('p');
|
||||
generationDetails = decoder.decode(exif.userComment);
|
||||
// Any annoying hack to deal with weirdness in the meta
|
||||
p.innerHTML = generationDetails;
|
||||
p.remove();
|
||||
generationDetails = p.innerHTML;
|
||||
} else if (exif?.parameters) {
|
||||
generationDetails = exif.parameters;
|
||||
}
|
||||
|
||||
const metadata = parseMetadata(generationDetails);
|
||||
const result = imageMetaSchema.safeParse(metadata);
|
||||
return result.success ? result.data : {};
|
||||
}
|
||||
|
||||
// #region [infra]
|
||||
function parseMetadata(meta: string): Record<string, unknown> {
|
||||
if (!meta) return {};
|
||||
|
||||
const { parse } = parsers.find((x) => x.canHandle(meta)) ?? {};
|
||||
if (!parse) return {};
|
||||
|
||||
return parse(meta);
|
||||
}
|
||||
|
||||
type MetadataParser = {
|
||||
canHandle: (meta: string) => boolean;
|
||||
parse: (meta: string) => ImageMetaProps;
|
||||
};
|
||||
|
||||
function createMetadataParser(
|
||||
canHandle: MetadataParser['canHandle'],
|
||||
parse: MetadataParser['parse']
|
||||
): MetadataParser {
|
||||
return {
|
||||
canHandle,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder('utf-8');
|
||||
// #endregion
|
||||
|
||||
// #region [parsers]
|
||||
const automaticSDKeyMap = new Map<string, keyof ImageMetaProps>([
|
||||
['Seed', 'seed'],
|
||||
['CFG scale', 'cfgScale'],
|
||||
['Sampler', 'sampler'],
|
||||
['Steps', 'steps'],
|
||||
]);
|
||||
const automaticSDParser = createMetadataParser(
|
||||
(meta: string) => meta.includes('Steps: '),
|
||||
(meta: string) => {
|
||||
const metadata: ImageMetaProps = {};
|
||||
if (!meta) return metadata;
|
||||
const metaLines = meta.split('\n');
|
||||
const fineDetails =
|
||||
metaLines
|
||||
.pop()
|
||||
?.split(',')
|
||||
.map((x) => x.split(':')) ?? [];
|
||||
for (const [k, v] of fineDetails) {
|
||||
const propKey = automaticSDKeyMap.get(k.trim()) ?? k.trim();
|
||||
metadata[propKey] = v.trim();
|
||||
}
|
||||
|
||||
const [prompt, negativePrompt] = metaLines
|
||||
.join('\n')
|
||||
.split('Negative prompt:')
|
||||
.map((x) => x.trim());
|
||||
metadata.prompt = prompt;
|
||||
metadata.negativePrompt = negativePrompt;
|
||||
return metadata;
|
||||
}
|
||||
);
|
||||
const parsers = [automaticSDParser];
|
||||
// #endregion
|
||||
|
||||
// #region [encoders]
|
||||
export function encodeMetadata(
|
||||
metadata: ImageMetaProps,
|
||||
encoder: keyof typeof encoders = 'automatic1111'
|
||||
) {
|
||||
return encoders[encoder](metadata);
|
||||
}
|
||||
|
||||
const automaticSDEncodeMap = new Map<keyof ImageMetaProps, string>(
|
||||
Array.from(automaticSDKeyMap, (a) => a.reverse()) as Iterable<readonly [string, string]>
|
||||
);
|
||||
function automaticEncoder({ prompt, negativePrompt, ...other }: ImageMetaProps) {
|
||||
const lines = [prompt];
|
||||
if (negativePrompt) lines.push(`Negative prompt: ${negativePrompt}`);
|
||||
const fineDetails = [];
|
||||
for (const [k, v] of Object.entries(other)) {
|
||||
const key = automaticSDEncodeMap.get(k) ?? k;
|
||||
fineDetails.push(`${key}: ${v}`);
|
||||
}
|
||||
if (fineDetails.length > 0) lines.push(fineDetails.join(', '));
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
const encoders = {
|
||||
automatic1111: automaticEncoder,
|
||||
};
|
||||
// #endregion
|
||||
Reference in New Issue
Block a user