mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-27 18:56:31 +08:00
Feat: Rewrite the Go-based extractor operator form. (#18600)
This commit is contained in:
balibabu
@@ -361,8 +361,8 @@ export const initialExtractorValues = {
|
||||
},
|
||||
};
|
||||
|
||||
// Defaults for the Go backend extractor (nested per-feature configs plus
|
||||
// legacy flat fields, which the Go schema still accepts).
|
||||
// Defaults for the Go backend extractor: the LLM settings plus the nested
|
||||
// per-feature groups the Go schema reads (schema.ExtractorParam).
|
||||
export const initialGoExtractorValues = {
|
||||
...initialLlmBaseValues,
|
||||
keywords: {
|
||||
@@ -381,23 +381,11 @@ export const initialGoExtractorValues = {
|
||||
enabled: false,
|
||||
system_prompt: '',
|
||||
},
|
||||
metadata_config: {
|
||||
metadata: {
|
||||
enabled: false,
|
||||
metadata: [],
|
||||
built_in_metadata: [],
|
||||
},
|
||||
metadata: [],
|
||||
built_in_metadata: [],
|
||||
field_name: '',
|
||||
auto_keywords: 0,
|
||||
auto_questions: 0,
|
||||
auto_tags: 0,
|
||||
tag_file_id: '',
|
||||
enable_summary: 0,
|
||||
enable_metadata: 0,
|
||||
keywords_sys_prompt: '',
|
||||
questions_sys_prompt: '',
|
||||
sys_prompt: '',
|
||||
outputs: {
|
||||
chunks: { type: 'Array<Object>', value: [] },
|
||||
},
|
||||
|
||||
@@ -20,13 +20,13 @@ import {
|
||||
} from '@/components/auto-keywords-form-field';
|
||||
import { LargeModelFormField } from '@/components/large-model-form-field';
|
||||
import { LlmSettingSchema } from '@/components/llm-setting-items/next';
|
||||
import { SelectWithSearch } from '@/components/originui/select-with-search';
|
||||
import { RAGFlowFormItem } from '@/components/ragflow-form';
|
||||
import { SliderInputFormField } from '@/components/slider-input-form-field';
|
||||
import { AsyncTreeSelect } from '@/components/ui/async-tree-select';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Form } from '@/components/ui/form';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import { FormLayout } from '@/constants/form';
|
||||
import { RAGFlowNodeType } from '@/interfaces/database/agent';
|
||||
import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';
|
||||
@@ -82,34 +82,21 @@ export const FormSchema = z.object({
|
||||
system_prompt: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
metadata_config: z
|
||||
// Matches schema.ExtractorParam.Metadata on the Go side
|
||||
// (internal/ingestion/component/schema/extractor.go).
|
||||
metadata: z
|
||||
.object({
|
||||
enabled: z.union([z.number(), z.boolean()]).optional(),
|
||||
metadata: z.any().optional(),
|
||||
built_in_metadata: z.any().optional(),
|
||||
})
|
||||
.optional(),
|
||||
|
||||
// Legacy flat fields for backward compatibility
|
||||
field_name: z.string().optional(),
|
||||
sys_prompt: z.string().optional(),
|
||||
prompts: z.string().optional(),
|
||||
keywords_sys_prompt: z.string().optional(),
|
||||
questions_sys_prompt: z.string().optional(),
|
||||
auto_keywords: z.number().optional(),
|
||||
auto_questions: z.number().optional(),
|
||||
auto_tags: z.number().optional(),
|
||||
tag_file_id: z.string().optional(),
|
||||
enable_summary: z.union([z.number(), z.boolean()]).optional(),
|
||||
enable_metadata: z.number().optional(),
|
||||
metadata: z.any().optional(),
|
||||
built_in_metadata: z.any().optional(),
|
||||
...LlmSettingSchema,
|
||||
});
|
||||
|
||||
export type ExtractorFormSchemaType = z.infer<typeof FormSchema>;
|
||||
|
||||
enum ExtractorSubTab {
|
||||
enum ExtractorSection {
|
||||
Keywords = 'keywords',
|
||||
Questions = 'questions',
|
||||
Tags = 'tags',
|
||||
@@ -118,9 +105,9 @@ enum ExtractorSubTab {
|
||||
}
|
||||
|
||||
// ExtractorAutoMetadata mirrors Python's dataset "Auto metadata" control: an
|
||||
// enable_metadata switch plus a field-schema editor (custom + built-in).
|
||||
// Values are stored on the node params (enable_metadata / metadata /
|
||||
// built_in_metadata) and drive the Go extractor's runEnableMetadata.
|
||||
// enable switch plus a field-schema editor (custom + built-in). Values are
|
||||
// stored as the nested `metadata` group ({enabled, metadata,
|
||||
// built_in_metadata}) that the Go extractor's runEnableMetadata reads.
|
||||
function ExtractorAutoMetadata() {
|
||||
const { t } = useTranslation();
|
||||
const form = useFormContext<ExtractorFormSchemaType>();
|
||||
@@ -135,16 +122,11 @@ function ExtractorAutoMetadata() {
|
||||
const handleOpen = useCallback(() => {
|
||||
showManageMetadataModal({
|
||||
metadata: util.metaDataSettingJSONToMetaDataTableData(
|
||||
form.getValues('metadata_config.metadata') ||
|
||||
form.getValues('metadata') ||
|
||||
[],
|
||||
form.getValues('metadata.metadata') || [],
|
||||
),
|
||||
isCanAdd: true,
|
||||
type: MetadataType.Setting,
|
||||
builtInMetadata:
|
||||
form.getValues('metadata_config.built_in_metadata') ||
|
||||
form.getValues('built_in_metadata') ||
|
||||
[],
|
||||
builtInMetadata: form.getValues('metadata.built_in_metadata') || [],
|
||||
});
|
||||
}, [form, showManageMetadataModal]);
|
||||
|
||||
@@ -153,19 +135,13 @@ function ExtractorAutoMetadata() {
|
||||
metadata?: IMetaDataReturnJSONSettings;
|
||||
builtInMetadata?: IBuiltInMetadataItem[];
|
||||
}) => {
|
||||
const metaList = data?.metadata || [];
|
||||
const builtInList = data?.builtInMetadata || [];
|
||||
form.setValue('metadata_config.metadata', metaList, {
|
||||
form.setValue('metadata.metadata', data?.metadata || [], {
|
||||
shouldDirty: true,
|
||||
});
|
||||
form.setValue('metadata_config.built_in_metadata', builtInList, {
|
||||
form.setValue('metadata.built_in_metadata', data?.builtInMetadata || [], {
|
||||
shouldDirty: true,
|
||||
});
|
||||
form.setValue('metadata_config.enabled', true, { shouldDirty: true });
|
||||
// Also keep flat fields for backward compatibility
|
||||
form.setValue('metadata', metaList, { shouldDirty: true });
|
||||
form.setValue('built_in_metadata', builtInList, { shouldDirty: true });
|
||||
form.setValue('enable_metadata', 1, { shouldDirty: true });
|
||||
form.setValue('metadata.enabled', true, { shouldDirty: true });
|
||||
},
|
||||
[form],
|
||||
);
|
||||
@@ -174,7 +150,7 @@ function ExtractorAutoMetadata() {
|
||||
<>
|
||||
<RAGFlowFormItem
|
||||
label={t('knowledgeConfiguration.autoMetadata')}
|
||||
name="metadata_config.enabled"
|
||||
name="metadata.enabled"
|
||||
>
|
||||
{(field) => (
|
||||
<div className="flex items-center justify-between">
|
||||
@@ -191,12 +167,7 @@ function ExtractorAutoMetadata() {
|
||||
</Button>
|
||||
<Switch
|
||||
checked={field.value === 1 || field.value === true}
|
||||
onCheckedChange={(checked) => {
|
||||
field.onChange(checked);
|
||||
form.setValue('enable_metadata', checked ? 1 : 0, {
|
||||
shouldDirty: true,
|
||||
});
|
||||
}}
|
||||
onCheckedChange={field.onChange}
|
||||
data-testid="extractor-metadata-switch"
|
||||
/>
|
||||
</div>
|
||||
@@ -249,12 +220,8 @@ const GoExtractorForm = ({
|
||||
resolver: zodResolver(FormSchema),
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
form.reset(defaultValues);
|
||||
}, [defaultValues, form]);
|
||||
|
||||
const [activeTab, setActiveTab] = useState<ExtractorSubTab>(
|
||||
ExtractorSubTab.Keywords,
|
||||
const [activeSection, setActiveSection] = useState<ExtractorSection>(
|
||||
ExtractorSection.Keywords,
|
||||
);
|
||||
|
||||
useWatchFormChange(node?.id, form);
|
||||
@@ -262,8 +229,7 @@ const GoExtractorForm = ({
|
||||
|
||||
const ownerTenantId = useOwnerTenantId();
|
||||
|
||||
const tagFileIdWatch =
|
||||
form.watch('tags.tag_file_id') || form.watch('tag_file_id');
|
||||
const tagFileIdWatch = form.watch('tags.tag_file_id');
|
||||
const { treeData, loadData } = useTagFileTree(tagFileIdWatch);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -284,8 +250,22 @@ const GoExtractorForm = ({
|
||||
}
|
||||
}, [form, t]);
|
||||
|
||||
const handleTabChange = useCallback((tab: string) => {
|
||||
setActiveTab(tab as ExtractorSubTab);
|
||||
const sectionOptions = useMemo(
|
||||
() => [
|
||||
{ label: t('flow.keywords'), value: ExtractorSection.Keywords },
|
||||
{ label: t('flow.questions'), value: ExtractorSection.Questions },
|
||||
{
|
||||
label: t('flow.tags') || t('knowledgeDetails.autoTags'),
|
||||
value: ExtractorSection.Tags,
|
||||
},
|
||||
{ label: t('flow.summary'), value: ExtractorSection.Summary },
|
||||
{ label: t('flow.metadata'), value: ExtractorSection.Metadata },
|
||||
],
|
||||
[t],
|
||||
);
|
||||
|
||||
const handleSectionChange = useCallback((section: string) => {
|
||||
setActiveSection(section as ExtractorSection);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
@@ -295,130 +275,107 @@ const GoExtractorForm = ({
|
||||
ownerTenantId={ownerTenantId}
|
||||
></LargeModelFormField>
|
||||
|
||||
<Tabs
|
||||
value={activeTab}
|
||||
onValueChange={handleTabChange}
|
||||
className="w-full"
|
||||
>
|
||||
<TabsList className="w-full justify-start">
|
||||
<TabsTrigger value={ExtractorSubTab.Keywords}>
|
||||
{t('flow.keywords')}
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value={ExtractorSubTab.Questions}>
|
||||
{t('flow.questions')}
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value={ExtractorSubTab.Tags}>
|
||||
{t('flow.tags') || t('knowledgeDetails.autoTags')}
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value={ExtractorSubTab.Summary}>
|
||||
{t('flow.summary')}
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value={ExtractorSubTab.Metadata}>
|
||||
{t('flow.metadata')}
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
<div>
|
||||
<SelectWithSearch
|
||||
value={activeSection}
|
||||
onChange={handleSectionChange}
|
||||
options={sectionOptions}
|
||||
/>
|
||||
|
||||
<TabsContent
|
||||
value={ExtractorSubTab.Keywords}
|
||||
className="space-y-4 pt-2"
|
||||
>
|
||||
<AutoKeywordsFormField name="keywords.top_n" />
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.systemPrompt')}
|
||||
name="keywords.system_prompt"
|
||||
>
|
||||
<PromptEditor
|
||||
placeholder={t('flow.messagePlaceholder')}
|
||||
showToolbar={false}
|
||||
showMergePath={false}
|
||||
/>
|
||||
</RAGFlowFormItem>
|
||||
</TabsContent>
|
||||
<div className="space-y-4 pt-4">
|
||||
{activeSection === ExtractorSection.Keywords && (
|
||||
<>
|
||||
<AutoKeywordsFormField name="keywords.top_n" />
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.systemPrompt')}
|
||||
name="keywords.system_prompt"
|
||||
>
|
||||
<PromptEditor
|
||||
placeholder={t('flow.messagePlaceholder')}
|
||||
showToolbar={false}
|
||||
showMergePath={false}
|
||||
/>
|
||||
</RAGFlowFormItem>
|
||||
</>
|
||||
)}
|
||||
|
||||
<TabsContent
|
||||
value={ExtractorSubTab.Questions}
|
||||
className="space-y-4 pt-2"
|
||||
>
|
||||
<AutoQuestionsFormField name="questions.top_n" />
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.systemPrompt')}
|
||||
name="questions.system_prompt"
|
||||
>
|
||||
<PromptEditor
|
||||
placeholder={t('flow.messagePlaceholder')}
|
||||
showToolbar={false}
|
||||
showMergePath={false}
|
||||
/>
|
||||
</RAGFlowFormItem>
|
||||
</TabsContent>
|
||||
{activeSection === ExtractorSection.Questions && (
|
||||
<>
|
||||
<AutoQuestionsFormField name="questions.top_n" />
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.systemPrompt')}
|
||||
name="questions.system_prompt"
|
||||
>
|
||||
<PromptEditor
|
||||
placeholder={t('flow.messagePlaceholder')}
|
||||
showToolbar={false}
|
||||
showMergePath={false}
|
||||
/>
|
||||
</RAGFlowFormItem>
|
||||
</>
|
||||
)}
|
||||
|
||||
<TabsContent value={ExtractorSubTab.Tags} className="space-y-4 pt-2">
|
||||
<SliderInputFormField
|
||||
name="tags.top_n"
|
||||
label={t('knowledgeDetails.autoTags')}
|
||||
min={0}
|
||||
max={10}
|
||||
defaultValue={0}
|
||||
layout={FormLayout.Vertical}
|
||||
/>
|
||||
<RAGFlowFormItem label={t('flow.tagFile')} name="tags.tag_file_id">
|
||||
{(field) => (
|
||||
<AsyncTreeSelect
|
||||
treeData={treeData}
|
||||
value={field.value}
|
||||
onChange={field.onChange}
|
||||
loadData={loadData}
|
||||
canSelect={canSelectTagFile}
|
||||
{activeSection === ExtractorSection.Tags && (
|
||||
<>
|
||||
<SliderInputFormField
|
||||
name="tags.top_n"
|
||||
label={t('knowledgeDetails.autoTags')}
|
||||
min={0}
|
||||
max={10}
|
||||
defaultValue={0}
|
||||
layout={FormLayout.Vertical}
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
</TabsContent>
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.tagFile')}
|
||||
name="tags.tag_file_id"
|
||||
>
|
||||
{(field) => (
|
||||
<AsyncTreeSelect
|
||||
treeData={treeData}
|
||||
value={field.value}
|
||||
onChange={field.onChange}
|
||||
loadData={loadData}
|
||||
canSelect={canSelectTagFile}
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
</>
|
||||
)}
|
||||
|
||||
<TabsContent
|
||||
value={ExtractorSubTab.Summary}
|
||||
className="space-y-4 pt-2"
|
||||
>
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.enableSummary')}
|
||||
name="summary.enabled"
|
||||
horizontal
|
||||
valueClassName="w-auto flex justify-end"
|
||||
>
|
||||
{(field) => (
|
||||
<Switch
|
||||
checked={field.value === 1 || field.value === true}
|
||||
onCheckedChange={(checked) => {
|
||||
field.onChange(checked);
|
||||
form.setValue('field_name', checked ? 'summary' : '', {
|
||||
shouldDirty: true,
|
||||
});
|
||||
form.setValue('enable_summary', checked ? 1 : 0, {
|
||||
shouldDirty: true,
|
||||
});
|
||||
}}
|
||||
data-testid="extractor-summary-switch"
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.systemPrompt')}
|
||||
name="summary.system_prompt"
|
||||
>
|
||||
<PromptEditor
|
||||
placeholder={t('flow.messagePlaceholder')}
|
||||
showToolbar={false}
|
||||
showMergePath={false}
|
||||
/>
|
||||
</RAGFlowFormItem>
|
||||
</TabsContent>
|
||||
{activeSection === ExtractorSection.Summary && (
|
||||
<>
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.enableSummary')}
|
||||
name="summary.enabled"
|
||||
horizontal
|
||||
valueClassName="w-auto flex justify-end"
|
||||
>
|
||||
{(field) => (
|
||||
<Switch
|
||||
checked={field.value === 1 || field.value === true}
|
||||
onCheckedChange={field.onChange}
|
||||
data-testid="extractor-summary-switch"
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
<RAGFlowFormItem
|
||||
label={t('flow.systemPrompt')}
|
||||
name="summary.system_prompt"
|
||||
>
|
||||
<PromptEditor
|
||||
placeholder={t('flow.messagePlaceholder')}
|
||||
showToolbar={false}
|
||||
showMergePath={false}
|
||||
/>
|
||||
</RAGFlowFormItem>
|
||||
</>
|
||||
)}
|
||||
|
||||
<TabsContent
|
||||
value={ExtractorSubTab.Metadata}
|
||||
className="space-y-4 pt-2"
|
||||
>
|
||||
<ExtractorAutoMetadata />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
{activeSection === ExtractorSection.Metadata && (
|
||||
<ExtractorAutoMetadata />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!hideOutputs && <Output list={outputList}></Output>}
|
||||
</FormWrapper>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';
|
||||
import { isGoBackend } from '@/utils/backend-runtime';
|
||||
import { Connection, Node, Position, ReactFlowInstance } from '@xyflow/react';
|
||||
import humanId from 'human-id';
|
||||
import { t } from 'i18next';
|
||||
@@ -184,8 +185,15 @@ export const useInitializeOperatorParams = () => {
|
||||
[Operator.Extractor]: {
|
||||
...getInitialExtractorValues(),
|
||||
llm_id: llmId,
|
||||
sys_prompt: t('flow.prompts.system.summary'),
|
||||
prompts: t('flow.prompts.user.summary'),
|
||||
// sys_prompt/prompts belong to the Python extractor form. The Go
|
||||
// form seeds summary.system_prompt itself, and the Go extractor
|
||||
// falls back to a built-in prompt when it is empty.
|
||||
...(isGoBackend()
|
||||
? {}
|
||||
: {
|
||||
sys_prompt: t('flow.prompts.system.summary'),
|
||||
prompts: t('flow.prompts.user.summary'),
|
||||
}),
|
||||
},
|
||||
[Operator.Compiler]: { ...initialCompilationValues, llm_id: llmId },
|
||||
[Operator.DataOperations]: initialDataOperationsValues,
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
isEmpty,
|
||||
isEqual,
|
||||
omit,
|
||||
pick,
|
||||
sample,
|
||||
} from 'lodash';
|
||||
import isObject from 'lodash/isObject';
|
||||
@@ -399,82 +400,99 @@ export function transformTitleChunkerParams(
|
||||
};
|
||||
}
|
||||
|
||||
// LLM setting keys the Go extractor DSL keeps besides the nested groups.
|
||||
// Mirrors LlmSettingSchema (components/llm-setting-items/next) — duplicated
|
||||
// here as a plain list so this module doesn't import the form components.
|
||||
const ExtractorLlmSettingKeys = [
|
||||
'llm_id',
|
||||
'temperature',
|
||||
'top_p',
|
||||
'presence_penalty',
|
||||
'frequency_penalty',
|
||||
'max_tokens',
|
||||
'parameter',
|
||||
'thinking',
|
||||
'temperatureEnabled',
|
||||
'topPEnabled',
|
||||
'presencePenaltyEnabled',
|
||||
'frequencyPenaltyEnabled',
|
||||
'maxTokensEnabled',
|
||||
];
|
||||
|
||||
export function transformExtractorParams(
|
||||
params: ExtractorFormSchemaType,
|
||||
): Record<string, any> {
|
||||
): Record<string, any> { const raw = params as Record<string, any>;
|
||||
// The Python extractor only reads the legacy flat fields; the nested
|
||||
// per-feature configs below are Go-only.
|
||||
if (!isGoBackend()) {
|
||||
return { ...params, prompts: [{ content: params.prompts, role: 'user' }] };
|
||||
return { ...params, prompts: [{ content: raw.prompts, role: 'user' }] };
|
||||
}
|
||||
|
||||
// An unopened legacy node can still flow through here with flat keys
|
||||
// (auto_keywords, keywords_sys_prompt, enable_metadata + metadata[],
|
||||
// the transitional "metadata_config", ...). Accept them as read
|
||||
// fallbacks — flat "metadata" is an array, which distinguishes it from
|
||||
// the group object — but never re-emit them.
|
||||
const metadataGroup =
|
||||
raw.metadata !== undefined && !Array.isArray(raw.metadata)
|
||||
? raw.metadata
|
||||
: raw.metadata_config;
|
||||
|
||||
const isMetadataEnabled =
|
||||
params.metadata_config?.enabled !== undefined
|
||||
? Boolean(params.metadata_config?.enabled)
|
||||
: params.enable_metadata === 1 || params.enable_metadata === true;
|
||||
metadataGroup?.enabled !== undefined
|
||||
? Boolean(metadataGroup.enabled)
|
||||
: raw.enable_metadata === 1 || raw.enable_metadata === true;
|
||||
|
||||
const isSummaryEnabled =
|
||||
params.summary?.enabled !== undefined
|
||||
? Boolean(params.summary?.enabled)
|
||||
: params.enable_summary === 1 ||
|
||||
params.enable_summary === true ||
|
||||
params.field_name === 'summary';
|
||||
: raw.enable_summary === 1 ||
|
||||
raw.enable_summary === true ||
|
||||
raw.field_name === 'summary';
|
||||
|
||||
const metadataList =
|
||||
params.metadata_config?.metadata ?? params.metadata ?? [];
|
||||
metadataGroup?.metadata ??
|
||||
(Array.isArray(raw.metadata) ? raw.metadata : []);
|
||||
const builtInMetadataList =
|
||||
params.metadata_config?.built_in_metadata ??
|
||||
params.built_in_metadata ??
|
||||
[];
|
||||
metadataGroup?.built_in_metadata ?? raw.built_in_metadata ?? [];
|
||||
|
||||
const summarySysPrompt =
|
||||
params.summary?.system_prompt ?? params.sys_prompt ?? '';
|
||||
params.summary?.system_prompt ?? raw.sys_prompt ?? '';
|
||||
|
||||
const keywordsTopN = params.keywords?.top_n ?? params.auto_keywords ?? 0;
|
||||
const keywordsTopN = params.keywords?.top_n ?? raw.auto_keywords ?? 0;
|
||||
const keywordsSysPrompt =
|
||||
params.keywords?.system_prompt ?? params.keywords_sys_prompt ?? '';
|
||||
params.keywords?.system_prompt ?? raw.keywords_sys_prompt ?? '';
|
||||
|
||||
const questionsTopN = params.questions?.top_n ?? params.auto_questions ?? 0;
|
||||
const questionsTopN = params.questions?.top_n ?? raw.auto_questions ?? 0;
|
||||
const questionsSysPrompt =
|
||||
params.questions?.system_prompt ?? params.questions_sys_prompt ?? '';
|
||||
params.questions?.system_prompt ?? raw.questions_sys_prompt ?? '';
|
||||
|
||||
const tagsTopN = params.tags?.top_n ?? params.auto_tags ?? 0;
|
||||
const tagFileId = params.tags?.tag_file_id ?? params.tag_file_id ?? '';
|
||||
const tagsTopN = params.tags?.top_n ?? raw.auto_tags ?? 0;
|
||||
const tagFileId = params.tags?.tag_file_id ?? raw.tag_file_id ?? '';
|
||||
|
||||
// The Go extractor (schema.ExtractorParam) reads only llm_id plus the
|
||||
// nested per-feature groups, so emit exactly that whitelist along with
|
||||
// the LLM settings the form defines — no legacy flat mirrors, no
|
||||
// display-only fields like outputs.
|
||||
return {
|
||||
...params,
|
||||
prompts: [{ content: params.prompts, role: 'user' }],
|
||||
auto_keywords: keywordsTopN,
|
||||
keywords_sys_prompt: keywordsSysPrompt,
|
||||
...pick(params, ExtractorLlmSettingKeys),
|
||||
keywords: {
|
||||
top_n: keywordsTopN,
|
||||
system_prompt: keywordsSysPrompt,
|
||||
},
|
||||
auto_questions: questionsTopN,
|
||||
questions_sys_prompt: questionsSysPrompt,
|
||||
questions: {
|
||||
top_n: questionsTopN,
|
||||
system_prompt: questionsSysPrompt,
|
||||
},
|
||||
auto_tags: tagsTopN,
|
||||
tag_file_id: tagFileId,
|
||||
tags: {
|
||||
top_n: tagsTopN,
|
||||
tag_file_id: tagFileId,
|
||||
},
|
||||
enable_summary: isSummaryEnabled ? 1 : 0,
|
||||
sys_prompt: summarySysPrompt,
|
||||
field_name: isSummaryEnabled
|
||||
? (params.field_name || 'summary')
|
||||
: (params.field_name === 'summary' ? '' : (params.field_name || '')),
|
||||
summary: {
|
||||
enabled: isSummaryEnabled,
|
||||
system_prompt: summarySysPrompt,
|
||||
},
|
||||
enable_metadata: isMetadataEnabled ? 1 : 0,
|
||||
metadata: metadataList,
|
||||
built_in_metadata: builtInMetadataList,
|
||||
metadata_config: {
|
||||
metadata: {
|
||||
enabled: isMetadataEnabled,
|
||||
metadata: metadataList,
|
||||
built_in_metadata: builtInMetadataList,
|
||||
|
||||
@@ -13,13 +13,13 @@ describe('Extractor parameter transformations & precedence', () => {
|
||||
});
|
||||
|
||||
describe('transformExtractorParams', () => {
|
||||
it('synchronizes nested modular configs to flat fields and preserves nested objects', () => {
|
||||
it('emits only the LLM settings and the nested groups the Go extractor reads', () => {
|
||||
const input: any = {
|
||||
summary: {
|
||||
enabled: true,
|
||||
system_prompt: 'Custom summary prompt',
|
||||
},
|
||||
metadata_config: {
|
||||
metadata: {
|
||||
enabled: true,
|
||||
metadata: [{ key: 'category', type: 'string' }],
|
||||
built_in_metadata: [{ key: 'update_time', type: 'time' }],
|
||||
@@ -37,22 +37,109 @@ describe('Extractor parameter transformations & precedence', () => {
|
||||
tag_file_id: 'tag-123',
|
||||
},
|
||||
llm_id: 'gpt-4',
|
||||
temperature: 0.5,
|
||||
temperatureEnabled: true,
|
||||
// Fields that must not leak into the DSL params
|
||||
outputs: { chunks: { type: 'Array<Object>', value: [] } },
|
||||
prompts: 'user prompt',
|
||||
sys_prompt: 'legacy sys',
|
||||
field_name: 'summary',
|
||||
auto_keywords: 9,
|
||||
keywords_sys_prompt: 'legacy kw',
|
||||
};
|
||||
|
||||
const result = transformExtractorParams(input);
|
||||
|
||||
expect(result.enable_summary).toBe(1);
|
||||
expect(result.summary).toEqual({
|
||||
enabled: true,
|
||||
system_prompt: 'Custom summary prompt',
|
||||
expect(result).toEqual({
|
||||
llm_id: 'gpt-4',
|
||||
temperature: 0.5,
|
||||
temperatureEnabled: true,
|
||||
keywords: {
|
||||
top_n: 5,
|
||||
system_prompt: 'KW prompt',
|
||||
},
|
||||
questions: {
|
||||
top_n: 3,
|
||||
system_prompt: 'Q prompt',
|
||||
},
|
||||
tags: {
|
||||
top_n: 2,
|
||||
tag_file_id: 'tag-123',
|
||||
},
|
||||
summary: {
|
||||
enabled: true,
|
||||
system_prompt: 'Custom summary prompt',
|
||||
},
|
||||
metadata: {
|
||||
enabled: true,
|
||||
metadata: [{ key: 'category', type: 'string' }],
|
||||
built_in_metadata: [{ key: 'update_time', type: 'time' }],
|
||||
},
|
||||
});
|
||||
expect(result.enable_metadata).toBe(1);
|
||||
expect(result.metadata_config.enabled).toBe(true);
|
||||
expect(result.metadata_config.metadata).toHaveLength(1);
|
||||
expect(result.auto_keywords).toBe(5);
|
||||
expect(result.auto_questions).toBe(3);
|
||||
expect(result.auto_tags).toBe(2);
|
||||
expect(result.tag_file_id).toBe('tag-123');
|
||||
});
|
||||
|
||||
it('maps legacy flat fields into the nested groups without re-emitting them', () => {
|
||||
const input: any = {
|
||||
auto_keywords: 4,
|
||||
keywords_sys_prompt: 'legacy kw prompt',
|
||||
auto_questions: 2,
|
||||
questions_sys_prompt: 'legacy q prompt',
|
||||
auto_tags: 1,
|
||||
tag_file_id: 'tag-legacy',
|
||||
enable_summary: 1,
|
||||
sys_prompt: 'legacy summary prompt',
|
||||
};
|
||||
|
||||
const result = transformExtractorParams(input);
|
||||
|
||||
expect(result).toEqual({
|
||||
keywords: { top_n: 4, system_prompt: 'legacy kw prompt' },
|
||||
questions: { top_n: 2, system_prompt: 'legacy q prompt' },
|
||||
tags: { top_n: 1, tag_file_id: 'tag-legacy' },
|
||||
summary: { enabled: true, system_prompt: 'legacy summary prompt' },
|
||||
metadata: { enabled: false, metadata: [], built_in_metadata: [] },
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts legacy flat metadata fields from unopened legacy nodes', () => {
|
||||
const input: any = {
|
||||
enable_metadata: 1,
|
||||
metadata: [{ key: 'author', type: 'string' }],
|
||||
built_in_metadata: [{ key: 'file_name', type: 'string' }],
|
||||
};
|
||||
|
||||
const result = transformExtractorParams(input);
|
||||
|
||||
expect(result).toEqual({
|
||||
keywords: { top_n: 0, system_prompt: '' },
|
||||
questions: { top_n: 0, system_prompt: '' },
|
||||
tags: { top_n: 0, tag_file_id: '' },
|
||||
summary: { enabled: false, system_prompt: '' },
|
||||
metadata: {
|
||||
enabled: true,
|
||||
metadata: [{ key: 'author', type: 'string' }],
|
||||
built_in_metadata: [{ key: 'file_name', type: 'string' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts the transitional metadata_config key', () => {
|
||||
const input: any = {
|
||||
metadata_config: {
|
||||
enabled: true,
|
||||
metadata: [{ key: 'category', type: 'string' }],
|
||||
built_in_metadata: [],
|
||||
},
|
||||
};
|
||||
|
||||
const result = transformExtractorParams(input);
|
||||
|
||||
expect(result.metadata).toEqual({
|
||||
enabled: true,
|
||||
metadata: [{ key: 'category', type: 'string' }],
|
||||
built_in_metadata: [],
|
||||
});
|
||||
expect(result).not.toHaveProperty('metadata_config');
|
||||
});
|
||||
|
||||
it('gives nested modular enabled: false precedence over legacy flat enable_*: 1', () => {
|
||||
@@ -62,7 +149,7 @@ describe('Extractor parameter transformations & precedence', () => {
|
||||
system_prompt: '',
|
||||
},
|
||||
enable_summary: 1,
|
||||
metadata_config: {
|
||||
metadata: {
|
||||
enabled: false,
|
||||
metadata: [],
|
||||
built_in_metadata: [],
|
||||
@@ -73,22 +160,10 @@ describe('Extractor parameter transformations & precedence', () => {
|
||||
const result = transformExtractorParams(input);
|
||||
|
||||
expect(result.summary.enabled).toBe(false);
|
||||
expect(result.enable_summary).toBe(0);
|
||||
expect(result.metadata_config.enabled).toBe(false);
|
||||
expect(result.enable_metadata).toBe(0);
|
||||
});
|
||||
|
||||
it('preserves custom field_name when summary is disabled', () => {
|
||||
const input: any = {
|
||||
summary: {
|
||||
enabled: false,
|
||||
system_prompt: '',
|
||||
},
|
||||
field_name: 'custom_chunk_field',
|
||||
};
|
||||
|
||||
const result = transformExtractorParams(input);
|
||||
expect(result.field_name).toBe('custom_chunk_field');
|
||||
expect(result.metadata.enabled).toBe(false);
|
||||
expect(result).not.toHaveProperty('enable_summary');
|
||||
expect(result).not.toHaveProperty('enable_metadata');
|
||||
expect(result).not.toHaveProperty('built_in_metadata');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -112,15 +187,36 @@ describe('Extractor parameter transformations & precedence', () => {
|
||||
enabled: true,
|
||||
system_prompt: 'Old summary prompt',
|
||||
});
|
||||
expect(result.metadata_config).toEqual({
|
||||
expect(result.metadata).toEqual({
|
||||
enabled: true,
|
||||
metadata: [{ key: 'author', type: 'string' }],
|
||||
built_in_metadata: [{ key: 'file_name', type: 'string' }],
|
||||
});
|
||||
expect(result).not.toHaveProperty('metadata_config');
|
||||
expect(result).not.toHaveProperty('enable_metadata');
|
||||
expect(result).not.toHaveProperty('built_in_metadata');
|
||||
expect(result.keywords.top_n).toBe(4);
|
||||
expect(result.questions.top_n).toBe(2);
|
||||
expect(result.tags.top_n).toBe(1);
|
||||
});
|
||||
|
||||
it('passes through the metadata group shape unchanged', () => {
|
||||
const config = {
|
||||
metadata: {
|
||||
enabled: true,
|
||||
metadata: [{ key: 'category', type: 'string', enum: ['科技'] }],
|
||||
built_in_metadata: [{ key: 'doc_name', type: 'string' }],
|
||||
},
|
||||
};
|
||||
|
||||
const result = transformExtractorConfigToForm(config);
|
||||
|
||||
expect(result.metadata).toEqual({
|
||||
enabled: true,
|
||||
metadata: [{ key: 'category', type: 'string', enum: ['科技'] }],
|
||||
built_in_metadata: [{ key: 'doc_name', type: 'string' }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('python backend keeps the legacy flat shape', () => {
|
||||
|
||||
@@ -109,9 +109,19 @@ export function transformExtractorConfigToForm(
|
||||
? Boolean(config.summary?.enabled)
|
||||
: config.enable_summary === 1 || config.enable_summary === true;
|
||||
|
||||
// The metadata group is stored under "metadata" — the shape the Go
|
||||
// Extractor reads (schema.ExtractorParam). Accept two historical shapes:
|
||||
// the transitional "metadata_config" key, and legacy flat nodes carrying
|
||||
// enable_metadata + metadata[] + built_in_metadata[] (flat "metadata" is
|
||||
// an array there, which distinguishes it from the group object).
|
||||
const metadataGroup =
|
||||
config.metadata !== undefined && !Array.isArray(config.metadata)
|
||||
? config.metadata
|
||||
: config.metadata_config;
|
||||
|
||||
const isMetadataEnabled =
|
||||
config.metadata_config?.enabled !== undefined
|
||||
? Boolean(config.metadata_config?.enabled)
|
||||
metadataGroup?.enabled !== undefined
|
||||
? Boolean(metadataGroup.enabled)
|
||||
: config.enable_metadata === 1 || config.enable_metadata === true;
|
||||
|
||||
result.keywords = {
|
||||
@@ -141,22 +151,20 @@ export function transformExtractorConfigToForm(
|
||||
enabled: isSummaryEnabled,
|
||||
system_prompt: config.summary?.system_prompt ?? config.sys_prompt ?? '',
|
||||
};
|
||||
result.enable_summary = isSummaryEnabled ? 1 : 0;
|
||||
result.field_name = isSummaryEnabled
|
||||
? (config.field_name || 'summary')
|
||||
: (config.field_name === 'summary' ? '' : (config.field_name || ''));
|
||||
|
||||
result.metadata_config = {
|
||||
result.metadata = {
|
||||
enabled: isMetadataEnabled,
|
||||
metadata: config.metadata_config?.metadata ?? config.metadata ?? [],
|
||||
metadata:
|
||||
metadataGroup?.metadata ??
|
||||
(Array.isArray(config.metadata) ? config.metadata : []),
|
||||
built_in_metadata:
|
||||
config.metadata_config?.built_in_metadata ??
|
||||
config.built_in_metadata ??
|
||||
[],
|
||||
metadataGroup?.built_in_metadata ?? config.built_in_metadata ?? [],
|
||||
};
|
||||
result.enable_metadata = isMetadataEnabled ? 1 : 0;
|
||||
result.metadata = result.metadata_config.metadata;
|
||||
result.built_in_metadata = result.metadata_config.built_in_metadata;
|
||||
// Drop the historical keys so they don't linger in form state and get
|
||||
// re-emitted into the DSL on save — the Go extractor only reads the group.
|
||||
delete result.metadata_config;
|
||||
delete result.enable_metadata;
|
||||
delete result.built_in_metadata;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user