mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-13 04:13:35 +08:00
fix(ingestion): remove broken built-in Resume pipeline (#18173)
This commit is contained in:
@@ -252,7 +252,7 @@ func (s *Service) CreateUser(ctx context.Context, username, password, role strin
|
||||
rerankModel := ""
|
||||
var ttsModel *string = nil
|
||||
var ocrModel *string = nil
|
||||
parserIDs := "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email"
|
||||
parserIDs := "naive:General,qa:Q&A,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email"
|
||||
|
||||
if cfg != nil {
|
||||
chatModel = cfg.GetDefaultChatModel().Name
|
||||
|
||||
@@ -70,7 +70,6 @@ func GetParserConfig(parserID string, parserConfig map[string]interface{}) map[s
|
||||
"topn_tags": 3,
|
||||
},
|
||||
"qa": nil,
|
||||
"resume": nil,
|
||||
"manual": nil,
|
||||
"paper": nil,
|
||||
"book": nil,
|
||||
|
||||
@@ -49,7 +49,6 @@ const (
|
||||
ParserTypeLaws ParserType = "laws"
|
||||
ParserTypeManual ParserType = "manual"
|
||||
ParserTypePaper ParserType = "paper"
|
||||
ParserTypeResume ParserType = "resume"
|
||||
ParserTypeBook ParserType = "book"
|
||||
ParserTypeQA ParserType = "qa"
|
||||
ParserTypeTable ParserType = "table"
|
||||
|
||||
@@ -88,7 +88,6 @@ func TestRegistryVsHardcodedList(t *testing.T) {
|
||||
"picture": true,
|
||||
"presentation": true,
|
||||
"qa": true,
|
||||
"resume": true,
|
||||
"table": true,
|
||||
}
|
||||
for h := range hardcoded {
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -36,7 +36,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/agent/runtime"
|
||||
"ragflow/internal/common"
|
||||
componentpkg "ragflow/internal/ingestion/component"
|
||||
_ "ragflow/internal/ingestion/component/chunker"
|
||||
"ragflow/internal/ingestion/testutil"
|
||||
@@ -690,102 +689,6 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineRun_TemplateResume_RealComponents(t *testing.T) {
|
||||
RequireTokenizerPool(t)
|
||||
apiKey := common.GetEnv(common.EnvOpenAIAPIKey)
|
||||
baseURL := common.GetEnv(common.EnvOpenAIBaseURL)
|
||||
model := common.GetEnv(common.EnvOpenAIModel)
|
||||
if apiKey == "" || baseURL == "" || model == "" {
|
||||
t.Skip("missing required env (OPENAI_API_KEY/OPENAI_BASE_URL/OPENAI_MODEL); skipping real resume extractor integration test")
|
||||
}
|
||||
|
||||
templatePath := filepath.Join(repoRootFromPipelineTest(t), "internal", "ingestion", "pipeline", "template", "ingestion_pipeline_resume.json")
|
||||
templateBytes, err := os.ReadFile(templatePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read template: %v", err)
|
||||
}
|
||||
terminalIDs := terminalComponentIDsFromTemplate(t, templateBytes)
|
||||
if len(terminalIDs) != 1 || terminalIDs[0] != "Tokenizer:KindHandsWin" {
|
||||
t.Fatalf("terminal ids = %v, want [Tokenizer:KindHandsWin]", terminalIDs)
|
||||
}
|
||||
|
||||
mem := withRealTemplateDeps(t)
|
||||
componentpkg.SetExtractorChatTargetResolverOverride(func(llmID string) (driver, modelName, apiKeyOut, baseURLOut string, ok bool) {
|
||||
return "openai", model, apiKey, baseURL, true
|
||||
})
|
||||
t.Cleanup(func() { componentpkg.SetExtractorChatTargetResolverOverride(nil) })
|
||||
|
||||
const (
|
||||
bucket = "test-bucket"
|
||||
path = "fixtures/template-resume.txt"
|
||||
filename = "template-resume.txt"
|
||||
)
|
||||
content := strings.Join([]string{
|
||||
"PERSONAL INFORMATION",
|
||||
"",
|
||||
"John Example",
|
||||
"Email: john.example@resume.test",
|
||||
"Phone: +1 555 000 1234",
|
||||
"City: Seattle",
|
||||
"",
|
||||
"EDUCATION",
|
||||
"",
|
||||
"Bachelor of Science in Computer Science",
|
||||
"Example University",
|
||||
"Graduation Year: 2024",
|
||||
"",
|
||||
"WORK EXPERIENCE",
|
||||
"",
|
||||
"Software Engineer",
|
||||
"Example Corp",
|
||||
"2024 - Present",
|
||||
"",
|
||||
"SKILLS",
|
||||
"",
|
||||
"Go",
|
||||
"Python",
|
||||
"Kubernetes",
|
||||
}, "\n")
|
||||
docID := seedTemplateDocument(t, mem, filename, bucket, path, content)
|
||||
|
||||
pipe, err := NewPipelineFromDSL(templateBytes, "template-resume-real")
|
||||
if err != nil {
|
||||
t.Fatalf("NewPipelineFromDSL: %v", err)
|
||||
}
|
||||
attachFixedEmbedderFactory(t, pipe)
|
||||
out, err := pipe.Run(context.Background(), map[string]any{
|
||||
"doc_id": docID,
|
||||
"llm_id": model + "@openai",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
payload := terminalPayloadFromRunOutput(t, out, terminalIDs[0])
|
||||
if got := payload["output_format"]; got != "chunks" {
|
||||
t.Fatalf("output_format = %v, want chunks", got)
|
||||
}
|
||||
chunks, ok := payload["chunks"].([]map[string]any)
|
||||
if !ok || len(chunks) == 0 {
|
||||
t.Fatalf("chunks = %T/%v, want non-empty []map[string]any", payload["chunks"], payload["chunks"])
|
||||
}
|
||||
|
||||
assertExtractedMetadataContains(t, chunks[0]["metadata"], "candidate_name", "John Example")
|
||||
assertExtractedMetadataContains(t, chunks[0]["metadata"], "email", "john.example@resume.test")
|
||||
assertExtractedMetadataContains(t, chunks[0]["metadata"], "phone", "+1 555 000 1234")
|
||||
|
||||
state := stateFromRunOutput(t, out)
|
||||
extractorState, ok := state["Extractor:ThreeDrinksAct"]
|
||||
if !ok {
|
||||
t.Fatal("missing Extractor:ThreeDrinksAct state")
|
||||
}
|
||||
extractorChunks, ok := extractorState["chunks"].([]map[string]any)
|
||||
if !ok || len(extractorChunks) == 0 {
|
||||
t.Fatalf("extractor chunks = %T/%v, want non-empty []map[string]any", extractorState["chunks"], extractorState["chunks"])
|
||||
}
|
||||
assertExtractedMetadataContains(t, extractorChunks[0]["metadata"], "candidate_name", "John Example")
|
||||
assertExtractedMetadataContains(t, extractorChunks[0]["metadata"], "email", "john.example@resume.test")
|
||||
}
|
||||
|
||||
func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {
|
||||
RequireTokenizerPool(t)
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -203,7 +203,7 @@ func (s *UserService) Register(ctx context.Context, req *RegisterRequest) (*enti
|
||||
RerankID: rerankID,
|
||||
TTSID: &ttsID,
|
||||
OCRID: &ocrID,
|
||||
ParserIDs: "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Research Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag",
|
||||
ParserIDs: "naive:General,qa:Q&A,manual:Manual,table:Table,paper:Research Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag",
|
||||
Status: &status,
|
||||
}
|
||||
userTenantID := utility.GenerateToken()
|
||||
|
||||
Reference in New Issue
Block a user