feat(goctl/rpc): support external proto imports with cross-package ty… (#5472)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

tools/goctl/rpc/parser/base.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package base;
+
+option go_package = "github.com/zeromicro/go-zero/tools/goctl/rpc/parser/base";
+
+message BaseMessage {
+  string id = 1;
+}

tools/goctl/rpc/parser/import.go

@@ -1,8 +1,194 @@
 package parser
 
-import "github.com/emicklei/proto"
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/emicklei/proto"
+)
 
 // Import embeds proto.Import
 type Import struct {
 	*proto.Import
 }
+
+// ImportedProto holds the package information of a transitively imported proto file.
+type ImportedProto struct {
+	// Src is the absolute path to the proto file.
+	Src string
+	// ProtoPackage is the value of the proto "package" declaration.
+	// It is the qualifier used in dotted type references, e.g. "ext" in "ext.ExtReq".
+	ProtoPackage string
+	// GoPackage is the value of the option go_package field, or the proto
+	// package name when go_package is absent.
+	GoPackage string
+	// PbPackage is the sanitized Go package name derived from GoPackage.
+	PbPackage string
+}
+
+// BuildProtoPackageMap returns a map from proto package name to ImportedProto,
+// enabling O(1) lookup of Go package info given a proto type qualifier like "ext".
+func BuildProtoPackageMap(importedProtos []ImportedProto) map[string]ImportedProto {
+	m := make(map[string]ImportedProto, len(importedProtos))
+	for _, imp := range importedProtos {
+		if imp.ProtoPackage != "" {
+			m[imp.ProtoPackage] = imp
+		}
+	}
+	return m
+}
+
+// ResolveImports returns the absolute paths of all transitively imported proto
+// files reachable from src, excluding well-known types (google/*).
+// It searches for imported files in protoPaths (equivalent to protoc -I flags).
+// Files that cannot be found in protoPaths are silently skipped so that
+// system-level or well-known protos do not cause errors.
+func ResolveImports(src string, protoPaths []string) ([]string, error) {
+	absSrc, err := filepath.Abs(src)
+	if err != nil {
+		return nil, err
+	}
+
+	visited := make(map[string]bool)
+	visited[absSrc] = true // exclude the source itself from the result
+	var result []string
+	if err := collectImports(absSrc, protoPaths, visited, &result); err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+// ParseImportedProtos resolves and parses all transitively imported proto
+// files, returning their package information for use in code generation.
+func ParseImportedProtos(src string, protoPaths []string) ([]ImportedProto, error) {
+	paths, err := ResolveImports(src, protoPaths)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make([]ImportedProto, 0, len(paths))
+	for _, p := range paths {
+		goPackage, pbPackage, protoPackage, err := parseGoPackage(p)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, ImportedProto{
+			Src:          p,
+			ProtoPackage: protoPackage,
+			GoPackage:    goPackage,
+			PbPackage:    pbPackage,
+		})
+	}
+	return result, nil
+}
+
+// collectImports recursively walks import declarations of src, appending newly
+// discovered absolute proto file paths to result.
+func collectImports(src string, protoPaths []string, visited map[string]bool, result *[]string) error {
+	importFilenames, err := parseImportFilenames(src)
+	if err != nil {
+		return err
+	}
+
+	for _, filename := range importFilenames {
+		if isWellKnownProto(filename) {
+			continue
+		}
+
+		abs, err := lookupProtoFile(filename, protoPaths)
+		if err != nil {
+			// Not found in the provided proto paths — may be a system-level proto.
+			// Skip rather than fail, mirroring protoc's own behaviour.
+			continue
+		}
+
+		if visited[abs] {
+			continue
+		}
+		visited[abs] = true
+		*result = append(*result, abs)
+
+		if err := collectImports(abs, protoPaths, visited, result); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// parseImportFilenames opens src and returns the Filename field of every
+// import statement without performing any file-system lookups.
+func parseImportFilenames(src string) ([]string, error) {
+	r, err := os.Open(src)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	p := proto.NewParser(r)
+	set, err := p.Parse()
+	if err != nil {
+		return nil, err
+	}
+
+	var imports []string
+	proto.Walk(set, proto.WithImport(func(i *proto.Import) {
+		imports = append(imports, i.Filename)
+	}))
+	return imports, nil
+}
+
+// parseGoPackage reads only the go_package option and package declaration from
+// src, returning the derived GoPackage, PbPackage, and ProtoPackage without
+// requiring a service definition (imported protos often have no service block).
+func parseGoPackage(src string) (goPackage, pbPackage, protoPackage string, err error) {
+	r, err := os.Open(src)
+	if err != nil {
+		return "", "", "", err
+	}
+	defer r.Close()
+
+	p := proto.NewParser(r)
+	set, err := p.Parse()
+	if err != nil {
+		return "", "", "", err
+	}
+
+	var packageName string
+	proto.Walk(set,
+		proto.WithOption(func(opt *proto.Option) {
+			if opt.Name == "go_package" {
+				goPackage = opt.Constant.Source
+			}
+		}),
+		proto.WithPackage(func(pkg *proto.Package) {
+			packageName = pkg.Name
+		}),
+	)
+
+	if len(goPackage) == 0 {
+		goPackage = packageName
+	}
+	pbPackage = GoSanitized(filepath.Base(goPackage))
+	protoPackage = packageName
+	return goPackage, pbPackage, protoPackage, nil
+}
+
+// lookupProtoFile searches for filename inside each directory of protoPaths,
+// returning its absolute path on the first match.
+func lookupProtoFile(filename string, protoPaths []string) (string, error) {
+	for _, dir := range protoPaths {
+		candidate := filepath.Join(dir, filename)
+		if _, err := os.Stat(candidate); err == nil {
+			return filepath.Abs(candidate)
+		}
+	}
+	return "", fmt.Errorf("proto file %q not found in proto paths %v", filename, protoPaths)
+}
+
+// isWellKnownProto reports whether filename refers to a well-known type
+// bundled with protoc (e.g. google/protobuf/timestamp.proto).
+func isWellKnownProto(filename string) bool {
+	return strings.HasPrefix(filename, "google/")
+}

tools/goctl/rpc/parser/import_test.go

@@ -0,0 +1,107 @@
+package parser
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestResolveImports_Basic(t *testing.T) {
+	// test.proto imports "base.proto", which lives in the same directory.
+	absDir, err := filepath.Abs(".")
+	assert.NoError(t, err)
+
+	paths, err := ResolveImports("./test.proto", []string{absDir})
+	assert.NoError(t, err)
+	assert.Len(t, paths, 1)
+	assert.True(t, strings.HasSuffix(paths[0], "base.proto"))
+	assert.True(t, filepath.IsAbs(paths[0]))
+}
+
+func TestResolveImports_SourceExcluded(t *testing.T) {
+	// The source file itself must not appear in the result.
+	absDir, err := filepath.Abs(".")
+	assert.NoError(t, err)
+
+	absSrc, err := filepath.Abs("./test.proto")
+	assert.NoError(t, err)
+
+	paths, err := ResolveImports("./test.proto", []string{absDir})
+	assert.NoError(t, err)
+	for _, p := range paths {
+		assert.NotEqual(t, absSrc, p)
+	}
+}
+
+func TestResolveImports_NotFound(t *testing.T) {
+	// Imports that cannot be located in protoPaths are silently skipped.
+	paths, err := ResolveImports("./test.proto", []string{"/nonexistent/path"})
+	assert.NoError(t, err)
+	assert.Empty(t, paths)
+}
+
+func TestResolveImports_NoDuplicates(t *testing.T) {
+	// Even if the same proto is found via multiple search paths, it should
+	// appear only once.
+	absDir, err := filepath.Abs(".")
+	assert.NoError(t, err)
+
+	paths, err := ResolveImports("./test.proto", []string{absDir, absDir})
+	assert.NoError(t, err)
+
+	seen := make(map[string]int)
+	for _, p := range paths {
+		seen[p]++
+	}
+	for p, count := range seen {
+		assert.Equal(t, 1, count, "duplicate path: %s", p)
+	}
+}
+
+func TestParseImportedProtos_Basic(t *testing.T) {
+	absDir, err := filepath.Abs(".")
+	assert.NoError(t, err)
+
+	protos, err := ParseImportedProtos("./test.proto", []string{absDir})
+	assert.NoError(t, err)
+	assert.Len(t, protos, 1)
+
+	imp := protos[0]
+	assert.Equal(t, "github.com/zeromicro/go-zero/tools/goctl/rpc/parser/base", imp.GoPackage)
+	assert.Equal(t, "base", imp.PbPackage)
+	assert.True(t, filepath.IsAbs(imp.Src))
+}
+
+func TestParseImportedProtos_EmptyWhenNoImports(t *testing.T) {
+	// test_option.proto has no imports, so the result should be empty.
+	absDir, err := filepath.Abs(".")
+	assert.NoError(t, err)
+
+	protos, err := ParseImportedProtos("./test_option.proto", []string{absDir})
+	assert.NoError(t, err)
+	assert.Empty(t, protos)
+}
+
+func TestIsWellKnownProto(t *testing.T) {
+	assert.True(t, isWellKnownProto("google/protobuf/timestamp.proto"))
+	assert.True(t, isWellKnownProto("google/protobuf/empty.proto"))
+	assert.False(t, isWellKnownProto("base.proto"))
+	assert.False(t, isWellKnownProto("common/types.proto"))
+}
+
+func TestLookupProtoFile_Found(t *testing.T) {
+	absDir, err := filepath.Abs(".")
+	assert.NoError(t, err)
+
+	got, err := lookupProtoFile("base.proto", []string{absDir})
+	assert.NoError(t, err)
+	assert.True(t, filepath.IsAbs(got))
+	assert.True(t, strings.HasSuffix(got, "base.proto"))
+}
+
+func TestLookupProtoFile_NotFound(t *testing.T) {
+	_, err := lookupProtoFile("nonexistent.proto", []string{"/no/such/dir"})
+	assert.Error(t, err)
+}

tools/goctl/rpc/parser/parser_test.go

@@ -2,7 +2,6 @@ package parser
 
 import (
 	"sort"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -44,20 +43,21 @@ func TestDefaultProtoParse(t *testing.T) {
 	}())
 }
 
-func TestDefaultProtoParseCaseInvalidRequestType(t *testing.T) {
+func TestDefaultProtoParseDottedRequestType(t *testing.T) {
+	// Dotted types (e.g. "base.Req") are now valid — they refer to messages in
+	// imported protos. Parsing should succeed.
 	p := NewDefaultProtoParser()
-	_, err := p.Parse("./test_invalid_request.proto")
-	assert.True(t, true, func() bool {
-		return strings.Contains(err.Error(), "request type must defined in")
-	}())
+	data, err := p.Parse("./test_invalid_request.proto")
+	assert.NoError(t, err)
+	assert.Equal(t, "base.Req", data.Service[0].RPC[0].RequestType)
 }
 
-func TestDefaultProtoParseCaseInvalidResponseType(t *testing.T) {
+func TestDefaultProtoParseDottedResponseType(t *testing.T) {
+	// Dotted return types (e.g. "base.Reply") are now valid.
 	p := NewDefaultProtoParser()
-	_, err := p.Parse("./test_invalid_response.proto")
-	assert.True(t, true, func() bool {
-		return strings.Contains(err.Error(), "response type must defined in")
-	}())
+	data, err := p.Parse("./test_invalid_response.proto")
+	assert.NoError(t, err)
+	assert.Equal(t, "base.Reply", data.Service[0].RPC[0].ReturnsType)
 }
 
 func TestDefaultProtoParseError(t *testing.T) {

tools/goctl/rpc/parser/proto.go

@@ -2,12 +2,15 @@ package parser
 
 // Proto describes a proto file,
 type Proto struct {
-	Src       string
-	Name      string
-	Package   Package
-	PbPackage string
-	GoPackage string
-	Import    []Import
-	Message   []Message
-	Service   Services
+	Src           string
+	Name          string
+	Package       Package
+	PbPackage     string
+	GoPackage     string
+	Import        []Import
+	Message       []Message
+	Service       Services
+	// ImportedProtos holds the metadata for all transitively imported proto files.
+	// Populated by the generator before code generation.
+	ImportedProtos []ImportedProto
 }

tools/goctl/rpc/parser/service.go

@@ -2,9 +2,6 @@ package parser
 
 import (
 	"errors"
-	"fmt"
-	"path/filepath"
-	"strings"
 
 	"github.com/emicklei/proto"
 )
@@ -35,20 +32,5 @@ func (s Services) validate(filename string, multipleOpt ...bool) error {
 		return errors.New("only one service expected")
 	}
 
-	name := filepath.Base(filename)
-	for _, service := range s {
-		for _, rpc := range service.RPC {
-			if strings.Contains(rpc.RequestType, ".") {
-				return fmt.Errorf("line %v:%v, request type must defined in %s",
-					rpc.Position.Line,
-					rpc.Position.Column, name)
-			}
-			if strings.Contains(rpc.ReturnsType, ".") {
-				return fmt.Errorf("line %v:%v, returns type must defined in %s",
-					rpc.Position.Line,
-					rpc.Position.Column, name)
-			}
-		}
-	}
 	return nil
 }