feat: add option to ignore unknown fields in gateway request parsing (#5058)

gateway/config.go

@@ -10,6 +10,10 @@ type (
 	GatewayConf struct {
 		rest.RestConf
 		Upstreams []Upstream
+		// IgnoreUnknownFields specifies whether to ignore unknown fields during proto unmarshaling.
+		// When set to true, extra query parameters or JSON fields that don't exist in the gRPC message
+		// will be ignored instead of causing an error.
+		IgnoreUnknownFields bool `json:",default=false"`
 	}
 
 	// HttpClientConf is the configuration for an HTTP client.

gateway/internal/requestparser.go

@@ -13,7 +13,7 @@ import (
 )
 
 // NewRequestParser creates a new request parser from the given http.Request and resolver.
-func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) {
+func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver, ignoreUnknownFields bool) (grpcurl.RequestParser, error) {
 	vars := pathvar.Vars(r)
 	params, err := httpx.GetFormValues(r)
 	if err != nil {
@@ -26,11 +26,14 @@ func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.Req
 
 	body, ok := getBody(r)
 	if !ok {
-		return buildJsonRequestParser(params, resolver)
+		return buildJsonRequestParserFromMap(params, resolver, ignoreUnknownFields)
 	}
 
 	if len(params) == 0 {
-		return grpcurl.NewJSONRequestParser(body, resolver), nil
+		if ignoreUnknownFields {
+			return buildJsonRequestParserWithUnknownFields(body, resolver)
+		}
+		return buildJsonRequestParser(body, resolver)
 	}
 
 	m := make(map[string]any)
@@ -42,17 +45,33 @@ func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.Req
 		m[k] = v
 	}
 
-	return buildJsonRequestParser(m, resolver)
+	return buildJsonRequestParserFromMap(m, resolver, ignoreUnknownFields)
 }
 
-func buildJsonRequestParser(m map[string]any, resolver jsonpb.AnyResolver) (
+func buildJsonRequestParserFromMap(data map[string]any, resolver jsonpb.AnyResolver, ignoreUnknownFields bool) (grpcurl.RequestParser, error) {
-	grpcurl.RequestParser, error) {
 	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(m); err != nil {
+	if err := json.NewEncoder(&buf).Encode(data); err != nil {
 		return nil, err
 	}
 
-	return grpcurl.NewJSONRequestParser(&buf, resolver), nil
+	if ignoreUnknownFields {
+		return buildJsonRequestParserWithUnknownFields(&buf, resolver)
+	}
+	return buildJsonRequestParser(&buf, resolver)
+}
+
+// buildJsonRequestParser creates a JSON request parser with default settings
+func buildJsonRequestParser(data io.Reader, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) {
+	return grpcurl.NewJSONRequestParser(data, resolver), nil
+}
+
+// buildJsonRequestParserWithUnknownFields creates a JSON request parser that ignores unknown fields
+func buildJsonRequestParserWithUnknownFields(data io.Reader, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) {
+	unmarshaler := jsonpb.Unmarshaler{
+		AllowUnknownFields: true,
+		AnyResolver:        resolver,
+	}
+	return grpcurl.NewJSONRequestParserWithUnmarshaler(data, unmarshaler), nil
 }
 
 func getBody(r *http.Request) (io.Reader, bool) {

gateway/internal/requestparser_test.go

@@ -7,13 +7,14 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/stretchr/testify/assert"
 	"github.com/zeromicro/go-zero/rest/pathvar"
 )
 
 func TestNewRequestParserNoVar(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", http.NoBody)
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
@@ -21,14 +22,14 @@ func TestNewRequestParserNoVar(t *testing.T) {
 func TestNewRequestParserWithVars(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", http.NoBody)
 	req = pathvar.WithVars(req, map[string]string{"a": "b"})
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
 
 func TestNewRequestParserNoVarWithBody(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`))
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
@@ -36,7 +37,7 @@ func TestNewRequestParserNoVarWithBody(t *testing.T) {
 func TestNewRequestParserWithNegativeContentLength(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`))
 	req.ContentLength = -1
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
@@ -44,7 +45,7 @@ func TestNewRequestParserWithNegativeContentLength(t *testing.T) {
 func TestNewRequestParserWithVarsWithBody(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`))
 	req = pathvar.WithVars(req, map[string]string{"c": "d"})
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
@@ -52,14 +53,14 @@ func TestNewRequestParserWithVarsWithBody(t *testing.T) {
 func TestNewRequestParserWithVarsWithWrongBody(t *testing.T) {
 	req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"`))
 	req = pathvar.WithVars(req, map[string]string{"c": "d"})
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.NotNil(t, err)
 	assert.Nil(t, parser)
 }
 
 func TestNewRequestParserWithForm(t *testing.T) {
 	req := httptest.NewRequest("GET", "/val?a=b", nil)
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
@@ -67,7 +68,7 @@ func TestNewRequestParserWithForm(t *testing.T) {
 func TestNewRequestParserWithNilBody(t *testing.T) {
 	req := httptest.NewRequest("GET", "/val?a=b", http.NoBody)
 	req.Body = nil
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
@@ -75,24 +76,135 @@ func TestNewRequestParserWithNilBody(t *testing.T) {
 func TestNewRequestParserWithBadBody(t *testing.T) {
 	req := httptest.NewRequest("GET", "/val?a=b", badBody{})
 	req.Body = badBody{}
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.Nil(t, err)
 	assert.NotNil(t, parser)
 }
 
 func TestNewRequestParserWithBadForm(t *testing.T) {
 	req := httptest.NewRequest("GET", "/val?a%1=b", http.NoBody)
-	parser, err := NewRequestParser(req, nil)
+	parser, err := NewRequestParser(req, nil, false)
 	assert.NotNil(t, err)
 	assert.Nil(t, parser)
 }
 
-func TestRequestParser_buildJsonRequestParser(t *testing.T) {
+func TestRequestParser_buildJsonRequestParserFromMap(t *testing.T) {
-	parser, err := buildJsonRequestParser(map[string]any{"a": make(chan int)}, nil)
+	parser, err := buildJsonRequestParserFromMap(map[string]any{"a": make(chan int)}, nil, false)
 	assert.NotNil(t, err)
 	assert.Nil(t, parser)
 }
 
+// mockAnyResolver is a simple implementation of jsonpb.AnyResolver for testing
+type mockAnyResolver struct{}
+
+func (m *mockAnyResolver) Resolve(typeUrl string) (proto.Message, error) {
+	return nil, nil
+}
+
+func TestNewRequestParserWithIgnoreUnknownFields(t *testing.T) {
+	// Create a concrete resolver for testing
+	resolver := &mockAnyResolver{}
+
+	// Test case 1: No body, no vars - should work with both true and false
+	req1 := httptest.NewRequest("GET", "/", http.NoBody)
+	parser1, err1 := NewRequestParser(req1, resolver, true)
+	assert.Nil(t, err1)
+	assert.NotNil(t, parser1)
+
+	req2 := httptest.NewRequest("GET", "/", http.NoBody)
+	parser2, err2 := NewRequestParser(req2, resolver, false)
+	assert.Nil(t, err2)
+	assert.NotNil(t, parser2)
+
+	// Test case 2: With JSON body - tests the body parsing path
+	req3 := httptest.NewRequest("POST", "/", strings.NewReader(`{"field": "value"}`))
+	parser3, err3 := NewRequestParser(req3, resolver, true)
+	assert.Nil(t, err3)
+	assert.NotNil(t, parser3)
+
+	req4 := httptest.NewRequest("POST", "/", strings.NewReader(`{"field": "value"}`))
+	parser4, err4 := NewRequestParser(req4, resolver, false)
+	assert.Nil(t, err4)
+	assert.NotNil(t, parser4)
+}
+
+func TestNewRequestParserWithVarsAndIgnoreUnknownFields(t *testing.T) {
+	resolver := &mockAnyResolver{}
+
+	// Test with path variables and ignoreUnknownFields = true
+	req := httptest.NewRequest("GET", "/", http.NoBody)
+	req = pathvar.WithVars(req, map[string]string{"a": "b"})
+	parser, err := NewRequestParser(req, resolver, true)
+	assert.Nil(t, err)
+	assert.NotNil(t, parser)
+
+	// Test with path variables and ignoreUnknownFields = false
+	req2 := httptest.NewRequest("GET", "/", http.NoBody)
+	req2 = pathvar.WithVars(req2, map[string]string{"c": "d"})
+	parser2, err2 := NewRequestParser(req2, resolver, false)
+	assert.Nil(t, err2)
+	assert.NotNil(t, parser2)
+}
+
+func TestNewRequestParserWithBodyAndIgnoreUnknownFields(t *testing.T) {
+	resolver := &mockAnyResolver{}
+
+	// Test with body and ignoreUnknownFields = true
+	req := httptest.NewRequest("POST", "/", strings.NewReader(`{"a": "b"}`))
+	parser, err := NewRequestParser(req, resolver, true)
+	assert.Nil(t, err)
+	assert.NotNil(t, parser)
+
+	// Test with body and ignoreUnknownFields = false
+	req2 := httptest.NewRequest("POST", "/", strings.NewReader(`{"c": "d"}`))
+	parser2, err2 := NewRequestParser(req2, resolver, false)
+	assert.Nil(t, err2)
+	assert.NotNil(t, parser2)
+}
+
+func TestNewRequestParserWithVarsBodyAndIgnoreUnknownFields(t *testing.T) {
+	resolver := &mockAnyResolver{}
+
+	// Test with both path variables and body, ignoreUnknownFields = true
+	req := httptest.NewRequest("POST", "/", strings.NewReader(`{"a": "b"}`))
+	req = pathvar.WithVars(req, map[string]string{"c": "d"})
+	parser, err := NewRequestParser(req, resolver, true)
+	assert.Nil(t, err)
+	assert.NotNil(t, parser)
+
+	// Test with both path variables and body, ignoreUnknownFields = false
+	req2 := httptest.NewRequest("POST", "/", strings.NewReader(`{"e": "f"}`))
+	req2 = pathvar.WithVars(req2, map[string]string{"g": "h"})
+	parser2, err2 := NewRequestParser(req2, resolver, false)
+	assert.Nil(t, err2)
+	assert.NotNil(t, parser2)
+}
+
+func TestBuildJsonRequestParserFromMapWithIgnoreUnknownFields(t *testing.T) {
+	resolver := &mockAnyResolver{}
+
+	// Test buildJsonRequestParserFromMap with ignoreUnknownFields = true
+	data := map[string]any{"key": "value"}
+	parser, err := buildJsonRequestParserFromMap(data, resolver, true)
+	assert.Nil(t, err)
+	assert.NotNil(t, parser)
+
+	// Test buildJsonRequestParserFromMap with ignoreUnknownFields = false
+	parser2, err2 := buildJsonRequestParserFromMap(data, resolver, false)
+	assert.Nil(t, err2)
+	assert.NotNil(t, parser2)
+}
+
+func TestBuildJsonRequestParserWithUnknownFields(t *testing.T) {
+	resolver := &mockAnyResolver{}
+
+	// Test buildJsonRequestParserWithUnknownFields
+	data := strings.NewReader(`{"test": "value"}`)
+	parser, err := buildJsonRequestParserWithUnknownFields(data, resolver)
+	assert.Nil(t, err)
+	assert.NotNil(t, parser)
+}
+
 type badBody struct{}
 
 func (badBody) Read([]byte) (int, error) { return 0, errors.New("something bad") }

gateway/server.go

@@ -30,11 +30,12 @@ type (
 	// Server is a gateway server.
 	Server struct {
 		*rest.Server
-		upstreams     []Upstream
+		upstreams           []Upstream
-		conns         []zrpc.Client
+		conns               []zrpc.Client
-		processHeader func(http.Header) []string
+		processHeader       func(http.Header) []string
-		dialer        func(conf zrpc.RpcClientConf) zrpc.Client
+		dialer              func(conf zrpc.RpcClientConf) zrpc.Client
-		middlewares   []rest.Middleware
+		middlewares         []rest.Middleware
+		ignoreUnknownFields bool
 	}
 
 	// Option defines the method to customize Server.
@@ -44,8 +45,9 @@ type (
 // MustNewServer creates a new gateway server.
 func MustNewServer(c GatewayConf, opts ...Option) *Server {
 	svr := &Server{
-		upstreams: c.Upstreams,
+		upstreams:           c.Upstreams,
-		Server:    rest.MustNewServer(c.RestConf),
+		Server:              rest.MustNewServer(c.RestConf),
+		ignoreUnknownFields: c.IgnoreUnknownFields,
 	}
 	for _, opt := range opts {
 		opt(svr)
@@ -114,7 +116,7 @@ func (s *Server) buildChainHandler(handler http.HandlerFunc) http.HandlerFunc {
 func (s *Server) buildGrpcHandler(source grpcurl.DescriptorSource, resolver jsonpb.AnyResolver,
 	cli zrpc.Client, rpcPath string) func(http.ResponseWriter, *http.Request) {
 	handler := func(w http.ResponseWriter, r *http.Request) {
-		parser, err := internal.NewRequestParser(r, resolver)
+		parser, err := internal.NewRequestParser(r, resolver, s.ignoreUnknownFields)
 		if err != nil {
 			httpx.ErrorCtx(r.Context(), w, err)
 			return
@@ -223,7 +225,6 @@ func (s *Server) buildHttpHandler(target *HttpClientConf) http.HandlerFunc {
 			logc.Error(r.Context(), err)
 		}
 	}
-
 	return s.buildChainHandler(handler)
 }