diff --git a/gateway/config.go b/gateway/config.go index 29e5f254d..b70408e14 100644 --- a/gateway/config.go +++ b/gateway/config.go @@ -10,6 +10,10 @@ type ( GatewayConf struct { rest.RestConf Upstreams []Upstream + // IgnoreUnknownFields specifies whether to ignore unknown fields during proto unmarshaling. + // When set to true, extra query parameters or JSON fields that don't exist in the gRPC message + // will be ignored instead of causing an error. + IgnoreUnknownFields bool `json:",default=false"` } // HttpClientConf is the configuration for an HTTP client. diff --git a/gateway/internal/requestparser.go b/gateway/internal/requestparser.go index 0a3559799..bcd0d4729 100644 --- a/gateway/internal/requestparser.go +++ b/gateway/internal/requestparser.go @@ -13,7 +13,7 @@ import ( ) // NewRequestParser creates a new request parser from the given http.Request and resolver. -func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) { +func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver, ignoreUnknownFields bool) (grpcurl.RequestParser, error) { vars := pathvar.Vars(r) params, err := httpx.GetFormValues(r) if err != nil { @@ -26,11 +26,14 @@ func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.Req body, ok := getBody(r) if !ok { - return buildJsonRequestParser(params, resolver) + return buildJsonRequestParserFromMap(params, resolver, ignoreUnknownFields) } if len(params) == 0 { - return grpcurl.NewJSONRequestParser(body, resolver), nil + if ignoreUnknownFields { + return buildJsonRequestParserWithUnknownFields(body, resolver) + } + return buildJsonRequestParser(body, resolver) } m := make(map[string]any) @@ -42,17 +45,33 @@ func NewRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.Req m[k] = v } - return buildJsonRequestParser(m, resolver) + return buildJsonRequestParserFromMap(m, resolver, ignoreUnknownFields) } -func buildJsonRequestParser(m map[string]any, resolver jsonpb.AnyResolver) ( - grpcurl.RequestParser, error) { +func buildJsonRequestParserFromMap(data map[string]any, resolver jsonpb.AnyResolver, ignoreUnknownFields bool) (grpcurl.RequestParser, error) { var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(m); err != nil { + if err := json.NewEncoder(&buf).Encode(data); err != nil { return nil, err } - return grpcurl.NewJSONRequestParser(&buf, resolver), nil + if ignoreUnknownFields { + return buildJsonRequestParserWithUnknownFields(&buf, resolver) + } + return buildJsonRequestParser(&buf, resolver) +} + +// buildJsonRequestParser creates a JSON request parser with default settings +func buildJsonRequestParser(data io.Reader, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) { + return grpcurl.NewJSONRequestParser(data, resolver), nil +} + +// buildJsonRequestParserWithUnknownFields creates a JSON request parser that ignores unknown fields +func buildJsonRequestParserWithUnknownFields(data io.Reader, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) { + unmarshaler := jsonpb.Unmarshaler{ + AllowUnknownFields: true, + AnyResolver: resolver, + } + return grpcurl.NewJSONRequestParserWithUnmarshaler(data, unmarshaler), nil } func getBody(r *http.Request) (io.Reader, bool) { diff --git a/gateway/internal/requestparser_test.go b/gateway/internal/requestparser_test.go index 709953cda..5ae0de654 100644 --- a/gateway/internal/requestparser_test.go +++ b/gateway/internal/requestparser_test.go @@ -7,13 +7,14 @@ import ( "strings" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/zeromicro/go-zero/rest/pathvar" ) func TestNewRequestParserNoVar(t *testing.T) { req := httptest.NewRequest("GET", "/", http.NoBody) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } @@ -21,14 +22,14 @@ func TestNewRequestParserNoVar(t *testing.T) { func TestNewRequestParserWithVars(t *testing.T) { req := httptest.NewRequest("GET", "/", http.NoBody) req = pathvar.WithVars(req, map[string]string{"a": "b"}) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } func TestNewRequestParserNoVarWithBody(t *testing.T) { req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`)) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } @@ -36,7 +37,7 @@ func TestNewRequestParserNoVarWithBody(t *testing.T) { func TestNewRequestParserWithNegativeContentLength(t *testing.T) { req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`)) req.ContentLength = -1 - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } @@ -44,7 +45,7 @@ func TestNewRequestParserWithNegativeContentLength(t *testing.T) { func TestNewRequestParserWithVarsWithBody(t *testing.T) { req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`)) req = pathvar.WithVars(req, map[string]string{"c": "d"}) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } @@ -52,14 +53,14 @@ func TestNewRequestParserWithVarsWithBody(t *testing.T) { func TestNewRequestParserWithVarsWithWrongBody(t *testing.T) { req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"`)) req = pathvar.WithVars(req, map[string]string{"c": "d"}) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.NotNil(t, err) assert.Nil(t, parser) } func TestNewRequestParserWithForm(t *testing.T) { req := httptest.NewRequest("GET", "/val?a=b", nil) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } @@ -67,7 +68,7 @@ func TestNewRequestParserWithForm(t *testing.T) { func TestNewRequestParserWithNilBody(t *testing.T) { req := httptest.NewRequest("GET", "/val?a=b", http.NoBody) req.Body = nil - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } @@ -75,24 +76,135 @@ func TestNewRequestParserWithNilBody(t *testing.T) { func TestNewRequestParserWithBadBody(t *testing.T) { req := httptest.NewRequest("GET", "/val?a=b", badBody{}) req.Body = badBody{} - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.Nil(t, err) assert.NotNil(t, parser) } func TestNewRequestParserWithBadForm(t *testing.T) { req := httptest.NewRequest("GET", "/val?a%1=b", http.NoBody) - parser, err := NewRequestParser(req, nil) + parser, err := NewRequestParser(req, nil, false) assert.NotNil(t, err) assert.Nil(t, parser) } -func TestRequestParser_buildJsonRequestParser(t *testing.T) { - parser, err := buildJsonRequestParser(map[string]any{"a": make(chan int)}, nil) +func TestRequestParser_buildJsonRequestParserFromMap(t *testing.T) { + parser, err := buildJsonRequestParserFromMap(map[string]any{"a": make(chan int)}, nil, false) assert.NotNil(t, err) assert.Nil(t, parser) } +// mockAnyResolver is a simple implementation of jsonpb.AnyResolver for testing +type mockAnyResolver struct{} + +func (m *mockAnyResolver) Resolve(typeUrl string) (proto.Message, error) { + return nil, nil +} + +func TestNewRequestParserWithIgnoreUnknownFields(t *testing.T) { + // Create a concrete resolver for testing + resolver := &mockAnyResolver{} + + // Test case 1: No body, no vars - should work with both true and false + req1 := httptest.NewRequest("GET", "/", http.NoBody) + parser1, err1 := NewRequestParser(req1, resolver, true) + assert.Nil(t, err1) + assert.NotNil(t, parser1) + + req2 := httptest.NewRequest("GET", "/", http.NoBody) + parser2, err2 := NewRequestParser(req2, resolver, false) + assert.Nil(t, err2) + assert.NotNil(t, parser2) + + // Test case 2: With JSON body - tests the body parsing path + req3 := httptest.NewRequest("POST", "/", strings.NewReader(`{"field": "value"}`)) + parser3, err3 := NewRequestParser(req3, resolver, true) + assert.Nil(t, err3) + assert.NotNil(t, parser3) + + req4 := httptest.NewRequest("POST", "/", strings.NewReader(`{"field": "value"}`)) + parser4, err4 := NewRequestParser(req4, resolver, false) + assert.Nil(t, err4) + assert.NotNil(t, parser4) +} + +func TestNewRequestParserWithVarsAndIgnoreUnknownFields(t *testing.T) { + resolver := &mockAnyResolver{} + + // Test with path variables and ignoreUnknownFields = true + req := httptest.NewRequest("GET", "/", http.NoBody) + req = pathvar.WithVars(req, map[string]string{"a": "b"}) + parser, err := NewRequestParser(req, resolver, true) + assert.Nil(t, err) + assert.NotNil(t, parser) + + // Test with path variables and ignoreUnknownFields = false + req2 := httptest.NewRequest("GET", "/", http.NoBody) + req2 = pathvar.WithVars(req2, map[string]string{"c": "d"}) + parser2, err2 := NewRequestParser(req2, resolver, false) + assert.Nil(t, err2) + assert.NotNil(t, parser2) +} + +func TestNewRequestParserWithBodyAndIgnoreUnknownFields(t *testing.T) { + resolver := &mockAnyResolver{} + + // Test with body and ignoreUnknownFields = true + req := httptest.NewRequest("POST", "/", strings.NewReader(`{"a": "b"}`)) + parser, err := NewRequestParser(req, resolver, true) + assert.Nil(t, err) + assert.NotNil(t, parser) + + // Test with body and ignoreUnknownFields = false + req2 := httptest.NewRequest("POST", "/", strings.NewReader(`{"c": "d"}`)) + parser2, err2 := NewRequestParser(req2, resolver, false) + assert.Nil(t, err2) + assert.NotNil(t, parser2) +} + +func TestNewRequestParserWithVarsBodyAndIgnoreUnknownFields(t *testing.T) { + resolver := &mockAnyResolver{} + + // Test with both path variables and body, ignoreUnknownFields = true + req := httptest.NewRequest("POST", "/", strings.NewReader(`{"a": "b"}`)) + req = pathvar.WithVars(req, map[string]string{"c": "d"}) + parser, err := NewRequestParser(req, resolver, true) + assert.Nil(t, err) + assert.NotNil(t, parser) + + // Test with both path variables and body, ignoreUnknownFields = false + req2 := httptest.NewRequest("POST", "/", strings.NewReader(`{"e": "f"}`)) + req2 = pathvar.WithVars(req2, map[string]string{"g": "h"}) + parser2, err2 := NewRequestParser(req2, resolver, false) + assert.Nil(t, err2) + assert.NotNil(t, parser2) +} + +func TestBuildJsonRequestParserFromMapWithIgnoreUnknownFields(t *testing.T) { + resolver := &mockAnyResolver{} + + // Test buildJsonRequestParserFromMap with ignoreUnknownFields = true + data := map[string]any{"key": "value"} + parser, err := buildJsonRequestParserFromMap(data, resolver, true) + assert.Nil(t, err) + assert.NotNil(t, parser) + + // Test buildJsonRequestParserFromMap with ignoreUnknownFields = false + parser2, err2 := buildJsonRequestParserFromMap(data, resolver, false) + assert.Nil(t, err2) + assert.NotNil(t, parser2) +} + +func TestBuildJsonRequestParserWithUnknownFields(t *testing.T) { + resolver := &mockAnyResolver{} + + // Test buildJsonRequestParserWithUnknownFields + data := strings.NewReader(`{"test": "value"}`) + parser, err := buildJsonRequestParserWithUnknownFields(data, resolver) + assert.Nil(t, err) + assert.NotNil(t, parser) +} + type badBody struct{} func (badBody) Read([]byte) (int, error) { return 0, errors.New("something bad") } diff --git a/gateway/server.go b/gateway/server.go index cc6f3db7b..3f60da9f4 100644 --- a/gateway/server.go +++ b/gateway/server.go @@ -30,11 +30,12 @@ type ( // Server is a gateway server. Server struct { *rest.Server - upstreams []Upstream - conns []zrpc.Client - processHeader func(http.Header) []string - dialer func(conf zrpc.RpcClientConf) zrpc.Client - middlewares []rest.Middleware + upstreams []Upstream + conns []zrpc.Client + processHeader func(http.Header) []string + dialer func(conf zrpc.RpcClientConf) zrpc.Client + middlewares []rest.Middleware + ignoreUnknownFields bool } // Option defines the method to customize Server. @@ -44,8 +45,9 @@ type ( // MustNewServer creates a new gateway server. func MustNewServer(c GatewayConf, opts ...Option) *Server { svr := &Server{ - upstreams: c.Upstreams, - Server: rest.MustNewServer(c.RestConf), + upstreams: c.Upstreams, + Server: rest.MustNewServer(c.RestConf), + ignoreUnknownFields: c.IgnoreUnknownFields, } for _, opt := range opts { opt(svr) @@ -114,7 +116,7 @@ func (s *Server) buildChainHandler(handler http.HandlerFunc) http.HandlerFunc { func (s *Server) buildGrpcHandler(source grpcurl.DescriptorSource, resolver jsonpb.AnyResolver, cli zrpc.Client, rpcPath string) func(http.ResponseWriter, *http.Request) { handler := func(w http.ResponseWriter, r *http.Request) { - parser, err := internal.NewRequestParser(r, resolver) + parser, err := internal.NewRequestParser(r, resolver, s.ignoreUnknownFields) if err != nil { httpx.ErrorCtx(r.Context(), w, err) return @@ -223,7 +225,6 @@ func (s *Server) buildHttpHandler(target *HttpClientConf) http.HandlerFunc { logc.Error(r.Context(), err) } } - return s.buildChainHandler(handler) }