From ec802e25a607737eaf2d2060983d414ff1ae4645 Mon Sep 17 00:00:00 2001 From: Kevin Wan Date: Sat, 14 Mar 2026 21:19:46 +0800 Subject: [PATCH] feat: add JSON5 configuration support (#5433) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- core/conf/config.go | 23 +++-- core/conf/config_test.go | 154 +++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 6 ++ internal/encoding/encoding.go | 51 ++++++++++ internal/encoding/encoding_test.go | 140 ++++++++++++++++++++++++++ 6 files changed, 369 insertions(+), 6 deletions(-) diff --git a/core/conf/config.go b/core/conf/config.go index 481b8ac12..7f7f920c0 100644 --- a/core/conf/config.go +++ b/core/conf/config.go @@ -21,10 +21,11 @@ const ( var ( fillDefaultUnmarshaler = mapping.NewUnmarshaler(jsonTagKey, mapping.WithDefault()) loaders = map[string]func([]byte, any) error{ - ".json": LoadFromJsonBytes, - ".toml": LoadFromTomlBytes, - ".yaml": LoadFromYamlBytes, - ".yml": LoadFromYamlBytes, + ".json": LoadFromJsonBytes, + ".json5": LoadFromJson5Bytes, + ".toml": LoadFromTomlBytes, + ".yaml": LoadFromYamlBytes, + ".yml": LoadFromYamlBytes, } ) @@ -41,7 +42,7 @@ func FillDefault(v any) error { return fillDefaultUnmarshaler.Unmarshal(map[string]any{}, v) } -// Load loads config into v from file, .json, .yaml and .yml are acceptable. +// Load loads config into v from file, .json, .json5, .toml, .yaml and .yml are acceptable. func Load(file string, v any, opts ...Option) error { content, err := os.ReadFile(file) if err != nil { @@ -65,7 +66,7 @@ func Load(file string, v any, opts ...Option) error { return loader(content, v) } -// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable. +// LoadConfig loads config into v from file, .json, .json5, .toml, .yaml and .yml are acceptable. // Deprecated: use Load instead. func LoadConfig(file string, v any, opts ...Option) error { return Load(file, v, opts...) @@ -119,6 +120,16 @@ func LoadFromYamlBytes(content []byte, v any) error { return LoadFromJsonBytes(b, v) } +// LoadFromJson5Bytes loads config into v from content json5 bytes. +func LoadFromJson5Bytes(content []byte, v any) error { + b, err := encoding.Json5ToJson(content) + if err != nil { + return err + } + + return LoadFromJsonBytes(b, v) +} + // LoadConfigFromYamlBytes loads config into v from content yaml bytes. // Deprecated: use LoadFromYamlBytes instead. func LoadConfigFromYamlBytes(content []byte, v any) error { diff --git a/core/conf/config_test.go b/core/conf/config_test.go index 7eae5964c..7c26a6493 100644 --- a/core/conf/config_test.go +++ b/core/conf/config_test.go @@ -75,6 +75,160 @@ func TestLoadFromJsonBytesArray(t *testing.T) { assert.EqualValues(t, []string{"foo", "bar"}, expect) } +func TestConfigJson5(t *testing.T) { + // JSON5 with comments, trailing commas, and unquoted keys + text := `{ + // This is a comment + a: 'foo', // single quotes + b: 1, + c: "${FOO}", + d: "abcd!@#$112", // trailing comma +}` + t.Setenv("FOO", "2") + + tmpfile, err := createTempFile(t, ".json5", text) + assert.Nil(t, err) + + var val struct { + A string `json:"a"` + B int `json:"b"` + C string `json:"c"` + D string `json:"d"` + } + MustLoad(tmpfile, &val) + assert.Equal(t, "foo", val.A) + assert.Equal(t, 1, val.B) + assert.Equal(t, "${FOO}", val.C) + assert.Equal(t, "abcd!@#$112", val.D) +} + +func TestConfigJsonStandardParser(t *testing.T) { + // Standard JSON uses standard JSON parser (not JSON5) for backward compatibility + text := `{ + "a": "foo", + "b": 1, + "c": "${FOO}", + "d": "abcd!@#$112" +}` + t.Setenv("FOO", "2") + + tmpfile, err := createTempFile(t, ".json", text) + assert.Nil(t, err) + + var val struct { + A string `json:"a"` + B int `json:"b"` + C string `json:"c"` + D string `json:"d"` + } + MustLoad(tmpfile, &val) + assert.Equal(t, "foo", val.A) + assert.Equal(t, 1, val.B) + assert.Equal(t, "${FOO}", val.C) + assert.Equal(t, "abcd!@#$112", val.D) +} + +func TestConfigJsonLargeIntegers(t *testing.T) { + // Test that .json files preserve large integer precision (backward compatibility) + text := `{ + "id": 1234567890123456789, + "timestamp": 9223372036854775807 +}` + + tmpfile, err := createTempFile(t, ".json", text) + assert.Nil(t, err) + + var val struct { + ID int64 `json:"id"` + Timestamp int64 `json:"timestamp"` + } + MustLoad(tmpfile, &val) + assert.Equal(t, int64(1234567890123456789), val.ID) + assert.Equal(t, int64(9223372036854775807), val.Timestamp) +} + +func TestConfigJson5Env(t *testing.T) { + text := `{ + // Comment with env variable + a: "foo", + b: 1, + c: "${FOO}", + d: "abcd!@#$a12 3", +}` + t.Setenv("FOO", "2") + + tmpfile, err := createTempFile(t, ".json5", text) + assert.Nil(t, err) + + var val struct { + A string `json:"a"` + B int `json:"b"` + C string `json:"c"` + D string `json:"d"` + } + MustLoad(tmpfile, &val, UseEnv()) + assert.Equal(t, "foo", val.A) + assert.Equal(t, 1, val.B) + assert.Equal(t, "2", val.C) + assert.Equal(t, "abcd!@# 3", val.D) +} + +func TestLoadFromJson5Bytes(t *testing.T) { + // Test JSON5 features: comments, trailing commas, single quotes, unquoted keys + input := []byte(`{ + // This is a comment + users: [ + {name: 'foo'}, // trailing comma + {Name: "bar"}, + ], + }`) + var val struct { + Users []struct { + Name string + } + } + + assert.NoError(t, LoadFromJson5Bytes(input, &val)) + var expect []string + for _, user := range val.Users { + expect = append(expect, user.Name) + } + assert.EqualValues(t, []string{"foo", "bar"}, expect) +} + +func TestLoadFromJson5BytesError(t *testing.T) { + // Invalid JSON5 syntax + input := []byte(`{a: foo}`) // unquoted string value (invalid) + var val struct { + A string + } + + assert.Error(t, LoadFromJson5Bytes(input, &val)) +} + +func TestConfigJson5LargeIntegersLimitation(t *testing.T) { + // Document that JSON5 has precision limitations for large integers (>2^53) + // due to JavaScript number semantics. Users should use .json for configs with large IDs. + text := `{ + // JSON5 converts numbers to float64, which loses precision for large integers + id: 1234567890123456789 +}` + + tmpfile, err := createTempFile(t, ".json5", text) + assert.Nil(t, err) + + var val struct { + ID int64 `json:"id"` + } + + // This will load; depending on the JSON5 implementation, large integers may lose precision. + // This test documents that behavior without requiring loss of precision as an invariant. + err = Load(tmpfile, &val) + assert.NoError(t, err) + + t.Logf("loaded JSON5 large integer id=%d (original 1234567890123456789)", val.ID) +} + func TestConfigToml(t *testing.T) { text := `a = "foo" b = 1 diff --git a/go.mod b/go.mod index 369fb936f..c04ac1b89 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/redis/go-redis/v9 v9.18.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.11.1 + github.com/titanous/json5 v1.0.0 go.etcd.io/etcd/api/v3 v3.5.15 go.etcd.io/etcd/client/v3 v3.5.15 go.mongodb.org/mongo-driver/v2 v2.5.0 diff --git a/go.sum b/go.sum index 0fdd932f2..6d788d7e3 100644 --- a/go.sum +++ b/go.sum @@ -164,6 +164,8 @@ github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfS github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0= +github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= @@ -186,6 +188,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/titanous/json5 v1.0.0 h1:hJf8Su1d9NuI/ffpxgxQfxh/UiBFZX7bMPid0rIL/7s= +github.com/titanous/json5 v1.0.0/go.mod h1:7JH1M8/LHKc6cyP5o5g3CSaRj+mBrIimTxzpvmckH8c= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= @@ -321,6 +325,8 @@ gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go index e1f82fb03..a9fe27237 100644 --- a/internal/encoding/encoding.go +++ b/internal/encoding/encoding.go @@ -3,12 +3,63 @@ package encoding import ( "bytes" "encoding/json" + "fmt" + "math" "github.com/pelletier/go-toml/v2" + "github.com/titanous/json5" "github.com/zeromicro/go-zero/core/lang" "gopkg.in/yaml.v2" ) +// Json5ToJson converts JSON5 data into its JSON representation. +func Json5ToJson(data []byte) ([]byte, error) { + var val any + if err := json5.Unmarshal(data, &val); err != nil { + return nil, err + } + + // Validate that there are no unsupported values like Infinity or NaN + if err := validateJSONCompatible(val); err != nil { + return nil, err + } + + return encodeToJSON(val) +} + +// validateJSONCompatible checks if the value can be represented in standard JSON. +// JSON5 allows Infinity and NaN, but standard JSON does not support these values. +func validateJSONCompatible(val any) error { + switch v := val.(type) { + case float64: + if math.IsInf(v, 0) { + return fmt.Errorf("JSON5 value Infinity cannot be represented in standard JSON") + } + if math.IsNaN(v) { + return fmt.Errorf("JSON5 value NaN cannot be represented in standard JSON") + } + case []any: + for _, item := range v { + if err := validateJSONCompatible(item); err != nil { + return err + } + } + case map[string]any: + for _, value := range v { + if err := validateJSONCompatible(value); err != nil { + return err + } + } + case map[any]any: + for _, value := range v { + if err := validateJSONCompatible(value); err != nil { + return err + } + } + } + return nil +} + // TomlToJson converts TOML data into its JSON representation. func TomlToJson(data []byte) ([]byte, error) { var val any diff --git a/internal/encoding/encoding_test.go b/internal/encoding/encoding_test.go index b7f8b858d..4cefe6532 100644 --- a/internal/encoding/encoding_test.go +++ b/internal/encoding/encoding_test.go @@ -1,6 +1,7 @@ package encoding import ( + "math" "testing" "github.com/stretchr/testify/assert" @@ -116,3 +117,142 @@ func TestYamlToJsonSlice(t *testing.T) { assert.Equal(t, `{"foo":["bar","baz"]} `, string(b)) } + +func TestJson5ToJson(t *testing.T) { + tests := []struct { + name string + input string + expect string + }{ + { + name: "standard json", + input: `{"a":"foo","b":1,"c":"${FOO}","d":"abcd!@#$112"}`, + expect: "{\"a\":\"foo\",\"b\":1,\"c\":\"${FOO}\",\"d\":\"abcd!@#$112\"}\n", + }, + { + name: "json5 with comments", + input: `{/*comment*/"a":"foo","b":1}`, + expect: "{\"a\":\"foo\",\"b\":1}\n", + }, + { + name: "json5 with trailing commas", + input: `{"a":"foo","b":1,}`, + expect: "{\"a\":\"foo\",\"b\":1}\n", + }, + { + name: "json5 with unquoted keys", + input: `{a:"foo",b:1}`, + expect: "{\"a\":\"foo\",\"b\":1}\n", + }, + { + name: "json5 with single quotes", + input: `{"a":'foo',"b":1}`, + expect: "{\"a\":\"foo\",\"b\":1}\n", + }, + { + name: "json5 with line comments", + input: "{\n// This is a comment\n\"a\":\"foo\",\n\"b\":1\n}", + expect: "{\"a\":\"foo\",\"b\":1}\n", + }, + { + name: "json5 all features combined", + input: "{\n// comment\na: 'foo', // trailing comma\nb: 1,\n}", + expect: "{\"a\":\"foo\",\"b\":1}\n", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := Json5ToJson([]byte(test.input)) + assert.NoError(t, err) + assert.Equal(t, test.expect, string(got)) + }) + } +} + +func TestJson5ToJsonError(t *testing.T) { + // Invalid JSON5: unquoted string value + _, err := Json5ToJson([]byte("{a: foo}")) + assert.Error(t, err) +} + +func TestJson5ToJsonInfinity(t *testing.T) { + // JSON5 allows Infinity but standard JSON does not + _, err := Json5ToJson([]byte(`{value: Infinity}`)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Infinity") + + // Negative infinity + _, err = Json5ToJson([]byte(`{value: -Infinity}`)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Infinity") + + // Infinity in array + _, err = Json5ToJson([]byte(`{values: [1, Infinity, 3]}`)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Infinity") +} + +func TestJson5ToJsonNaN(t *testing.T) { + // JSON5 allows NaN but standard JSON does not + _, err := Json5ToJson([]byte(`{value: NaN}`)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "NaN") + + // NaN in nested structure + _, err = Json5ToJson([]byte(`{nested: {value: NaN}}`)) + assert.Error(t, err) + assert.Contains(t, err.Error(), "NaN") +} + +func TestJson5ToJsonSlice(t *testing.T) { + b, err := Json5ToJson([]byte(`{ + // comment + foo: [ + 'bar', + "baz", // trailing comma + ], + }`)) + assert.NoError(t, err) + assert.Equal(t, `{"foo":["bar","baz"]} +`, string(b)) +} + +func TestValidateJSONCompatible(t *testing.T) { + // Test float64 types + assert.NoError(t, validateJSONCompatible(float64(1.5))) + assert.Error(t, validateJSONCompatible(math.Inf(1))) + assert.Error(t, validateJSONCompatible(math.Inf(-1))) + assert.Error(t, validateJSONCompatible(math.NaN())) + + // Test arrays with invalid values + assert.Error(t, validateJSONCompatible([]any{1, math.Inf(1), 3})) + assert.Error(t, validateJSONCompatible([]any{1, math.NaN(), 3})) + assert.NoError(t, validateJSONCompatible([]any{1, 2, 3})) + + // Test map[string]any with invalid values + assert.Error(t, validateJSONCompatible(map[string]any{"value": math.Inf(1)})) + assert.Error(t, validateJSONCompatible(map[string]any{"value": math.NaN()})) + assert.NoError(t, validateJSONCompatible(map[string]any{"value": 1.5})) + + // Test map[any]any with invalid values + assert.Error(t, validateJSONCompatible(map[any]any{"value": math.Inf(1)})) + assert.Error(t, validateJSONCompatible(map[any]any{"value": math.NaN()})) + assert.NoError(t, validateJSONCompatible(map[any]any{"value": 1.5})) + + // Test nested structures + assert.Error(t, validateJSONCompatible(map[string]any{ + "nested": map[string]any{"value": math.Inf(1)}, + })) + assert.Error(t, validateJSONCompatible([]any{ + map[string]any{"value": math.NaN()}, + })) + + // Test valid values of various types + assert.NoError(t, validateJSONCompatible("string")) + assert.NoError(t, validateJSONCompatible(42)) + assert.NoError(t, validateJSONCompatible(true)) + assert.NoError(t, validateJSONCompatible(nil)) +}