From c8f3f1f032dfa30dd4c5bc98acfad269b861e006 Mon Sep 17 00:00:00 2001 From: Jonathan Duck Date: Tue, 28 Apr 2026 10:50:36 -0600 Subject: [PATCH] Allow parsing a int array into bytes --- jlexer/lexer.go | 57 ++++++++++++++++++------ jlexer/lexer_test.go | 13 +++++- tests/basic_test.go | 104 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 15 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index a27705b..2ed47af 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -703,30 +703,59 @@ func (r *Lexer) StringIntern() string { return ret } -// Bytes reads a string literal and base64 decodes it into a byte slice. +// Bytes reads a byte slice. Accepts either a base64-encoded JSON string +// (default form) or a JSON array of integers in [0,255]. func (r *Lexer) Bytes() []byte { if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != TokenString { - r.errInvalidToken("string") - return nil - } - if err := r.unescapeStringToken(); err != nil { + if !r.Ok() { r.errInvalidToken("string") return nil } - ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) - n, err := base64.StdEncoding.Decode(ret, r.token.byteValue) - if err != nil { - r.fatalError = &LexerError{ - Reason: err.Error(), + + switch r.token.kind { + case TokenString: + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return nil + } + ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) + n, err := base64.StdEncoding.Decode(ret, r.token.byteValue) + if err != nil { + r.fatalError = &LexerError{ + Reason: err.Error(), + } + return nil } + r.consume() + return ret[:n] + + case TokenDelim: + if r.token.delimValue != '[' { + r.errInvalidToken("string") + return nil + } + startErrors := len(r.multipleErrors) + r.Delim('[') + ret := []byte{} + for !r.IsDelim(']') { + ret = append(ret, r.Uint8()) + r.WantComma() + } + r.Delim(']') + if !r.Ok() { + return nil + } + if !r.UseMultipleErrors && len(r.multipleErrors) > startErrors { + return nil + } + return ret + + default: + r.errInvalidToken("string") return nil } - - r.consume() - return ret[:n] } // Bool reads a true or false boolean keyword. diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index b2bbad7..d88b8fd 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -94,11 +94,22 @@ func TestBytes(t *testing.T) { {toParse: `5`, wantError: true}, // not a JSON string {toParse: `"foobar"`, wantError: true}, // not base64 encoded {toParse: `"c2ltcGxlIHN0cmluZw="`, wantError: true}, // invalid base64 padding + + // Array-of-int form, matching encoding/json's reflective []byte path. + {toParse: `[]`, want: ""}, + {toParse: `[97,98,99]`, want: "abc"}, + {toParse: ` [ 97 , 98 , 99 ] `, want: "abc"}, // whitespace tolerated + {toParse: `[0,255]`, want: "\x00\xff"}, // boundary values + {toParse: `[256]`, wantError: true}, // out of uint8 range + {toParse: `[-1]`, wantError: true}, // negative + {toParse: `[1,"x",3]`, wantError: true}, // non-numeric element + {toParse: `[1,2,3,]`, wantError: true}, // trailing comma + {toParse: `[1,2`, wantError: true}, // unterminated } { l := Lexer{Data: []byte(test.toParse)} got := l.Bytes() - if bytes.Compare(got, []byte(test.want)) != 0 { + if !bytes.Equal(got, []byte(test.want)) { t.Errorf("[%d, %q] Bytes() = %v; want: %v", i, test.toParse, got, []byte(test.want)) } err := l.Error() diff --git a/tests/basic_test.go b/tests/basic_test.go index cdc7e8a..1e4a954 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -124,6 +124,110 @@ func TestRawMessageSTD(t *testing.T) { } } +// TestByteSliceAsIntArray verifies that generated unmarshalers accept +// the JSON array-of-integers form for []byte fields, matching the +// reflective decoder in encoding/json. Tools like Goa emit []byte for +// List types and some clients serialize them as int arrays. +func TestByteSliceAsIntArray(t *testing.T) { + tests := []struct { + name string + json string + want Slices + }{ + { + name: "all int arrays", + json: `{` + + `"ByteSlice":[97,98,99],` + + `"EmptyByteSlice":[],` + + `"NilByteSlice":null,` + + `"IntSlice":[1,2,3,4,5],` + + `"EmptyIntSlice":[],` + + `"NilIntSlice":null` + + `}`, + want: Slices{ + ByteSlice: []byte("abc"), + EmptyByteSlice: []byte{}, + NilByteSlice: nil, + IntSlice: []int{1, 2, 3, 4, 5}, + EmptyIntSlice: []int{}, + NilIntSlice: nil, + }, + }, + { + name: "boundary values 0 and 255", + json: `{` + + `"ByteSlice":[0,255,128],` + + `"EmptyByteSlice":"",` + + `"NilByteSlice":null,` + + `"IntSlice":[],` + + `"EmptyIntSlice":[],` + + `"NilIntSlice":null` + + `}`, + want: Slices{ + ByteSlice: []byte{0, 255, 128}, + EmptyByteSlice: []byte{}, + NilByteSlice: nil, + IntSlice: []int{}, + EmptyIntSlice: []int{}, + NilIntSlice: nil, + }, + }, + { + name: "base64 string still works", + json: `{` + + `"ByteSlice":"YWJj",` + + `"EmptyByteSlice":"",` + + `"NilByteSlice":null,` + + `"IntSlice":[],` + + `"EmptyIntSlice":[],` + + `"NilIntSlice":null` + + `}`, + want: Slices{ + ByteSlice: []byte("abc"), + EmptyByteSlice: []byte{}, + NilByteSlice: nil, + IntSlice: []int{}, + EmptyIntSlice: []int{}, + NilIntSlice: nil, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var got Slices + if err := got.UnmarshalJSON([]byte(tc.json)); err != nil { + t.Fatalf("UnmarshalJSON() error: %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("UnmarshalJSON() = %+v; want %+v", got, tc.want) + } + }) + } +} + +func TestByteSliceAsIntArrayErrors(t *testing.T) { + tests := []struct { + name string + json string + }{ + {"out of range", `{"ByteSlice":[256]}`}, + {"negative", `{"ByteSlice":[-1]}`}, + {"non-integer element", `{"ByteSlice":[1,"x",3]}`}, + {"trailing comma", `{"ByteSlice":[1,2,3,]}`}, + {"unterminated", `{"ByteSlice":[1,2`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var got Slices + if err := got.UnmarshalJSON([]byte(tc.json)); err == nil { + t.Errorf("UnmarshalJSON(%q) = nil; want error (got value %+v)", tc.json, got) + } + }) + } +} + func TestParseNull(t *testing.T) { var got, want SubStruct if err := easyjson.Unmarshal([]byte("null"), &got); err != nil {