decode.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530
  1. // Copyright 2015 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package jsonpb
  5. import (
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "io"
  10. "math"
  11. "reflect"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "github.com/golang/protobuf/proto"
  16. "google.golang.org/protobuf/encoding/protojson"
  17. protoV2 "google.golang.org/protobuf/proto"
  18. "google.golang.org/protobuf/reflect/protoreflect"
  19. "google.golang.org/protobuf/reflect/protoregistry"
  20. )
  21. const wrapJSONUnmarshalV2 = false
  22. // UnmarshalNext unmarshals the next JSON object from d into m.
  23. func UnmarshalNext(d *json.Decoder, m proto.Message) error {
  24. return new(Unmarshaler).UnmarshalNext(d, m)
  25. }
  26. // Unmarshal unmarshals a JSON object from r into m.
  27. func Unmarshal(r io.Reader, m proto.Message) error {
  28. return new(Unmarshaler).Unmarshal(r, m)
  29. }
  30. // UnmarshalString unmarshals a JSON object from s into m.
  31. func UnmarshalString(s string, m proto.Message) error {
  32. return new(Unmarshaler).Unmarshal(strings.NewReader(s), m)
  33. }
  34. // Unmarshaler is a configurable object for converting from a JSON
  35. // representation to a protocol buffer object.
  36. type Unmarshaler struct {
  37. // AllowUnknownFields specifies whether to allow messages to contain
  38. // unknown JSON fields, as opposed to failing to unmarshal.
  39. AllowUnknownFields bool
  40. // AnyResolver is used to resolve the google.protobuf.Any well-known type.
  41. // If unset, the global registry is used by default.
  42. AnyResolver AnyResolver
  43. }
  44. // JSONPBUnmarshaler is implemented by protobuf messages that customize the way
  45. // they are unmarshaled from JSON. Messages that implement this should also
  46. // implement JSONPBMarshaler so that the custom format can be produced.
  47. //
  48. // The JSON unmarshaling must follow the JSON to proto specification:
  49. // https://developers.google.com/protocol-buffers/docs/proto3#json
  50. //
  51. // Deprecated: Custom types should implement protobuf reflection instead.
  52. type JSONPBUnmarshaler interface {
  53. UnmarshalJSONPB(*Unmarshaler, []byte) error
  54. }
  55. // Unmarshal unmarshals a JSON object from r into m.
  56. func (u *Unmarshaler) Unmarshal(r io.Reader, m proto.Message) error {
  57. return u.UnmarshalNext(json.NewDecoder(r), m)
  58. }
  59. // UnmarshalNext unmarshals the next JSON object from d into m.
  60. func (u *Unmarshaler) UnmarshalNext(d *json.Decoder, m proto.Message) error {
  61. if m == nil {
  62. return errors.New("invalid nil message")
  63. }
  64. // Parse the next JSON object from the stream.
  65. raw := json.RawMessage{}
  66. if err := d.Decode(&raw); err != nil {
  67. return err
  68. }
  69. // Check for custom unmarshalers first since they may not properly
  70. // implement protobuf reflection that the logic below relies on.
  71. if jsu, ok := m.(JSONPBUnmarshaler); ok {
  72. return jsu.UnmarshalJSONPB(u, raw)
  73. }
  74. mr := proto.MessageReflect(m)
  75. // NOTE: For historical reasons, a top-level null is treated as a noop.
  76. // This is incorrect, but kept for compatibility.
  77. if string(raw) == "null" && mr.Descriptor().FullName() != "google.protobuf.Value" {
  78. return nil
  79. }
  80. if wrapJSONUnmarshalV2 {
  81. // NOTE: If input message is non-empty, we need to preserve merge semantics
  82. // of the old jsonpb implementation. These semantics are not supported by
  83. // the protobuf JSON specification.
  84. isEmpty := true
  85. mr.Range(func(protoreflect.FieldDescriptor, protoreflect.Value) bool {
  86. isEmpty = false // at least one iteration implies non-empty
  87. return false
  88. })
  89. if !isEmpty {
  90. // Perform unmarshaling into a newly allocated, empty message.
  91. mr = mr.New()
  92. // Use a defer to copy all unmarshaled fields into the original message.
  93. dst := proto.MessageReflect(m)
  94. defer mr.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
  95. dst.Set(fd, v)
  96. return true
  97. })
  98. }
  99. // Unmarshal using the v2 JSON unmarshaler.
  100. opts := protojson.UnmarshalOptions{
  101. DiscardUnknown: u.AllowUnknownFields,
  102. }
  103. if u.AnyResolver != nil {
  104. opts.Resolver = anyResolver{u.AnyResolver}
  105. }
  106. return opts.Unmarshal(raw, mr.Interface())
  107. } else {
  108. if err := u.unmarshalMessage(mr, raw); err != nil {
  109. return err
  110. }
  111. return protoV2.CheckInitialized(mr.Interface())
  112. }
  113. }
  114. func (u *Unmarshaler) unmarshalMessage(m protoreflect.Message, in []byte) error {
  115. md := m.Descriptor()
  116. fds := md.Fields()
  117. if jsu, ok := proto.MessageV1(m.Interface()).(JSONPBUnmarshaler); ok {
  118. return jsu.UnmarshalJSONPB(u, in)
  119. }
  120. if string(in) == "null" && md.FullName() != "google.protobuf.Value" {
  121. return nil
  122. }
  123. switch wellKnownType(md.FullName()) {
  124. case "Any":
  125. var jsonObject map[string]json.RawMessage
  126. if err := json.Unmarshal(in, &jsonObject); err != nil {
  127. return err
  128. }
  129. rawTypeURL, ok := jsonObject["@type"]
  130. if !ok {
  131. return errors.New("Any JSON doesn't have '@type'")
  132. }
  133. typeURL, err := unquoteString(string(rawTypeURL))
  134. if err != nil {
  135. return fmt.Errorf("can't unmarshal Any's '@type': %q", rawTypeURL)
  136. }
  137. m.Set(fds.ByNumber(1), protoreflect.ValueOfString(typeURL))
  138. var m2 protoreflect.Message
  139. if u.AnyResolver != nil {
  140. mi, err := u.AnyResolver.Resolve(typeURL)
  141. if err != nil {
  142. return err
  143. }
  144. m2 = proto.MessageReflect(mi)
  145. } else {
  146. mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
  147. if err != nil {
  148. if err == protoregistry.NotFound {
  149. return fmt.Errorf("could not resolve Any message type: %v", typeURL)
  150. }
  151. return err
  152. }
  153. m2 = mt.New()
  154. }
  155. if wellKnownType(m2.Descriptor().FullName()) != "" {
  156. rawValue, ok := jsonObject["value"]
  157. if !ok {
  158. return errors.New("Any JSON doesn't have 'value'")
  159. }
  160. if err := u.unmarshalMessage(m2, rawValue); err != nil {
  161. return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
  162. }
  163. } else {
  164. delete(jsonObject, "@type")
  165. rawJSON, err := json.Marshal(jsonObject)
  166. if err != nil {
  167. return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
  168. }
  169. if err = u.unmarshalMessage(m2, rawJSON); err != nil {
  170. return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
  171. }
  172. }
  173. rawWire, err := protoV2.Marshal(m2.Interface())
  174. if err != nil {
  175. return fmt.Errorf("can't marshal proto %v into Any.Value: %v", typeURL, err)
  176. }
  177. m.Set(fds.ByNumber(2), protoreflect.ValueOfBytes(rawWire))
  178. return nil
  179. case "BoolValue", "BytesValue", "StringValue",
  180. "Int32Value", "UInt32Value", "FloatValue",
  181. "Int64Value", "UInt64Value", "DoubleValue":
  182. fd := fds.ByNumber(1)
  183. v, err := u.unmarshalValue(m.NewField(fd), in, fd)
  184. if err != nil {
  185. return err
  186. }
  187. m.Set(fd, v)
  188. return nil
  189. case "Duration":
  190. v, err := unquoteString(string(in))
  191. if err != nil {
  192. return err
  193. }
  194. d, err := time.ParseDuration(v)
  195. if err != nil {
  196. return fmt.Errorf("bad Duration: %v", err)
  197. }
  198. sec := d.Nanoseconds() / 1e9
  199. nsec := d.Nanoseconds() % 1e9
  200. m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
  201. m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
  202. return nil
  203. case "Timestamp":
  204. v, err := unquoteString(string(in))
  205. if err != nil {
  206. return err
  207. }
  208. t, err := time.Parse(time.RFC3339Nano, v)
  209. if err != nil {
  210. return fmt.Errorf("bad Timestamp: %v", err)
  211. }
  212. sec := t.Unix()
  213. nsec := t.Nanosecond()
  214. m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
  215. m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
  216. return nil
  217. case "Value":
  218. switch {
  219. case string(in) == "null":
  220. m.Set(fds.ByNumber(1), protoreflect.ValueOfEnum(0))
  221. case string(in) == "true":
  222. m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(true))
  223. case string(in) == "false":
  224. m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(false))
  225. case hasPrefixAndSuffix('"', in, '"'):
  226. s, err := unquoteString(string(in))
  227. if err != nil {
  228. return fmt.Errorf("unrecognized type for Value %q", in)
  229. }
  230. m.Set(fds.ByNumber(3), protoreflect.ValueOfString(s))
  231. case hasPrefixAndSuffix('[', in, ']'):
  232. v := m.Mutable(fds.ByNumber(6))
  233. return u.unmarshalMessage(v.Message(), in)
  234. case hasPrefixAndSuffix('{', in, '}'):
  235. v := m.Mutable(fds.ByNumber(5))
  236. return u.unmarshalMessage(v.Message(), in)
  237. default:
  238. f, err := strconv.ParseFloat(string(in), 0)
  239. if err != nil {
  240. return fmt.Errorf("unrecognized type for Value %q", in)
  241. }
  242. m.Set(fds.ByNumber(2), protoreflect.ValueOfFloat64(f))
  243. }
  244. return nil
  245. case "ListValue":
  246. var jsonArray []json.RawMessage
  247. if err := json.Unmarshal(in, &jsonArray); err != nil {
  248. return fmt.Errorf("bad ListValue: %v", err)
  249. }
  250. lv := m.Mutable(fds.ByNumber(1)).List()
  251. for _, raw := range jsonArray {
  252. ve := lv.NewElement()
  253. if err := u.unmarshalMessage(ve.Message(), raw); err != nil {
  254. return err
  255. }
  256. lv.Append(ve)
  257. }
  258. return nil
  259. case "Struct":
  260. var jsonObject map[string]json.RawMessage
  261. if err := json.Unmarshal(in, &jsonObject); err != nil {
  262. return fmt.Errorf("bad StructValue: %v", err)
  263. }
  264. mv := m.Mutable(fds.ByNumber(1)).Map()
  265. for key, raw := range jsonObject {
  266. kv := protoreflect.ValueOf(key).MapKey()
  267. vv := mv.NewValue()
  268. if err := u.unmarshalMessage(vv.Message(), raw); err != nil {
  269. return fmt.Errorf("bad value in StructValue for key %q: %v", key, err)
  270. }
  271. mv.Set(kv, vv)
  272. }
  273. return nil
  274. }
  275. var jsonObject map[string]json.RawMessage
  276. if err := json.Unmarshal(in, &jsonObject); err != nil {
  277. return err
  278. }
  279. // Handle known fields.
  280. for i := 0; i < fds.Len(); i++ {
  281. fd := fds.Get(i)
  282. if fd.IsWeak() && fd.Message().IsPlaceholder() {
  283. continue // weak reference is not linked in
  284. }
  285. // Search for any raw JSON value associated with this field.
  286. var raw json.RawMessage
  287. name := string(fd.Name())
  288. if fd.Kind() == protoreflect.GroupKind {
  289. name = string(fd.Message().Name())
  290. }
  291. if v, ok := jsonObject[name]; ok {
  292. delete(jsonObject, name)
  293. raw = v
  294. }
  295. name = string(fd.JSONName())
  296. if v, ok := jsonObject[name]; ok {
  297. delete(jsonObject, name)
  298. raw = v
  299. }
  300. field := m.NewField(fd)
  301. // Unmarshal the field value.
  302. if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
  303. continue
  304. }
  305. v, err := u.unmarshalValue(field, raw, fd)
  306. if err != nil {
  307. return err
  308. }
  309. m.Set(fd, v)
  310. }
  311. // Handle extension fields.
  312. for name, raw := range jsonObject {
  313. if !strings.HasPrefix(name, "[") || !strings.HasSuffix(name, "]") {
  314. continue
  315. }
  316. // Resolve the extension field by name.
  317. xname := protoreflect.FullName(name[len("[") : len(name)-len("]")])
  318. xt, _ := protoregistry.GlobalTypes.FindExtensionByName(xname)
  319. if xt == nil && isMessageSet(md) {
  320. xt, _ = protoregistry.GlobalTypes.FindExtensionByName(xname.Append("message_set_extension"))
  321. }
  322. if xt == nil {
  323. continue
  324. }
  325. delete(jsonObject, name)
  326. fd := xt.TypeDescriptor()
  327. if fd.ContainingMessage().FullName() != m.Descriptor().FullName() {
  328. return fmt.Errorf("extension field %q does not extend message %q", xname, m.Descriptor().FullName())
  329. }
  330. field := m.NewField(fd)
  331. // Unmarshal the field value.
  332. if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
  333. continue
  334. }
  335. v, err := u.unmarshalValue(field, raw, fd)
  336. if err != nil {
  337. return err
  338. }
  339. m.Set(fd, v)
  340. }
  341. if !u.AllowUnknownFields && len(jsonObject) > 0 {
  342. for name := range jsonObject {
  343. return fmt.Errorf("unknown field %q in %v", name, md.FullName())
  344. }
  345. }
  346. return nil
  347. }
  348. func isSingularWellKnownValue(fd protoreflect.FieldDescriptor) bool {
  349. if fd.Cardinality() == protoreflect.Repeated {
  350. return false
  351. }
  352. if md := fd.Message(); md != nil {
  353. return md.FullName() == "google.protobuf.Value"
  354. }
  355. if ed := fd.Enum(); ed != nil {
  356. return ed.FullName() == "google.protobuf.NullValue"
  357. }
  358. return false
  359. }
  360. func isSingularJSONPBUnmarshaler(v protoreflect.Value, fd protoreflect.FieldDescriptor) bool {
  361. if fd.Message() != nil && fd.Cardinality() != protoreflect.Repeated {
  362. _, ok := proto.MessageV1(v.Interface()).(JSONPBUnmarshaler)
  363. return ok
  364. }
  365. return false
  366. }
  367. func (u *Unmarshaler) unmarshalValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
  368. switch {
  369. case fd.IsList():
  370. var jsonArray []json.RawMessage
  371. if err := json.Unmarshal(in, &jsonArray); err != nil {
  372. return v, err
  373. }
  374. lv := v.List()
  375. for _, raw := range jsonArray {
  376. ve, err := u.unmarshalSingularValue(lv.NewElement(), raw, fd)
  377. if err != nil {
  378. return v, err
  379. }
  380. lv.Append(ve)
  381. }
  382. return v, nil
  383. case fd.IsMap():
  384. var jsonObject map[string]json.RawMessage
  385. if err := json.Unmarshal(in, &jsonObject); err != nil {
  386. return v, err
  387. }
  388. kfd := fd.MapKey()
  389. vfd := fd.MapValue()
  390. mv := v.Map()
  391. for key, raw := range jsonObject {
  392. var kv protoreflect.MapKey
  393. if kfd.Kind() == protoreflect.StringKind {
  394. kv = protoreflect.ValueOf(key).MapKey()
  395. } else {
  396. v, err := u.unmarshalSingularValue(kfd.Default(), []byte(key), kfd)
  397. if err != nil {
  398. return v, err
  399. }
  400. kv = v.MapKey()
  401. }
  402. vv, err := u.unmarshalSingularValue(mv.NewValue(), raw, vfd)
  403. if err != nil {
  404. return v, err
  405. }
  406. mv.Set(kv, vv)
  407. }
  408. return v, nil
  409. default:
  410. return u.unmarshalSingularValue(v, in, fd)
  411. }
  412. }
  413. var nonFinite = map[string]float64{
  414. `"NaN"`: math.NaN(),
  415. `"Infinity"`: math.Inf(+1),
  416. `"-Infinity"`: math.Inf(-1),
  417. }
  418. func (u *Unmarshaler) unmarshalSingularValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
  419. switch fd.Kind() {
  420. case protoreflect.BoolKind:
  421. return unmarshalValue(in, new(bool))
  422. case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
  423. return unmarshalValue(trimQuote(in), new(int32))
  424. case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
  425. return unmarshalValue(trimQuote(in), new(int64))
  426. case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
  427. return unmarshalValue(trimQuote(in), new(uint32))
  428. case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
  429. return unmarshalValue(trimQuote(in), new(uint64))
  430. case protoreflect.FloatKind:
  431. if f, ok := nonFinite[string(in)]; ok {
  432. return protoreflect.ValueOfFloat32(float32(f)), nil
  433. }
  434. return unmarshalValue(trimQuote(in), new(float32))
  435. case protoreflect.DoubleKind:
  436. if f, ok := nonFinite[string(in)]; ok {
  437. return protoreflect.ValueOfFloat64(float64(f)), nil
  438. }
  439. return unmarshalValue(trimQuote(in), new(float64))
  440. case protoreflect.StringKind:
  441. return unmarshalValue(in, new(string))
  442. case protoreflect.BytesKind:
  443. return unmarshalValue(in, new([]byte))
  444. case protoreflect.EnumKind:
  445. if hasPrefixAndSuffix('"', in, '"') {
  446. vd := fd.Enum().Values().ByName(protoreflect.Name(trimQuote(in)))
  447. if vd == nil {
  448. return v, fmt.Errorf("unknown value %q for enum %s", in, fd.Enum().FullName())
  449. }
  450. return protoreflect.ValueOfEnum(vd.Number()), nil
  451. }
  452. return unmarshalValue(in, new(protoreflect.EnumNumber))
  453. case protoreflect.MessageKind, protoreflect.GroupKind:
  454. err := u.unmarshalMessage(v.Message(), in)
  455. return v, err
  456. default:
  457. panic(fmt.Sprintf("invalid kind %v", fd.Kind()))
  458. }
  459. }
  460. func unmarshalValue(in []byte, v interface{}) (protoreflect.Value, error) {
  461. err := json.Unmarshal(in, v)
  462. return protoreflect.ValueOf(reflect.ValueOf(v).Elem().Interface()), err
  463. }
  464. func unquoteString(in string) (out string, err error) {
  465. err = json.Unmarshal([]byte(in), &out)
  466. return out, err
  467. }
  468. func hasPrefixAndSuffix(prefix byte, in []byte, suffix byte) bool {
  469. if len(in) >= 2 && in[0] == prefix && in[len(in)-1] == suffix {
  470. return true
  471. }
  472. return false
  473. }
  474. // trimQuote is like unquoteString but simply strips surrounding quotes.
  475. // This is incorrect, but is behavior done by the legacy implementation.
  476. func trimQuote(in []byte) []byte {
  477. if len(in) >= 2 && in[0] == '"' && in[len(in)-1] == '"' {
  478. in = in[1 : len(in)-1]
  479. }
  480. return in
  481. }