460 lines
15 KiB
Go
460 lines
15 KiB
Go
// Package yaml provides a wrapper around go-yaml designed to enable a better
|
|
// way of handling YAML when marshaling to and from structs.
|
|
//
|
|
// In short, this package first converts YAML to JSON using go-yaml and then
|
|
// uses json.Marshal and json.Unmarshal to convert to or from the struct. This
|
|
// means that it effectively reuses the JSON struct tags as well as the custom
|
|
// JSON methods MarshalJSON and UnmarshalJSON unlike go-yaml.
|
|
package yaml // import "github.com/invopop/yaml"
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"reflect"
|
|
"strconv"
|
|
|
|
yaml "github.com/oasdiff/yaml3"
|
|
)
|
|
|
|
// Marshal the object into JSON then converts JSON to YAML and returns the
|
|
// YAML.
|
|
func Marshal(o interface{}) ([]byte, error) {
|
|
j, err := json.Marshal(o)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error marshaling into JSON: %v", err)
|
|
}
|
|
|
|
y, err := JSONToYAML(j)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error converting JSON to YAML: %v", err)
|
|
}
|
|
|
|
return y, nil
|
|
}
|
|
|
|
// JSONOpt is a decoding option for decoding from JSON format.
|
|
type JSONOpt func(*json.Decoder) *json.Decoder
|
|
|
|
// OriginOpt controls origin-tracking behavior. When Enabled is false the
|
|
// OriginTree returned by Unmarshal is nil.
|
|
type OriginOpt struct {
|
|
// Enabled adds __origin__ metadata to maps during unmarshaling.
|
|
Enabled bool
|
|
// File is the source file name recorded in origin metadata.
|
|
File string
|
|
}
|
|
|
|
// DecodeOpts groups options that apply to the YAML decoder side, as opposed
|
|
// to JSONOpt which configures the JSON unmarshal step.
|
|
type DecodeOpts struct {
|
|
// Origin controls origin-tracking behavior. When Origin.Enabled is
|
|
// false the OriginTree returned by Unmarshal is nil.
|
|
Origin OriginOpt
|
|
// DisableTimestamps suppresses YAML 1.1 implicit-timestamp resolution.
|
|
// When true, untagged date-shaped scalars (e.g. "1344-08-22") resolve
|
|
// to strings instead of time.Time, which keeps map keys stable for
|
|
// real-world specs that use date-shaped strings as keys. Explicit
|
|
// "!!timestamp" tags in the source still resolve to time.Time.
|
|
DisableTimestamps bool
|
|
}
|
|
|
|
// OriginTree holds __origin__ data extracted from a YAML-decoded map tree.
|
|
// It mirrors the structure of the spec: Fields tracks map children by key,
|
|
// Items tracks slice children by index.
|
|
type OriginTree struct {
|
|
// File is the source file for all origins in this subtree.
|
|
// Set once at the root of each decode call; all nodes in the same
|
|
// decode share the same file.
|
|
File string
|
|
// Origin is the raw __origin__ value ([]any compact sequence) for this node.
|
|
// Format: [key_name, key_line, key_col, nf, f1_name, f1_delta, f1_col, ..., ns, ...]
|
|
Origin any
|
|
// Fields holds child trees keyed by map key name.
|
|
Fields map[string]*OriginTree
|
|
// Items holds child trees for slice elements (index-aligned).
|
|
Items []*OriginTree
|
|
}
|
|
|
|
// Unmarshal converts YAML to JSON then uses JSON to unmarshal
|
|
// into o. It is the single public unmarshal entry point: pass DecodeOpts{}
|
|
// for the simple case, or set Origin / DisableTimestamps to opt into
|
|
// origin tracking or YAML 1.1 timestamp-resolution suppression. The
|
|
// variadic JSONOpt list configures the JSON unmarshal step. The returned
|
|
// OriginTree is nil when origin tracking is disabled.
|
|
func Unmarshal(y []byte, o interface{}, decode DecodeOpts, opts ...JSONOpt) (*OriginTree, error) {
|
|
dec := yaml.NewDecoder(bytes.NewReader(y))
|
|
dec.Origin(decode.Origin.Enabled, decode.Origin.File)
|
|
if decode.DisableTimestamps {
|
|
dec.DisableTimestamps(true)
|
|
}
|
|
|
|
// Decode YAML into a generic object.
|
|
var yamlObj interface{}
|
|
if err := dec.Decode(&yamlObj); err != nil {
|
|
if !errors.Is(err, io.EOF) {
|
|
return nil, fmt.Errorf("error converting YAML to JSON: %v", err)
|
|
}
|
|
}
|
|
|
|
// Extract __origin__ before JSON conversion so the JSON stays small.
|
|
var tree *OriginTree
|
|
if decode.Origin.Enabled {
|
|
tree = extractOrigins(yamlObj, decode.Origin.File)
|
|
}
|
|
|
|
// Convert to JSON (without __origin__) and unmarshal into the target struct.
|
|
vo := reflect.ValueOf(o)
|
|
jsonObj, err := convertToJSONableObject(yamlObj, &vo)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error converting YAML to JSON: %v", err)
|
|
}
|
|
j, err := json.Marshal(jsonObj)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error converting YAML to JSON: %v", err)
|
|
}
|
|
if err := jsonUnmarshal(bytes.NewReader(j), o, opts...); err != nil {
|
|
return nil, fmt.Errorf("error unmarshaling JSON: %v", err)
|
|
}
|
|
|
|
return tree, nil
|
|
}
|
|
|
|
const originKey = "__origin__"
|
|
|
|
// extractOrigins recursively extracts and removes __origin__ entries from a
|
|
// YAML-decoded map tree, returning the origin data as an OriginTree.
|
|
// file is the source file for all nodes in this decode call.
|
|
func extractOrigins(v any, file string) *OriginTree {
|
|
switch val := v.(type) {
|
|
case map[string]any:
|
|
return extractOriginsFromStringMap(val, file)
|
|
case map[interface{}]interface{}:
|
|
// yaml3 produces map[interface{}]interface{} when map keys are non-string
|
|
// (e.g. integer HTTP status codes like 200). __origin__ is always a string
|
|
// key, so we must handle this case to strip it from mixed-key maps.
|
|
tree := &OriginTree{File: file}
|
|
if orig, ok := val[originKey]; ok {
|
|
tree.Origin = orig
|
|
delete(val, originKey)
|
|
}
|
|
for k, child := range val {
|
|
if childTree := extractOrigins(child, file); childTree != nil {
|
|
if tree.Fields == nil {
|
|
tree.Fields = make(map[string]*OriginTree)
|
|
}
|
|
// Convert key to string: mirrors convertToJSONableObject behaviour.
|
|
// String keys pass through; int/int64/float64 keys are formatted.
|
|
var ks string
|
|
switch kt := k.(type) {
|
|
case string:
|
|
ks = kt
|
|
case int:
|
|
ks = strconv.Itoa(kt)
|
|
case int64:
|
|
ks = strconv.FormatInt(kt, 10)
|
|
case float64:
|
|
ks = strconv.FormatFloat(kt, 'g', -1, 64)
|
|
default:
|
|
ks = fmt.Sprintf("%v", k)
|
|
}
|
|
tree.Fields[ks] = childTree
|
|
}
|
|
}
|
|
if tree.Origin == nil && tree.Fields == nil {
|
|
return nil
|
|
}
|
|
return tree
|
|
case []any:
|
|
var items []*OriginTree
|
|
hasChild := false
|
|
for _, child := range val {
|
|
childTree := extractOrigins(child, file)
|
|
items = append(items, childTree) // may be nil; preserves index alignment
|
|
if childTree != nil {
|
|
hasChild = true
|
|
}
|
|
}
|
|
if !hasChild {
|
|
return nil
|
|
}
|
|
return &OriginTree{File: file, Items: items}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func extractOriginsFromStringMap(val map[string]any, file string) *OriginTree {
|
|
tree := &OriginTree{File: file}
|
|
if orig, ok := val[originKey]; ok {
|
|
tree.Origin = orig
|
|
delete(val, originKey)
|
|
}
|
|
for k, child := range val {
|
|
if childTree := extractOrigins(child, file); childTree != nil {
|
|
if tree.Fields == nil {
|
|
tree.Fields = make(map[string]*OriginTree)
|
|
}
|
|
tree.Fields[k] = childTree
|
|
}
|
|
}
|
|
if tree.Origin == nil && tree.Fields == nil {
|
|
return nil
|
|
}
|
|
return tree
|
|
}
|
|
|
|
// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the
|
|
// object, optionally applying decoder options prior to decoding. We are not
|
|
// using json.Unmarshal directly as we want the chance to pass in non-default
|
|
// options.
|
|
func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error {
|
|
d := json.NewDecoder(r)
|
|
for _, opt := range opts {
|
|
d = opt(d)
|
|
}
|
|
if err := d.Decode(&o); err != nil {
|
|
return fmt.Errorf("while decoding JSON: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// JSONToYAML converts JSON to YAML.
|
|
func JSONToYAML(j []byte) ([]byte, error) {
|
|
// Convert the JSON to an object.
|
|
var jsonObj interface{}
|
|
// We are using yaml.Unmarshal here (instead of json.Unmarshal) because the
|
|
// Go JSON library doesn't try to pick the right number type (int, float,
|
|
// etc.) when unmarshalling to interface{}, it just picks float64
|
|
// universally. go-yaml does go through the effort of picking the right
|
|
// number type, so we can preserve number type throughout this process.
|
|
err := yaml.Unmarshal(j, &jsonObj)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Marshal this object into YAML.
|
|
return yaml.Marshal(jsonObj)
|
|
}
|
|
|
|
// YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML,
|
|
// passing JSON through this method should be a no-op.
|
|
//
|
|
// Things YAML can do that are not supported by JSON:
|
|
// - In YAML you can have binary and null keys in your maps. These are invalid
|
|
// in JSON. (int and float keys are converted to strings.)
|
|
// - Binary data in YAML with the !!binary tag is not supported. If you want to
|
|
// use binary data with this library, encode the data as base64 as usual but do
|
|
// not use the !!binary tag in your YAML. This will ensure the original base64
|
|
// encoded data makes it all the way through to the JSON.
|
|
func YAMLToJSON(y []byte) ([]byte, error) { //nolint:revive
|
|
dec := yaml.NewDecoder(bytes.NewReader(y))
|
|
return yamlToJSON(dec, nil)
|
|
}
|
|
|
|
func yamlToJSON(dec *yaml.Decoder, jsonTarget *reflect.Value) ([]byte, error) {
|
|
// Convert the YAML to an object.
|
|
var yamlObj interface{}
|
|
if err := dec.Decode(&yamlObj); err != nil {
|
|
// Functionality changed in v3 which means we need to ignore EOF error.
|
|
// See https://github.com/go-yaml/yaml/issues/639
|
|
if !errors.Is(err, io.EOF) {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// YAML objects are not completely compatible with JSON objects (e.g. you
|
|
// can have non-string keys in YAML). So, convert the YAML-compatible object
|
|
// to a JSON-compatible object, failing with an error if irrecoverable
|
|
// incompatibilities happen along the way.
|
|
jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Convert this object to JSON and return the data.
|
|
return json.Marshal(jsonObj)
|
|
}
|
|
|
|
// convertToJSONableObject converts a YAML object to a JSON-compatible object.
|
|
func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { //nolint:gocyclo
|
|
var err error
|
|
|
|
// Resolve jsonTarget to a concrete value (i.e. not a pointer or an
|
|
// interface). We pass decodingNull as false because we're not actually
|
|
// decoding into the value, we're just checking if the ultimate target is a
|
|
// string.
|
|
if jsonTarget != nil {
|
|
ju, tu, pv := indirect(*jsonTarget, false)
|
|
// We have a JSON or Text Umarshaler at this level, so we can't be trying
|
|
// to decode into a string.
|
|
if ju != nil || tu != nil {
|
|
jsonTarget = nil
|
|
} else {
|
|
jsonTarget = &pv
|
|
}
|
|
}
|
|
|
|
// go-yaml v3 changed from v2 and now will provide map[string]interface{} by
|
|
// default and map[interface{}]interface{} when none of the keys strings.
|
|
// To get around this, we run a pre-loop to convert the map.
|
|
// JSON only supports strings as keys, so we must convert.
|
|
|
|
switch typedYAMLObj := yamlObj.(type) {
|
|
case map[interface{}]interface{}:
|
|
// From my reading of go-yaml v2 (specifically the resolve function),
|
|
// keys can only have the types string, int, int64, float64, binary
|
|
// (unsupported), or null (unsupported).
|
|
strMap := make(map[string]interface{})
|
|
for k, v := range typedYAMLObj {
|
|
// Resolve the key to a string first.
|
|
var keyString string
|
|
switch typedKey := k.(type) {
|
|
case string:
|
|
keyString = typedKey
|
|
case int:
|
|
keyString = strconv.Itoa(typedKey)
|
|
case int64:
|
|
// go-yaml will only return an int64 as a key if the system
|
|
// architecture is 32-bit and the key's value is between 32-bit
|
|
// and 64-bit. Otherwise the key type will simply be int.
|
|
keyString = strconv.FormatInt(typedKey, 10)
|
|
case float64:
|
|
// Float64 is now supported in keys
|
|
keyString = strconv.FormatFloat(typedKey, 'g', -1, 64)
|
|
case bool:
|
|
if typedKey {
|
|
keyString = "true"
|
|
} else {
|
|
keyString = "false"
|
|
}
|
|
default:
|
|
return nil, fmt.Errorf("unsupported map key of type: %s, key: %+#v, value: %+#v",
|
|
reflect.TypeOf(k), k, v)
|
|
}
|
|
strMap[keyString] = v
|
|
}
|
|
// replace yamlObj with our new string map
|
|
yamlObj = strMap
|
|
}
|
|
|
|
// If yamlObj is a number or a boolean, check if jsonTarget is a string -
|
|
// if so, coerce. Else return normal.
|
|
// If yamlObj is a map or array, find the field that each key is
|
|
// unmarshaling to, and when you recurse pass the reflect.Value for that
|
|
// field back into this function.
|
|
switch typedYAMLObj := yamlObj.(type) {
|
|
case map[string]interface{}:
|
|
for k, v := range typedYAMLObj {
|
|
|
|
// jsonTarget should be a struct or a map. If it's a struct, find
|
|
// the field it's going to map to and pass its reflect.Value. If
|
|
// it's a map, find the element type of the map and pass the
|
|
// reflect.Value created from that type. If it's neither, just pass
|
|
// nil - JSON conversion will error for us if it's a real issue.
|
|
if jsonTarget != nil {
|
|
t := *jsonTarget
|
|
if t.Kind() == reflect.Struct {
|
|
keyBytes := []byte(k)
|
|
// Find the field that the JSON library would use.
|
|
var f *field
|
|
fields := cachedTypeFields(t.Type())
|
|
for i := range fields {
|
|
ff := &fields[i]
|
|
if bytes.Equal(ff.nameBytes, keyBytes) {
|
|
f = ff
|
|
break
|
|
}
|
|
// Do case-insensitive comparison.
|
|
if f == nil && ff.equalFold(ff.nameBytes, keyBytes) {
|
|
f = ff
|
|
}
|
|
}
|
|
if f != nil {
|
|
// Find the reflect.Value of the most preferential
|
|
// struct field.
|
|
jtf := t.Field(f.index[0])
|
|
typedYAMLObj[k], err = convertToJSONableObject(v, &jtf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
continue
|
|
}
|
|
} else if t.Kind() == reflect.Map {
|
|
// Create a zero value of the map's element type to use as
|
|
// the JSON target.
|
|
jtv := reflect.Zero(t.Type().Elem())
|
|
typedYAMLObj[k], err = convertToJSONableObject(v, &jtv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
typedYAMLObj[k], err = convertToJSONableObject(v, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return typedYAMLObj, nil
|
|
case []interface{}:
|
|
// We need to recurse into arrays in case there are any
|
|
// map[interface{}]interface{}'s inside and to convert any
|
|
// numbers to strings.
|
|
|
|
// If jsonTarget is a slice (which it really should be), find the
|
|
// thing it's going to map to. If it's not a slice, just pass nil
|
|
// - JSON conversion will error for us if it's a real issue.
|
|
var jsonSliceElemValue *reflect.Value
|
|
if jsonTarget != nil {
|
|
t := *jsonTarget
|
|
if t.Kind() == reflect.Slice {
|
|
// By default slices point to nil, but we need a reflect.Value
|
|
// pointing to a value of the slice type, so we create one here.
|
|
ev := reflect.Indirect(reflect.New(t.Type().Elem()))
|
|
jsonSliceElemValue = &ev
|
|
}
|
|
}
|
|
|
|
// Make and use a new array.
|
|
arr := make([]interface{}, len(typedYAMLObj))
|
|
for i, v := range typedYAMLObj {
|
|
arr[i], err = convertToJSONableObject(v, jsonSliceElemValue)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return arr, nil
|
|
default:
|
|
// If the target type is a string and the YAML type is a number,
|
|
// convert the YAML type to a string.
|
|
if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String {
|
|
// Based on my reading of go-yaml, it may return int, int64,
|
|
// float64, or uint64.
|
|
var s string
|
|
switch typedVal := typedYAMLObj.(type) {
|
|
case int:
|
|
s = strconv.FormatInt(int64(typedVal), 10)
|
|
case int64:
|
|
s = strconv.FormatInt(typedVal, 10)
|
|
case float64:
|
|
s = strconv.FormatFloat(typedVal, 'g', -1, 64)
|
|
case uint64:
|
|
s = strconv.FormatUint(typedVal, 10)
|
|
case bool:
|
|
if typedVal {
|
|
s = "true"
|
|
} else {
|
|
s = "false"
|
|
}
|
|
}
|
|
if len(s) > 0 {
|
|
yamlObj = interface{}(s)
|
|
}
|
|
}
|
|
return yamlObj, nil
|
|
}
|
|
}
|