AllKeys() includes all keys / AllSettings() includes overridden neste…

…d values

* Function AllKeys() now returns all keys holding a value (for nested values,
  the nested key is the full path, i.e., a sequence of dot-separated keys).
  Previously, returned only depth-1 keys, as well as flags and environment
  variables: this is more generic and may be used widely.
  Besides, it takes into account shadowed keys (key ignored if shadowed by
  a path at a higher-priority level).

* Function AllSettings() now returns nested maps for all keys holding a value,
  as specified by AllKeys().
  The value stored in the map is the one with highest priority, as returned
  by the Get() function (taking into account aliases, environment variables,
  flags, etc.).

This fixes Unmarshal(): it fills in correct values for nested configuration
elements overridden by flags or env variables.

+ tests fixed accordingly
+ test added to TestShadowedNestedValue(), to test Unmarshalling of shadowed keys

viper.go

@@ -1282,55 +1282,114 @@ func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface
 	return v.kvstore, err
 }
 
-// AllKeys returns all keys regardless where they are set.
+// AllKeys returns all keys holding a value, regardless of where they are set.
+// Nested keys are returned with a v.keyDelim (= ".") separator
 func AllKeys() []string { return v.AllKeys() }
 func (v *Viper) AllKeys() []string {
-	m := map[string]struct{}{}
-
-	for key := range v.defaults {
-		m[strings.ToLower(key)] = struct{}{}
-	}
-
-	for key := range v.pflags {
-		m[strings.ToLower(key)] = struct{}{}
-	}
-
-	for key := range v.env {
-		m[strings.ToLower(key)] = struct{}{}
+	m := map[string]bool{}
+	// add all paths, by order of descending priority to ensure correct shadowing
+	m = v.flattenAndMergeMap(m, castMapStringToMapInterface(v.aliases), "")
+	m = v.flattenAndMergeMap(m, v.override, "")
+	m = v.mergeFlatMap(m, v.pflags)
+	m = v.mergeFlatMap(m, v.env)
+	m = v.flattenAndMergeMap(m, v.config, "")
+	m = v.flattenAndMergeMap(m, v.kvstore, "")
+	m = v.flattenAndMergeMap(m, v.defaults, "")
+
+	// convert set of paths to list
+	a := []string{}
+	for x := range m {
+		a = append(a, x)
 	}
+	return a
+}
 
-	for key := range v.config {
-		m[strings.ToLower(key)] = struct{}{}
+// flattenAndMergeMap recursively flattens the given map into a map[string]bool
+// of key paths (used as a set, easier to manipulate than a []string):
+// - each path is merged into a single key string, delimited with v.keyDelim (= ".")
+// - if a path is shadowed by an earlier value in the initial shadow map,
+//   it is skipped.
+// The resulting set of paths is merged to the given shadow set at the same time.
+func (v *Viper) flattenAndMergeMap(shadow map[string]bool, m map[string]interface{}, prefix string) map[string]bool {
+	if shadow != nil && prefix != "" && shadow[prefix] {
+		// prefix is shadowed => nothing more to flatten
+		return shadow
 	}
-
-	for key := range v.kvstore {
-		m[strings.ToLower(key)] = struct{}{}
+	if shadow == nil {
+		shadow = make(map[string]bool)
 	}
 
-	for key := range v.override {
-		m[strings.ToLower(key)] = struct{}{}
+	var m2 map[string]interface{}
+	if prefix != "" {
+		prefix += v.keyDelim
 	}
-
-	for key := range v.aliases {
-		m[strings.ToLower(key)] = struct{}{}
+	for k, val := range m {
+		fullKey := prefix + k
+		switch val.(type) {
+		case map[string]interface{}:
+			m2 = val.(map[string]interface{})
+		case map[interface{}]interface{}:
+			m2 = cast.ToStringMap(val)
+		default:
+			// immediate value
+			shadow[strings.ToLower(fullKey)] = true
+			continue
+		}
+		// recursively merge to shadow map
+		shadow = v.flattenAndMergeMap(shadow, m2, fullKey)
 	}
+	return shadow
+}
 
-	a := []string{}
-	for x := range m {
-		a = append(a, x)
+// mergeFlatMap merges the given maps, excluding values of the second map
+// shadowed by values from the first map.
+func (v *Viper) mergeFlatMap(shadow map[string]bool, mi interface{}) map[string]bool {
+	// unify input map
+	var m map[string]interface{}
+	switch mi.(type) {
+	case map[string]string, map[string]FlagValue:
+		m = cast.ToStringMap(mi)
+	default:
+		return shadow
+	}
+
+	// scan keys
+outer:
+	for k, _ := range m {
+		path := strings.Split(k, v.keyDelim)
+		// scan intermediate paths
+		var parentKey string
+		for i := 1; i < len(path); i++ {
+			parentKey = strings.Join(path[0:i], v.keyDelim)
+			if shadow[parentKey] {
+				// path is shadowed, continue
+				continue outer
+			}
+		}
+		// add key
+		shadow[strings.ToLower(k)] = true
 	}
-
-	return a
+	return shadow
 }
 
-// AllSettings returns all settings as a map[string]interface{}.
+// AllSettings merges all settings and returns them as a map[string]interface{}.
 func AllSettings() map[string]interface{} { return v.AllSettings() }
 func (v *Viper) AllSettings() map[string]interface{} {
 	m := map[string]interface{}{}
-	for _, x := range v.AllKeys() {
-		m[x] = v.Get(x)
+	// start from the list of keys, and construct the map one value at a time
+	for _, k := range v.AllKeys() {
+		value := v.Get(k)
+		if value == nil {
+			// should not happen, since AllKeys() returns only keys holding a value,
+			// check just in case anything changes
+			continue
+		}
+		path := strings.Split(k, v.keyDelim)
+		lastKey := strings.ToLower(path[len(path)-1])
+		deepestMap := deepSearch(m, path[0:len(path)-1])
+		// set innermost value
+		deepestMap[lastKey] = value
 	}
-
 	return m
 }
 

viper_test.go

@@ -422,9 +422,9 @@ func TestSetEnvReplacer(t *testing.T) {
 func TestAllKeys(t *testing.T) {
 	initConfigs()
 
-	ks := sort.StringSlice{"title", "newkey", "owner", "name", "beard", "ppu", "batters", "hobbies", "clothing", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters", "p_type", "p_name", "foos"}
+	ks := sort.StringSlice{"title", "newkey", "owner.organization", "owner.dob", "owner.bio", "name", "beard", "ppu", "batters.batter", "hobbies", "clothing.jacket", "clothing.trousers", "clothing.pants.size", "age", "hacker", "id", "type", "eyes", "p_id", "p_ppu", "p_batters.batter.type", "p_type", "p_name", "foos"}
 	dob, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z")
-	all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "Bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[string]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[interface{}]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters": map[string]interface{}{"batter": map[string]interface{}{"type": "Regular"}}, "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}}
+	all := map[string]interface{}{"owner": map[string]interface{}{"organization": "MongoDB", "bio": "MongoDB Chief Developer Advocate & Hacker at Large", "dob": dob}, "title": "TOML Example", "ppu": 0.55, "eyes": "brown", "clothing": map[string]interface{}{"trousers": "denim", "jacket": "leather", "pants": map[string]interface{}{"size": "large"}}, "id": "0001", "batters": map[string]interface{}{"batter": []interface{}{map[string]interface{}{"type": "Regular"}, map[string]interface{}{"type": "Chocolate"}, map[string]interface{}{"type": "Blueberry"}, map[string]interface{}{"type": "Devil's Food"}}}, "hacker": true, "beard": true, "hobbies": []interface{}{"skateboarding", "snowboarding", "go"}, "age": 35, "type": "donut", "newkey": "remote", "name": "Cake", "p_id": "0001", "p_ppu": "0.55", "p_name": "Cake", "p_batters": map[string]interface{}{"batter": map[string]interface{}{"type": "Regular"}}, "p_type": "donut", "foos": []map[string]interface{}{map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"key": 1}, map[string]interface{}{"key": 2}, map[string]interface{}{"key": 3}, map[string]interface{}{"key": 4}}}}}
 
 	var allkeys sort.StringSlice
 	allkeys = AllKeys()
@@ -883,13 +883,14 @@ func TestMergeConfigNoMerge(t *testing.T) {
 }
 
 func TestUnmarshalingWithAliases(t *testing.T) {
-	SetDefault("ID", 1)
-	Set("name", "Steve")
-	Set("lastname", "Owen")
+	v := New()
+	v.SetDefault("ID", 1)
+	v.Set("name", "Steve")
+	v.Set("lastname", "Owen")
 
-	RegisterAlias("UserID", "ID")
-	RegisterAlias("Firstname", "name")
-	RegisterAlias("Surname", "lastname")
+	v.RegisterAlias("UserID", "ID")
+	v.RegisterAlias("Firstname", "name")
+	v.RegisterAlias("Surname", "lastname")
 
 	type config struct {
 		ID        int
@@ -898,8 +899,7 @@ func TestUnmarshalingWithAliases(t *testing.T) {
 	}
 
 	var C config
-
-	err := Unmarshal(&C)
+	err := v.Unmarshal(&C)
 	if err != nil {
 		t.Fatalf("unable to decode into struct, %v", err)
 	}
@@ -922,6 +922,10 @@ func TestShadowedNestedValue(t *testing.T) {
 	assert.Equal(t, "leather", GetString("clothing.jacket"))
 	assert.Nil(t, Get("clothing.jacket.price"))
 	assert.Equal(t, polyester, GetString("clothing.shirt"))
+
+	clothingSettings := AllSettings()["clothing"].(map[string]interface{})
+	assert.Equal(t, "leather", clothingSettings["jacket"])
+	assert.Equal(t, polyester, clothingSettings["shirt"])
 }
 
 func TestDotParameter(t *testing.T) {