Skip to content

Commit

Permalink
Exposing memory.numa_stats
Browse files Browse the repository at this point in the history 
Making information on page usage by type and NUMA node available

Signed-off-by: Maciej "Iwan" Iwanowski <maciej.iwanowski@intel.com>
  • Loading branch information
iwankgb committed Mar 30, 2020
1 parent f1eea90 commit 8a031e3
Show file tree
Hide file tree
Showing 5 changed files with 212 additions and 61 deletions.
96 changes: 94 additions & 2 deletions libcontainer/cgroups/fs/memory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ package fs
import (
"bufio"
"fmt"
"math"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"

Expand All @@ -16,10 +19,19 @@ import (
)

const (
cgroupMemorySwapLimit = "memory.memsw.limit_in_bytes"
cgroupMemoryLimit = "memory.limit_in_bytes"
numaStatColumnSeparator = " "
numaStatKeyValueSeparator = "="
numaStatMaxColumns = math.MaxUint8 + 1
numaStatValueIndex = 1
numaStatTypeIndex = 0
numaStatColumnSliceLength = 2
cgroupMemorySwapLimit = "memory.memsw.limit_in_bytes"
cgroupMemoryLimit = "memory.limit_in_bytes"
cgroupMemoryPagesByNuma = "memory.numa_stat"
)

var isNUMANode = regexp.MustCompile("N[0-9]+")

type MemoryGroup struct {
}

Expand Down Expand Up @@ -209,6 +221,13 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
if value == 1 {
stats.MemoryStats.UseHierarchy = true
}

pagesByNUMA, err := getPageUsageByNUMA(path)
if err != nil {
return err
}
stats.MemoryStats.PageUsageByNUMA = pagesByNUMA

return nil
}

Expand Down Expand Up @@ -269,3 +288,76 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) {

return memoryData, nil
}

func getPageUsageByNUMA(cgroupPath string) (cgroups.PageUsageByNUMA, error) {
stats := cgroups.PageUsageByNUMA{}

file, err := os.Open(path.Join(cgroupPath, cgroupMemoryPagesByNuma))
if err != nil {
return stats, err
}

scanner := bufio.NewScanner(file)
for scanner.Scan() {
var statsType string
statsByType := cgroups.PageStats{Nodes: map[uint8]uint64{}}
columns := strings.SplitN(scanner.Text(), numaStatColumnSeparator, numaStatMaxColumns)

for _, column := range columns {
pagesByNode := strings.SplitN(column, numaStatKeyValueSeparator, numaStatColumnSliceLength)
if len(pagesByNode) != numaStatColumnSliceLength {
return cgroups.PageUsageByNUMA{}, fmt.Errorf("wrong data format, found %d fields instead of 2", len(pagesByNode))
}

if isNUMANode.MatchString(pagesByNode[numaStatTypeIndex]) {
nodeID, err := strconv.ParseUint(pagesByNode[numaStatTypeIndex][1:], 10, 8)
if err != nil {
return cgroups.PageUsageByNUMA{}, err
}

statsByType.Nodes[uint8(nodeID)], err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
if err != nil {
return cgroups.PageUsageByNUMA{}, err
}
} else {
statsByType.Total, err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
if err != nil {
return cgroups.PageUsageByNUMA{}, err
}

statsType = pagesByNode[numaStatTypeIndex]
}

err := addNUMAStatsByType(&stats, statsByType, statsType)
if err != nil {
return cgroups.PageUsageByNUMA{}, err
}
}
}

return stats, nil
}

func addNUMAStatsByType(stats *cgroups.PageUsageByNUMA, byTypeStats cgroups.PageStats, statsType string) error {
switch statsType {
case "total":
stats.Total = byTypeStats
case "file":
stats.File = byTypeStats
case "anon":
stats.Anon = byTypeStats
case "unevictable":
stats.Unevictable = byTypeStats
case "hierarchical_total":
stats.Hierarchical.Total = byTypeStats
case "hierarchical_file":
stats.Hierarchical.File = byTypeStats
case "hierarchical_anon":
stats.Hierarchical.Anon = byTypeStats
case "hierarchical_unevictable":
stats.Hierarchical.Unevictable = byTypeStats
default:
return fmt.Errorf("unsupported NUMA page type found: %s", statsType)
}
return nil
}
52 changes: 51 additions & 1 deletion libcontainer/cgroups/fs/memory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,18 @@ rss 1024`
memoryFailcnt = "100\n"
memoryLimitContents = "8192\n"
memoryUseHierarchyContents = "1\n"
memoryNUMAStatContents = `total=44611 N0=32631 N1=7501 N2=1982 N3=2497
file=44428 N0=32614 N1=7335 N2=1982 N3=2497
anon=183 N0=17 N1=166 N2=0 N3=0
unevictable=0 N0=0 N1=0 N2=0 N3=0
hierarchical_total=768133 N0=509113 N1=138887 N2=20464 N3=99669
hierarchical_file=722017 N0=496516 N1=119997 N2=20181 N3=85323
hierarchical_anon=46096 N0=12597 N1=18890 N2=283 N3=14326
hierarchical_unevictable=20 N0=0 N1=0 N2=0 N3=20`
memoryNUMAStatNoHierarchyContents = `total=44611 N0=32631 N1=7501 N2=1982 N3=2497
file=44428 N0=32614 N1=7335 N2=1982 N3=2497
anon=183 N0=17 N1=166 N2=0 N3=0
unevictable=0 N0=0 N1=0 N2=0 N3=0`
)

func TestMemorySetMemory(t *testing.T) {
Expand Down Expand Up @@ -276,6 +288,7 @@ func TestMemoryStats(t *testing.T) {
"memory.kmem.failcnt": memoryFailcnt,
"memory.kmem.limit_in_bytes": memoryLimitContents,
"memory.use_hierarchy": memoryUseHierarchyContents,
"memory.numa_stat": memoryNUMAStatContents,
})

memory := &MemoryGroup{}
Expand All @@ -284,7 +297,21 @@ func TestMemoryStats(t *testing.T) {
if err != nil {
t.Fatal(err)
}
expectedStats := cgroups.MemoryStats{Cache: 512, Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, Stats: map[string]uint64{"cache": 512, "rss": 1024}, UseHierarchy: true}
expectedStats := cgroups.MemoryStats{Cache: 512, Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, Stats: map[string]uint64{"cache": 512, "rss": 1024}, UseHierarchy: true,
PageUsageByNUMA: cgroups.PageUsageByNUMA{
PageUsageByNUMAInner: cgroups.PageUsageByNUMAInner{
Total: cgroups.PageStats{Total: 44611, Nodes: map[uint8]uint64{0: 32631, 1: 7501, 2: 1982, 3: 2497}},
File: cgroups.PageStats{Total: 44428, Nodes: map[uint8]uint64{0: 32614, 1: 7335, 2: 1982, 3: 2497}},
Anon: cgroups.PageStats{Total: 183, Nodes: map[uint8]uint64{0: 17, 1: 166, 2: 0, 3: 0}},
Unevictable: cgroups.PageStats{Total: 0, Nodes: map[uint8]uint64{0: 0, 1: 0, 2: 0, 3: 0}},
},
Hierarchical: cgroups.PageUsageByNUMAInner{
Total: cgroups.PageStats{Total: 768133, Nodes: map[uint8]uint64{0: 509113, 1: 138887, 2: 20464, 3: 99669}},
File: cgroups.PageStats{Total: 722017, Nodes: map[uint8]uint64{0: 496516, 1: 119997, 2: 20181, 3: 85323}},
Anon: cgroups.PageStats{Total: 46096, Nodes: map[uint8]uint64{0: 12597, 1: 18890, 2: 283, 3: 14326}},
Unevictable: cgroups.PageStats{Total: 20, Nodes: map[uint8]uint64{0: 0, 1: 0, 2: 0, 3: 20}},
},
}}
expectMemoryStatEquals(t, expectedStats, actualStats.MemoryStats)
}

Expand Down Expand Up @@ -454,3 +481,26 @@ func TestMemorySetOomControl(t *testing.T) {
t.Fatalf("Got the wrong value, set memory.oom_control failed.")
}
}

func TestNoHierarchicalNumaStat(t *testing.T) {
helper := NewCgroupTestUtil("memory", t)
defer helper.cleanup()
helper.writeFileContents(map[string]string{
"memory.numa_stat": memoryNUMAStatNoHierarchyContents,
})

actualStats, err := getPageUsageByNUMA(helper.CgroupPath)
if err != nil {
t.Fatal(err)
}
pageUsageByNUMA := cgroups.PageUsageByNUMA{
PageUsageByNUMAInner: cgroups.PageUsageByNUMAInner{
Total: cgroups.PageStats{Total: 44611, Nodes: map[uint8]uint64{0: 32631, 1: 7501, 2: 1982, 3: 2497}},
File: cgroups.PageStats{Total: 44428, Nodes: map[uint8]uint64{0: 32614, 1: 7335, 2: 1982, 3: 2497}},
Anon: cgroups.PageStats{Total: 183, Nodes: map[uint8]uint64{0: 17, 1: 166, 2: 0, 3: 0}},
Unevictable: cgroups.PageStats{Total: 0, Nodes: map[uint8]uint64{0: 0, 1: 0, 2: 0, 3: 0}},
},
Hierarchical: cgroups.PageUsageByNUMAInner{},
}
expectPageUsageByNUMAEquals(t, pageUsageByNUMA, actualStats)
}
82 changes: 46 additions & 36 deletions libcontainer/cgroups/fs/stats_util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@ package fs

import (
"fmt"
"reflect"
"testing"

"github.com/opencontainers/runc/libcontainer/cgroups"

"github.com/sirupsen/logrus"
)

func blkioStatEntryEquals(expected, actual []cgroups.BlkioStatEntry) error {
Expand All @@ -26,98 +25,109 @@ func blkioStatEntryEquals(expected, actual []cgroups.BlkioStatEntry) error {

func expectBlkioStatsEquals(t *testing.T, expected, actual cgroups.BlkioStats) {
if err := blkioStatEntryEquals(expected.IoServiceBytesRecursive, actual.IoServiceBytesRecursive); err != nil {
logrus.Printf("blkio IoServiceBytesRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio IoServiceBytesRecursive do not match - %s\n", err)
}

if err := blkioStatEntryEquals(expected.IoServicedRecursive, actual.IoServicedRecursive); err != nil {
logrus.Printf("blkio IoServicedRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio IoServicedRecursive do not match - %s\n", err)
}

if err := blkioStatEntryEquals(expected.IoQueuedRecursive, actual.IoQueuedRecursive); err != nil {
logrus.Printf("blkio IoQueuedRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio IoQueuedRecursive do not match - %s\n", err)
}

if err := blkioStatEntryEquals(expected.SectorsRecursive, actual.SectorsRecursive); err != nil {
logrus.Printf("blkio SectorsRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio SectorsRecursive do not match - %s\n", err)
}

if err := blkioStatEntryEquals(expected.IoServiceTimeRecursive, actual.IoServiceTimeRecursive); err != nil {
logrus.Printf("blkio IoServiceTimeRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio IoServiceTimeRecursive do not match - %s\n", err)
}

if err := blkioStatEntryEquals(expected.IoWaitTimeRecursive, actual.IoWaitTimeRecursive); err != nil {
logrus.Printf("blkio IoWaitTimeRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio IoWaitTimeRecursive do not match - %s\n", err)
}

if err := blkioStatEntryEquals(expected.IoMergedRecursive, actual.IoMergedRecursive); err != nil {
logrus.Printf("blkio IoMergedRecursive do not match - %v vs %v\n", expected.IoMergedRecursive, actual.IoMergedRecursive)
t.Fail()
t.Errorf("blkio IoMergedRecursive do not match - %v vs %v\n", expected.IoMergedRecursive, actual.IoMergedRecursive)
}

if err := blkioStatEntryEquals(expected.IoTimeRecursive, actual.IoTimeRecursive); err != nil {
logrus.Printf("blkio IoTimeRecursive do not match - %s\n", err)
t.Fail()
t.Errorf("blkio IoTimeRecursive do not match - %s\n", err)
}
}

func expectThrottlingDataEquals(t *testing.T, expected, actual cgroups.ThrottlingData) {
if expected != actual {
logrus.Printf("Expected throttling data %v but found %v\n", expected, actual)
t.Fail()
t.Errorf("Expected throttling data %v but found %v\n", expected, actual)
}
}

func expectHugetlbStatEquals(t *testing.T, expected, actual cgroups.HugetlbStats) {
if expected != actual {
logrus.Printf("Expected hugetlb stats %v but found %v\n", expected, actual)
t.Fail()
t.Errorf("Expected hugetlb stats %v but found %v\n", expected, actual)
}
}

func expectMemoryStatEquals(t *testing.T, expected, actual cgroups.MemoryStats) {
expectMemoryDataEquals(t, expected.Usage, actual.Usage)
expectMemoryDataEquals(t, expected.SwapUsage, actual.SwapUsage)
expectMemoryDataEquals(t, expected.KernelUsage, actual.KernelUsage)
expectPageUsageByNUMAEquals(t, expected.PageUsageByNUMA, actual.PageUsageByNUMA)

if expected.UseHierarchy != actual.UseHierarchy {
logrus.Printf("Expected memory use hierarchy %v, but found %v\n", expected.UseHierarchy, actual.UseHierarchy)
t.Fail()
t.Errorf("Expected memory use hierarchy %v, but found %v\n", expected.UseHierarchy, actual.UseHierarchy)
}

for key, expValue := range expected.Stats {
actValue, ok := actual.Stats[key]
if !ok {
logrus.Printf("Expected memory stat key %s not found\n", key)
t.Fail()
t.Errorf("Expected memory stat key %s not found\n", key)
}
if expValue != actValue {
logrus.Printf("Expected memory stat value %d but found %d\n", expValue, actValue)
t.Fail()
t.Errorf("Expected memory stat value %d but found %d\n", expValue, actValue)
}
}
}

func expectMemoryDataEquals(t *testing.T, expected, actual cgroups.MemoryData) {
if expected.Usage != actual.Usage {
logrus.Printf("Expected memory usage %d but found %d\n", expected.Usage, actual.Usage)
t.Fail()
t.Errorf("Expected memory usage %d but found %d\n", expected.Usage, actual.Usage)
}
if expected.MaxUsage != actual.MaxUsage {
logrus.Printf("Expected memory max usage %d but found %d\n", expected.MaxUsage, actual.MaxUsage)
t.Fail()
t.Errorf("Expected memory max usage %d but found %d\n", expected.MaxUsage, actual.MaxUsage)
}
if expected.Failcnt != actual.Failcnt {
logrus.Printf("Expected memory failcnt %d but found %d\n", expected.Failcnt, actual.Failcnt)
t.Fail()
t.Errorf("Expected memory failcnt %d but found %d\n", expected.Failcnt, actual.Failcnt)
}
if expected.Limit != actual.Limit {
logrus.Printf("Expected memory limit %d but found %d\n", expected.Limit, actual.Limit)
t.Fail()
t.Errorf("Expected memory limit %d but found %d\n", expected.Limit, actual.Limit)
}
}

func expectPageUsageByNUMAEquals(t *testing.T, expected, actual cgroups.PageUsageByNUMA) {
if !reflect.DeepEqual(expected.Total, actual.Total) {
t.Errorf("Expected total page usage by NUMA %#v but found %#v", expected.Total, actual.Total)
}
if !reflect.DeepEqual(expected.File, actual.File) {
t.Errorf("Expected file page usage by NUMA %#v but found %#v", expected.File, actual.File)
}
if !reflect.DeepEqual(expected.Anon, actual.Anon) {
t.Errorf("Expected anon page usage by NUMA %#v but found %#v", expected.Anon, actual.Anon)
}
if !reflect.DeepEqual(expected.Unevictable, actual.Unevictable) {
t.Errorf("Expected unevictable page usage by NUMA %#v but found %#v", expected.Unevictable, actual.Unevictable)
}
if !reflect.DeepEqual(expected.Hierarchical.Total, actual.Hierarchical.Total) {
t.Errorf("Expected hierarchical total page usage by NUMA %#v but found %#v", expected.Hierarchical.Total, actual.Hierarchical.Total)
}
if !reflect.DeepEqual(expected.Hierarchical.File, actual.Hierarchical.File) {
t.Errorf("Expected hierarchical file page usage by NUMA %#v but found %#v", expected.Hierarchical.File, actual.Hierarchical.File)
}
if !reflect.DeepEqual(expected.Hierarchical.Anon, actual.Hierarchical.Anon) {
t.Errorf("Expected hierarchical anon page usage by NUMA %#v but found %#v", expected.Hierarchical.Anon, actual.Hierarchical.Anon)
}
if !reflect.DeepEqual(expected.Hierarchical.Unevictable, actual.Hierarchical.Unevictable) {
t.Errorf("Expected hierarchical total page usage by NUMA %#v but found %#v", expected.Hierarchical.Unevictable, actual.Hierarchical.Unevictable)
}
}
21 changes: 21 additions & 0 deletions libcontainer/cgroups/stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,33 @@ type MemoryStats struct {
KernelUsage MemoryData `json:"kernel_usage,omitempty"`
// usage of kernel TCP memory
KernelTCPUsage MemoryData `json:"kernel_tcp_usage,omitempty"`
// usage of memory pages by NUMA node
// see chapter 5.6 of memory controller documentation
PageUsageByNUMA PageUsageByNUMA `json:"page_usage_by_numa,omitempty"`
// if true, memory usage is accounted for throughout a hierarchy of cgroups.
UseHierarchy bool `json:"use_hierarchy"`

Stats map[string]uint64 `json:"stats,omitempty"`
}

type PageUsageByNUMA struct {
// Embedding is used as types can't be recursive.
PageUsageByNUMAInner
Hierarchical PageUsageByNUMAInner `json:"hierarchical,omitempty"`
}

type PageUsageByNUMAInner struct {
Total PageStats `json:"total,omitempty"`
File PageStats `json:"file,omitempty"`
Anon PageStats `json:"anon,omitempty"`
Unevictable PageStats `json:"unevictable,omitempty"`
}

type PageStats struct {
Total uint64 `json:"total,omitempty"`
Nodes map[uint8]uint64 `json:"nodes,omitempty"`
}

type PidsStats struct {
// number of pids in the cgroup
Current uint64 `json:"current,omitempty"`
Expand Down
Loading

0 comments on commit 8a031e3

Please sign in to comment.