api,dap: fix hexadecimal printing of vars with symbolic const values (#…

…3487)

Fix hexadecimal printing of variables that can be represented using
symbolic const values in DAP as well as the command line interface.

Fixes #3485

pkg/proc/variables_test.go

@@ -1104,16 +1104,16 @@ func TestPackageRenames(t *testing.T) {
 
 func TestConstants(t *testing.T) {
 	testcases := []varTest{
-		{"a", true, "constTwo (2)", "", "main.ConstType", nil},
-		{"b", true, "constThree (3)", "", "main.ConstType", nil},
-		{"c", true, "bitZero|bitOne (3)", "", "main.BitFieldType", nil},
-		{"d", true, "33", "", "main.BitFieldType", nil},
-		{"e", true, "10", "", "main.ConstType", nil},
-		{"f", true, "0", "", "main.BitFieldType", nil},
-		{"bitZero", true, "1", "", "main.BitFieldType", nil},
-		{"bitOne", true, "2", "", "main.BitFieldType", nil},
-		{"constTwo", true, "2", "", "main.ConstType", nil},
-		{"pkg.SomeConst", false, "2", "", "int", nil},
+		{"a", true, "constTwo (2)", "0x2", "main.ConstType", nil},
+		{"b", true, "constThree (3)", "0x3", "main.ConstType", nil},
+		{"c", true, "bitZero|bitOne (3)", "0x3", "main.BitFieldType", nil},
+		{"d", true, "33", "0x21", "main.BitFieldType", nil},
+		{"e", true, "10", "0xa", "main.ConstType", nil},
+		{"f", true, "0", "0x0", "main.BitFieldType", nil},
+		{"bitZero", true, "1", "0x1", "main.BitFieldType", nil},
+		{"bitOne", true, "2", "0x2", "main.BitFieldType", nil},
+		{"constTwo", true, "2", "0x2", "main.ConstType", nil},
+		{"pkg.SomeConst", false, "2", "0x2", "int", nil},
 	}
 	ver, _ := goversion.Parse(runtime.Version())
 	if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{Major: 1, Minor: 10, Rev: -1}) {
@@ -1126,6 +1126,11 @@ func TestConstants(t *testing.T) {
 			variable, err := evalVariableWithCfg(p, testcase.name, pnormalLoadConfig)
 			assertNoError(err, t, fmt.Sprintf("EvalVariable(%s)", testcase.name))
 			assertVariable(t, variable, testcase)
+			cv := api.ConvertVar(variable)
+			str := cv.SinglelineStringFormatted("%#x")
+			if str != testcase.alternate {
+				t.Errorf("for %s expected %q got %q when formatting in hexadecimal", testcase.name, testcase.alternate, str)
+			}
 		}
 	})
 }

service/api/prettyprint.go

@@ -173,15 +173,15 @@ func (v *Variable) writeBasicType(buf io.Writer, fmtstr string) {
 			buf.Write([]byte(v.Value))
 			return
 		}
-		n, _ := strconv.ParseInt(v.Value, 10, 64)
+		n, _ := strconv.ParseInt(ExtractIntValue(v.Value), 10, 64)
 		fmt.Fprintf(buf, fmtstr, n)
 
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
 		if fmtstr == "" {
 			buf.Write([]byte(v.Value))
 			return
 		}
-		n, _ := strconv.ParseUint(v.Value, 10, 64)
+		n, _ := strconv.ParseUint(ExtractIntValue(v.Value), 10, 64)
 		fmt.Fprintf(buf, fmtstr, n)
 
 	case reflect.Float32, reflect.Float64:
@@ -215,6 +215,17 @@ func (v *Variable) writeBasicType(buf io.Writer, fmtstr string) {
 	}
 }
 
+func ExtractIntValue(s string) string {
+	if s == "" || s[len(s)-1] != ')' {
+		return s
+	}
+	open := strings.LastIndex(s, "(")
+	if open < 0 {
+		return s
+	}
+	return s[open+1 : len(s)-1]
+}
+
 func (v *Variable) writeSliceTo(buf io.Writer, newlines, includeType bool, indent, fmtstr string) {
 	if includeType {
 		fmt.Fprintf(buf, "%s len: %d, cap: %d, ", v.Type, v.Len, v.Cap)

service/dap/server.go

@@ -2593,7 +2593,7 @@ func (s *Session) convertVariableWithOpts(v *proc.Variable, qualifiedNameOrExpr
 
 	switch v.Kind {
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		n, _ := strconv.ParseUint(api.ConvertVar(v).Value, 10, 64)
+		n, _ := strconv.ParseUint(api.ExtractIntValue(api.ConvertVar(v).Value), 10, 64)
 		value = fmt.Sprintf("%s = %#x", value, n)
 	case reflect.UnsafePointer:
 		// Skip child reference