From bfdbc0453260ebf56c5a718594de243d9131ac82 Mon Sep 17 00:00:00 2001 From: Szabolcs Toth Date: Fri, 8 Sep 2023 10:49:19 +0100 Subject: [PATCH 1/4] Update dependencies --- go.mod | 2 +- go.sum | 4 +- .../go-utils/v2/redactwriter/range.go | 41 ++ .../go-utils/v2/redactwriter/redactwriter.go | 369 ++++++++++++++++++ vendor/modules.txt | 3 +- 5 files changed, 415 insertions(+), 4 deletions(-) create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go diff --git a/go.mod b/go.mod index d4a74d9e4..550a22ba4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/bitrise-io/envman v0.0.0-20221010094751-a03ce30a5316 github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.19 github.com/bitrise-io/go-utils v1.0.8 - github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16 + github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 github.com/bitrise-io/gows v0.0.0-20211005113107-14f65e686b88 github.com/bitrise-io/stepman v0.0.0-20221010110437-a88e9a915b58 diff --git a/go.sum b/go.sum index cef0ecffc..485b55a4b 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/bitrise-io/go-utils v1.0.3/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0 github.com/bitrise-io/go-utils v1.0.8 h1:ekXH6FK5V8UxkyHm2FsQL8yk9Wqd5fjg1NGlD7jK2kc= github.com/bitrise-io/go-utils v1.0.8/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.13/go.mod h1:gZWtM7PLn1VOroa4gN1La/24aRVc0jg5R701jTsPaO8= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16 h1:y+Yo0d8pYIjZiKhQuPM/Z5FY9/mu+wrWkyQlch8f9Po= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 h1:55as5Iv0N4btuRP3YwRzN+BCMtKO210MnJ8mpxmeI7o= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= github.com/bitrise-io/goinp v0.0.0-20210504152833-8559b0680ab1/go.mod h1:iRbd8zAXLeNy+0gic0eqNCxXvDGe8ZEY/uYX2CCeAoo= github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 h1:ytUxnO7iSGHlNpbdjhDUefEM5WRy1kD2ElGfBA7r1PE= github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4/go.mod h1:iRbd8zAXLeNy+0gic0eqNCxXvDGe8ZEY/uYX2CCeAoo= diff --git a/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go new file mode 100644 index 000000000..71b9e815f --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/range.go @@ -0,0 +1,41 @@ +package redactwriter + +import ( + "bytes" + "sort" +) + +type matchRange struct{ first, last int } + +// allRanges returns every indexes of instance of pattern in b, or nil if pattern is not present in b. +func allRanges(b, pattern []byte) (ranges []matchRange) { + i := 0 + for { + sub := b[i:] + idx := bytes.Index(sub, pattern) + if idx == -1 { + return + } + + ranges = append(ranges, matchRange{first: idx + i, last: idx + i + len(pattern)}) + + i += idx + 1 + if i > len(b)-1 { + return + } + } +} + +// mergeAllRanges merges every overlapping ranges in r. +func mergeAllRanges(r []matchRange) []matchRange { + sort.Slice(r, func(i, j int) bool { return r[i].first < r[j].first }) + for i := 0; i < len(r)-1; i++ { + for i+1 < len(r) && r[i+1].first <= r[i].last { + if r[i+1].last > r[i].last { + r[i].last = r[i+1].last + } + r = append(r[:i+1], r[i+2:]...) + } + } + return r +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go new file mode 100644 index 000000000..a140e3a37 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/redactwriter/redactwriter.go @@ -0,0 +1,369 @@ +package redactwriter + +import ( + "bytes" + "io" + "sort" + "strings" + "sync" + "time" + + "github.com/bitrise-io/go-utils/v2/log" +) + +// RedactStr ... +const RedactStr = "[REDACTED]" + +var newLine = []byte("\n") + +// Writer ... +type Writer struct { + writer io.Writer + secrets [][][]byte + + chunk []byte + store [][]byte + mux sync.Mutex + timer *time.Timer + logger log.Logger +} + +// New ... +func New(secrets []string, target io.Writer, logger log.Logger) *Writer { + extendedSecrets := secrets + // adding transformed secrets with escaped newline characters to ensure that these are also obscured if found in logs + for _, secret := range secrets { + if strings.Contains(secret, "\n") { + extendedSecrets = append(extendedSecrets, strings.ReplaceAll(secret, "\n", `\n`)) + } + } + + return &Writer{ + writer: target, + secrets: secretsByteList(extendedSecrets), + logger: logger, + } +} + +// Write implements io.Writer interface. +// Splits p into lines, the lines are matched against the secrets, +// this determines which lines can be redacted and passed to the next writer (target). +// There might be lines that need to be buffered since they partially match a secret. +// We do not know the last Write call, so Close needs to be called to flush the buffer. +func (w *Writer) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + w.mux.Lock() + defer w.mux.Unlock() + + // previous bytes may not end with newline + data := append(w.chunk, p...) + + lastLines, chunk := splitAfterNewline(data) + w.chunk = chunk + if len(chunk) > 0 { + // we have remaining bytes, do not swallow them + if w.timer != nil { + w.timer.Stop() + w.timer.C = nil + } + w.timer = time.AfterFunc(100*time.Millisecond, func() { + if _, err := w.flush(); err != nil { + w.logger.Errorf("Failed to print last lines: %s", err) + } + }) + } + + if len(lastLines) == 0 { + // it is necessary to return the count of incoming bytes + return len(p), nil + } + + for _, line := range lastLines { + lines := append(w.store, line) + matchMap, partialMatchIndexes := w.matchSecrets(lines) + + var linesToPrint [][]byte + linesToPrint, w.store = w.matchLines(lines, partialMatchIndexes) + if linesToPrint == nil { + continue + } + + redactedLines := w.redact(linesToPrint, matchMap) + redactedBytes := bytes.Join(redactedLines, nil) + if c, err := w.writer.Write(redactedBytes); err != nil { + return c, err + } + } + + // it is necessary to return the count of incoming bytes + // to let the exec.Command work properly + return len(p), nil +} + +// Close implements io.Writer interface +func (w *Writer) Close() error { + _, err := w.flush() + return err +} + +// flush writes the remaining bytes. +func (w *Writer) flush() (int, error) { + defer func() { + w.mux.Unlock() + }() + w.mux.Lock() + + if len(w.chunk) > 0 { + // lines are containing newline, chunk may not + chunk := w.chunk + w.chunk = nil + w.store = append(w.store, chunk) + } + + // we only need to care about the full matches in the remaining lines + // (no more lines were come, why care about the partial matches?) + matchMap, _ := w.matchSecrets(w.store) + redactedLines := w.redact(w.store, matchMap) + w.store = nil + + return w.writer.Write(bytes.Join(redactedLines, nil)) +} + +// matchSecrets collects which secrets matches from which line indexes +// and which secrets matches partially from which line indexes. +// matchMap: matching line chunk's first line indexes by secret index +// partialMatchIndexes: line indexes from which secrets matching but not fully contained in lines +func (w *Writer) matchSecrets(lines [][]byte) (matchMap map[int][]int, partialMatchIndexes map[int]bool) { + matchMap = make(map[int][]int) + partialMatchIndexes = make(map[int]bool) + + for secretIdx, secret := range w.secrets { + secretLine := secret[0] // every match should begin from the secret first line + var lineIndexes []int // the indexes of lines which contains the secret's first line + + for i, line := range lines { + if bytes.Contains(line, secretLine) { + lineIndexes = append(lineIndexes, i) + } + } + + if len(lineIndexes) == 0 { + // this secret can not be found in the lines + continue + } + + for _, lineIdx := range lineIndexes { + if len(secret) == 1 { + // the single line secret found in the lines + indexes := matchMap[secretIdx] + matchMap[secretIdx] = append(indexes, lineIdx) + continue + } + + // lineIdx. line matches to a multi line secret's first line + // if lines has more line, every subsequent line must match to the secret's subsequent lines + partialMatch := true + match := false + + for i := lineIdx + 1; i < len(lines); i++ { + secretLineIdx := i - lineIdx + + secretLine = secret[secretLineIdx] + line := lines[i] + + if !bytes.Contains(line, secretLine) { + partialMatch = false + break + } + + if secretLineIdx == len(secret)-1 { + // multi line secret found in the lines + match = true + break + } + } + + if match { + // multi line secret found in the lines + indexes := matchMap[secretIdx] + matchMap[secretIdx] = append(indexes, lineIdx) + continue + } + + if partialMatch { + // this secret partially can be found in the lines + partialMatchIndexes[lineIdx] = true + } + } + } + + return +} + +// linesToKeepRange returns the first line index needs to be observed +// since they contain partially matching secrets. +func (w *Writer) linesToKeepRange(partialMatchIndexes map[int]bool) int { + first := -1 + + for lineIdx := range partialMatchIndexes { + if first == -1 { + first = lineIdx + continue + } + + if first > lineIdx { + first = lineIdx + } + } + + return first +} + +// matchLines return which lines can be printed and which should be kept for further observing. +func (w *Writer) matchLines(lines [][]byte, partialMatchIndexes map[int]bool) ([][]byte, [][]byte) { + first := w.linesToKeepRange(partialMatchIndexes) + switch first { + case -1: + // no lines need to be kept + return lines, nil + case 0: + // partial match is always longer then the lines + return nil, lines + default: + return lines[:first], lines[first:] + } +} + +// secretLinesToRedact returns which secret lines should be redacted +func (w *Writer) secretLinesToRedact(lineIdxToRedact int, matchMap map[int][]int) [][]byte { + // which line is which secrets first matching line + secretIdxsByLine := make(map[int][]int) + for secretIdx, lineIndexes := range matchMap { + for _, lineIdx := range lineIndexes { + secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) + } + } + + var secretChunks [][]byte + for firstMatchingLineIdx, secretIndexes := range secretIdxsByLine { + if lineIdxToRedact < firstMatchingLineIdx { + continue + } + + for _, secretIdx := range secretIndexes { + secret := w.secrets[secretIdx] + + if lineIdxToRedact > firstMatchingLineIdx+len(secret)-1 { + continue + } + + secretLineIdx := lineIdxToRedact - firstMatchingLineIdx + secretChunks = append(secretChunks, secret[secretLineIdx]) + } + } + + sort.Slice(secretChunks, func(i, j int) bool { return len(secretChunks[i]) < len(secretChunks[j]) }) + return secretChunks +} + +// redact hides the given ranges in the given line. +func redact(line []byte, ranges []matchRange) []byte { + var offset int // the offset of ranges generated by replacing x bytes by the RedactStr + for _, r := range ranges { + length := r.last - r.first + first := r.first + offset + last := first + length + + toRedact := line[first:last] + redactStr := RedactStr + if bytes.HasSuffix(toRedact, []byte("\n")) { + // if string to redact ends with newline redact message should also + redactStr += "\n" + } + + newLine := append([]byte{}, line[:first]...) + newLine = append(newLine, redactStr...) + newLine = append(newLine, line[last:]...) + + offset += len(redactStr) - length + + line = newLine + } + return line +} + +// redact hides the given secrets in the given lines. +func (w *Writer) redact(lines [][]byte, matchMap map[int][]int) [][]byte { + secretIdxsByLine := map[int][]int{} + for secretIdx, lineIndexes := range matchMap { + for _, lineIdx := range lineIndexes { + secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) + } + } + + for i, line := range lines { + linesToRedact := w.secretLinesToRedact(i, matchMap) + if linesToRedact == nil { + continue + } + + var ranges []matchRange + for _, lineToRedact := range linesToRedact { + ranges = append(ranges, allRanges(line, lineToRedact)...) + } + + lines[i] = redact(line, mergeAllRanges(ranges)) + } + + return lines +} + +// secretsByteList returns the list of secret byte lines. +func secretsByteList(secrets []string) [][][]byte { + var s [][][]byte + for _, secret := range secrets { + lines, lastLine := splitAfterNewline([]byte(secret)) + if lines == nil && lastLine == nil { + continue + } + + var secretLines [][]byte + if lines != nil { + secretLines = append(secretLines, lines...) + } + if lastLine != nil { + secretLines = append(secretLines, lastLine) + } + s = append(s, secretLines) + } + return s +} + +// splitAfterNewline splits p after "\n", the split is assigned to lines +// if last line has no "\n" it is assigned to the chunk. +// If p is nil both lines and chunk is set to nil. +func splitAfterNewline(p []byte) ([][]byte, []byte) { + chunk := p + var lines [][]byte + + for len(chunk) > 0 { + idx := bytes.Index(chunk, newLine) + if idx == -1 { + return lines, chunk + } + + lines = append(lines, chunk[:idx+1]) + + if idx == len(chunk)-1 { + chunk = nil + break + } + + chunk = chunk[idx+1:] + } + + return lines, chunk +} diff --git a/vendor/modules.txt b/vendor/modules.txt index cad71fcef..7964261be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -29,12 +29,13 @@ github.com/bitrise-io/go-utils/sliceutil github.com/bitrise-io/go-utils/stringutil github.com/bitrise-io/go-utils/urlutil github.com/bitrise-io/go-utils/versions -# github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.16 +# github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 ## explicit; go 1.17 github.com/bitrise-io/go-utils/v2/analytics github.com/bitrise-io/go-utils/v2/env github.com/bitrise-io/go-utils/v2/log github.com/bitrise-io/go-utils/v2/log/colorstring +github.com/bitrise-io/go-utils/v2/redactwriter github.com/bitrise-io/go-utils/v2/retryhttp # github.com/bitrise-io/goinp v0.0.0-20211005113137-305e91b481f4 ## explicit; go 1.16 From ffe9345ea8d3e8c65f2fbba71d3ac250be6c3df9 Mon Sep 17 00:00:00 2001 From: Szabolcs Toth Date: Fri, 8 Sep 2023 10:49:40 +0100 Subject: [PATCH 2/4] Remove old implementation --- stepruncmd/filterwriter/filterwriter.go | 366 ------------- stepruncmd/filterwriter/filterwriter_test.go | 544 ------------------- stepruncmd/filterwriter/range.go | 41 -- stepruncmd/filterwriter/range_test.go | 77 --- 4 files changed, 1028 deletions(-) delete mode 100644 stepruncmd/filterwriter/filterwriter.go delete mode 100644 stepruncmd/filterwriter/filterwriter_test.go delete mode 100644 stepruncmd/filterwriter/range.go delete mode 100644 stepruncmd/filterwriter/range_test.go diff --git a/stepruncmd/filterwriter/filterwriter.go b/stepruncmd/filterwriter/filterwriter.go deleted file mode 100644 index 49c5e461d..000000000 --- a/stepruncmd/filterwriter/filterwriter.go +++ /dev/null @@ -1,366 +0,0 @@ -package filterwriter - -import ( - "bytes" - "io" - "sort" - "strings" - "sync" - "time" - - "github.com/bitrise-io/bitrise/log" -) - -// RedactStr ... -const RedactStr = "[REDACTED]" - -var newLine = []byte("\n") - -// Writer ... -type Writer struct { - writer io.Writer - secrets [][][]byte - - chunk []byte - store [][]byte - mux sync.Mutex - timer *time.Timer -} - -// New ... -func New(secrets []string, target io.Writer) *Writer { - extendedSecrets := secrets - // adding transformed secrets with escaped newline characters to ensure that these are also obscured if found in logs - for _, secret := range secrets { - if strings.Contains(secret, "\n") { - extendedSecrets = append(extendedSecrets, strings.ReplaceAll(secret, "\n", `\n`)) - } - } - - return &Writer{ - writer: target, - secrets: secretsByteList(extendedSecrets), - } -} - -// Write implements io.Writer interface. -// Splits p into lines, the lines are matched against the secrets, -// this determines which lines can be redacted and passed to the next writer (target). -// There might be lines that need to be buffered since they partially match a secret. -// We do not know the last Write call, so Close needs to be called to flush the buffer. -func (w *Writer) Write(p []byte) (int, error) { - if len(p) == 0 { - return 0, nil - } - - w.mux.Lock() - defer w.mux.Unlock() - - // previous bytes may not end with newline - data := append(w.chunk, p...) - - lastLines, chunk := splitAfterNewline(data) - w.chunk = chunk - if len(chunk) > 0 { - // we have remaining bytes, do not swallow them - if w.timer != nil { - w.timer.Stop() - w.timer.C = nil - } - w.timer = time.AfterFunc(100*time.Millisecond, func() { - if _, err := w.flush(); err != nil { - log.Errorf("Failed to print last lines: %s", err) - } - }) - } - - if len(lastLines) == 0 { - // it is necessary to return the count of incoming bytes - return len(p), nil - } - - for _, line := range lastLines { - lines := append(w.store, line) - matchMap, partialMatchIndexes := w.matchSecrets(lines) - - var linesToPrint [][]byte - linesToPrint, w.store = w.matchLines(lines, partialMatchIndexes) - if linesToPrint == nil { - continue - } - - redactedLines := w.redact(linesToPrint, matchMap) - redactedBytes := bytes.Join(redactedLines, nil) - if c, err := w.writer.Write(redactedBytes); err != nil { - return c, err - } - } - - // it is necessary to return the count of incoming bytes - // to let the exec.Command work properly - return len(p), nil -} - -func (w *Writer) Close() error { - _, err := w.flush() - return err -} - -// flush writes the remaining bytes. -func (w *Writer) flush() (int, error) { - defer func() { - w.mux.Unlock() - }() - w.mux.Lock() - - if len(w.chunk) > 0 { - // lines are containing newline, chunk may not - chunk := w.chunk - w.chunk = nil - w.store = append(w.store, chunk) - } - - // we only need to care about the full matches in the remaining lines - // (no more lines were come, why care about the partial matches?) - matchMap, _ := w.matchSecrets(w.store) - redactedLines := w.redact(w.store, matchMap) - w.store = nil - - return w.writer.Write(bytes.Join(redactedLines, nil)) -} - -// matchSecrets collects which secrets matches from which line indexes -// and which secrets matches partially from which line indexes. -// matchMap: matching line chunk's first line indexes by secret index -// partialMatchIndexes: line indexes from which secrets matching but not fully contained in lines -func (w *Writer) matchSecrets(lines [][]byte) (matchMap map[int][]int, partialMatchIndexes map[int]bool) { - matchMap = make(map[int][]int) - partialMatchIndexes = make(map[int]bool) - - for secretIdx, secret := range w.secrets { - secretLine := secret[0] // every match should begin from the secret first line - var lineIndexes []int // the indexes of lines which contains the secret's first line - - for i, line := range lines { - if bytes.Contains(line, secretLine) { - lineIndexes = append(lineIndexes, i) - } - } - - if len(lineIndexes) == 0 { - // this secret can not be found in the lines - continue - } - - for _, lineIdx := range lineIndexes { - if len(secret) == 1 { - // the single line secret found in the lines - indexes := matchMap[secretIdx] - matchMap[secretIdx] = append(indexes, lineIdx) - continue - } - - // lineIdx. line matches to a multi line secret's first line - // if lines has more line, every subsequent line must match to the secret's subsequent lines - partialMatch := true - match := false - - for i := lineIdx + 1; i < len(lines); i++ { - secretLineIdx := i - lineIdx - - secretLine = secret[secretLineIdx] - line := lines[i] - - if !bytes.Contains(line, secretLine) { - partialMatch = false - break - } - - if secretLineIdx == len(secret)-1 { - // multi line secret found in the lines - match = true - break - } - } - - if match { - // multi line secret found in the lines - indexes := matchMap[secretIdx] - matchMap[secretIdx] = append(indexes, lineIdx) - continue - } - - if partialMatch { - // this secret partially can be found in the lines - partialMatchIndexes[lineIdx] = true - } - } - } - - return -} - -// linesToKeepRange returns the first line index needs to be observed -// since they contain partially matching secrets. -func (w *Writer) linesToKeepRange(partialMatchIndexes map[int]bool) int { - first := -1 - - for lineIdx := range partialMatchIndexes { - if first == -1 { - first = lineIdx - continue - } - - if first > lineIdx { - first = lineIdx - } - } - - return first -} - -// matchLines return which lines can be printed and which should be kept for further observing. -func (w *Writer) matchLines(lines [][]byte, partialMatchIndexes map[int]bool) ([][]byte, [][]byte) { - first := w.linesToKeepRange(partialMatchIndexes) - switch first { - case -1: - // no lines need to be kept - return lines, nil - case 0: - // partial match is always longer then the lines - return nil, lines - default: - return lines[:first], lines[first:] - } -} - -// secretLinesToRedact returns which secret lines should be redacted -func (w *Writer) secretLinesToRedact(lineIdxToRedact int, matchMap map[int][]int) [][]byte { - // which line is which secrets first matching line - secretIdxsByLine := make(map[int][]int) - for secretIdx, lineIndexes := range matchMap { - for _, lineIdx := range lineIndexes { - secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) - } - } - - var secretChunks [][]byte - for firstMatchingLineIdx, secretIndexes := range secretIdxsByLine { - if lineIdxToRedact < firstMatchingLineIdx { - continue - } - - for _, secretIdx := range secretIndexes { - secret := w.secrets[secretIdx] - - if lineIdxToRedact > firstMatchingLineIdx+len(secret)-1 { - continue - } - - secretLineIdx := lineIdxToRedact - firstMatchingLineIdx - secretChunks = append(secretChunks, secret[secretLineIdx]) - } - } - - sort.Slice(secretChunks, func(i, j int) bool { return len(secretChunks[i]) < len(secretChunks[j]) }) - return secretChunks -} - -// redact hides the given ranges in the given line. -func redact(line []byte, ranges []matchRange) []byte { - var offset int // the offset of ranges generated by replacing x bytes by the RedactStr - for _, r := range ranges { - length := r.last - r.first - first := r.first + offset - last := first + length - - toRedact := line[first:last] - redactStr := RedactStr - if bytes.HasSuffix(toRedact, []byte("\n")) { - // if string to redact ends with newline redact message should also - redactStr += "\n" - } - - newLine := append([]byte{}, line[:first]...) - newLine = append(newLine, redactStr...) - newLine = append(newLine, line[last:]...) - - offset += len(redactStr) - length - - line = newLine - } - return line -} - -// redact hides the given secrets in the given lines. -func (w *Writer) redact(lines [][]byte, matchMap map[int][]int) [][]byte { - secretIdxsByLine := map[int][]int{} - for secretIdx, lineIndexes := range matchMap { - for _, lineIdx := range lineIndexes { - secretIdxsByLine[lineIdx] = append(secretIdxsByLine[lineIdx], secretIdx) - } - } - - for i, line := range lines { - linesToRedact := w.secretLinesToRedact(i, matchMap) - if linesToRedact == nil { - continue - } - - var ranges []matchRange - for _, lineToRedact := range linesToRedact { - ranges = append(ranges, allRanges(line, lineToRedact)...) - } - - lines[i] = redact(line, mergeAllRanges(ranges)) - } - - return lines -} - -// secretsByteList returns the list of secret byte lines. -func secretsByteList(secrets []string) [][][]byte { - var s [][][]byte - for _, secret := range secrets { - lines, lastLine := splitAfterNewline([]byte(secret)) - if lines == nil && lastLine == nil { - continue - } - - var secretLines [][]byte - if lines != nil { - secretLines = append(secretLines, lines...) - } - if lastLine != nil { - secretLines = append(secretLines, lastLine) - } - s = append(s, secretLines) - } - return s -} - -// splitAfterNewline splits p after "\n", the split is assigned to lines -// if last line has no "\n" it is assigned to the chunk. -// If p is nil both lines and chunk is set to nil. -func splitAfterNewline(p []byte) ([][]byte, []byte) { - chunk := p - var lines [][]byte - - for len(chunk) > 0 { - idx := bytes.Index(chunk, newLine) - if idx == -1 { - return lines, chunk - } - - lines = append(lines, chunk[:idx+1]) - - if idx == len(chunk)-1 { - chunk = nil - break - } - - chunk = chunk[idx+1:] - } - - return lines, chunk -} diff --git a/stepruncmd/filterwriter/filterwriter_test.go b/stepruncmd/filterwriter/filterwriter_test.go deleted file mode 100644 index 2d01701d1..000000000 --- a/stepruncmd/filterwriter/filterwriter_test.go +++ /dev/null @@ -1,544 +0,0 @@ -package filterwriter - -import ( - "bytes" - "fmt" - "runtime" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSecretsByteList(t *testing.T) { - { - secrets := []string{"secret value"} - byteList := secretsByteList(secrets) - require.Equal(t, [][][]byte{ - [][]byte{ - []byte("secret value"), - }, - }, byteList) - } - - { - secrets := []string{"secret value1", "secret value2"} - byteList := secretsByteList(secrets) - require.Equal(t, [][][]byte{ - [][]byte{ - []byte("secret value1"), - }, - [][]byte{ - []byte("secret value2"), - }, - }, byteList) - } - - { - secrets := []string{"multi\nline\nsecret"} - byteList := secretsByteList(secrets) - require.Equal(t, [][][]byte{ - [][]byte{ - []byte("multi\n"), - []byte("line\n"), - []byte("secret"), - }, - }, byteList) - } - - { - secrets := []string{"ending\nwith\nnewline\n"} - byteList := secretsByteList(secrets) - require.Equal(t, [][][]byte{ - [][]byte{ - []byte("ending\n"), - []byte("with\n"), - []byte("newline\n"), - }, - }, byteList) - } - - { - secrets := []string{"\nstarting\nwith\nnewline"} - byteList := secretsByteList(secrets) - require.Equal(t, [][][]byte{ - [][]byte{ - []byte("\n"), - []byte("starting\n"), - []byte("with\n"), - []byte("newline"), - }, - }, byteList) - } - - { - secrets := []string{"newlines\nin\n\nthe\n\n\nmiddle"} - byteList := secretsByteList(secrets) - require.Equal(t, [][][]byte{ - [][]byte{ - []byte("newlines\n"), - []byte("in\n"), - []byte("\n"), - []byte("the\n"), - []byte("\n"), - []byte("\n"), - []byte("middle"), - }, - }, byteList) - } -} - -func TestWrite(t *testing.T) { - t.Log("trivial test") - { - var buff bytes.Buffer - out := New([]string{"abc", "a\nb\nc"}, &buff) - log := []byte("test with\nnew line\nand single line secret:abc\nand multiline secret:a\nb\nc") - wc, err := out.Write(log) - require.NoError(t, err) - require.Equal(t, len(log), wc) - - err = out.Close() - require.NoError(t, err) - require.Equal(t, "test with\nnew line\nand single line secret:[REDACTED]\nand multiline secret:[REDACTED]\n[REDACTED]\n[REDACTED]", buff.String()) - } - - t.Log("chunk without newline") - { - var buff bytes.Buffer - out := New([]string{"ab", "a\nb"}, &buff) - log := []byte("test without newline, secret:ab") - wc, err := out.Write(log) - require.NoError(t, err) - require.Equal(t, len(log), wc) - - err = out.Close() - require.NoError(t, err) - require.Equal(t, "test without newline, secret:[REDACTED]", buff.String()) - } - - t.Log("multiple secret in the same line") - { - var buff bytes.Buffer - out := New([]string{"x1", "x\n2"}, &buff) - log := []byte("multiple secrets like: x1 and x\n2 and some extra text") - wc, err := out.Write(log) - require.NoError(t, err) - require.Equal(t, len(log), wc) - - err = out.Close() - require.NoError(t, err) - require.Equal(t, "multiple secrets like: [REDACTED] and [REDACTED]\n[REDACTED] and some extra text", buff.String()) - } - - maxRun := 150000 - t.Log("multiple secret in the same line with multiple gorutine ") - { - cherr := make(chan error, maxRun) - chStr := make(chan string, maxRun) - - var buff bytes.Buffer - out := New([]string{"x1", "x\n2"}, &buff) - log := []byte("multiple secrets like: x1 and x\n2 and some extra text") - for i := 0; i < maxRun; i++ { - go func(buff bytes.Buffer, out *Writer, log []byte) { - runtime.Gosched() - buff.Reset() - - wc, err := out.Write(log) - require.NoError(t, err) - require.Equal(t, len(log), wc) - - err = out.Close() - - cherr <- err - chStr <- buff.String() - }(buff, out, log) - } - - errCounter := 0 - for err := range cherr { - require.NoError(t, err) - - errCounter++ - if errCounter == maxRun { - close(cherr) - } - } - - strCounter := 0 - for str := range chStr { - fmt.Println(str) - - strCounter++ - if strCounter == maxRun { - close(chStr) - } - } - } -} - -func TestSecrets(t *testing.T) { - secrets := []string{ - "a\nb\nc", - "b", - "c\nb", - "x\nc\nb\nd", - "f", - } - - var buff bytes.Buffer - out := New(secrets, &buff) - require.Equal(t, [][][]byte{ - [][]byte{[]byte("a\n"), []byte("b\n"), []byte("c")}, - [][]byte{[]byte("b")}, - [][]byte{[]byte("c\n"), []byte("b")}, - [][]byte{[]byte("x\n"), []byte("c\n"), []byte("b\n"), []byte("d")}, - [][]byte{[]byte("f")}, - [][]byte{[]byte(`a\nb\nc`)}, - [][]byte{[]byte(`c\nb`)}, - [][]byte{[]byte(`x\nc\nb\nd`)}, - }, out.secrets) -} - -func TestMatchSecrets(t *testing.T) { - secrets := []string{ - "a\nb\nc", - "b", - "c\nb", - "x\nc\nb\nd", - "f", - } - lines := [][]byte{ - []byte("x\n"), - []byte("a\n"), - []byte("a\n"), - []byte("b\n"), - []byte("c\n"), - []byte("x\n"), - []byte("c\n"), - []byte("b\n")} - - var buff bytes.Buffer - out := New(secrets, &buff) - - matchMap, partialMatchMap := out.matchSecrets(lines) - require.Equal(t, map[int][]int{ - 0: []int{2}, - 1: []int{3, 7}, - 2: []int{6}, - }, matchMap) - require.Equal(t, map[int]bool{5: true}, partialMatchMap) -} - -func TestLinesToKeepRange(t *testing.T) { - t.Log() - secrets := []string{ - "a\nb\nc", - "b", - "c\nb", - "x\nc\nb\nd", - } - // lines := [][]byte{ - // []byte("x\n"), - // []byte("a\n"), - // []byte("a\n"), - // []byte("b\n"), - // []byte("c\n"), - // []byte("x\n"), 5.line - // []byte("c\n"), - // []byte("b\n")} - - var buff bytes.Buffer - out := New(secrets, &buff) - - partialMatchMap := map[int]bool{6: true, 2: true, 5: true, 7: true} - first := out.linesToKeepRange(partialMatchMap) // minimal index in the partialMatchMap - require.Equal(t, 2, first) -} - -func TestMatchLine(t *testing.T) { - secrets := []string{ - "a\nb\nc", - "b", - "c\nb", - "x\nc\nb\nd", - "f", - } - lines := [][]byte{ - []byte("x\n"), // 0. - []byte("a\n"), - []byte("a\n"), // 2. - []byte("b\n"), - []byte("c\n"), // 4. - []byte("x\n"), - []byte("c\n"), // 6. - []byte("b\n")} - - var buff bytes.Buffer - out := New(secrets, &buff) - - _, partialMatchMap := out.matchSecrets(lines) - print, remaining := out.matchLines(lines, partialMatchMap) - require.Equal(t, [][]byte{ - []byte("x\n"), - []byte("a\n"), - []byte("a\n"), - []byte("b\n"), - []byte("c\n"), - }, print) - require.Equal(t, [][]byte{ - []byte("x\n"), - []byte("c\n"), - []byte("b\n"), - }, remaining) -} - -func TestSecretLinesToRedact(t *testing.T) { - secrets := []string{ - "a\nb\nc", - "b", - } - lines := [][]byte{ - []byte("x\n"), - []byte("a\n"), - []byte("b\n"), - []byte("c\n"), - []byte("b\n"), - } - - var buff bytes.Buffer - out := New(secrets, &buff) - - matchMap, _ := out.matchSecrets(lines) - require.Equal(t, map[int][]int{ - 0: []int{1}, - 1: []int{2, 4}, - }, matchMap) - - secretLines := out.secretLinesToRedact(0, matchMap) - require.Equal(t, ([][]byte)(nil), secretLines, fmt.Sprintf("%s\n", secretLines)) - - secretLines = out.secretLinesToRedact(1, matchMap) - require.Equal(t, [][]byte{[]byte("a\n")}, secretLines, fmt.Sprintf("%s\n", secretLines)) - - secretLines = out.secretLinesToRedact(2, matchMap) - require.Equal(t, [][]byte{[]byte("b"), []byte("b\n")}, secretLines, fmt.Sprintf("%s\n", secretLines)) - - secretLines = out.secretLinesToRedact(3, matchMap) - require.Equal(t, [][]byte{[]byte("c")}, secretLines, fmt.Sprintf("%s\n", secretLines)) - - secretLines = out.secretLinesToRedact(4, matchMap) - require.Equal(t, [][]byte{[]byte("b")}, secretLines, fmt.Sprintf("%s\n", secretLines)) -} - -func TestRedactLine(t *testing.T) { - t.Log("redacts the middle of the line") - { - line := []byte("asdfabcasdf") - ranges := []matchRange{ // asdfabcasdf - {first: 4, last: 7}, // ****abc**** - } - redacted := redact(line, ranges) - require.Equal(t, []byte("asdf[REDACTED]asdf"), redacted, string(redacted)) - } - - t.Log("redacts the begining of the line") - { - line := []byte("asdfabcasdf") - ranges := []matchRange{ // asdfabcasdf - {first: 0, last: 5}, // asdfa****** - } - redacted := redact(line, ranges) - require.Equal(t, []byte("[REDACTED]bcasdf"), redacted, string(redacted)) - } - - t.Log("redacts the end of the line") - { - line := []byte("asdfabcasdf") - ranges := []matchRange{ // asdfabcasdf - {first: 9, last: 11}, // *********df - } - redacted := redact(line, ranges) - require.Equal(t, []byte("asdfabcas[REDACTED]"), redacted, string(redacted)) - } - - t.Log("redacts multiple secrets") - { - line := []byte("asdfabcasdf") - ranges := []matchRange{ // asdfabcasdf - {first: 4, last: 7}, // ****abc**** - {first: 8, last: 10}, // ********sd* - } - redacted := redact(line, ranges) - require.Equal(t, []byte("asdf[REDACTED]a[REDACTED]f"), redacted, string(redacted)) - } - - t.Log("redacts the whole line") - { - line := []byte("asdfabcasdf") - ranges := []matchRange{ // asdfabcasdf - {first: 0, last: 4}, // asdf******* - {first: 7, last: 11}, // *******asdf - {first: 3, last: 9}, // ***fabcas** - } - ranges = mergeAllRanges(ranges) - redacted := redact(line, ranges) - require.Equal(t, []byte("[REDACTED]"), redacted, string(redacted)) - } -} - -func TestRedact(t *testing.T) { - secrets := []string{ - "a\nb\nc", - "b", - } - lines := [][]byte{ - []byte("x\n"), - []byte("a\n"), - []byte("a\n"), - []byte("b\n"), - []byte("c\n"), - } - - var buff bytes.Buffer - out := New(secrets, &buff) - - matchMap := map[int][]int{0: []int{2}, 1: []int{3}} - redacted := out.redact(lines, matchMap) - require.Equal(t, [][]byte{ - []byte("x\n"), - []byte("a\n"), - []byte(RedactStr + "\n"), - []byte(RedactStr + "\n"), - []byte(RedactStr + "\n"), - }, redacted) - - { - secrets := []string{ - "106\n105", - "99", - } - lines := [][]byte{ - []byte("106\n"), - []byte("105\n"), - []byte("104\n"), - []byte("103\n"), - []byte("102\n"), - []byte("101\n"), - []byte("100\n"), - []byte("99\n")} - - var buff bytes.Buffer - out := New(secrets, &buff) - - matchMap := map[int][]int{ - 0: []int{0}, - 1: []int{7}, - } - redacted := out.redact(lines, matchMap) - require.Equal(t, [][]byte{ - []byte(RedactStr + "\n"), - []byte(RedactStr + "\n"), - []byte("104" + "\n"), - []byte("103" + "\n"), - []byte("102" + "\n"), - []byte("101" + "\n"), - []byte("100" + "\n"), - []byte(RedactStr + "\n"), - }, redacted, fmt.Sprintf("%s", redacted)) - } -} - -func TestSplitAfterNewline(t *testing.T) { - t.Log("bytes") - { - require.Equal(t, []byte{}, []byte("")) - } - - t.Log("empty test") - { - b := []byte{} - lines, chunk := splitAfterNewline(b) - require.Equal(t, [][]byte(nil), lines) - require.Equal(t, []byte{}, chunk) - } - - t.Log("empty test - empty string bytes") - { - b := []byte("") - lines, chunk := splitAfterNewline(b) - require.Equal(t, [][]byte(nil), lines) - require.Equal(t, []byte{}, chunk) - } - - t.Log("newline test") - { - b := []byte("\n") - lines, chunk := splitAfterNewline(b) - require.Equal(t, [][]byte{[]byte("\n")}, lines) - require.Equal(t, []byte(nil), chunk) - } - - t.Log("multi line test") - { - b := []byte(`line 1 -line 2 -line 3 -`) - lines, chunk := splitAfterNewline(b) - require.Equal(t, 3, len(lines)) - require.Equal(t, []byte("line 1\n"), lines[0]) - require.Equal(t, []byte("line 2\n"), lines[1]) - require.Equal(t, []byte("line 3\n"), lines[2]) - require.Equal(t, []byte(nil), chunk) - } - - t.Log("multi line test - newlines") - { - b := []byte(` - -line 1 - -line 2 -`) - - lines, chunk := splitAfterNewline(b) - require.Equal(t, 5, len(lines)) - require.Equal(t, []byte("\n"), lines[0]) - require.Equal(t, []byte("\n"), lines[1]) - require.Equal(t, []byte("line 1\n"), lines[2]) - require.Equal(t, []byte("\n"), lines[3]) - require.Equal(t, []byte("line 2\n"), lines[4]) - require.Equal(t, []byte(nil), chunk) - } - - t.Log("chunk test") - { - b := []byte("line 1") - lines, chunk := splitAfterNewline(b) - require.Equal(t, [][]byte(nil), lines) - require.Equal(t, []byte("line 1"), chunk) - } - - t.Log("chunk test") - { - b := []byte(`line 1 -line 2`) - - lines, chunk := splitAfterNewline(b) - require.Equal(t, 1, len(lines)) - require.Equal(t, []byte("line 1\n"), lines[0]) - require.Equal(t, []byte("line 2"), chunk) - } - t.Log("chunk test") - { - b := []byte("test\n\ntest\n") - - lines, chunk := splitAfterNewline(b) - require.Equal(t, 3, len(lines)) - require.Equal(t, []byte("test\n"), lines[0]) - require.Equal(t, []byte("\n"), lines[1]) - require.Equal(t, []byte("test\n"), lines[2]) - require.Equal(t, []byte(nil), chunk) - } -} diff --git a/stepruncmd/filterwriter/range.go b/stepruncmd/filterwriter/range.go deleted file mode 100644 index 015311ad4..000000000 --- a/stepruncmd/filterwriter/range.go +++ /dev/null @@ -1,41 +0,0 @@ -package filterwriter - -import ( - "bytes" - "sort" -) - -type matchRange struct{ first, last int } - -// allRanges returns every indexes of instance of pattern in b, or nil if pattern is not present in b. -func allRanges(b, pattern []byte) (ranges []matchRange) { - i := 0 - for { - sub := b[i:] - idx := bytes.Index(sub, pattern) - if idx == -1 { - return - } - - ranges = append(ranges, matchRange{first: idx + i, last: idx + i + len(pattern)}) - - i += idx + 1 - if i > len(b)-1 { - return - } - } -} - -// mergeAllRanges merges every overlapping ranges in r. -func mergeAllRanges(r []matchRange) []matchRange { - sort.Slice(r, func(i, j int) bool { return r[i].first < r[j].first }) - for i := 0; i < len(r)-1; i++ { - for i+1 < len(r) && r[i+1].first <= r[i].last { - if r[i+1].last > r[i].last { - r[i].last = r[i+1].last - } - r = append(r[:i+1], r[i+2:]...) - } - } - return r -} diff --git a/stepruncmd/filterwriter/range_test.go b/stepruncmd/filterwriter/range_test.go deleted file mode 100644 index e29cd3986..000000000 --- a/stepruncmd/filterwriter/range_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package filterwriter - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestAllRanges(t *testing.T) { - { - ranges := allRanges([]byte("test"), []byte("t")) - require.Equal(t, []matchRange{{first: 0, last: 1}, {first: 3, last: 4}}, ranges) - } - - { - ranges := allRanges([]byte("test rangetest"), []byte("test")) - require.Equal(t, []matchRange{{first: 0, last: 4}, {first: 10, last: 14}}, ranges) - } - - { - ranges := allRanges([]byte("\n"), []byte("\n")) - require.Equal(t, []matchRange{{first: 0, last: 1}}, ranges) - } - - { - ranges := allRanges([]byte("test\n"), []byte("\n")) - require.Equal(t, []matchRange{{first: 4, last: 5}}, ranges) - } - - { - ranges := allRanges([]byte("\n\ntest\n"), []byte("\n")) - require.Equal(t, []matchRange{{first: 0, last: 1}, {first: 1, last: 2}, {first: 6, last: 7}}, ranges) - } - - { - ranges := allRanges([]byte("\n\ntest\n"), []byte("test\n")) - require.Equal(t, []matchRange{{first: 2, last: 7}}, ranges) - } -} - -func TestMergeAllRanges(t *testing.T) { - var testCases = []struct { - name string - ranges []matchRange - want []matchRange - }{ - { - name: "merges overlapping ranges", - ranges: []matchRange{{0, 2}, {1, 3}}, - want: []matchRange{{0, 3}}, - }, - { - name: "does not merge distinct ranges", - ranges: []matchRange{{0, 2}, {3, 5}}, - want: []matchRange{{0, 2}, {3, 5}}, - }, - { - name: "returns the wider range", - ranges: []matchRange{{0, 2}, {1, 2}}, - want: []matchRange{{0, 2}}, - }, - { - name: "complex test", - ranges: []matchRange{{11, 15}, {0, 2}, {11, 13}, {2, 4}, {6, 9}, {5, 10}}, - want: []matchRange{{0, 4}, {5, 10}, {11, 15}}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if got := mergeAllRanges(tc.ranges); !reflect.DeepEqual(got, tc.want) { - t.Errorf("got %v, want %v", got, tc.want) - } - }) - } - -} From cdc24eeb3ac24e96874cedfe144eb9c3bffad586 Mon Sep 17 00:00:00 2001 From: Szabolcs Toth Date: Fri, 8 Sep 2023 10:52:03 +0100 Subject: [PATCH 3/4] Use shared secret redacting --- cli/analytics.go | 9 +++++---- cli/run_util.go | 3 ++- stepruncmd/stdout.go | 19 ++++++++++--------- stepruncmd/stdout_test.go | 19 ++++++++++--------- stepruncmd/stepruncmd.go | 11 ++++++----- stepruncmd/stepruncmd_test.go | 5 +++-- 6 files changed, 36 insertions(+), 30 deletions(-) diff --git a/cli/analytics.go b/cli/analytics.go index 29b1fd836..5d6254f28 100644 --- a/cli/analytics.go +++ b/cli/analytics.go @@ -5,8 +5,9 @@ import ( "fmt" "io" - "github.com/bitrise-io/bitrise/stepruncmd/filterwriter" "github.com/bitrise-io/envman/models" + "github.com/bitrise-io/go-utils/v2/log" + "github.com/bitrise-io/go-utils/v2/redactwriter" ) func redactStepInputs(environment map[string]string, inputs []models.EnvironmentItemModel, secrets []string) (map[string]string, map[string]string, error) { @@ -49,12 +50,12 @@ func redactStepInputs(environment map[string]string, inputs []models.Environment func redactWithSecrets(inputValue string, secrets []string) (string, error) { src := bytes.NewReader([]byte(inputValue)) dstBuf := new(bytes.Buffer) - secretFilterDst := filterwriter.New(secrets, dstBuf) + redactWriterDst := redactwriter.New(secrets, dstBuf, log.NewLogger()) - if _, err := io.Copy(secretFilterDst, src); err != nil { + if _, err := io.Copy(redactWriterDst, src); err != nil { return "", fmt.Errorf("failed to redact secrets, stream copy failed: %s", err) } - if err := secretFilterDst.Close(); err != nil { + if err := redactWriterDst.Close(); err != nil { return "", fmt.Errorf("failed to redact secrets, closing the stream failed: %s", err) } diff --git a/cli/run_util.go b/cli/run_util.go index 49368528e..febaf446e 100644 --- a/cli/run_util.go +++ b/cli/run_util.go @@ -33,6 +33,7 @@ import ( "github.com/bitrise-io/go-utils/pointers" "github.com/bitrise-io/go-utils/retry" coreanalytics "github.com/bitrise-io/go-utils/v2/analytics" + logV2 "github.com/bitrise-io/go-utils/v2/log" "github.com/bitrise-io/go-utils/versions" stepmanModels "github.com/bitrise-io/stepman/models" ) @@ -433,7 +434,7 @@ func (r WorkflowRunner) executeStep( return 1, fmt.Errorf("failed to read command environment: %w", err) } - cmd := stepruncmd.New(name, args, bitriseSourceDir, envs, stepSecrets, timeout, noOutputTimeout, stdout) + cmd := stepruncmd.New(name, args, bitriseSourceDir, envs, stepSecrets, timeout, noOutputTimeout, stdout, logV2.NewLogger()) return cmd.Run() } diff --git a/stepruncmd/stdout.go b/stepruncmd/stdout.go index dda5e183a..3fae6cee4 100644 --- a/stepruncmd/stdout.go +++ b/stepruncmd/stdout.go @@ -4,34 +4,35 @@ import ( "io" "github.com/bitrise-io/bitrise/stepruncmd/errorfinder" - "github.com/bitrise-io/bitrise/stepruncmd/filterwriter" + "github.com/bitrise-io/go-utils/v2/log" + "github.com/bitrise-io/go-utils/v2/redactwriter" ) type StdoutWriter struct { writer io.Writer - secretWriter *filterwriter.Writer + redactWriter *redactwriter.Writer errorWriter *errorfinder.ErrorFinder destWriter io.Writer } -func NewStdoutWriter(secrets []string, dest io.Writer) StdoutWriter { +func NewStdoutWriter(secrets []string, dest io.Writer, logger log.Logger) StdoutWriter { var outWriter io.Writer outWriter = dest errorWriter := errorfinder.NewErrorFinder(outWriter) outWriter = errorWriter - var secretWriter *filterwriter.Writer + var redactWriter *redactwriter.Writer if len(secrets) > 0 { - secretWriter = filterwriter.New(secrets, outWriter) - outWriter = secretWriter + redactWriter = redactwriter.New(secrets, outWriter, logger) + outWriter = redactWriter } return StdoutWriter{ writer: outWriter, - secretWriter: secretWriter, + redactWriter: redactWriter, errorWriter: errorWriter, destWriter: dest, } @@ -42,8 +43,8 @@ func (w StdoutWriter) Write(p []byte) (n int, err error) { } func (w StdoutWriter) Close() error { - if w.secretWriter != nil { - if err := w.secretWriter.Close(); err != nil { + if w.redactWriter != nil { + if err := w.redactWriter.Close(); err != nil { return err } } diff --git a/stepruncmd/stdout_test.go b/stepruncmd/stdout_test.go index e2b0959cd..74c0742d7 100644 --- a/stepruncmd/stdout_test.go +++ b/stepruncmd/stdout_test.go @@ -2,12 +2,13 @@ package stepruncmd import ( "bytes" + "github.com/bitrise-io/go-utils/v2/redactwriter" "testing" "time" "github.com/bitrise-io/bitrise/log" "github.com/bitrise-io/bitrise/log/logwriter" - "github.com/bitrise-io/bitrise/stepruncmd/filterwriter" + logV2 "github.com/bitrise-io/go-utils/v2/log" "github.com/stretchr/testify/require" ) @@ -43,7 +44,7 @@ func Test_GivenWriter_WhenConsoleLogging_ThenTransmitsLogs(t *testing.T) { logger := log.NewLogger(opts) writer := logwriter.NewLogWriter(logger) - w := NewStdoutWriter(nil, writer) + w := NewStdoutWriter(nil, writer, logV2.NewLogger()) for _, message := range tt.messages { gotN, err := w.Write([]byte(message)) require.NoError(t, err) @@ -90,7 +91,7 @@ func Test_GivenWriter_WhenConsoleLoggingAndSecretFiltering_ThenRedactsSecrets(t logger := log.NewLogger(opts) writer := logwriter.NewLogWriter(logger) - w := NewStdoutWriter(tt.secrets, writer) + w := NewStdoutWriter(tt.secrets, writer, logV2.NewLogger()) for _, message := range tt.messages { gotN, err := w.Write([]byte(message)) require.NoError(t, err) @@ -143,7 +144,7 @@ func Test_GivenWriter_WhenJSONLogging_ThenWritesJSON(t *testing.T) { logger := log.NewLogger(opts) writer := logwriter.NewLogWriter(logger) - w := NewStdoutWriter(nil, writer) + w := NewStdoutWriter(nil, writer, logV2.NewLogger()) for _, message := range tt.messages { gotN, err := w.Write([]byte(message)) require.NoError(t, err) @@ -209,7 +210,7 @@ func Test_GivenWriter_WhenJSONLoggingAndSecretFiltering_ThenWritesJSON(t *testin logger := log.NewLogger(opts) writer := logwriter.NewLogWriter(logger) - w := NewStdoutWriter([]string{"secret value"}, writer) + w := NewStdoutWriter([]string{"secret value"}, writer, logV2.NewLogger()) for _, message := range tt.messages { gotN, err := w.Write([]byte(message)) require.NoError(t, err) @@ -258,7 +259,7 @@ func Test_GivenWriter_WhenJSONLoggingAndSecretFiltering_ThenReturnsError(t *test logger := log.NewLogger(opts) writer := logwriter.NewLogWriter(logger) - w := NewStdoutWriter([]string{"secret value"}, writer) + w := NewStdoutWriter([]string{"secret value"}, writer, logV2.NewLogger()) for _, message := range tt.messages { gotN, err := w.Write([]byte(message)) require.NoError(t, err) @@ -306,7 +307,7 @@ func Test_GivenWriter_WhenJSONLoggingAndSecretFiltering_ThenRedactsSecrets(t *te logger := log.NewLogger(opts) writer := logwriter.NewLogWriter(logger) - w := NewStdoutWriter(tt.secrets, writer) + w := NewStdoutWriter(tt.secrets, writer, logV2.NewLogger()) for _, message := range tt.messages { gotN, err := w.Write([]byte(message)) require.NoError(t, err) @@ -325,7 +326,7 @@ func Test_GivenWriter_WhenJSONLoggingAndSecretFiltering_ThenRedactsSecrets(t *te } func Test_WhenSecretsProvided_ThenRootWriterIsFilterWriter(t *testing.T) { - w := NewStdoutWriter([]string{"secret"}, nil) - _, isFilterWriter := w.writer.(*filterwriter.Writer) + w := NewStdoutWriter([]string{"secret"}, nil, logV2.NewLogger()) + _, isFilterWriter := w.writer.(*redactwriter.Writer) require.True(t, isFilterWriter) } diff --git a/stepruncmd/stepruncmd.go b/stepruncmd/stepruncmd.go index 91dc5b241..cff65674b 100644 --- a/stepruncmd/stepruncmd.go +++ b/stepruncmd/stepruncmd.go @@ -8,17 +8,18 @@ import ( "os/exec" "time" - "github.com/bitrise-io/bitrise/log" "github.com/bitrise-io/bitrise/stepruncmd/timeoutcmd" + "github.com/bitrise-io/go-utils/v2/log" ) type Cmd struct { cmd timeoutcmd.Command stdout StdoutWriter + logger log.Logger } -func New(name string, args []string, workDir string, envs, secrets []string, timeout, noOutputTimeout time.Duration, stdout io.Writer) Cmd { - outWriter := NewStdoutWriter(secrets, stdout) +func New(name string, args []string, workDir string, envs, secrets []string, timeout, noOutputTimeout time.Duration, stdout io.Writer, logger log.Logger) Cmd { + outWriter := NewStdoutWriter(secrets, stdout, logger) cmd := timeoutcmd.New(workDir, name, args...) cmd.SetTimeout(timeout) @@ -26,14 +27,14 @@ func New(name string, args []string, workDir string, envs, secrets []string, tim cmd.SetStandardIO(os.Stdin, outWriter, outWriter) cmd.SetEnv(append(envs, "PWD="+workDir)) - return Cmd{cmd: cmd, stdout: outWriter} + return Cmd{cmd: cmd, stdout: outWriter, logger: logger} } func (c *Cmd) Run() (int, error) { cmdErr := c.cmd.Start() if err := c.stdout.Close(); err != nil { - log.Warnf("Failed to close command output writer: %s", err) + c.logger.Warnf("Failed to close command output writer: %s", err) } if cmdErr == nil { diff --git a/stepruncmd/stepruncmd_test.go b/stepruncmd/stepruncmd_test.go index 278bba4eb..50dd506d3 100644 --- a/stepruncmd/stepruncmd_test.go +++ b/stepruncmd/stepruncmd_test.go @@ -3,13 +3,14 @@ package stepruncmd import ( "testing" + "github.com/bitrise-io/go-utils/v2/log" "github.com/stretchr/testify/require" ) func TestCmdSecretRedactionEnabled(t *testing.T) { failingBashCmd := `echo -e "\033[31;1mInvalid password: 1234\033[0m"; exit 1` secrets := []string{"1234"} - cmd := New("bash", []string{"-c", failingBashCmd}, "", nil, secrets, 0, 0, nil) + cmd := New("bash", []string{"-c", failingBashCmd}, "", nil, secrets, 0, 0, nil, log.NewLogger()) _, err := cmd.Run() require.EqualError(t, err, "Invalid password: [REDACTED]") } @@ -17,7 +18,7 @@ func TestCmdSecretRedactionEnabled(t *testing.T) { func TestCmdSecretRedactionDisabled(t *testing.T) { failingBashCmd := `echo -e "\033[31;1mInvalid password: 1234\033[0m"; exit 1` secrets := []string(nil) - cmd := New("bash", []string{"-c", failingBashCmd}, "", nil, secrets, 0, 0, nil) + cmd := New("bash", []string{"-c", failingBashCmd}, "", nil, secrets, 0, 0, nil, log.NewLogger()) _, err := cmd.Run() require.EqualError(t, err, "Invalid password: 1234") } From e47c5f580098927847223126b7e9ae135bda4307 Mon Sep 17 00:00:00 2001 From: Szabolcs Toth Date: Wed, 13 Sep 2023 11:02:00 +0100 Subject: [PATCH 4/4] Use log adapter --- analytics/logger.go | 47 -------------------------------------------- analytics/tracker.go | 3 ++- cli/analytics.go | 5 +++-- log/adapter.go | 43 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 50 deletions(-) delete mode 100644 analytics/logger.go create mode 100644 log/adapter.go diff --git a/analytics/logger.go b/analytics/logger.go deleted file mode 100644 index 6b75ea466..000000000 --- a/analytics/logger.go +++ /dev/null @@ -1,47 +0,0 @@ -package analytics - -import ( - "github.com/bitrise-io/bitrise/log" -) - -// utilsLogAdapter extends the bitrise/log.Logger to meet the go-utils/v2/log.Logger interface. -type utilsLogAdapter struct { - debug bool - log.Logger -} - -func newUtilsLogAdapter() utilsLogAdapter { - opts := log.GetGlobalLoggerOpts() - return utilsLogAdapter{ - Logger: log.NewLogger(opts), - debug: opts.DebugLogEnabled, - } -} - -func (l *utilsLogAdapter) TInfof(format string, v ...interface{}) { - log.Infof(format, v...) -} -func (l *utilsLogAdapter) TWarnf(format string, v ...interface{}) { - log.Warnf(format, v...) -} -func (l *utilsLogAdapter) TPrintf(format string, v ...interface{}) { - log.Printf(format, v...) -} -func (l *utilsLogAdapter) TDonef(format string, v ...interface{}) { - log.Donef(format, v...) -} -func (l *utilsLogAdapter) TDebugf(format string, v ...interface{}) { - if !l.debug { - return - } - log.Debugf(format, v...) -} -func (l *utilsLogAdapter) TErrorf(format string, v ...interface{}) { - log.Errorf(format, v...) -} -func (l *utilsLogAdapter) Println() { - log.Print() -} -func (l *utilsLogAdapter) EnableDebugLog(enable bool) { - l.debug = enable -} diff --git a/analytics/tracker.go b/analytics/tracker.go index 75a42fa51..2446fd0f9 100644 --- a/analytics/tracker.go +++ b/analytics/tracker.go @@ -7,6 +7,7 @@ import ( "time" "github.com/bitrise-io/bitrise/configs" + "github.com/bitrise-io/bitrise/log" "github.com/bitrise-io/bitrise/models" "github.com/bitrise-io/bitrise/version" "github.com/bitrise-io/go-utils/v2/analytics" @@ -123,7 +124,7 @@ func NewDefaultTracker() Tracker { envRepository := env.NewRepository() stateChecker := NewStateChecker(envRepository) - logger := newUtilsLogAdapter() + logger := log.NewUtilsLogAdapter() tracker := analytics.NewDefaultTracker(&logger) if stateChecker.UseAsync() { tracker = analytics.NewDefaultTracker(&logger) diff --git a/cli/analytics.go b/cli/analytics.go index 5d6254f28..281a4976a 100644 --- a/cli/analytics.go +++ b/cli/analytics.go @@ -5,8 +5,8 @@ import ( "fmt" "io" + "github.com/bitrise-io/bitrise/log" "github.com/bitrise-io/envman/models" - "github.com/bitrise-io/go-utils/v2/log" "github.com/bitrise-io/go-utils/v2/redactwriter" ) @@ -50,7 +50,8 @@ func redactStepInputs(environment map[string]string, inputs []models.Environment func redactWithSecrets(inputValue string, secrets []string) (string, error) { src := bytes.NewReader([]byte(inputValue)) dstBuf := new(bytes.Buffer) - redactWriterDst := redactwriter.New(secrets, dstBuf, log.NewLogger()) + logger := log.NewUtilsLogAdapter() + redactWriterDst := redactwriter.New(secrets, dstBuf, &logger) if _, err := io.Copy(redactWriterDst, src); err != nil { return "", fmt.Errorf("failed to redact secrets, stream copy failed: %s", err) diff --git a/log/adapter.go b/log/adapter.go new file mode 100644 index 000000000..252222d60 --- /dev/null +++ b/log/adapter.go @@ -0,0 +1,43 @@ +package log + +// UtilsLogAdapter extends the bitrise/log.Logger to meet the go-utils/v2/log.Logger interface. +type UtilsLogAdapter struct { + debug bool + Logger +} + +func NewUtilsLogAdapter() UtilsLogAdapter { + opts := GetGlobalLoggerOpts() + return UtilsLogAdapter{ + Logger: NewLogger(opts), + debug: opts.DebugLogEnabled, + } +} + +func (l *UtilsLogAdapter) TInfof(format string, v ...interface{}) { + Infof(format, v...) +} +func (l *UtilsLogAdapter) TWarnf(format string, v ...interface{}) { + Warnf(format, v...) +} +func (l *UtilsLogAdapter) TPrintf(format string, v ...interface{}) { + Printf(format, v...) +} +func (l *UtilsLogAdapter) TDonef(format string, v ...interface{}) { + Donef(format, v...) +} +func (l *UtilsLogAdapter) TDebugf(format string, v ...interface{}) { + if !l.debug { + return + } + Debugf(format, v...) +} +func (l *UtilsLogAdapter) TErrorf(format string, v ...interface{}) { + Errorf(format, v...) +} +func (l *UtilsLogAdapter) Println() { + Print() +} +func (l *UtilsLogAdapter) EnableDebugLog(enable bool) { + l.debug = enable +}