Bugfix: watch_usn() was not flushing the mft LRU properly (#2032)

This caused it to stop emitting rows after a while because it was unable to see new data.
Velocidex · Aug 30, 2022 · 0cd9c83 · 0cd9c83
1 parent a1c6351
commit 0cd9c83
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 11 deletions.
diff --git a/artifacts/definitions/Windows/Detection/Usn.yaml b/artifacts/definitions/Windows/Detection/Usn.yaml
@@ -23,15 +23,14 @@ parameters:
   - name: Device
     description: The NTFS drive to watch
     default: C:\\
+  - name: USN_FREQUENCY
+    type: int
+    description: How many seconds before rechecking the USN journal.
+    default: "30"
 
 precondition: SELECT OS from info() where OS = "windows"
 
 sources:
   - query: |
-      -- We need to make sure the NTFS cache is not too old to make
-      -- sure we can pick up changes quickly.
-      LET NTFS_CACHE_TIME = 30
-      LET USN_FREQUENCY = 60
-
       SELECT * FROM watch_usn(device=Device)
       WHERE FullPath =~ PathRegex
diff --git a/go.mod b/go.mod
@@ -116,7 +116,7 @@ require (
 	howett.net/plist v0.0.0-20201203080718-1454fab16a06
 	www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e
 	www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671
-	www.velocidex.com/golang/go-ntfs v0.1.2-0.20220616022946-852572498c13
+	www.velocidex.com/golang/go-ntfs v0.1.2-0.20220830062011-3a8f0f1aeae9
 	www.velocidex.com/golang/go-pe v0.1.1-0.20220506020923-9fac492a9b0d
 	www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a
 	www.velocidex.com/golang/oleparse v0.0.0-20220617011920-94df2342d0b7

diff --git a/go.sum b/go.sum
@@ -1030,8 +1030,8 @@ www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e h1:AhcXPgNKhJ
 www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e/go.mod h1:ykEQ7AUF9AL+mfCefDmLwmZOnU2So6wM3qKM8xdsHhU=
 www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671 h1:pfvo7NFo0eJj6Zr7d+4vMx/Zr2JriMMPEWRHUf1YjUw=
 www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671/go.mod h1:qnzHyB9yD2khtYO+wf3ck9FQxX3wFhXeJHFBnuUIZcc=
-www.velocidex.com/golang/go-ntfs v0.1.2-0.20220616022946-852572498c13 h1:FHcUvOaYKJTjW94hIVBfes1+VXerEF7x1VHw3sRJQpg=
-www.velocidex.com/golang/go-ntfs v0.1.2-0.20220616022946-852572498c13/go.mod h1:Ha0bxCjZeLVvNM7zU6B7bj5nWkWn2UyHs2QWSj3OspQ=
+www.velocidex.com/golang/go-ntfs v0.1.2-0.20220830062011-3a8f0f1aeae9 h1:7Xm/mOURZiZt+lJTOZwgSPILJ5dYGPmzi1o8+ne67k0=
+www.velocidex.com/golang/go-ntfs v0.1.2-0.20220830062011-3a8f0f1aeae9/go.mod h1:Ha0bxCjZeLVvNM7zU6B7bj5nWkWn2UyHs2QWSj3OspQ=
 www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de/go.mod h1:j9Xy8Z9wxzY2SCB8CqDkkoSzy+eUwevnOrRm/XM2q/A=
 www.velocidex.com/golang/go-pe v0.1.1-0.20220506020923-9fac492a9b0d h1:OQKwxK0O4a/8YTmfkQNzUspyrvlpRbLi318L08DC0oY=
 www.velocidex.com/golang/go-pe v0.1.1-0.20220506020923-9fac492a9b0d/go.mod h1:TPJ3phbAuZIu7XuPyNqgoP2k3P+eNHfHHGcivhcsxaA=

diff --git a/utils/read_seek_reader_adapter.go b/utils/read_seek_reader_adapter.go
@@ -5,12 +5,30 @@ import (
 	"io"
 )
 
+type Flusher interface {
+	Flush()
+}
+
+type Closer interface {
+	Close() error
+}
+
 type ReadSeekReaderAdapter struct {
 	reader io.ReaderAt
 	offset int64
 }
 
 func (self ReadSeekReaderAdapter) Close() error {
+	// Try to close our delegate if possible
+	switch t := self.reader.(type) {
+	case Flusher:
+		t.Flush()
+
+	case Closer:
+		t.Close()
+
+	default:
+	}
 	return nil
 }
 

diff --git a/vql/constants/ntfs.go b/vql/constants/ntfs.go
@@ -26,8 +26,6 @@ func GetNTFSCacheTime(ctx context.Context, scope vfilter.Scope) time.Duration {
 	}
 	if cache_life == 0 {
 		cache_life = 60
-	} else {
-		scope.Log("Will expire NTFS cache every %v sec\n", cache_life)
 	}
 
 	return time.Duration(cache_life) * time.Second

diff --git a/vql/readers/paged.go b/vql/readers/paged.go
@@ -121,7 +121,12 @@ func (self *AccessorReader) Size() int {
 func (self *AccessorReader) Key() string {
 	return self.key
 }
-func (self *AccessorReader) Close() {
+
+func (self *AccessorReader) Flush() {
+	self.Close()
+}
+
+func (self *AccessorReader) Close() error {
 	self.mu.Lock()
 
 	cancel := self.cancel
@@ -141,6 +146,8 @@ func (self *AccessorReader) Close() {
 	if reader != nil {
 		reader.Close()
 	}
+
+	return nil
 }
 
 func (self *AccessorReader) ReadAt(buf []byte, offset int64) (int, error) {