Added basic support for vectorscan

docker/build/entrypoint.sh

@@ -95,6 +95,8 @@ if [ "$PACKAGE_TYPE" = 'standalone' ]; then
     --hidden-import wordfence.cli.remediate.definition \
     --hidden-import wordfence.cli.countsites.countsites \
     --hidden-import wordfence.cli.countsites.definition \
+    --hidden-import wordfence.scanning.matching.pcre \
+    --hidden-import wordfence.scanning.matching.vectorscan \
     main.py
 
   # compress and copy to output volume

wordfence/cli/context.py

@@ -3,6 +3,7 @@
 
 from ..version import __version__, __version_name__
 from ..util import pcre
+from ..util.library import OptionalUtility
 from ..api import noc1, intelligence
 from ..util.caching import Cache, CacheDirectory, RuntimeCache, \
         InvalidCachedValueException, CacheException
@@ -37,6 +38,7 @@ def __init__(
         self._mailer = None
         self.configurer = None
         self._log_settings = None
+        self._vectorscan = None
 
     def get_log_level(self) -> LogLevel:
         if self.config.log_level is not None:
@@ -156,6 +158,14 @@ def get_mailer(self) -> Mailer:
             self._mailer = Mailer(self.config)
         return self._mailer
 
+    def get_vectorscan(self) -> OptionalUtility:
+        if self._vectorscan is None:
+            self._vectorscan = OptionalUtility('vectorscan')
+        return self._vectorscan
+
+    def has_vectorscan(self) -> bool:
+        return self.get_vectorscan().is_available()
+
     def display_version(self) -> None:
         if __version_name__ is None:
             name_suffix = ''
@@ -167,6 +177,13 @@ def display_version(self) -> None:
                 f"PCRE Version: {pcre.VERSION} - "
                 f"JIT Supported: {jit_support_text}"
             )
+        vectorscan = self.get_vectorscan()
+        vectorscan_supported = vectorscan.is_available()
+        vectorscan_support_text = 'Yes' if vectorscan_supported else 'No'
+        if vectorscan_supported:
+            vectorscan_support_text += \
+                f' - Version: {vectorscan.module.VERSION}'
+        print(f'Vectorscan Supported: {vectorscan_support_text}')
 
     def has_terminal_output(self) -> bool:
         return has_terminal_output()

wordfence/cli/malwarescan/definition.py

@@ -2,6 +2,7 @@
         PCRE_DEFAULT_MATCH_LIMIT_RECURSION
 from wordfence.util.units import byte_length
 
+from ...scanning.matching import MatchEngine
 from ..subcommands import SubcommandDefinition, UsageExample
 from ..config.typing import ConfigDefinitions
 from .reporting import SCAN_REPORT_CONFIG_OPTIONS
@@ -146,6 +147,15 @@
             "value_type": byte_length
         }
     },
+    "match-engine": {
+        "description": "The regex engine to use for malware scanning.",
+        "context": "ALL",
+        "argument_type": "OPTION",
+        "default": MatchEngine.get_default_option(),
+        "meta": {
+            "valid_options": MatchEngine.get_options()
+        }
+    },
     "match-all": {
         "description": "If set, all possible signatures will be checked "
                        "against each scanned file. Otherwise, only the "

wordfence/cli/malwarescan/malwarescan.py

@@ -6,6 +6,7 @@
 
 from wordfence import scanning
 from wordfence.scanning import filtering
+from wordfence.scanning.matching import MatchEngine
 from wordfence.util import caching, pcre
 from wordfence.intel.signatures import SignatureSet
 from wordfence.logging import (log, remove_initial_handler,
@@ -165,7 +166,8 @@ def invoke(self) -> int:
                 pcre_options=self._get_pcre_options(),
                 allow_io_errors=self.config.allow_io_errors,
                 debug=self.config.debug,
-                logging_initializer=self.context.get_log_settings().apply
+                logging_initializer=self.context.get_log_settings().apply,
+                match_engine=MatchEngine.for_option(self.config.match_engine)
             )
         if io_manager.should_read_stdin():
             options.path_source = io_manager.get_input_reader()

wordfence/scanning/matching/__init__.py

@@ -0,0 +1 @@
+from .matching import *  # noqa: 401, 403

wordfence/scanning/matching/matching.py

@@ -0,0 +1,137 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import List, Optional
+from importlib import import_module
+from contextlib import AbstractContextManager
+
+from ...intel.signatures import SignatureSet
+from ...util.pcre import PcreOptions
+
+DEFAULT_TIMEOUT = 1  # Seconds
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class MatchResult:
+
+    def __init__(self, matches: list):
+        self.matches = matches
+
+    def matches(self) -> bool:
+        return len(self.matches) > 0
+
+
+class MatchWorkspace(AbstractContextManager):
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        return self
+
+
+class MatcherContext(AbstractContextManager):
+
+    def __init__(self):
+        self.matches = {}
+        self.timeouts = set()
+
+    def process_chunk(
+                self,
+                chunk: bytes,
+                start: bool = False,
+                workspace: Optional[MatchWorkspace] = None
+            ):
+        raise NotImplementedError()
+
+    def _record_match(self, identifier: str, matched: str) -> None:
+        self.matches[identifier] = matched
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        pass
+
+
+class Matcher:
+
+    def __init__(
+                self,
+                signature_set: SignatureSet,
+                timeout: int = DEFAULT_TIMEOUT,
+                match_all: bool = False,
+                lazy: bool = False
+            ):
+        self.signature_set = signature_set
+        self.timeout = timeout
+        self.match_all = match_all
+        self.lazy = lazy
+        self.prepared = False
+
+    def prepare(self) -> None:
+        if self.prepared:
+            return
+        self._prepare()
+        self.prepared = True
+
+    def _prepare(self) -> None:
+        raise NotImplementedError()
+
+    def create_workspace(self) -> Optional[MatchWorkspace]:
+        return MatchWorkspace()
+
+    def create_context(self) -> MatcherContext:
+        raise NotImplementedError()
+
+
+class BaseMatcherContext(MatcherContext):
+
+    def __init__(self, matcher: Matcher):
+        self.matcher = matcher
+        super().__init__()
+
+
+@dataclass
+class MatchEngineOptions:
+    signature_set: SignatureSet
+    match_all: bool
+    lazy: bool
+    pcre_options: PcreOptions
+
+
+class MatchEngine(Enum):
+    PCRE = 'pcre'
+    VECTORSCAN = 'vectorscan'
+
+    def __init__(self, option: str, module: Optional[str] = None):
+        self.option = option
+        self.module = option if module is None else module
+
+    @classmethod
+    def get_options(cls) -> List:
+        return [engine.option for engine in cls]
+
+    @classmethod
+    def for_option(cls, option: str):
+        for engine in cls:
+            if engine.option == option:
+                return engine
+        raise ValueError(f'Unrecognized engine option: {option}')
+
+    @classmethod
+    def get_default(cls):
+        return cls.PCRE
+
+    @classmethod
+    def get_default_option(cls):
+        return cls.get_default().option
+
+    def create_matcher(self, options: MatchEngineOptions) -> Matcher:
+        module = import_module(
+                f'.{self.module}',
+                'wordfence.scanning.matching'
+            )
+        return module.create_matcher(options)