From 77ae54f1cec57d84ec3993e288561d489f001832 Mon Sep 17 00:00:00 2001 From: Shawn Hurley Date: Wed, 25 Sep 2024 13:23:01 -0400 Subject: [PATCH] :sparkles: adding the intial analizer-lsp RPC client and validator --- playpen/client/rpc.py | 14 +++ playpen/repo_level_awareness/api.py | 7 +- playpen/repo_level_awareness/codeplan.py | 25 +++++- .../{agents => task_runner}/__init__.py | 0 .../task_runner/analyzer_lsp/__init__.py | 0 .../task_runner/analyzer_lsp/agent.py | 0 .../task_runner/analyzer_lsp/api.py | 12 +++ .../task_runner/analyzer_lsp/validator.py | 88 +++++++++++++++++++ .../task_runner/compiler/__init__.py | 0 .../compiler}/compiler_agent.py | 0 10 files changed, 143 insertions(+), 3 deletions(-) rename playpen/repo_level_awareness/{agents => task_runner}/__init__.py (100%) create mode 100644 playpen/repo_level_awareness/task_runner/analyzer_lsp/__init__.py create mode 100644 playpen/repo_level_awareness/task_runner/analyzer_lsp/agent.py create mode 100644 playpen/repo_level_awareness/task_runner/analyzer_lsp/api.py create mode 100644 playpen/repo_level_awareness/task_runner/analyzer_lsp/validator.py create mode 100644 playpen/repo_level_awareness/task_runner/compiler/__init__.py rename playpen/repo_level_awareness/{agents => task_runner/compiler}/compiler_agent.py (100%) diff --git a/playpen/client/rpc.py b/playpen/client/rpc.py index 62a6b53a..522cc01c 100644 --- a/playpen/client/rpc.py +++ b/playpen/client/rpc.py @@ -124,6 +124,20 @@ def recv_response(self): log.debug(f"read data from stdout {repr(jsonrpc_res)}") return json.loads(jsonrpc_res) +class BaseRPCEndpoint(JsonRpcEndpoint): + def send_request(self, message): + json_string = json.dumps(message, cls=MyEncoder) + log.debug(f"sending data over stdin {repr(json_string)}") + with self.write_lock: + self.stdin.buffer.write(json_string.encode()) + self.stdin.flush() + + def recv_response(self): + with self.read_lock: + jsonrpc_res = self.stdout.readline().decode("utf-8") + log.debug(f"read data from stdout {repr(jsonrpc_res)}") + return json.loads(jsonrpc_res) + class RPCParams: def __init__(self, **kwargs): diff --git a/playpen/repo_level_awareness/api.py b/playpen/repo_level_awareness/api.py index a63c5bed..4522c335 100644 --- a/playpen/repo_level_awareness/api.py +++ b/playpen/repo_level_awareness/api.py @@ -1,7 +1,7 @@ from abc import ABC from dataclasses import dataclass from pathlib import Path -from typing import Any, Generator, Iterator, Optional +from typing import Any, List, Generator, Iterator, Optional from pydantic import BaseModel @@ -9,6 +9,11 @@ @dataclass class RpcClientConfig: repo_directory: Path + analyzer_lsp_server_binary: Path + rules_directory: Path + label_selector: Optional[str] + incident_selector: Optional[str] + included_paths: Optional[List[str]] # FIXME: Oh god oh no oh jeez oh man diff --git a/playpen/repo_level_awareness/codeplan.py b/playpen/repo_level_awareness/codeplan.py index 86b09721..c7d177e1 100755 --- a/playpen/repo_level_awareness/codeplan.py +++ b/playpen/repo_level_awareness/codeplan.py @@ -6,6 +6,7 @@ from playpen.repo_level_awareness.api import Agent, RpcClientConfig, Task, TaskResult, ValidationStep from playpen.repo_level_awareness.maven_validator import MavenCompileStep +from playpen.repo_level_awareness.task_runner.analyzer_lsp.validator import AnlayzerLSPStep def main(): @@ -18,9 +19,17 @@ def main(): "source_directory", help="The root directory of the project to be fixed" ) + parser.add_argument( + "rules_directory", help="The root directory of the rules to use during analysis" + ) + + parser.add_argument( + "analyzer_lsp_server_binary", help="The binary for running analyzer-lsp RPC server" + ) + args = parser.parse_args() - config = RpcClientConfig(args.source_directory) + config = RpcClientConfig(args.source_directory, args.analyzer_lsp_server_binary, args.rules_directory, None, None, None) codeplan(config, None) @@ -33,7 +42,7 @@ def codeplan( task_manager = TaskManager( config, updated_file_content, - validators=[MavenCompileStep(config)], + validators=[MavenCompileStep(config), AnlayzerLSPStep(config)], agents=[whatever_agent], ) # has a list of files affected and unprocessed @@ -41,11 +50,13 @@ def codeplan( # has a list of current validation errors for task in task_manager.get_next_task(): + print(f"Next task is {task}") task_manager.supply_result(task_manager.execute_task(task)) # all current failing validations and all currently affected AND UNDEALT # WITH files # Can do revalidation, or use cached results or whatever + task_manager.stop() class TaskManager: @@ -150,6 +161,16 @@ def get_next_task(self) -> Generator[Task, Any, None]: # continue break + + def stop(self): + """For all agents or validators, if they have a running thread stop them.""" + for a in self.agents: + if hasattr(a, "stop"): + a.stop() + + for v in self.validators: + if hasattr(v, "stop"): + v.stop() if __name__ == "__main__": diff --git a/playpen/repo_level_awareness/agents/__init__.py b/playpen/repo_level_awareness/task_runner/__init__.py similarity index 100% rename from playpen/repo_level_awareness/agents/__init__.py rename to playpen/repo_level_awareness/task_runner/__init__.py diff --git a/playpen/repo_level_awareness/task_runner/analyzer_lsp/__init__.py b/playpen/repo_level_awareness/task_runner/analyzer_lsp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/playpen/repo_level_awareness/task_runner/analyzer_lsp/agent.py b/playpen/repo_level_awareness/task_runner/analyzer_lsp/agent.py new file mode 100644 index 00000000..e69de29b diff --git a/playpen/repo_level_awareness/task_runner/analyzer_lsp/api.py b/playpen/repo_level_awareness/task_runner/analyzer_lsp/api.py new file mode 100644 index 00000000..172d0c7c --- /dev/null +++ b/playpen/repo_level_awareness/task_runner/analyzer_lsp/api.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Type +from kai.models.report_types import AnalysisReport, Incident, RuleSet, Violation +from playpen.repo_level_awareness.api import ValidationError, ValidationResult, ValidationStep + +@dataclass +class AnalyzerRuleViolation(ValidationError): + incident: Incident + violation: Violation + ruleset: RuleSet + + diff --git a/playpen/repo_level_awareness/task_runner/analyzer_lsp/validator.py b/playpen/repo_level_awareness/task_runner/analyzer_lsp/validator.py new file mode 100644 index 00000000..09dab668 --- /dev/null +++ b/playpen/repo_level_awareness/task_runner/analyzer_lsp/validator.py @@ -0,0 +1,88 @@ +from dataclasses import dataclass, field +from typing import List, Dict, Optional, Type +import subprocess + +from playpen.repo_level_awareness.api import RpcClientConfig, ValidationError, ValidationResult, ValidationStep +from playpen.repo_level_awareness.task_runner.analyzer_lsp.api import AnalyzerRuleViolation +from playpen.client import anlalyzer_rpc as analyzer_rpc + +from kai.models.report import Report + +@dataclass +class RuleSet: + pass + +class AnlayzerLSPStep(ValidationStep): + + rpc: analyzer_rpc.AnalyzerRpcServer + + def __init__(self, RpcClientConfig: RpcClientConfig) -> None: + """This will start and analyzer-lsp jsonrpc server""" + + rpc_server = subprocess.Popen( + [RpcClientConfig.analyzer_lsp_server_binary, "-source-directory", RpcClientConfig.repo_directory, "-rules-directory", RpcClientConfig.rules_directory], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + # trunk-ignore-end(bandit/B603) + + self.rpc = analyzer_rpc.AnalyzerRpcServer( + json_rpc_endpoint=analyzer_rpc.AnlayzerRPCEndpoint( + rpc_server.stdin, rpc_server.stdout + ), + timeout=60, + ) + self.rpc.start() + + super().__init__(RpcClientConfig) + + def run(self) -> ValidationResult: + analyzer_output = self.__run_analyzer_lsp() + print(type(analyzer_output)) + errors = self.__parse_analyzer_lsp_output(analyzer_output) + return ValidationResult(passed=not errors, errors=errors) + + def __run_analyzer_lsp(self) -> List[RuleSet]: + + request_params = { + "label_selector": "konveyor.io/target=cloud-readiness", + "included_paths": [], + "incident_selector": "" + } + if self.config.label_selector is not None: + request_params.LabelSelector = self.config.label_selector + + if self.config.included_paths is not None: + request_params.IncludedPaths = self.config.included_paths + + if self.config.incident_selector is not None: + request_params.IncidentSelector = self.config.incident_selector + + return self.rpc.call_method( + "analysis_engine.Analyze", + kwargs=request_params, + ) + + def __parse_analyzer_lsp_output(self, analyzer_output: Dict[str, any]) -> List[AnalyzerRuleViolation]: + rulesets = analyzer_output.get("Rulesets") + + if not rulesets or not isinstance(rulesets, list): + print(f"here rulesets: {rulesets}") + return [] + + r = Report.load_report_from_object(rulesets, "analysis_run_task_runner") + + validation_errors: List[AnalyzerRuleViolation] = [] + for k, v in r.rulesets.items(): + print(k) + + for vk, vio in v.violations.items(): + print(vk) + for i in vio.incidents: + validation_errors.append(AnalyzerRuleViolation(i, vio, v)) + + return validation_errors + + def stop(self): + self.rpc.stop() \ No newline at end of file diff --git a/playpen/repo_level_awareness/task_runner/compiler/__init__.py b/playpen/repo_level_awareness/task_runner/compiler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/playpen/repo_level_awareness/agents/compiler_agent.py b/playpen/repo_level_awareness/task_runner/compiler/compiler_agent.py similarity index 100% rename from playpen/repo_level_awareness/agents/compiler_agent.py rename to playpen/repo_level_awareness/task_runner/compiler/compiler_agent.py