feat: generate report for cloned git project

.gitignore

@@ -1,6 +1,7 @@
 node_modules
 report.md
-.vscode/tasks.json
+.vscode/
 scope.txt
 repos
-reports
+reports
+github-projects/

package.json

@@ -7,27 +7,32 @@
   "license": "Unlicensed",
   "private": true,
   "scripts": {
-    "analyze": "ts-node src/index.ts"
+    "analyze": "ts-node src/index.ts",
+    "worker": "ts-node src/worker.ts"
   },
   "dependencies": {
     "@types/node": "^7.0.5",
     "@types/semver": "^7.3.13",
     "@typescript-eslint/eslint-plugin": "4.28.4",
     "@typescript-eslint/parser": "4.28.4",
+    "bull": "^4.12.2",
+    "commander": "^11.0.0",
     "eslint": "^8.25.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-config-standard": "16.0.3",
     "eslint-plugin-import": "2.23.4",
     "eslint-plugin-mocha": "^10.0.3",
     "eslint-plugin-mocha-no-only": "1.1.1",
     "eslint-plugin-node": "11.1.0",
-    "commander": "^11.0.0",
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-promise": "^5.1.0",
     "eslint-plugin-simple-import-sort": "^7.0.0",
     "eslint-plugin-standard": "5.0.0",
+    "glob": "^10.3.10",
     "prettier": "^2.7.1",
+    "rimraf": "^5.0.5",
     "semver": "^7.3.8",
+    "simple-git": "^3.22.0",
     "solc": "^0.8.17",
     "solc-0.6.10": "npm:solc@0.6.10",
     "solc-0.6.11": "npm:solc@0.6.11",

src/main.ts

@@ -4,12 +4,13 @@ import compileAndBuildAST from './compile';
 import issues from './issues';
 import { InputType, IssueTypes } from './types';
 import { recursiveExploration } from './utils';
+const path = require('node:path');
 
-/*   .---. ,--.  ,--  / ,---.   ,--.   ,--.'  ,-. .----. ,------.,------, 
-    / .  | |   \ |  | | \ /`.\  |  |   `\ . '.' /\_.-,  ||  .---'|   /`. ' 
-   / /|  | |  . '|  |)'-'|_.' | |  |     \     /   |_  <(|  '--. |  |_.' | 
-  / '-'  |||  |\    |(|  .-.  |(|  '_     /   /) .-. \  ||  .--' |  .   .' 
-  `---|  |'|  | \   | |  | |  | |     |`-/   /`  \ `-'  /|  `---.|  |\  \  
+/*   .---. ,--.  ,--  / ,---.   ,--.   ,--.'  ,-. .----. ,------.,------,
+    / .  | |   \ |  | | \ /`.\  |  |   `\ . '.' /\_.-,  ||  .---'|   /`. '
+   / /|  | |  . '|  |)'-'|_.' | |  |     \     /   |_  <(|  '--. |  |_.' |
+  / '-'  |||  |\    |(|  .-.  |(|  '_     /   /) .-. \  ||  .--' |  .   .'
+  `---|  |'|  | \   | |  | |  | |     |`-/   /`  \ `-'  /|  `---.|  |\  \
     `--' `--'  `--' `--' `--' `-----'  `--'     `---'' `------'`--' '--' */
 
 // ============================== GENERATE REPORT ==============================
@@ -25,7 +26,7 @@ const main = async (
   basePath: string,
   scopeFile: string | null,
   githubLink: string | null,
-  out: string,
+  out?: string,
   scope?: string,
 ) => {
   let result = '# Report\n\n';
@@ -45,21 +46,21 @@ const main = async (
     fileNames = recursiveExploration(basePath);
   }
 
-  console.log('Scope: ', fileNames);
+  // console.log('Scope: ', fileNames);
 
   // Uncomment next lines to have the list of analyzed files in the report
 
-  // result += '## Files analyzed\n\n';
-  // fileNames.forEach(fileName => {
-  //   result += ` - ${fileName}\n`;
-  // });
+  result += '## Files analyzed\n\n';
+  fileNames.forEach(fileName => {
+    result += ` - ${fileName}\n`;
+  });
 
   // Read file contents and build AST
   const files: InputType = [];
   const asts = await compileAndBuildAST(basePath, fileNames);
   fileNames.forEach((fileName, index) => {
     files.push({
-      content: fs.readFileSync(`${basePath}${fileName}`, { encoding: 'utf8', flag: 'r' }),
+      content: fs.readFileSync(path.join(basePath, fileName), { encoding: 'utf8', flag: 'r' }),
       name: fileName,
       ast: asts[index],
     });
@@ -73,6 +74,9 @@ const main = async (
     );
   }
 
+  if (!out) {
+    return result;
+  }
   fs.writeFileSync(out, result);
 };
 

src/utils.ts

@@ -3,6 +3,7 @@ import fs from 'fs';
 import { findAll } from 'solidity-ast/utils';
 import { ContractDefinition, SourceUnit } from 'solidity-ast';
 import { exec } from 'child_process';
+const path = require('node:path');
 
 /**
  * @notice Returns the line corresponding to a character index in a file
@@ -50,12 +51,15 @@ export const recursiveExploration = (basePath: string, extension = '.sol'): stri
   let directoryQueue = [''];
   while (directoryQueue.length > 0) {
     let dir = directoryQueue.pop();
-    let tempFileNames = fs.readdirSync(`${basePath}${dir}`);
-    for (let fileName of tempFileNames) {
+    let tempFileNames = fs.readdirSync(path.join(basePath, dir));
+    for (let  fileName of tempFileNames) {
       fileName = `${dir}${fileName}`;
+      if (fileName === 'node_modules') {
+        continue;
+      }
       if (fileName.endsWith(extension)) {
         fileNames.push(fileName);
-      } else if (fs.statSync(`${basePath}${fileName}`).isDirectory()) {
+      } else if (fs.statSync(path.join(basePath, fileName)).isDirectory()) {
         directoryQueue.push(fileName + '/');
       }
     }
@@ -112,3 +116,23 @@ export const getStorageVariable = (contract: ContractDefinition): string[] => {
   }
   return storageVariables;
 };
+
+export const gitUrlToSsh = (url: string) => {
+  const regex = /^(https?:\/\/)?github\.com\/([a-zA-Z0-9-_.]+\/[a-zA-Z0-9-_.]+)(\/?)$/;
+
+  const match = url.match(regex);
+  if (match && match.length === 4) {
+    const [_, protocol, repoPath, trailingSlash] = match;
+
+    const formattedRepoPath = repoPath.endsWith('/') ? repoPath.slice(0, -1) : repoPath;
+    const [username, projectName] = formattedRepoPath.split('/');
+
+    return [
+      `git@github.com:${formattedRepoPath}.git`,
+      username,
+      projectName,
+    ];
+  } else {
+    return [];
+  }
+};

src/worker.ts

@@ -0,0 +1,63 @@
+const Queue = require('bull');
+const path = require('node:path');
+const util = require('util');
+const exec = util.promisify(require('child_process').exec);
+import { rimrafSync } from 'rimraf';
+import { gitUrlToSsh } from './utils';
+import { simpleGit } from 'simple-git';
+import main from './main';
+import { Job } from 'bull';
+
+const REDIS_URL = 'redis://localhost:6379';
+const QUEUE_NAME = '4naly3er_analyze';
+const RETURN_QUEUE_NAME = 'auditor_platform:auto_analyze';
+
+interface IAutoAnalyzeJobData {
+  id: string;
+  github: string;
+  report?: string;
+}
+
+const queue = new Queue(QUEUE_NAME, { redis: REDIS_URL });
+const returnQueue = new Queue(RETURN_QUEUE_NAME, { redis: REDIS_URL });
+
+queue.process(async (job: Job<IAutoAnalyzeJobData>) => {
+  const data = job.data;
+
+  const [gitSsh, gitUser, gitProject] = gitUrlToSsh(data.github);
+  const projectPath = path.join(__dirname, 'github-projects', gitUser, gitProject);
+
+  try {
+    await simpleGit().clone(gitSsh, projectPath);
+    console.log('Repository cloned successfully!');
+  } catch (error) {
+    console.error('Error cloning repository:', error);
+    return;
+  }
+
+  const npmInstallCommand = 'npm install';
+  try {
+    const { stdout, stderr } = await exec(npmInstallCommand, { cwd: projectPath });
+    console.log(`stdout: ${stdout}`);
+    console.error(`stderr: ${stderr}`);
+  } catch (error: any) {
+    console.error(`Error: ${error.message}`);
+    return;
+  }
+  const result = await main(projectPath, ``, data.github);
+
+  rimrafSync(projectPath);
+  console.log('Remove project folder successfully');
+
+  returnQueue.add('auditor_platform:auto_analyze::HandleSaveAnalyzeJob', { id: data.id, result });
+});
+
+queue.on('completed', (job: any) => {
+  console.log('Job completed:', job.id);
+});
+
+queue.on('error', (error: any) => {
+  console.error('Error processing job:', error);
+});
+
+console.log(`Worker listening to queue "${QUEUE_NAME}" on Redis...`);