CyberBuddy - Add the ability to upload pdf/doc/docx/txt files to feed…

… the vector database with tech documents.

app/Http/Controllers/HomeController.php

@@ -10,6 +10,7 @@
 use App\Models\YnhOsqueryRule;
 use App\Models\YnhServer;
 use App\Models\YnhSshTraces;
+use App\Modules\CyberBuddy\Http\Controllers\CyberBuddyController;
 use App\User;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
@@ -154,6 +155,12 @@ public function index(Request $request)
             ];
         }
 
+        $knowledge_base = collect();
+
+        if ($tab === 'knowledge_base') {
+            $knowledge_base = (new CyberBuddyController())->files();
+        }
+
         $notifications = $user->unreadNotifications
             ->map(function ($notification) {
                 return [
@@ -187,7 +194,8 @@ public function index(Request $request)
             'backups',
             'os_infos',
             'notifications',
-            'overview'
+            'overview',
+            'knowledge_base'
         ));
     }
 }

app/Modules/CyberBuddy/Http/Controllers/CyberBuddyController.php

@@ -13,6 +13,7 @@
 use App\Modules\CyberBuddy\Http\Requests\UploadOneFileRequest;
 use App\Modules\CyberBuddy\Models\ChunkCollection;
 use App\Modules\CyberBuddy\Models\File;
+use App\Modules\CyberBuddy\Rules\IsValidCollectionName;
 use App\User;
 use BotMan\BotMan\BotMan;
 use Illuminate\Http\UploadedFile;
@@ -62,6 +63,42 @@ public function showChat()
         return view('modules.cyber-buddy.chat');
     }
 
+    public function files()
+    {
+        return File::select('cb_files.*')
+            ->join('cb_chunks_collections', 'cb_chunks_collections.id', '=', 'cb_files.collection_id')
+            ->orderBy('cb_chunks_collections.name')
+            ->orderBy('name_normalized')
+            ->get()
+            ->map(function (File $file) {
+                $nbChunks = $file->chunks()->count();
+                $nbVectors = $file->chunks()->where('is_embedded', true)->count();
+                $nbNotVectors = $file->chunks()->where('is_embedded', false)->count();
+                return [
+                    'collection' => $file->collection->name,
+                    'filename' => "{$file->name_normalized}.{$file->extension}",
+                    'size' => $file->size,
+                    'nb_chunks' => $nbChunks,
+                    'nb_vectors' => $nbVectors,
+                    'nb_not_vectors' => $nbNotVectors,
+                    'status' => $nbVectors != 0 && $nbNotVectors === 0 ? 'processed' : 'processing',
+                    'download_url' => $file->downloadUrl(),
+                ];
+            });
+    }
+
+    public function collections()
+    {
+        return ChunkCollection::orderBy('name', 'asc')
+            ->get()
+            ->map(function (ChunkCollection $collection) {
+                return [
+                    'id' => $collection->id,
+                    'name' => $collection->name,
+                ];
+            });
+    }
+
     public function streamFile(string $secret, StreamOneFileRequest $request)
     {
         /** @var File $file */
@@ -124,7 +161,10 @@ public function uploadOneFile(UploadOneFileRequest $request)
         $collection = ChunkCollection::where('name', $request->string('collection'))->where('is_deleted', false)->first();
 
         if (!$collection) {
-            return response()->json(['error' => 'Invalid collection name.'], 500);
+            if (!IsValidCollectionName::test($request->string('collection'))) {
+                return response()->json(['error' => 'Invalid collection name.'], 500);
+            }
+            $collection = ChunkCollection::create(['name' => $request->string('collection')]);
         }
         if (!$request->hasFile('file')) {
             return response()->json(['error' => 'Missing file content.'], 500);
@@ -148,7 +188,10 @@ public function uploadManyFiles(UploadManyFilesRequest $request)
         $collection = ChunkCollection::where('name', $request->string('collection'))->where('is_deleted', false)->first();
 
         if (!$collection) {
-            return response()->json(['error' => 'Invalid collection name.'], 500);
+            if (!IsValidCollectionName::test($request->string('collection'))) {
+                return response()->json(['error' => 'Invalid collection name.'], 500);
+            }
+            $collection = ChunkCollection::create(['name' => $request->string('collection')]);
         }
 
         $files = $request->file('files');

app/Modules/CyberBuddy/web.php

@@ -2,6 +2,8 @@
 
 use Illuminate\Support\Facades\Route;
 
+Route::get('/files', 'CyberBuddyController@files')->middleware('auth');
+
 Route::get('/files/stream/{secret}', 'CyberBuddyController@streamFile');
 
 Route::get('/files/download/{secret}', 'CyberBuddyController@downloadFile');
@@ -10,6 +12,8 @@
 
 Route::post('/files/many', 'CyberBuddyController@uploadManyFiles')->middleware('auth:sanctum');
 
+Route::get('/collections', 'CyberBuddyController@collections')->middleware('auth');
+
 Route::get('/cyber-buddy', 'CyberBuddyController@showPage')->middleware('auth');
 
 Route::get('/cyber-buddy/chat', 'CyberBuddyController@showChat');

public/css/custom.css

@@ -114,6 +114,14 @@ body {
   padding: .25em .6em;
 }
 
+.tw-pill-blue-text {
+  font-size: 75%;
+  color: #00264b;
+  font-weight: 600;
+  line-height: 1;
+  padding: .25em .6em;
+}
+
 .cursor-pointer {
   cursor: pointer
 }

resources/lang/fr.json

@@ -130,5 +130,19 @@
   "User's name": "Nom de l'utilisateur",
   "Create Invitation": "Créer une invitation",
   "Excited to get started ?": "Impatient de commencer ?",
-  "Enter a domain name or an IP address belonging to you below :": "Saisissez ci-dessous un nom de domaine ou une adresse IP vous appartenant :"
+  "Enter a domain name or an IP address belonging to you below :": "Saisissez ci-dessous un nom de domaine ou une adresse IP vous appartenant :",
+  "Knowledge Base": "Base de connaissances",
+  "Import your documents !": "Importez vos documents !",
+  "Select or create collection...": "Sélectionnez ou créez une collection...",
+  "Submit": "Soumettre",
+  "Browse": "Parcourir",
+  "Filename": "Nom du fichier",
+  "File Size": "Taille du fichier",
+  "Number of Chunks": "Nombre de chunks",
+  "Number of Vectors": "Nombre de vecteurs",
+  "Integration Status": "Statut de l'intégration",
+  "processing": "en cours de traitement",
+  "processed": "traité",
+  "An error occurred. Try again in a moment or contact the support.": "Une erreur est survenue. Réessayez dans un instant ou contactez le support.",
+  "Your file has been successfully uploaded. It will be available shortly.": "Votre fichier a bien été importé. Il sera disponible dans quelques instants."
 }

resources/views/home/cards/_knowledge_base.blade.php

@@ -0,0 +1,130 @@
+@if(Auth::user()->canUseCyberBuddy())
+<div class="card card-accent-secondary tw-card">
+  <div class="card-header">
+    <h3 class="m-0"><b>{{ __('Import your documents !') }}</b></h3>
+  </div>
+  <div class="card-body p-3">
+    <div class="row">
+      <div class="col-3">
+        <div id="collections"></div>
+      </div>
+      <div class="col">
+        <div id="files"></div>
+      </div>
+      <div class="col-3">
+        <div id="submit">
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+<div class="card card-accent-secondary tw-card mt-3">
+  <div class="card-header">
+    <h3 class="m-0"><b>{{ __('Knowledge Base') }}</b></h3>
+  </div>
+  @if($files->isEmpty())
+  <div class="card-body">
+    <div class="row">
+      <div class="col">
+        {{ __('None.') }}
+      </div>
+    </div>
+  </div>
+  @else
+  <div class="card-body p-0">
+    <table class="table table-hover">
+      <thead>
+      <tr>
+        <th>{{ __('Collection') }}</th>
+        <th><i class="zmdi zmdi-long-arrow-down"></i>&nbsp;{{ __('Filename') }}</th>
+        <th style="text-align:right">{{ __('File Size') }}</th>
+        <th style="text-align:right">{{ __('Number of Chunks') }}</th>
+        <th style="text-align:right">{{ __('Number of Vectors') }}</th>
+        <th style="text-align:right">{{ __('Integration Status') }}</th>
+      </tr>
+      </thead>
+      <tbody>
+      @foreach($files as $file)
+      <tr>
+        <td>
+          <span class="tw-pill pill bg-primary">{{ $file['collection'] }}</span>
+        </td>
+        <td>
+          <a href="{{ $file['download_url'] }}">
+            {{ $file['filename'] }}
+          </a>
+        </td>
+        <td style="text-align:right">
+          {{ Illuminate\Support\Number::format($file['size'], locale:'sv') }}
+        </td>
+        <td style="text-align:right">
+          {{ Illuminate\Support\Number::format($file['nb_chunks'], locale:'sv') }}
+        </td>
+        <td style="text-align:right">
+          {{ Illuminate\Support\Number::format($file['nb_vectors'], locale:'sv') }}
+        </td>
+        <td style="text-align:right">
+          @if($file['status'] === 'processed')
+          <span class="tw-pill rounded-pill bg-success">{{ __($file['status']) }}</span>
+          @else
+          <span class="tw-pill-blue-text rounded-pill bg-warning">{{ __($file['status']) }}</span>
+          @endif
+        </td>
+      </tr>
+      @endforeach
+      </tbody>
+    </table>
+  </div>
+  @endif
+</div>
+<script>
+
+  let file = null;
+  let collection = null;
+
+  const elSubmit = new com.computablefacts.blueprintjs.MinimalButton(document.getElementById('submit'),
+    "{{ __('Submit') }}");
+  elSubmit.disabled = true;
+  elSubmit.onClick(() => {
+
+    elSubmit.loading = true;
+    elSubmit.disabled = true;
+
+    const formData = new FormData();
+    formData.append('file', file);
+    formData.append('collection', collection);
+
+    axios.post("{{ url('/cb/web/files/one') }}", formData, {
+      headers: {
+        'Content-Type': 'multipart/form-data',
+      }
+    }).then(response => {
+      toaster.toastSuccess("{{ __('Your file has been successfully uploaded. It will be available shortly.') }}");
+    }).catch(error => toaster.toastAxiosError(error)).finally(() => {
+      elSubmit.loading = false;
+      elSubmit.disabled = false;
+    });
+  });
+
+  const elFile = new com.computablefacts.blueprintjs.MinimalFileInput(document.getElementById('files'));
+  elFile.onSelectionChange(item => {
+    file = item;
+    elSubmit.disabled = !file || !collection;
+  });
+  elFile.buttonText = "{{ __('Browse') }}";
+
+  const elCollections = new com.computablefacts.blueprintjs.MinimalSelect(document.getElementById('collections'));
+  elCollections.onSelectionChange(item => {
+    collection = item;
+    elSubmit.disabled = !file || !collection;
+  });
+  elCollections.defaultText = "{{ __('Select or create collection...') }}";
+
+  document.addEventListener('DOMContentLoaded', function (event) {
+    axios.get("{{ url('/cb/web/collections') }}")
+    .then(response => {
+      elCollections.items = response.data.map(collection => collection.name);
+    }).catch(error => toaster.toastAxiosError(error));
+  });
+</script>
+@endif

resources/views/home/cards/_overview.blade.php

@@ -224,21 +224,14 @@ function createAsset() {
         'Authorization': 'Bearer {{Auth::user()->adversaryMeterApiToken()}}'
       }
     }).then(function (asset) {
-      toaster.toast(`La surveillance de ${asset.data.asset.asset} a commencé.`, 'success');
+      toaster.toastSuccess(`La surveillance de ${asset.data.asset.asset} a commencé.`);
       if (asset.data.asset.type === 'IP') {
         const div = document.getElementById('ip-monitored');
         div.innerText = parseInt(div.innerText) + 1;
       } else if (asset.data.asset.type === 'DNS') {
         const div = document.getElementById('dns-monitored');
         div.innerText = parseInt(div.innerText, 10) + 1;
       }
-    }).catch(function (error) {
-      console.error('Error:', error.response.data);
-      if (error.response && error.response.data && error.response.data.errors) {
-        toaster.toast(error.response.data.message || 'Une erreur est survenue.', 'danger');
-      } else {
-        toaster.toast('Une erreur est survenue.', 'danger');
-      }
-    });
+    }).catch((error) => toaster.toastAxiosError(error));
   }
 </script>

resources/views/home/index.blade.php

@@ -24,6 +24,8 @@
   <span class="breadcrumb-item active">{{ __('Security') }}</span>
   @elseif($tab === 'security_rules')
   <span class="breadcrumb-item active">{{ __('Security Rules') }}</span>
+  @elseif($tab === 'knowledge_base')
+  <span class="breadcrumb-item active">{{ __('Knowledge Base') }}</span>
   @elseif($tab === 'orders')
   <span class="breadcrumb-item active">{{ __('Orders') }}</span>
   @elseif($tab === 'users')
@@ -127,6 +129,12 @@
         {{ __('Security Rules') }}
       </a>
     </li>
+    <li class="nav-item">
+      <a class="nav-link {{ $tab === 'knowledge_base' ? 'active' : '' }}"
+         href="/home?tab=knowledge_base">
+        {{ __('Knowledge Base') }}
+      </a>
+    </li>
     @if(Auth::user()->canListOrders() && !Auth::user()->isCywiseUser())
     <li class="nav-item">
       <a class="nav-link {{ $tab === 'orders' ? 'active' : '' }}"
@@ -188,6 +196,9 @@
   @if($tab === 'security_rules')
   @include('home.cards._osquery_rules', [ 'rules' => $security_rules ])
   @endif
+  @if($tab === 'knowledge_base')
+  @include('home.cards._knowledge_base', [ 'files' => $knowledge_base ])
+  @endif
   @if($tab === 'interdependencies')
   @include('home.cards._interdependencies')
   @endif

resources/views/layouts/app.blade.php

@@ -198,7 +198,17 @@
 
   const toaster = {
     el: new com.computablefacts.blueprintjs.MinimalToaster(document.getElementById('toaster')),
-    toast: (msg, intent) => toaster.el.toast(msg, intent)
+    toast: (msg, intent) => toaster.el.toast(msg, intent),
+    toastSuccess: (msg) => toaster.toast(msg, 'success'),
+    toastError: (msg) => toaster.toast(msg, 'danger'),
+    toastAxiosError: (error) => {
+      console.error('Error:', error.response.data);
+      if (error.response && error.response.data && error.response.data.message) {
+        toaster.toastError(error.response.data.message);
+      } else {
+        toaster.toastError("{{ __('An error occurred. Try again in a moment or contact the support.') }}");
+      }
+    },
   };
   const drawer33 = {
     el: new com.computablefacts.blueprintjs.MinimalDrawer(document.getElementById('drawer-33'), '33%'),