Setup and docs

README.md

@@ -1,5 +1,39 @@
+![image](https://github.com/iNewLegend/web-crawler/assets/10234691/830714f3-c0ec-4b51-add7-d44b0598cc98)
+![image](https://github.com/iNewLegend/web-crawler/assets/10234691/279734f9-88ae-4729-9175-742be5b38b7c)
+
 # Requirements
-- PHP Version: `8.2`
+- MongoDB
+- PHP 
+  - Min Version: `8.2`
+  - MongoDB extension
+  - Composer
+- Node
+    - NPM
+    - Angular CLI
+
+# Installation
+- Clone the repository
+    - $ `git clone https://github.com/iNewLegend/web-crawler.git`
+    - $ `cd web-crawler`
+    - $ `composer install`
+
+
+- Setup mongodb connection
+    - $ `cd crawler-backend`
+    - $ `cp .env.example .env`
+    - $ `vim .env`
+        - Set `MONGODB_URI` to your mongodb connection string
+        - Set `MONGODB_DB` to your mongodb database name
+        - Save and exit
+    - $ Create database collections
+      - $ `php artisan migrate`
+    - $ `cd ..`
+
+
+  - Run backend server
+      - $ `composer dev-start-backend`
+  - Run frontend server (in another terminal)
+      - $ `composer dev-start-frontend`
 
 # Major commits
 -  2023-08-31 08:00:00 ~ 14:30:00 `New: CrawlerLib - Initial Commit`

composer.json

@@ -8,6 +8,16 @@
         "dev-start-frontend": [
             "Composer\\Config::disableProcessTimeout",
             "cd crawler-frontend && ng serve"
+        ],
+        "clean-vendor": [
+            "rm -rf crawler-lib/vendor",
+            "rm -rf crawler-backend/vendor",
+            "rm -rf crawler-frontend/node_modules"
+        ],
+        "pre-install-cmd": [
+            "cd crawler-lib && composer install",
+            "cd crawler-backend && composer install",
+            "cd crawler-frontend && npm install"
         ]
     }
 }

crawler-backend/.env.example

@@ -8,12 +8,9 @@ LOG_CHANNEL=stack
 LOG_DEPRECATIONS_CHANNEL=null
 LOG_LEVEL=debug
 
-DB_CONNECTION=mysql
-DB_HOST=127.0.0.1
-DB_PORT=3306
+DB_CONNECTION=mongodb
+DB_DSN=mongodb://127.0.0.1:27017/?directConnection=true
 DB_DATABASE=laravel
-DB_USERNAME=root
-DB_PASSWORD=
 
 BROADCAST_DRIVER=log
 CACHE_DRIVER=file

crawler-frontend/src/app/crawler/crawler-control/crawler-control.component.scss

@@ -1,4 +0,0 @@
-// Submit button
-button[type="submit"] {
-    max-width: 379px;
-}

crawler-frontend/src/app/crawler/crawler-display-index/crawler-display-index.component.html

@@ -1,18 +1,19 @@
-<table class="table">
-    <thead>
-    <tr>
-        <th scope="col">#</th>
-        <th scope="col">URL</th>
-        <th scope="col">Text associated</th>
-        <th scope="col">Depth</th>
-        <th scope="col">Links count</th>
-        <th scope="col">Created at</th>
-        <th scope="col">Updated at</th>
-        <th scope="col">Actions</th>
-    </tr>
-    </thead>
+<div class="table-responsive">
+    <table class="table">
+        <thead>
+        <tr>
+            <th scope="col">#</th>
+            <th scope="col">URL</th>
+            <th scope="col">Text associated</th>
+            <th scope="col">Depth</th>
+            <th scope="col">Links count</th>
+            <th scope="col">Created at</th>
+            <th scope="col">Updated at</th>
+            <th scope="col">Actions</th>
+        </tr>
+        </thead>
 
-    <tbody>
+        <tbody>
         <tr *ngFor="let item of items" class="cursor-pointer" (click)="onClick(item)">
             <th scope="row" >{{ items.indexOf( item ) + 1 }}</th>
             <td class="text-start">{{ item.url }}</td>
@@ -26,5 +27,6 @@
                 <span class="mx-2" title="Delete" (click)="onDeleteClick(item); $event.stopPropagation()">❌</span>
             </th>
         </tr>
-    </tbody>
-</table>
+        </tbody>
+    </table>
+</div>

crawler-frontend/src/app/crawler/crawler-display/crawler-display.component.html

@@ -1,21 +1,23 @@
-<table class="table">
-    <thead>
-    <tr>
-        <th scope="col">#</th>
-        <th scope="col">URL</th>
-        <th scope="col">Text associated</th>
-        <th scope="col">Created at</th>
-        <th scope="col">Updated at</th>
-    </tr>
-    </thead>
+<div class="table-responsive">
+    <table class="table">
+        <thead>
+        <tr>
+            <th scope="col">#</th>
+            <th scope="col">URL</th>
+            <th scope="col">Text associated</th>
+            <th scope="col">Created at</th>
+            <th scope="col">Updated at</th>
+        </tr>
+        </thead>
 
-    <tbody>
+        <tbody>
         <tr *ngFor="let item of items" class="cursor-pointer" (click)="onClick(item)">
-            <th scope="row" >{{ items.indexOf( item ) + 1 }}</th>
+            <th scope="row">{{ items.indexOf(item) + 1 }}</th>
             <td class="text-start">{{ item.url }}</td>
             <td>{{ item.text }}</td>
             <td>{{ item.createdAt | date:'yyyy-mm-dd HH:mm' }}</td>
             <td>{{ item.updatedAt | date:'yyyy-mm-dd HH:mm' }}</td>
         </tr>
-    </tbody>
-</table>
+        </tbody>
+    </table>
+</div>

crawler-frontend/src/app/crawler/crawler.component.html

@@ -1,6 +1,6 @@
-<div class="container mt-5">
+<div class="container-fluid mt-5">
     <div class="row justify-content-center text-center">
-        <div class="col-md-8 shadow-sm mb-5 pt-3 pb-2">
+        <div id="crawler-control-holder" class="col-md-8 shadow-sm mb-5 pt-3 pb-2">
             <crawler-control
                     [selectedCrawler]="selectedCrawler"
                     [shouldDisableCrawlerControl]="shouldDisableCrawlerControl"

crawler-frontend/src/styles.scss

@@ -4,10 +4,23 @@
     cursor: pointer;
 }
 
+#crawler-control-holder {
+    max-width: 900px;
+
+    // Submit button
+    button[type="submit"] {
+        max-width: 379px;
+    }
+}
+
 crawler-display, crawler-display-index {
     tr:hover {
         td, th {
             background-color: var(--bs-gray-100);
         }
     }
 }
+
+.container-fluid {
+    max-width: 2000px;
+}