Merge branch 'VueJS' into 'master'

Include Wowa Training

See merge request !1

.env.example

@@ -4,7 +4,7 @@ APP_KEY=
 APP_DEBUG=true
 APP_URL=http://localhost:8000
 
-LOG_CHANNEL=stack
+LOG_CHANNEL=custom
 
 DB_CONNECTION=mysql
 DB_HOST=127.0.0.1

.gitignore

@@ -6,6 +6,7 @@
 /.idea
 .env
 .env.backup
+.idea/
 .phpunit.result.cache
 Homestead.json
 Homestead.yaml

.gitlab-ci-envoy.yml

+stages:
+  - test
+  - build
+  - deploy
+
+## Cache composer packages between all jobs and all branches
+## of this project...
+cache:
+  key: one-key-to-rull-them-all
+  paths:
+    - composer-cache/
+
+before_script:
+  - cp env.gitlab-ci .env
+  - touch storage/app/db.sqlite
+  - COMPOSER_CACHE_DIR=composer-cache composer install --ignore-platform-reqs
+  - php artisan key:generate
+  - php artisan migrate
+
+# Test with PHP7.4
+test:php74:
+  image: cylab/php74
+  stage: test
+  script:
+    #- vendor/bin/phpunit --coverage-text --colors=never --coverage-html build/coverage
+    - vendor/bin/phpcs
+  artifacts:
+    paths:
+    - build/
+    expire_in: 1 month
+
+build:
+  image: docker:19.03.1
+  stage: build
+  tags:
+    - dind
+  services:
+    - docker:19.03.1-dind
+  variables:
+    DOCKER_TLS_CERTDIR: "/certs"
+  before_script:
+    - docker info
+  script:
+    - docker build -t cylab/ctf .
+
+build:release:
+  image: docker:19.03.1
+  stage: build
+  tags:
+    - dind
+  services:
+    - docker:19.03.1-dind
+  variables:
+    DOCKER_TLS_CERTDIR: "/certs"
+    IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
+  before_script:
+    - docker info
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -t gitlab.cylab.be:8081/cylab/mark-web:$CI_COMMIT_REF_SLUG .
+    - docker tag gitlab.cylab.be:8081/cylab/mark-web:$CI_COMMIT_REF_SLUG gitlab.cylab.be:8081/cylab/mark-web:latest
+    - docker push gitlab.cylab.be:8081/cylab/mark-web:$CI_COMMIT_REF_SLUG
+    - docker push gitlab.cylab.be:8081/cylab/mark-web:latest
+
+build:tagged:
+  only:
+    - tags
+  image: docker:19.03.1
+  stage: build
+  tags:
+    - dind
+  services:
+    - docker:19.03.1-dind
+  variables:
+    DOCKER_TLS_CERTDIR: "/certs"
+  before_script:
+    - docker info
+  script:
+    - docker build -t cylab/mark-web:$CI_COMMIT_TAG .
+    - docker tag cylab/mark-web:$CI_COMMIT_TAG cylab/mark-web:latest
+    - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+    - docker push cylab/mark-web:$CI_COMMIT_TAG
+    - docker push cylab/mark-web:latest
+
+deploy:
+  image: cylab/php74
+  stage: deploy
+  script:
+    # import ssh private key
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - mkdir -p ~/.ssh
+    - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
+    # in cylab/php72, envoy is already installed...
+    - ~/.composer/vendor/bin/envoy run deploy --commit="$CI_COMMIT_SHA"

.gitlab-ci.yml

@@ -25,7 +25,7 @@ test:php74:
     - vendor/bin/phpcs
   artifacts:
     paths:
-    - build/
+      - build/
     expire_in: 1 month
 
 build:

Dockerfile

 #### Step 1 : composer
 
-FROM cylab/php72 AS composer
+FROM cylab/php74 AS composer
 
 COPY . /var/www/html
 WORKDIR /var/www/html
-RUN composer install --no-dev --optimize-autoloader
+# RUN composer install --no-dev --optimize-autoloader
+RUN composer install --optimize-autoloader --ignore-platform-reqs
+# RUN pecl install redis && docker-php-ext-enable redis
 
 #### Step 2 : node
 
@@ -20,10 +22,13 @@ FROM php:7.4-apache
 
 ### PHP
 
-# we may need some other php modules, but we can first check the enabled 
+# we may need some other php modules, but we can first check the enabled
 # modules with
 # docker run -it --rm php:7.4-apache php -m
-# RUN docker-php-ext-install mbstring 
+# RUN docker-php-ext-install mbstring
+RUN pecl install -o -f redis \
+    &&  rm -rf /tmp/pear \
+    &&  docker-php-ext-enable redis
 
 ### Apache
 
@@ -56,7 +61,9 @@ RUN chown -R www-data:www-data /var/www/html/storage \
     /var/www/html/bootstrap/cache
 
 # clear config cache
-RUN php artisan config:clear 
+RUN php artisan config:clear
+RUN php artisan route:clear
+
 
 ### Docker image metadata
 

Envoy.blade.php

+@servers(['web' => 'deployer@172.20.116.154'])
+
+@setup
+    $repository = 'git@gitlab.cylab.be:a.croix/mark-web-webshells-test.git';
+    $root = '~';
+    $current_dir = $root . '/current';
+    $releases_dir = $root . '/releases';
+    $release = date('YmdHis');
+    $release_dir = $releases_dir .'/'. $release;
+    $env = $root . '/env.prod';
+@endsetup
+
+@story('deploy')
+    pwd
+    docker_stop
+    clone
+    composer
+    maven
+    docker
+@endstory
+
+@task('pwd')
+pwd
+@endtask
+
+@task('docker_stop')
+    echo 'Stop docker container'
+    docker stop $(docker ps -aq)
+    docker rm $(docker ps -aq)
+    docker image rm gitlab.cylab.be:8081/cylab/mark-web:latest
+@endtask
+@task('clone')
+    echo 'clone {{ $repository }} ...'
+    [ -d {{ $releases_dir }} ] || mkdir {{ $releases_dir }}
+    git clone --depth 1 {{ $repository }} {{ $release_dir }}
+    cd {{ $release_dir }}
+@endtask
+
+@task('composer')
+    echo "install composer dependencies ..."
+    cd {{ $release_dir }}
+    composer install --prefer-dist --no-dev -o
+@endtask
+
+@task('maven')
+    echo "Maven compile"
+    cd {{ $release_dir }}
+    mvn clean package
+    cp target/webshells-tests-1.0-SNAPSHOT.jar modules/
+@endtask
+
+@task('docker')
+    echo "Docker-compose"
+    cd {{ $release_dir }}
+    docker-compose up -d
+@endtask
+
+@task('database')
+    echo "Copy sqlite db into mark-web container"
+    docker cp db.sqlite.bak mark-web:/var/www/html/storage/app/db.sqlite
+    docker exec mark-web chown -R www-data:www-data /var/www/html/storage/app/db.sqlite
+@endtask

app/Events/LogTrainingEvent.php

+<?php
+
+
+namespace App\Events;
+
+use Illuminate\Queue\SerializesModels;
+
+class LogTrainingEvent
+{
+    use SerializesModels;
+
+    public $records;
+
+    public function __construct(array $records)
+    {
+        $this->records = $records;
+    }
+}

app/Exceptions/Handler.php

@@ -36,7 +36,13 @@ class Handler extends ExceptionHandler
      */
     public function report(Throwable $exception)
     {
-        parent::report($exception);
+        if ($this->shouldntReport($exception)) {
+            return;
+        }
+        \Log::channel('single')->error(
+            $exception->getMessage(),
+            array_merge($this->context(), ['exception' => $exception])
+        );
     }
 
     /**

app/Http/Controllers/FeedbackController.php

@@ -2,9 +2,14 @@
 namespace App\Http\Controllers;
 
 use App\Feedback;
+use App\Jobs\WowaJob;
+use App\Mark;
+use Barryvdh\Debugbar\Facade;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\Validator;
 use Illuminate\Http\Request;
+use Monolog\Handler\StreamHandler;
+use Monolog\Logger;
 
 class FeedbackController extends Controller
 {
@@ -72,6 +77,7 @@ class FeedbackController extends Controller
      */
     public function show(Feedback $feedback)
     {
+        Facade::info($feedback);
         return view("feedback.show", ["feedback" => $feedback]);
     }
 

app/Http/Controllers/WowaController.php

+<?php
+namespace App\Http\Controllers;
+
+use App\Jobs\WowaJob;
+use App\Log;
+use App\Wowa;
+use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\Validator;
+use Illuminate\Http\Request;
+
+class WowaController extends Controller
+{
+
+    public function __construct()
+    {
+        // Uncomment to require authentication
+        $this->middleware('auth');
+    }
+
+    /**
+     * Get a validator for an incoming registration request.
+     *
+     * @param  array  $data
+     * @return \Illuminate\Contracts\Validation\Validator
+     */
+    protected function validator(array $data)
+    {
+        return Validator::make($data, [
+            //'name' => 'required|string|regex:/^[a-zA-Z0-9\s-\.]+$/|max:255'
+            'population' => 'required|integer|min:10|max:250',
+            'crossover' => 'required|integer|min:1|max:99',
+            'mutation' => 'required|integer|min:1|max:99',
+            'generation' => 'required|integer|min:10|max:500'
+        ]);
+    }
+
+    /**
+     * Display a listing of the resource.
+     *
+     * @return \Illuminate\Http\Response
+     */
+    public function index()
+    {
+        return view("wowa.index", ["wowas" => Wowa::all()->sortBy("id")]);
+    }
+
+    /**
+     * Show the form for creating a new resource.
+     * We use the same view for create and update => provide an empty Wowa.
+     *
+     * @return \Illuminate\Http\Response
+     */
+    public function create()
+    {
+        $wowa = new Wowa();
+        return view("wowa.edit", ["wowa" => $wowa]);
+        //return $this->saveAndQueue($wowa);
+    }
+
+    /**
+     * Store a newly created resource in storage.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @return \Illuminate\Http\Response
+     */
+    public function store(Request $request)
+    {
+        $wowa = new Wowa();
+        return $this->saveAndQueue($wowa, $request);
+    }
+
+    /**
+     * Display the specified resource.
+     *
+     * @param  Wowa $wowa     * @return \Illuminate\Http\Response
+     */
+    public function show(Wowa $wowa)
+    {
+        return view("wowa.show", ["wowa" => $wowa]);
+    }
+
+    /**
+     * Show the form for editing the specified resource.
+     *
+     * @param  Wowa $wowa     * @return \Illuminate\Http\Response
+     */
+    public function edit(Wowa $wowa)
+    {
+        return view("wowa.edit", ["wowa" => $wowa]);
+    }
+
+    /**
+     * Update the specified resource in storage.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @param  Wowa $wowa     * @return \Illuminate\Http\Response
+     */
+    public function update(Request $request, Wowa $wowa)
+    {
+        $this->validator($request->all())->validate();
+
+        $wowa->name = $request->name;
+        $wowa->save();
+        return redirect(action('WowaController@index'));
+    }
+
+    /**
+     * Remove the specified resource from storage.
+     *
+     * @param  int  $id
+     * @return \Illuminate\Http\Response
+     */
+    public function destroy($id)
+    {
+        Log::where('Job_id', $id)->delete();
+        Wowa::find($id)->delete();
+        return redirect(action("WowaController@index"));
+    }
+
+    public function saveAndQueue(Wowa $wowa, Request $request)
+    {
+        $this->validator($request->all())->validate();
+        $wowa->user_id = Auth::user()->id;
+        $wowa->status = Wowa::STATE_QUEUED;
+        $wowa->population = $request->population;
+        $wowa->crossover_rate = $request->crossover;
+        $wowa->mutation_rate = $request->mutation;
+        $wowa->generation_number = $request->generation;
+        $wowa->save();
+        WowaJob::dispatch($wowa);
+        return redirect(action('WowaController@index'));
+    }
+
+    public function dropWowas()
+    {
+        Wowa::truncate();
+        return redirect(action("WowaController@index"));
+    }
+}

app/Jobs/WowaJob.php

+<?php
+
+namespace App\Jobs;
+
+use App\Feedback;
+use App\Log;
+use App\Logging\LogProcessor;
+use App\Mark;
+use App\Wowa;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use RUCD\Training\SolutionDistance;
+use RUCD\Training\Trainer;
+use RUCD\Training\TrainerParameters;
+
+class WowaJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $feedback_number;
+    protected $wowa;
+    protected $is_true_alert = [];
+    protected $evidences = [];
+    protected $bool_auc;
+    protected $bool_pr;
+    protected $chunk_size = 100;
+
+    /**
+     * Create a new job instance.
+     *
+     * @param Wowa $wowa
+     * @param bool $auc
+     * @param bool $pr
+     */
+    public function __construct(Wowa $wowa)
+    {
+        $this->wowa = $wowa;
+        $this->feedback_number = Feedback::count();
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $logg = \Log::channel('custom');
+        //$logger->pushHandler(new StreamHandler(__DIR__ . '/../../storage/logs/debug.log', Logger::INFO));
+        $this->wowa->status = Wowa::STATE_RUNNING;
+        $this->wowa->start_time = time();
+        $this->wowa->save();
+        if ($this->feedback_number == 0) {
+            $this->wowa->status = Wowa::STATE_FAILED;
+            $this->wowa->end_time = time();
+            $this->wowa->save();
+            return;
+        }
+        $processor = new LogProcessor();
+        $processor->setJobId($this->wowa->id);
+        $logg->pushProcessor($processor);
+
+
+        try {
+            $this->wowa->status = Wowa::STATE_COLLECTION;
+            $this->wowa->save();
+            Feedback::chunk($this->chunk_size, function ($feedbacks) {
+                foreach ($feedbacks as $feed) {
+                    $evidence_references = $feed->report()->references;
+                    $this->is_true_alert[] = $feed->is_true_alert;
+                    $scores_references = [];
+                    foreach ($evidence_references as $evidence_reference) {
+                        $scores_references[] = Mark::get()->findEvidenceById($evidence_reference)->score;
+                    }
+                    $this->evidences[] = $scores_references;
+                }
+                \Log::info(($feed->id * 100) / $this->feedback_number . " | Evidences collection");
+            });
+        } catch (\Exception $e) {
+            $this->wowa->status = Wowa::STATE_FAILED;
+            $logg->error("Error during retrieving evidences");
+            $this->wowa->save();
+        }
+        $processor->setTrainingBooleanTrue();
+        $processor->setJobId($this->wowa->id);
+        $logg->pushProcessor($processor);
+        $training_prameters = new TrainerParameters(
+            $logg,
+            $this->wowa->population,
+            $this->wowa->crossover_rate,
+            $this->wowa->mutation_rate,
+            TrainerParameters::SELECTION_METHOD_RWS,
+            $this->wowa->generation_number,
+            TrainerParameters::INITIAL_POPULATION_GENERATION_RANDOM
+        );
+        $trainer = new Trainer($training_prameters, new SolutionDistance(count($this->evidences[0])));
+        $this->wowa->status = Wowa::STATE_RUNNING;
+        $this->wowa->save();
+        try {
+            $solution = $trainer->run($this->evidences, $this->is_true_alert);
+        } catch (\Exception $e) {
+            $this->wowa->status = Wowa::STATE_FAILED;
+            $logg->error("Error during wowa training");
+            $this->wowa->save();
+        }
+        $this->wowa->w_weights = $solution->weights_w;
+        $this->wowa->p_weights = $solution->weights_p;
+        $this->wowa->score = $solution->getDistance();
+        $this->wowa->end_time = time();
+        $solution->computeAUC($this->evidences, $this->is_true_alert);
+        $solution->computePRAUC($this->evidences, $this->is_true_alert);
+        $this->wowa->roc = $solution->getAucRoc();
+        $this->wowa->pr = $solution->getAucPr();
+        $this->wowa->status = Wowa::STATE_SUCCESS;
+        $this->wowa->save();
+        Log::where('job_id', $this->wowa->id)->delete();
+    }
+
+    public function failed()
+    {
+        $this->wowa->status = Wowa::STATE_FAILED;
+        $this->wowa->save();
+    }
+}

app/Listeners/LogTrainingEventListener.php

+<?php
+
+
+namespace App\Listeners;
+
+use App\Events\LogTrainingEvent;
+use App\Log;
+use Illuminate\Contracts\Queue\ShouldQueue;
+
+class LogTrainingEventListener implements ShouldQueue
+{
+    //public $queue = 'default';
+    protected $log;
+
+    public function __construct(Log $log)
+    {
+        $this->log = $log;
+    }
+
+    public function handle($event)
+    {
+        $log = new Log();
+        $log->fill($event->records['formatted']);
+        $log->save();
+    }
+
+    public function subscribe($events)
+    {
+        $events->listen(
+            LogTrainingEvent::class,
+            'App\Listeners\LogTrainingEventListener@handle'
+        );
+    }
+}

app/Log.php

+<?php
+
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class Log extends Model
+{
+    protected $table = 'logs';
+
+    protected $guarded = ['id'];
+
+    public static function getJobProgression(Wowa $wowa)
+    {
+        if ($wowa->status == Wowa::STATE_COLLECTION) {
+            $log = self::where('job_id', '=', $wowa->id)
+                ->where('training', '=', false)->orderBy('progression', 'desc')->first();
+            if (!is_object($log)) {
+                return "Computation in progress...";
+            }
+            return $log->progression;
+        } elseif ($wowa->status == Wowa::STATE_RUNNING) {
+            $log = self::where('job_id', '=', $wowa->id)
+                ->where('training', '=', true)->orderBy('progression', 'desc')->first();
+            if (!is_object($log)) {
+                return "Calculation progress...";
+            }
+            return ((($log->progression + 1) / $wowa->generation_number) * 100);
+        } elseif ($wowa->status == Wowa::STATE_SUCCESS) {
+            return "100";
+        } elseif ($wowa->status == Wowa::STATE_QUEUED) {
+            return "Computation in progress";
+        } else {
+            return 'Error';
+        }
+    }
+}

app/Logging/LogFormatter.php

+<?php
+
+
+namespace App\Logging;
+
+use Monolog\Formatter\NormalizerFormatter;
+
+class LogFormatter extends NormalizerFormatter
+{
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    public function format(array $record)
+    {
+        $record = parent::format($record);
+        return $this->convertToSQLite($record);
+    }
+
+    protected function convertToSQLite(array $record)
+    {
+        $el = $record['extra'];
+        $el['level'] = strtolower($record['level_name']);
+        $el['message'] = $record['message'];
+        $progression = substr($record['message'], 0, strpos($record['message'], ' | '));
+        $el['progression'] = (int)$progression;
+
+        return $el;
+    }
+}

app/Logging/LogProcessor.php

+<?php
+
+
+namespace App\Logging;
+
+class LogProcessor
+{
+
+    protected $job_id;
+    protected $training_bool = false;
+    public function __invoke(array $record): array
+    {
+        $record['extra'] = [
+            'user_id' => auth()->user() ? auth()->user()->id : null,
+            'job_id' => $this->job_id,
+            'training' => $this->training_bool
+        ];
+        return $record;
+    }
+
+    public function setJobId(int $job_id)
+    {
+        $this->job_id = $job_id;
+    }
+
+    public function setTrainingBooleanTrue()
+    {
+        $this->training_bool = true;
+    }
+}

app/Logging/LogTraining.php

+<?php
+
+
+namespace App\Logging;
+
+use Monolog\Logger;
+
+class LogTraining
+{
+    public function __invoke(): Logger
+    {
+        $logger = new Logger('training_logger');
+        $handler = new LogTrainingHandler();
+        //$processor = new LogProcessor();
+        //$processor->setJobId(26);
+        $logger->pushHandler($handler);
+        //$logger->pushProcessor($processor);
+        return $logger;
+    }
+}

app/Logging/LogTrainingHandler.php

+<?php
+
+
+namespace App\Logging;
+
+use App\Log;
+use Monolog\Formatter\FormatterInterface;
+use Monolog\Handler\AbstractProcessingHandler;
+use Monolog\Logger;
+
+class LogTrainingHandler extends AbstractProcessingHandler
+{
+    public function __construct($level = Logger::DEBUG, bool $bubble = true)
+    {
+        parent::__construct($level, $bubble);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    protected function write(array $record): void
+    {
+        $log = new Log();
+        $log->fill($record['formatted']);
+        $log->save();
+
+        //event(new LogTrainingEvent($record));
+    }
+
+    protected function getDefaultFormatter(): FormatterInterface
+    {
+        return new LogFormatter();
+    }
+}

app/Providers/EventServiceProvider.php

@@ -17,7 +17,7 @@ class EventServiceProvider extends ServiceProvider
     protected $listen = [
         Registered::class => [
             SendEmailVerificationNotification::class,
-        ],
+        ]
     ];
 
     /**

app/User.php

@@ -48,4 +48,10 @@ class User extends Authenticatable
     {
         return $this->admin;
     }
+
+    public static function findUserNameById(Wowa $wowa)
+    {
+        $user = self::where("id", "=", $wowa->user_id)->pluck('name');
+        return $user;
+    }
 }

app/Wowa.php

+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class Wowa extends Model
+{
+    const STATE_CREATED = "CREATED";
+    const STATE_QUEUED = "QUEUED";
+    const STATE_COLLECTION = "DATA_COLLECTION";
+    const STATE_RUNNING = "RUNNING";
+    const STATE_SUCCESS = "SUCCESS";
+    const STATE_FAILED = "FAILED";
+
+    public static $status_classes = [
+        self::STATE_SUCCESS => "btn-success",
+        self::STATE_FAILED => "btn-danger",
+        self::STATE_RUNNING => "btn-warning",
+        self::STATE_COLLECTION => "btn-warning"
+    ];
+
+
+    protected $casts = [
+        'w_weights' => 'array',
+        'p_weights' => 'array',
+    ];
+    private function getStatusBadgeClass()
+    {
+        $status = $this->status;
+        if (!isset(self::$status_classes[$status])) {
+            return "btn-primary";
+        }
+
+        return self::$status_classes[$status];
+    }
+
+    public function statusBadge()
+    {
+        if ($this->status == self::STATE_RUNNING || $this->status == self::STATE_QUEUED) {
+            return "<span class='btn btn-badge btn-sm "
+                . $this->getStatusBadgeClass() . "'>" . $this->status
+                . " : " . Log::getJobProgression($this) . "</span>";
+        } else {
+            return "<span class='btn btn-badge btn-sm "
+                . $this->getStatusBadgeClass() . "'>" . $this->status . "</span>";
+        }
+    }
+
+    public function weightsP()
+    {
+        return $this->w_weights->toArray();
+    }
+}