Merge remote-tracking branch 'origin/main' into stable

.devcontainer/machine-learning/devcontainer.json

@@ -21,7 +21,7 @@
 	},
 	"runArgs": [
 		"--gpus=all",
-		"--shm-size=16g",
+		"--shm-size=8g",
 		"--env-file", ".devcontainer/dev.env"
 	]
 }

.devcontainer/python/devcontainer.json

@@ -19,6 +19,7 @@
 		}
 	},
 	"runArgs": [
+		"--shm-size=4g",
 		"--env-file", ".devcontainer/dev.env"
 	]
 }

.devcontainer/requirements-cpu.in

@@ -24,3 +24,5 @@ ruff
 dpkt
 pypcap
 textual
+isort
+imbalanced-learn

.devcontainer/requirements-cpu.txt

@@ -199,14 +199,20 @@ idna==3.8
     # via
     #   requests
     #   yarl
+imbalanced-learn==0.12.4
+    # via -r .devcontainer/requirements-cpu.in
 iniconfig==2.0.0
     # via pytest
+isort==5.13.2
+    # via -r .devcontainer/requirements-cpu.in
 iterative-telemetry==0.0.8
     # via dvc
 jinja2==3.1.4
     # via torch
 joblib==1.4.2
-    # via scikit-learn
+    # via
+    #   imbalanced-learn
+    #   scikit-learn
 kiwisolver==1.4.5
     # via matplotlib
 kombu==5.4.0
@@ -252,6 +258,7 @@ numpy==1.26.4
     # via
     #   -r .devcontainer/requirements-cpu.in
     #   contourpy
+    #   imbalanced-learn
     #   matplotlib
     #   nfstream
     #   pandas
@@ -392,9 +399,13 @@ ruamel-yaml-clib==0.2.8
 ruff==0.6.4
     # via -r .devcontainer/requirements-cpu.in
 scikit-learn==1.5.1
-    # via -r .devcontainer/requirements-cpu.in
+    # via
+    #   -r .devcontainer/requirements-cpu.in
+    #   imbalanced-learn
 scipy==1.14.1
-    # via scikit-learn
+    # via
+    #   imbalanced-learn
+    #   scikit-learn
 scmrepo==3.3.7
     # via
     #   dvc
@@ -434,7 +445,9 @@ tabulate==0.9.0
 textual==0.82.0
     # via -r .devcontainer/requirements-cpu.in
 threadpoolctl==3.5.0
-    # via scikit-learn
+    # via
+    #   imbalanced-learn
+    #   scikit-learn
 tomlkit==0.13.2
     # via dvc
 torch==2.4.1

.devcontainer/requirements.in

@@ -24,3 +24,5 @@ ruff
 dpkt
 pypcap
 textual
+isort
+imbalanced-learn

.gitignore

@@ -167,7 +167,7 @@ DvcLiveLogger/
 *acc.tsv
 *loss.tsv
 *epoch.tsv
-*.png 
+*.png
 !docs/*.png
 !src/app/client/src/components/logo/nids_logo.png
 scripts/setup_git_personal.sh
@@ -176,3 +176,4 @@ data/CIC/*.csv
 .devcontainer/wandb.env
 wandb/
 dvclive/plots
+.DS_Store

compose.yml

@@ -1,5 +1,6 @@
 include:
   - deploy/compose.yml
+  - sdk/compose.yml
 
 services:
   mongo:
@@ -24,7 +25,6 @@ services:
     command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017", "--quiet"]
     attach: false # wont monitor mongo logs
 
-
   mongo-express:
     container_name: mongo-express
     image: mongo-express:latest
@@ -53,8 +53,8 @@ services:
       dockerfile: deploy/webserver/Dockerfile
     network_mode: host
     ports:
-      - "8000:8000"  # server
-      - "5073:5073"  # client
+      - "8000:8000" # server
+      - "5173:5173" # client
     depends_on:
       mongo:
         condition: service_healthy

deploy/feeder/Dockerfile

@@ -1,21 +1,47 @@
 FROM python:3.11-slim
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    build-essential libpcap-dev 
+    build-essential \
+    libpcap-dev \
+    python3-dev \
+    autoconf \
+    automake \
+    libtool \
+    pkg-config \
+    flex \
+    bison \
+    gettext \
+    libjson-c-dev \
+    libusb-1.0-0-dev \
+    libdbus-glib-1-dev \
+    libbluetooth-dev \
+    libnl-genl-3-dev \
+    git && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY deploy/feeder/requirements.txt /app/requirements.txt
+RUN pip3 install --no-cache-dir -r /app/requirements.txt && rm /app/requirements.txt
 
 ENV PORT=50053
 ENV INTERFACE="eth0"
+# ENV INTERFACE="en0"
+
 ENV FILE_NAME="capture.pcap"
-ENV DURATION=10
+ENV DURATION=5
 
-LABEL ENV_VARS="INTERFACE,FILE_NAME,DURATION"
 
-COPY deploy/feeder/requirements.txt /app/requirements.txt
-RUN pip3 install --no-cache-dir -r /app/requirements.txt && rm /app/requirements.txt
+
+ENV FILE_NAME="capture.pcap"
+ENV DURATION=5
+
+LABEL ENV_VARS="INTERFACE,FILE_NAME,DURATION"
 
 COPY src/grpc_ /app/src/grpc_
 COPY src/services/feeder/feeder.py /app/main.py
 
+COPY src/ai/BasicModule.py /app/src/ai/BasicModule.py
+COPY data/checkpoints/MLP.ckpt /app/model.ckpt
+
 ENV PYTHONPATH=/app/src
 WORKDIR /app
 

deploy/feeder/compose.yml

@@ -8,6 +8,7 @@ services:
     network_mode: host
     ports:
       - "50053:50053"
+    # environment:
+    #   GRPC_VERBOSITY: ERROR
+    #   GRPC_LOG_SEVERITY: ERROR
     container_name: feeder
-    profiles:
-      - feeder

deploy/feeder/requirements.in

@@ -5,7 +5,10 @@ icecream
 click
 numpy<2
 scikit-learn
-nfstream
+# nfstream
+git+https://github.com/praetoriannero/nfstream.git
 pymongo
 dpkt
 pypcap
+lightning
+torch

deploy/feeder/requirements.txt

@@ -4,8 +4,16 @@
 #
 #    pip-compile deploy/feeder/requirements.in
 #
+aiohappyeyeballs==2.4.4
+    # via aiohttp
+aiohttp==3.11.10
+    # via fsspec
+aiosignal==1.3.1
+    # via aiohttp
 asttokens==2.4.1
     # via icecream
+attrs==24.2.0
+    # via aiohttp
 cffi==1.17.0
     # via nfstream
 click==8.1.7
@@ -20,6 +28,17 @@ dpkt==1.9.8
     #   nfstream
 executing==2.0.1
     # via icecream
+filelock==3.16.1
+    # via torch
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2024.10.0
+    # via
+    #   lightning
+    #   pytorch-lightning
+    #   torch
 grpcio==1.66.0
     # via
     #   -r deploy/feeder/requirements.in
@@ -28,9 +47,30 @@ grpcio-tools==1.66.0
     # via -r deploy/feeder/requirements.in
 icecream==2.1.3
     # via -r deploy/feeder/requirements.in
+idna==3.10
+    # via yarl
+jinja2==3.1.4
+    # via torch
 joblib==1.4.2
     # via scikit-learn
-nfstream==6.5.3
+lightning==2.4.0
+    # via -r deploy/feeder/requirements.in
+lightning-utilities==0.11.9
+    # via
+    #   lightning
+    #   pytorch-lightning
+    #   torchmetrics
+markupsafe==3.0.2
+    # via jinja2
+mpmath==1.3.0
+    # via sympy
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+networkx==3.4.2
+    # via torch
+nfstream @ git+https://github.com/praetoriannero/nfstream.git
     # via -r deploy/feeder/requirements.in
 numpy==1.26.4
     # via
@@ -39,10 +79,21 @@ numpy==1.26.4
     #   pandas
     #   scikit-learn
     #   scipy
+    #   torchmetrics
+packaging==24.2
+    # via
+    #   lightning
+    #   lightning-utilities
+    #   pytorch-lightning
+    #   torchmetrics
 pandas==2.2.2
     # via
     #   -r deploy/feeder/requirements.in
     #   nfstream
+propcache==0.2.1
+    # via
+    #   aiohttp
+    #   yarl
 protobuf==5.27.3
     # via grpcio-tools
 psutil==6.0.0
@@ -57,8 +108,14 @@ pypcap==1.3.0
     # via -r deploy/feeder/requirements.in
 python-dateutil==2.9.0.post0
     # via pandas
+pytorch-lightning==2.4.0
+    # via lightning
 pytz==2024.1
     # via pandas
+pyyaml==6.0.2
+    # via
+    #   lightning
+    #   pytorch-lightning
 scikit-learn==1.5.1
     # via -r deploy/feeder/requirements.in
 scipy==1.14.1
@@ -67,10 +124,34 @@ six==1.16.0
     # via
     #   asttokens
     #   python-dateutil
+sympy==1.13.1
+    # via torch
 threadpoolctl==3.5.0
     # via scikit-learn
+torch==2.5.1
+    # via
+    #   -r deploy/feeder/requirements.in
+    #   lightning
+    #   pytorch-lightning
+    #   torchmetrics
+torchmetrics==1.6.0
+    # via
+    #   lightning
+    #   pytorch-lightning
+tqdm==4.67.1
+    # via
+    #   lightning
+    #   pytorch-lightning
+typing-extensions==4.12.2
+    # via
+    #   lightning
+    #   lightning-utilities
+    #   pytorch-lightning
+    #   torch
 tzdata==2024.1
     # via pandas
+yarl==1.18.3
+    # via aiohttp
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

deploy/model/compose.yml

@@ -5,7 +5,7 @@ services:
       dockerfile: deploy/model/Dockerfile
       args:
         # - MODEL_PATH=data/checkpoints/ResidualNetwork.ckpt
-        - MODEL_PATH=data/checkpoints/MLP-v1.ckpt
+        - MODEL_PATH=data/checkpoints/MLP.ckpt
     ports:
       - "50052:50052"
     container_name: neural-network

deploy/offline_feeder/requirements.in

@@ -9,3 +9,4 @@ scikit-learn
 pymongo
 torch
 lightning
+imblearn