From 102113e950849a66d9be67b9ec9fbfb72b38deae Mon Sep 17 00:00:00 2001 From: marco370 <48531002-marco370@users.noreply.replit.com> Date: Mon, 24 Nov 2025 17:02:15 +0000 Subject: [PATCH] Improve ML dependency installation script for robust deployment Update deployment script to correctly activate virtual environment, install Cython and numpy as build dependencies before eif, and ensure sequential installation for the ML hybrid detector. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: 8b4c76c7-3a42-4713-8396-40f5db530225 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/2lUhxO2 --- ...TALLAZIONE-1764003628537_1764003628537.txt | 40 +++++++++++++++ deployment/CHECKLIST_ML_HYBRID.md | 20 +++++--- deployment/install_ml_deps.sh | 51 ++++++++++++++++--- replit.md | 10 ++-- 4 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 attached_assets/Pasted--deployment-install-ml-deps-sh-INSTALLAZIONE-1764003628537_1764003628537.txt diff --git a/attached_assets/Pasted--deployment-install-ml-deps-sh-INSTALLAZIONE-1764003628537_1764003628537.txt b/attached_assets/Pasted--deployment-install-ml-deps-sh-INSTALLAZIONE-1764003628537_1764003628537.txt new file mode 100644 index 0000000..289964e --- /dev/null +++ b/attached_assets/Pasted--deployment-install-ml-deps-sh-INSTALLAZIONE-1764003628537_1764003628537.txt @@ -0,0 +1,40 @@ + ./deployment/install_ml_deps.sh +╔═══════════════════════════════════════════════╗ +║ INSTALLAZIONE DIPENDENZE ML HYBRID ║ +╚═══════════════════════════════════════════════╝ + +📍 Directory corrente: /opt/ids/python_ml + +📦 Step 1/2: Installazione Cython (richiesto per compilare eif)... +Collecting Cython==3.0.5 + Downloading Cython-3.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB) + |████████████████████████████████| 3.6 MB 6.2 MB/s +Installing collected packages: Cython +Successfully installed Cython-3.0.5 +WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv +✅ Cython installato con successo + +📦 Step 2/2: Installazione dipendenze ML (xgboost, joblib, eif)... +Collecting xgboost==2.0.3 + Downloading xgboost-2.0.3-py3-none-manylinux2014_x86_64.whl (297.1 MB) + |████████████████████████████████| 297.1 MB 13 kB/s +Collecting joblib==1.3.2 + Downloading joblib-1.3.2-py3-none-any.whl (302 kB) + |████████████████████████████████| 302 kB 41.7 MB/s +Collecting eif==2.0.2 + Downloading eif-2.0.2.tar.gz (1.6 MB) + |████████████████████████████████| 1.6 MB 59.4 MB/s + Preparing metadata (setup.py) ... error + ERROR: Command errored out with exit status 1: + command: /usr/bin/python3 -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-xpd6jc3z/eif_1c539132fe1d4772ada0979407304392/setup.py'"'"'; __file__='"'"'/tmp/pip-install-xpd6jc3z/eif_1c539132fe1d4772ada0979407304392/setup.py'"'"';f = getattr(tokenize, '"'"'open'"'"', open)(__file__) if os.path.exists(__file__) else io.StringIO('"'"'from setuptools import setup; setup()'"'"');code = f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' egg_info --egg-base /tmp/pip-pip-egg-info-lg0m0ish + cwd: /tmp/pip-install-xpd6jc3z/eif_1c539132fe1d4772ada0979407304392/ + Complete output (5 lines): + Traceback (most recent call last): + File "", line 1, in + File "/tmp/pip-install-xpd6jc3z/eif_1c539132fe1d4772ada0979407304392/setup.py", line 3, in + import numpy + ModuleNotFoundError: No module named 'numpy' + ---------------------------------------- +WARNING: Discarding https://files.pythonhosted.org/packages/83/b2/d87d869deeb192ab599c899b91a9ad1d3775d04f5b7adcaf7ff6daa54c24/eif-2.0.2.tar.gz#sha256=86e2c98caf530ae73d8bc7153c1bf6b9684c905c9dfc7bdab280846ada1e45ab (from https://pypi.org/simple/eif/). Command errored out with exit status 1: python setup.py egg_info Check the logs for full command output. +ERROR: Could not find a version that satisfies the requirement eif==2.0.2 (from versions: 1.0.0, 1.0.1, 1.0.2, 2.0.2) +ERROR: No matching distribution found for eif==2.0.2 \ No newline at end of file diff --git a/deployment/CHECKLIST_ML_HYBRID.md b/deployment/CHECKLIST_ML_HYBRID.md index 572e74d..80a5272 100644 --- a/deployment/CHECKLIST_ML_HYBRID.md +++ b/deployment/CHECKLIST_ML_HYBRID.md @@ -14,7 +14,7 @@ Sistema ML avanzato per riduzione falsi positivi 80-90% con Extended Isolation F ## 🔧 Step 1: Installazione Dipendenze -⚠️ **IMPORTANTE**: Usare lo script dedicato per risolvere dipendenza Cython +⚠️ **IMPORTANTE**: Usare lo script dedicato che attiva venv e gestisce build dependencies ```bash # SSH al server @@ -26,20 +26,26 @@ chmod +x deployment/install_ml_deps.sh ./deployment/install_ml_deps.sh # Output atteso: +# 🔧 Attivazione virtual environment... +# 📍 Python in uso: /opt/ids/python_ml/venv/bin/python # ✅ Cython installato con successo +# ✅ numpy 1.26.2 già installato # ✅ Dipendenze ML installate con successo # ✅ eif importato correttamente # ✅ TUTTO OK! Hybrid ML Detector pronto per l'uso ``` **Dipendenze nuove**: -- `Cython==3.0.5` - Build dependency per eif (installato per primo) -- `xgboost==2.0.3` - Gradient Boosting per ensemble classifier -- `eif==2.0.2` - Extended Isolation Forest -- `joblib==1.3.2` - Model persistence +- `Cython==3.0.5` - Build dependency per eif (installato Step 1) +- `numpy==1.26.2` - Build dependency per eif (verificato Step 2) +- `xgboost==2.0.3` - Gradient Boosting per ensemble classifier (Step 3) +- `eif==2.0.2` - Extended Isolation Forest (Step 3) +- `joblib==1.3.2` - Model persistence (Step 3) -**Perché lo script separato?** -`eif` richiede Cython per compilare il codice durante l'installazione. Lo script installa Cython PRIMA, poi le altre dipendenze. +**Perché lo script in 3 fasi?** +1. `eif` si compila da source e richiede **Cython** durante setup.py +2. `eif` setup.py fa `import numpy` → richiede **numpy pre-installato** +3. Lo script attiva **venv** e installa sequenzialmente: Cython → verifica numpy → eif --- diff --git a/deployment/install_ml_deps.sh b/deployment/install_ml_deps.sh index 44f86dc..2ff7d72 100755 --- a/deployment/install_ml_deps.sh +++ b/deployment/install_ml_deps.sh @@ -1,7 +1,7 @@ #!/bin/bash # Script per installare dipendenze ML Hybrid Detector -# Risolve il problema di Cython richiesto come build dependency da eif +# Risolve il problema di build dependencies (Cython + numpy) richieste da eif set -e @@ -16,9 +16,29 @@ cd "$(dirname "$0")/../python_ml" || exit 1 echo "📍 Directory corrente: $(pwd)" echo "" -# STEP 1: Installa Cython PRIMA (build dependency per eif) -echo "📦 Step 1/2: Installazione Cython (richiesto per compilare eif)..." -pip install --user Cython==3.0.5 +# Verifica venv +if [ ! -d "venv" ]; then + echo "❌ ERRORE: Virtual environment non trovato in $(pwd)/venv" + echo " Esegui prima: python3 -m venv venv" + exit 1 +fi + +# Attiva venv +echo "🔧 Attivazione virtual environment..." +source venv/bin/activate + +# Verifica che stiamo usando il venv +PYTHON_PATH=$(which python) +echo "📍 Python in uso: $PYTHON_PATH" +if [[ ! "$PYTHON_PATH" =~ "venv" ]]; then + echo "⚠️ WARNING: Non stiamo usando il venv correttamente!" +fi + +echo "" + +# STEP 1: Installa build dependencies PRIMA (Cython + numpy) +echo "📦 Step 1/3: Installazione build dependencies (Cython + numpy)..." +pip install Cython==3.0.5 if [ $? -eq 0 ]; then echo "✅ Cython installato con successo" @@ -29,9 +49,20 @@ fi echo "" -# STEP 2: Installa tutte le altre dipendenze -echo "📦 Step 2/2: Installazione dipendenze ML (xgboost, joblib, eif)..." -pip install --user xgboost==2.0.3 joblib==1.3.2 eif==2.0.2 +# STEP 2: Verifica che numpy sia disponibile (già nei requirements base) +echo "📦 Step 2/3: Verifica numpy disponibile..." +python -c "import numpy; print(f'✅ numpy {numpy.__version__} già installato')" 2>/dev/null + +if [ $? -ne 0 ]; then + echo "⚠️ numpy non trovato, installo..." + pip install numpy==1.26.2 +fi + +echo "" + +# STEP 3: Installa dipendenze ML (ora Cython e numpy sono disponibili) +echo "📦 Step 3/3: Installazione dipendenze ML (xgboost, joblib, eif)..." +pip install xgboost==2.0.3 joblib==1.3.2 eif==2.0.2 if [ $? -eq 0 ]; then echo "✅ Dipendenze ML installate con successo" @@ -44,11 +75,15 @@ echo "" echo "✅ INSTALLAZIONE COMPLETATA!" echo "" echo "🧪 Test import eif..." -python3 -c "from eif import iForest; print('✅ eif importato correttamente')" +python -c "from eif import iForest; print('✅ eif importato correttamente')" if [ $? -eq 0 ]; then echo "" echo "✅ TUTTO OK! Hybrid ML Detector pronto per l'uso" + echo "" + echo "📋 Prossimi step:" + echo " 1. Testa sistema: python train_hybrid.py --mode test" + echo " 2. Training reale: python train_hybrid.py --mode train" else echo "❌ Errore durante test import eif" exit 1 diff --git a/replit.md b/replit.md index 680b2ea..f591931 100644 --- a/replit.md +++ b/replit.md @@ -103,8 +103,10 @@ The IDS employs a React-based frontend for real-time monitoring, detection visua - **Backward Compatibility**: USE_HYBRID_DETECTOR env var (default=true) - **Target Metrics**: Precision≥90%, Recall≥80%, FPR≤5%, F1≥85% - **Deploy**: Vedere `deployment/CHECKLIST_ML_HYBRID.md` -- **Fix Deploy (24 Nov 2025 - 19:30)**: - - Corretto `eif==2.0.0` → `eif==2.0.2` (versione 2.0.0 non disponibile) +- **Fix Deploy (24 Nov 2025 - 20:00)**: + - Corretto `eif==2.0.0` → `eif==2.0.2` (versione 2.0.0 non disponibile su PyPI) - Aggiunto `Cython==3.0.5` come build dependency (eif richiede compilazione) - - Creato `deployment/install_ml_deps.sh` per installazione in 2 fasi (Cython → eif) - - **Soluzione**: pip non installa Cython in tempo per eif → script separa installazione \ No newline at end of file + - Creato `deployment/install_ml_deps.sh` con installazione in 3 fasi sequenziali + - **Problema 1**: pip installa pacchetti in parallelo → Cython non disponibile per eif + - **Problema 2**: eif setup.py richiede numpy import → numpy deve essere pre-installato + - **Soluzione**: Script attiva venv + installa (1) Cython, (2) verifica numpy, (3) eif \ No newline at end of file