0bfe3258b5
Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: 1c71ce6e-1a3e-4f53-bb5d-77cdd22b8ea3
315 lines
11 KiB
Python
315 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
=================================================================
|
|
TEST GPU READINESS per 1M+ RECORD - TESLA M60
|
|
=================================================================
|
|
🔧 Test rapido per verificare se il sistema è pronto per processing
|
|
di 1.000.000+ record completamente su GPU Tesla M60
|
|
=================================================================
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
# ⚡ CONFIGURAZIONE GPU CRITICA ⚡
|
|
os.environ['TF_GPU_ALLOCATOR'] = 'legacy' # CRITICO per Tesla M60 CC 5.2
|
|
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
|
|
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
|
|
|
|
def test_cudf_cupy():
|
|
"""Test CuDF + CuPy per DataFrame GPU-native"""
|
|
print("\n🔧 TEST 1: CuDF + CuPy (DataFrame 100% GPU)")
|
|
try:
|
|
import cudf
|
|
import cupy as cp
|
|
|
|
# Test CuDF DataFrame
|
|
print("⚡ Creazione CuDF DataFrame test...")
|
|
test_data = {
|
|
'id': range(100000),
|
|
'ip': [f"192.168.{i%256}.{(i*7)%256}" for i in range(100000)],
|
|
'timestamp': ['2024-01-01 12:00:00'] * 100000,
|
|
'message': [f"test_msg_{i%1000}" for i in range(100000)]
|
|
}
|
|
df_gpu = cudf.DataFrame(test_data)
|
|
print(f"✅ CuDF DataFrame: {len(df_gpu):,} record su GPU")
|
|
|
|
# Test CuPy arrays
|
|
print("⚡ Test CuPy arrays...")
|
|
arr_gpu = cp.random.random((100000, 50))
|
|
result = cp.sum(arr_gpu, axis=1)
|
|
print(f"✅ CuPy compute: {result.shape} array processed")
|
|
|
|
# Memory info
|
|
pool = cp.get_default_memory_pool()
|
|
memory_mb = pool.used_bytes() / 1024**2
|
|
print(f"✅ GPU Memory used: {memory_mb:.1f}MB")
|
|
|
|
# Cleanup
|
|
del df_gpu, arr_gpu, result
|
|
pool.free_all_blocks()
|
|
|
|
return True, f"CuDF + CuPy OK (Memory: {memory_mb:.1f}MB)"
|
|
|
|
except ImportError as e:
|
|
return False, f"CuDF/CuPy non installati: {e}"
|
|
except Exception as e:
|
|
return False, f"Errore CuDF/CuPy: {e}"
|
|
|
|
def test_cuml():
|
|
"""Test CuML per ML GPU-native"""
|
|
print("\n🔧 TEST 2: CuML (ML 100% GPU)")
|
|
try:
|
|
import cuml
|
|
import cupy as cp
|
|
from cuml.ensemble import IsolationForest as IFGPU
|
|
from cuml.neighbors import LocalOutlierFactor as LOFGPU
|
|
from cuml.svm import OneClassSVM as SVMGPU
|
|
|
|
print("⚡ Test Isolation Forest GPU...")
|
|
X_test = cp.random.random((10000, 20))
|
|
if_gpu = IFGPU(n_estimators=50, max_samples=5000)
|
|
if_gpu.fit(X_test)
|
|
predictions = if_gpu.predict(X_test)
|
|
print(f"✅ Isolation Forest GPU: {len(predictions):,} predictions")
|
|
|
|
print("⚡ Test LOF GPU...")
|
|
lof_gpu = LOFGPU(n_neighbors=20)
|
|
# LOF con subset per performance
|
|
X_small = X_test[:5000]
|
|
lof_gpu.fit(X_small)
|
|
lof_pred = lof_gpu.predict(X_small)
|
|
print(f"✅ LOF GPU: {len(lof_pred):,} predictions")
|
|
|
|
print("⚡ Test SVM GPU...")
|
|
svm_gpu = SVMGPU(nu=0.05)
|
|
# SVM con subset ancora più piccolo
|
|
X_tiny = X_test[:2000]
|
|
svm_gpu.fit(X_tiny)
|
|
svm_pred = svm_gpu.predict(X_tiny)
|
|
print(f"✅ SVM GPU: {len(svm_pred):,} predictions")
|
|
|
|
# Memory cleanup
|
|
del X_test, X_small, X_tiny, if_gpu, lof_gpu, svm_gpu
|
|
|
|
return True, "CuML ML completo OK"
|
|
|
|
except ImportError as e:
|
|
return False, f"CuML non installato: {e}"
|
|
except Exception as e:
|
|
return False, f"Errore CuML: {e}"
|
|
|
|
def test_tensorflow_gpu():
|
|
"""Test TensorFlow GPU"""
|
|
print("\n🔧 TEST 3: TensorFlow GPU")
|
|
try:
|
|
import tensorflow as tf
|
|
|
|
# Check GPU devices
|
|
gpus = tf.config.list_physical_devices('GPU')
|
|
if not gpus:
|
|
return False, "Nessuna GPU TensorFlow rilevata"
|
|
|
|
gpu = gpus[0]
|
|
print(f"⚡ GPU rilevata: {gpu}")
|
|
|
|
# Configure GPU
|
|
tf.config.experimental.set_memory_growth(gpu, True)
|
|
print("✅ Memory growth configurato")
|
|
|
|
# Test GPU operations
|
|
with tf.device('/GPU:0'):
|
|
print("⚡ Test matrix operations GPU...")
|
|
A = tf.random.normal((5000, 500))
|
|
B = tf.random.normal((500, 200))
|
|
C = tf.matmul(A, B)
|
|
print(f"✅ Matrix GPU: {C.shape} = {A.shape} @ {B.shape}")
|
|
|
|
print("⚡ Test neural network GPU...")
|
|
model = tf.keras.Sequential([
|
|
tf.keras.layers.Dense(128, activation='relu', input_shape=(500,)),
|
|
tf.keras.layers.Dense(64, activation='relu'),
|
|
tf.keras.layers.Dense(1, activation='sigmoid')
|
|
])
|
|
|
|
# Compile and test
|
|
model.compile(optimizer='adam', loss='binary_crossentropy')
|
|
|
|
# Test data
|
|
X_test = tf.random.normal((10000, 500))
|
|
y_test = tf.random.uniform((10000, 1))
|
|
|
|
# Quick training test
|
|
model.fit(X_test, y_test, epochs=1, batch_size=1000, verbose=0)
|
|
predictions = model.predict(X_test[:1000], verbose=0)
|
|
print(f"✅ Neural Network GPU: {len(predictions):,} predictions")
|
|
|
|
return True, f"TensorFlow GPU OK ({gpu})"
|
|
|
|
except ImportError as e:
|
|
return False, f"TensorFlow non disponibile: {e}"
|
|
except Exception as e:
|
|
return False, f"Errore TensorFlow GPU: {e}"
|
|
|
|
def test_memory_capacity():
|
|
"""Test capacità memoria GPU per 1M+ record"""
|
|
print("\n🔧 TEST 4: Capacità Memoria GPU per 1M+ record")
|
|
try:
|
|
import cupy as cp
|
|
|
|
# Simula 1M record processing
|
|
print("⚡ Simulazione 1M record processing...")
|
|
|
|
# Test 1: DataFrame equivalent (1M rows x 50 cols)
|
|
print("⚡ Test 1M x 50 array (simula DataFrame)...")
|
|
test_df = cp.random.random((1000000, 50), dtype=cp.float32)
|
|
print(f"✅ 1M x 50 array: {test_df.nbytes / 1024**2:.1f}MB GPU memory")
|
|
|
|
# Test 2: Feature matrix (1M rows x 500 features)
|
|
print("⚡ Test 1M x 500 array (simula feature matrix)...")
|
|
test_features = cp.random.random((1000000, 500), dtype=cp.float32)
|
|
print(f"✅ 1M x 500 array: {test_features.nbytes / 1024**2:.1f}MB GPU memory")
|
|
|
|
# Test 3: Operations simulation
|
|
print("⚡ Test operazioni massive...")
|
|
result = cp.sum(test_features, axis=1)
|
|
result2 = cp.mean(test_features, axis=1)
|
|
result3 = cp.std(test_features, axis=1)
|
|
print(f"✅ Operazioni massive: 3 calcoli su 1M record")
|
|
|
|
# Memory status
|
|
pool = cp.get_default_memory_pool()
|
|
used_gb = pool.used_bytes() / 1024**3
|
|
|
|
print(f"📊 Memoria GPU utilizzata: {used_gb:.2f}GB")
|
|
|
|
# Cleanup
|
|
del test_df, test_features, result, result2, result3
|
|
pool.free_all_blocks()
|
|
|
|
if used_gb < 7.0: # Tesla M60 ha 8GB
|
|
return True, f"Memoria OK: {used_gb:.2f}GB/8GB utilizzati"
|
|
else:
|
|
return False, f"Memoria limite: {used_gb:.2f}GB/8GB"
|
|
|
|
except Exception as e:
|
|
return False, f"Errore test memoria: {e}"
|
|
|
|
def test_analisys_04_import():
|
|
"""Test import analisys_04 with GPU support"""
|
|
print("\n🔧 TEST 5: Import analisys_04 con supporto GPU")
|
|
try:
|
|
# Test import modules
|
|
from analisys_04 import AdvancedFeatureExtractor, AdvancedEnsemble
|
|
print("✅ Import analisys_04 OK")
|
|
|
|
# Test feature extractor
|
|
extractor = AdvancedFeatureExtractor()
|
|
print("✅ AdvancedFeatureExtractor creato")
|
|
|
|
# Test ensemble
|
|
ensemble = AdvancedEnsemble()
|
|
print("✅ AdvancedEnsemble creato")
|
|
|
|
return True, "analisys_04 modules OK"
|
|
|
|
except ImportError as e:
|
|
return False, f"Import analisys_04 fallito: {e}"
|
|
except Exception as e:
|
|
return False, f"Errore analisys_04: {e}"
|
|
|
|
def main():
|
|
"""Test completo GPU readiness"""
|
|
print("="*80)
|
|
print("🚀 TEST GPU READINESS per 1M+ RECORD - TESLA M60")
|
|
print("="*80)
|
|
|
|
results = []
|
|
total_start = time.time()
|
|
|
|
# Run all tests
|
|
tests = [
|
|
("CuDF + CuPy", test_cudf_cupy),
|
|
("CuML", test_cuml),
|
|
("TensorFlow GPU", test_tensorflow_gpu),
|
|
("Memoria 1M+ record", test_memory_capacity),
|
|
("analisys_04 import", test_analisys_04_import)
|
|
]
|
|
|
|
for test_name, test_func in tests:
|
|
try:
|
|
start_time = time.time()
|
|
success, message = test_func()
|
|
elapsed = time.time() - start_time
|
|
|
|
results.append({
|
|
'name': test_name,
|
|
'success': success,
|
|
'message': message,
|
|
'time': elapsed
|
|
})
|
|
|
|
status = "✅ PASS" if success else "❌ FAIL"
|
|
print(f"{status} {test_name}: {message} ({elapsed:.1f}s)")
|
|
|
|
except Exception as e:
|
|
results.append({
|
|
'name': test_name,
|
|
'success': False,
|
|
'message': f"Exception: {e}",
|
|
'time': 0
|
|
})
|
|
print(f"❌ FAIL {test_name}: Exception {e}")
|
|
traceback.print_exc()
|
|
|
|
total_time = time.time() - total_start
|
|
|
|
# Summary
|
|
print("\n" + "="*80)
|
|
print("📊 SUMMARY RESULTS")
|
|
print("="*80)
|
|
|
|
passed = sum(1 for r in results if r['success'])
|
|
total = len(results)
|
|
|
|
print(f"📈 TESTS PASSED: {passed}/{total}")
|
|
print(f"⏱️ TOTAL TIME: {total_time:.1f}s")
|
|
|
|
# Detailed results
|
|
for result in results:
|
|
status = "✅" if result['success'] else "❌"
|
|
print(f"{status} {result['name']}: {result['message']}")
|
|
|
|
# GPU Readiness verdict
|
|
print("\n" + "="*80)
|
|
if passed == total:
|
|
print("🎉 GPU SYSTEM READY per 1M+ RECORD!")
|
|
print("✅ Tutte le librerie GPU sono installate e funzionanti")
|
|
print("✅ Memoria GPU sufficiente per 1M+ record")
|
|
print("✅ Performance attese: 5-10x superiori vs CPU")
|
|
print("\n🚀 NEXT STEPS:")
|
|
print(" python train_gpu_native_1M.py --max-records 1000000")
|
|
elif passed >= 3:
|
|
print("⚠️ GPU SYSTEM PARZIALMENTE READY")
|
|
print("✅ Componenti base GPU funzionanti")
|
|
print("⚠️ Alcune librerie mancanti - performance ridotte")
|
|
print("\n💡 AZIONI:")
|
|
print(" - Installa librerie mancanti (vedi INSTALL_GPU_LIBRARIES.md)")
|
|
print(" - Testa con: python train_gpu_native_1M.py --max-records 500000")
|
|
else:
|
|
print("❌ GPU SYSTEM NOT READY")
|
|
print("❌ Troppe librerie mancanti per 1M+ record")
|
|
print("\n💡 AZIONI NECESSARIE:")
|
|
print(" 1. Installa CUDA toolkit")
|
|
print(" 2. pip install cudf-cu11 cupy-cuda11x")
|
|
print(" 3. pip install cuml-cu11")
|
|
print(" 4. Verifica GPU drivers")
|
|
|
|
print("="*80)
|
|
|
|
return passed == total
|
|
|
|
if __name__ == "__main__":
|
|
main() |