diff --git a/attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-12-5_1767354943964.txt b/attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-12-5_1767354943964.txt new file mode 100644 index 0000000..64df002 --- /dev/null +++ b/attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-12-5_1767354943964.txt @@ -0,0 +1,51 @@ +journalctl -u ids-list-fetcher -n 50 --no-pager +Jan 02 12:50:02 ids.alfacom.it ids-list-fetcher[5900]: ============================================================ +Jan 02 12:50:02 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully. +Jan 02 12:50:02 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service. +Jan 02 12:54:56 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service... +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: [2026-01-02 12:54:56] PUBLIC LISTS SYNC +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: Found 2 enabled lists +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: [12:54:56] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json... +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: [12:54:56] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json... +Jan 02 12:54:56 ids.alfacom.it ids-list-fetcher[6290]: [12:54:56] Parsing AWS... +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: [12:54:57] Found 9548 IPs, syncing to database... +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: [12:54:57] ✓ AWS: +0 -0 ~9511 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: [12:54:57] Parsing Spamhaus... +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: [12:54:57] Found 1468 IPs, syncing to database... +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: [12:54:57] ✗ Spamhaus: ON CONFLICT DO UPDATE command cannot affect row a second time +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: HINT: Ensure that no rows proposed for insertion within the same command have duplicate constrained values. +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: SYNC SUMMARY +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Success: 1/2 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Errors: 1/2 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Total IPs Added: 0 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Total IPs Removed: 0 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: RUNNING MERGE LOGIC +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ERROR:merge_logic:Failed to cleanup detections: operator does not exist: inet = text +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: LINE 9: d.source_ip::inet = wl.ip_inet +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ^ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ERROR:merge_logic:Failed to sync detections: operator does not exist: text <<= text +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: LINE 30: OR bl.ip_inet <<= wl.ip_inet +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ^ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Traceback (most recent call last): +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: cur.execute(""" +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: psycopg2.errors.UndefinedFunction: operator does not exist: text <<= text +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: LINE 30: OR bl.ip_inet <<= wl.ip_inet +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ^ +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Merge Logic Stats: +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Created detections: 0 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Cleaned invalid detections: 0 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: Skipped (whitelisted): 0 +Jan 02 12:54:57 ids.alfacom.it ids-list-fetcher[6290]: ============================================================ +Jan 02 12:54:57 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully. +Jan 02 12:54:57 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service. \ No newline at end of file diff --git a/python_ml/list_fetcher/parsers.py b/python_ml/list_fetcher/parsers.py index 50c94b0..714c8fb 100644 --- a/python_ml/list_fetcher/parsers.py +++ b/python_ml/list_fetcher/parsers.py @@ -21,13 +21,15 @@ class ListParser: def normalize_cidr(ip_str: str) -> tuple[str, Optional[str]]: """ Normalize IP/CIDR to (ip_address, cidr_range) - Example: '1.2.3.0/24' -> ('1.2.3.0', '1.2.3.0/24') + For CIDR ranges, use the full CIDR notation as ip_address to ensure uniqueness + Example: '1.2.3.0/24' -> ('1.2.3.0/24', '1.2.3.0/24') '1.2.3.4' -> ('1.2.3.4', None) """ try: network = ipaddress.ip_network(ip_str, strict=False) if '/' in ip_str: - return (str(network.network_address), str(network)) + normalized_cidr = str(network) + return (normalized_cidr, normalized_cidr) else: return (ip_str, None) except ValueError: