From 64edf0e996bc83e25ddd325f2922f8c661d6430f Mon Sep 17 00:00:00 2001 From: alirezade Date: Wed, 30 Oct 2024 12:21:05 +0100 Subject: [PATCH] Automotive data compatible --- requirements.txt | 15 ++-- src/AgentExtractor.py | 2 +- src/AgentProcessor.py | 10 +-- src/Flow.py | 26 +++---- src/Helper.py | 3 +- src/PacketInfo.py | 80 ++++++++++++++++++++ src/PacketParameter.py | 131 ++++++++++++++++++++------------- src/ProcessFlowSenderMQTT.py | 2 +- src/ProcessStatusSenderMQTT.py | 2 +- 9 files changed, 189 insertions(+), 82 deletions(-) create mode 100644 src/PacketInfo.py diff --git a/requirements.txt b/requirements.txt index 4bd5afa..103cda7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,8 @@ -pytest~=7.4.3 scapy~=2.5.0 paho-mqtt~=1.6.1 -keras>=2.15,<2.16 -numpy~=1.26.3 -pandas~=2.2.0 +numpy~=1.26.2 +pandas~=2.1.3 joblib~=1.3.2 -scikit-learn~=1.4.0 -scikeras~=0.12.0 -tensorflow~=2.15.0 -pip~=23.3.2 -wheel~=0.42.0 -setuptools~=60.2.0 +scikit-learn~=1.3.2 +scikeras~=0.11.0 +keras~=2.14.0 \ No newline at end of file diff --git a/src/AgentExtractor.py b/src/AgentExtractor.py index d9031f4..c0c7722 100644 --- a/src/AgentExtractor.py +++ b/src/AgentExtractor.py @@ -39,7 +39,7 @@ def process_packets(self, ether_pkt, pkt_time): packet_para = PacketParameter(ether_pkt, pkt_time) flow_src = min(packet_para.get_src(), packet_para.get_dst()) flow_dst = max(packet_para.get_src(), packet_para.get_dst()) - flow_proto = packet_para.protocol_name + flow_proto = packet_para.type_protocol_name key = (flow_src, flow_dst, flow_proto) if not self.processing_dict.keys().__contains__(key): diff --git a/src/AgentProcessor.py b/src/AgentProcessor.py index b802744..e2bba4e 100644 --- a/src/AgentProcessor.py +++ b/src/AgentProcessor.py @@ -1,4 +1,4 @@ -from ProcessAnnotator import ProcessAnnotator +#from ProcessAnnotator import ProcessAnnotator from ProcessFlowFileWriter import ProcessFlowFileWriter from ProcessStatusSenderMQTT import ProcessStatusSenderMQTT from Config import Config @@ -9,15 +9,15 @@ class AgentProcessor: def __init__(self, predictor_address, attacks_address, file_address, server_connection): self.__processes = [] - annotator = ProcessAnnotator(predictor_address, attacks_address) + #annotator = ProcessAnnotator(predictor_address, attacks_address) flow_file_writer = ProcessFlowFileWriter(file_address) status_sender = ProcessStatusSenderMQTT(server_connection, voting_interval=Config.StatusSender.voting_interval) flow_sender = ProcessFlowSenderMQTT(server_connection) - self.__processes.append(annotator) + # self.__processes.append(annotator) self.__processes.append(flow_file_writer) - if annotator.is_prediction_enabled(): - self.__processes.append(status_sender) + # if annotator.is_prediction_enabled(): + # self.__processes.append(status_sender) #self.__processes.append(flow_sender) def process(self, flow): diff --git a/src/Flow.py b/src/Flow.py index 1ebdbc3..b34f642 100644 --- a/src/Flow.py +++ b/src/Flow.py @@ -34,9 +34,9 @@ def start_time(self): s_start = sys.float_info.max r_start = sys.float_info.max if len(self.sen_list) != 0: - s_start = self.sen_list[0].packet_time + s_start = self.sen_list[0].time_stamp if len(self.rec_list) != 0: - r_start = self.rec_list[0].packet_time + r_start = self.rec_list[0].time_stamp return min(s_start, r_start) @@ -44,9 +44,9 @@ def end_time(self): s_end = 0 r_end = 0 if len(self.sen_list) != 0: - s_end = self.sen_list[-1].packet_time + s_end = self.sen_list[-1].time_stamp if len(self.rec_list) != 0: - r_end = self.rec_list[-1].packet_time + r_end = self.rec_list[-1].time_stamp return max(s_end, r_end) @@ -67,13 +67,13 @@ def add_packet(self, packet_parameter): if packet_parameter.get_src() == self.src: self.src_mac_list.add(packet_parameter.src_mac) self.dst_mac_list.add(packet_parameter.dst_mac) - if packet_parameter.is_ip(): + if packet_parameter.is_ip_based(): self.src_ip_list.add(packet_parameter.src_ip) self.dst_ip_list.add(packet_parameter.dst_ip) else: self.src_mac_list.add(packet_parameter.dst_mac) self.dst_mac_list.add(packet_parameter.src_mac) - if packet_parameter.is_ip(): + if packet_parameter.is_ip_based(): self.src_ip_list.add(packet_parameter.dst_ip) self.dst_ip_list.add(packet_parameter.src_ip) @@ -83,13 +83,13 @@ def compute_delay(self, packet_parameter): return if packet_parameter.get_src() == self.src: - self.acc_sen_dic[packet_parameter.ack] = packet_parameter.packet_time + self.acc_sen_dic[packet_parameter.ack] = packet_parameter.time_stamp if self.acc_rec_dic.keys().__contains__(packet_parameter.seq): - self.sen_delay.append(packet_parameter.packet_time - self.acc_rec_dic[packet_parameter.seq]) + self.sen_delay.append(packet_parameter.time_stamp - self.acc_rec_dic[packet_parameter.seq]) else: - self.acc_rec_dic[packet_parameter.ack] = packet_parameter.packet_time + self.acc_rec_dic[packet_parameter.ack] = packet_parameter.time_stamp if self.acc_sen_dic.keys().__contains__(packet_parameter.seq): - self.rec_delay.append(packet_parameter.packet_time - self.acc_sen_dic[packet_parameter.seq]) + self.rec_delay.append(packet_parameter.time_stamp - self.acc_sen_dic[packet_parameter.seq]) def compute_parameters(self): @@ -254,7 +254,7 @@ def inter_packets_avg(packets): if Flow.packets_cnt(packets) == 1: return '' - return (packets[-1].packet_time - packets[0].packet_time) / (Flow.packets_cnt(packets) - 1) + return (packets[-1].time_stamp - packets[0].time_stamp) / (Flow.packets_cnt(packets) - 1) @staticmethod def ttl_avg(packets): @@ -264,7 +264,7 @@ def ttl_avg(packets): if not packets[0].is_tcp(): return '' - if not packets[0].is_ip(): + if not packets[0].is_ip_based(): return '' else: value = sum([pkt.ttl for pkt in packets]) / Flow.packets_cnt(packets) @@ -311,7 +311,7 @@ def fragmentation_rate(packets): if Flow.packets_cnt(packets) == 0: return '' - if not packets[0].is_ip(): + if not packets[0].is_ip_based(): return '' return sum([int(pkt.fragment) for pkt in packets]) / Flow.packets_cnt(packets) diff --git a/src/Helper.py b/src/Helper.py index c2bdfa5..34b0c6e 100644 --- a/src/Helper.py +++ b/src/Helper.py @@ -14,7 +14,8 @@ def get_packet_time(pkt_metadata): :param pkt_metadata: meta data received from PCAP file. :return: formatted packet time. """ - return pkt_metadata.sec + pkt_metadata.usec / pow(10, 6) + first_pkt_timestamp = (pkt_metadata.tshigh << 32) | pkt_metadata.tslow + return first_pkt_timestamp / pkt_metadata.tsresol def format_decimal(value, rnd=3): diff --git a/src/PacketInfo.py b/src/PacketInfo.py new file mode 100644 index 0000000..7abb22f --- /dev/null +++ b/src/PacketInfo.py @@ -0,0 +1,80 @@ +class PacketInfo: + TYPE_IP = "IP" # Internet Protocol (IPv4) + TYPE_ARP = "ARP" # Address Resolution Protocol + TYPE_IPv6 = "IPv6" # Internet Protocol (IPv6) + TYPE_IPX = "IPX" # Internetwork Packet Exchange + TYPE_VLAN = "VLAN" # IEEE 802.1Q (VLAN tagging) + TYPE_PPP = "PPP" # Point-to-Point Protocol + TYPE_MPLS = "MPLS" # Multiprotocol Label Switching + TYPE_MPLS = "MPLS" # MPLS with downstream-assigned label + TYPE_PPPoE = "PPPoE" # PPP over Ethernet (Discovery stage) + TYPE_PPPoE = "PPPoE" # PPP over Ethernet (Session stage) + TYPE_QinQ = "QinQ" # 802.1ad (Q-in-Q VLAN tagging) + TYPE_Realtek = "Realtek" # Realtek protocol + TYPE_LLDP = "LLDP" # Link Layer Discovery Protocol + TYPE_FCoE = "FCoE" # Fibre Channel over Ethernet + TYPE_FCoE = "FCoE" # FCoE Initialization Protocol + + # Define packet types (Ethertypes) + TYPES = { + 0x0800: TYPE_IP, # Internet Protocol (IPv4) + 0x0806: TYPE_ARP, # Address Resolution Protocol + 0x86DD: TYPE_IPv6, # Internet Protocol (IPv6) + 0x8137: TYPE_IPX, # Internetwork Packet Exchange + 0x8100: TYPE_VLAN, # IEEE 802.1Q (VLAN tagging) + 0x880B: TYPE_PPP, # Point-to-Point Protocol + 0x8847: TYPE_MPLS, # Multiprotocol Label Switching + 0x8848: TYPE_MPLS, # MPLS with downstream-assigned label + 0x8863: TYPE_PPPoE, # PPP over Ethernet (Discovery stage) + 0x8864: TYPE_PPPoE, # PPP over Ethernet (Session stage) + 0x88A8: TYPE_QinQ, # 802.1ad (Q-in-Q VLAN tagging) + 0x8899: TYPE_Realtek, # Realtek protocol + 0x88CC: TYPE_LLDP, # Link Layer Discovery Protocol + 0x8906: TYPE_FCoE, # Fibre Channel over Ethernet + 0x8914: TYPE_FCoE, # FCoE Initialization Protocol + } + + PROTOCOL_ICMP = "ICMP" # Internet Control Message Proto + PROTOCOL_IGMP = "IGMP" # Internet Group Management Prot + PROTOCOL_TCP = "TCP" # Transmission Control Protocol + PROTOCOL_UDP = "UDP" # User Datagram Protocol + PROTOCOL_IPv6 = "IPv6" # IPv6 encapsulation + PROTOCOL_GRE = "GRE" # Generic Routing Encapsulation + PROTOCOL_ESP = "ESP" # Encapsulating Security Payload + PROTOCOL_AH = "AH" # Authentication Header + PROTOCOL_ICMPv6 = "ICMPv6" # Internet Control Message Pr + PROTOCOL_OSPF = "OSPF" # Open Shortest Path First + PROTOCOL_SCTP = "SCTP" # Stream Control Transmission + PROTOCOL_MPLS = "MPLS" # MPLS-in-IP + PROTOCOL_FCoE = "FCoE" # Fibre Channel over Ethernet + + # Define IP-based protocols + PROTOCOLS = { + 1: PROTOCOL_ICMP, # Internet Control Message Protocol + 2: PROTOCOL_IGMP, # Internet Group Management Protocol + 6: PROTOCOL_TCP, # Transmission Control Protocol + 17: PROTOCOL_UDP, # User Datagram Protocol + 41: PROTOCOL_IPv6, # IPv6 encapsulation + 47: PROTOCOL_GRE, # Generic Routing Encapsulation + 50: PROTOCOL_ESP, # Encapsulating Security Payload + 51: PROTOCOL_AH, # Authentication Header + 58: PROTOCOL_ICMPv6, # Internet Control Message Protocol for IPv6 + 89: PROTOCOL_OSPF, # Open Shortest Path First + 132: PROTOCOL_SCTP, # Stream Control Transmission Protocol + 137: PROTOCOL_MPLS, # MPLS-in-IP + 138: PROTOCOL_FCoE, # Fibre Channel over Ethernet + } + + @classmethod + def get_packet_type(cls, eth_type): + if eth_type in cls.TYPES: + return cls.TYPES[eth_type] + else: + return f'{hex(eth_type)}' + + @classmethod + def get_packet_protocol(cls, protocol): + if protocol in cls.PROTOCOLS: + return cls.PROTOCOLS[protocol] + else: + return f'{protocol}' diff --git a/src/PacketParameter.py b/src/PacketParameter.py index 65ef186..87720b7 100644 --- a/src/PacketParameter.py +++ b/src/PacketParameter.py @@ -2,91 +2,122 @@ from Helper import Log from scapy.layers.inet import TCP, UDP, IP +from scapy.layers.inet6 import IPv6, IPv6ExtHdrFragment, IPv6ExtHdrHopByHop +from scapy.all import * +from PacketInfo import PacketInfo class PacketParameter: + def __init__(self, ether_pkt, pkt_time): + + # get ether packet info self.src_mac = ether_pkt.src self.dst_mac = ether_pkt.dst + self.time_stamp = pkt_time + + self.type = PacketInfo.get_packet_type(ether_pkt.type) self.length = len(ether_pkt) - self.packet_time = pkt_time - self.type = ether_pkt.type - self.protocol_name = ether_pkt.type - if self.is_ip(): - ip_pkt = ether_pkt[IP] + self.type_protocol_name = self.type + self.protocol_length = self.length - 14 + + if self.type == PacketInfo.TYPE_ARP: # process ARP messages + self.protocol_length -= 18 # 18 is padding size for ARP messages + self.payload = 0 + + elif self.type == PacketInfo.TYPE_Realtek: # process Realtek Messages + self.payload = 0 # actually the payload is unknown + + elif self.is_ip_based(): + ip_pkt = ether_pkt[IP] if self.type == PacketInfo.TYPE_IP else ether_pkt[IPv6] + self.protocol_length -= (ip_pkt.ihl * 4 if self.type == PacketInfo.TYPE_IP else 40) + + if self.type == PacketInfo.TYPE_IP: + proto = ip_pkt.proto + else: + if not ip_pkt.nh ==0: + proto = ip_pkt.nh + else: + if IPv6ExtHdrHopByHop in ip_pkt: + hop_by_hop_header = ip_pkt[IPv6ExtHdrHopByHop] + proto = hop_by_hop_header.nh + else: + proto = 0 + + self.protocol = PacketInfo.get_packet_protocol(proto) + self.type_protocol_name += ':' + self.protocol + + self.ttl = ip_pkt.ttl if self.type == PacketInfo.TYPE_IP else ip_pkt.hlim + self.fragment = ip_pkt.flags == 'MF' or ip_pkt.frag != 0 if self.type == PacketInfo.TYPE_IP else (IPv6ExtHdrFragment in ether_pkt) self.src_ip = ip_pkt.src self.dst_ip = ip_pkt.dst - self.proto = ip_pkt.proto - self.length = ip_pkt.len - self.ttl = ip_pkt.ttl - self.fragment = ip_pkt.flags == 'MF' or ip_pkt.frag != 0 - if self.is_tcp(): # tcp + if self.protocol == PacketInfo.PROTOCOL_TCP: tcp_pkt = ip_pkt[TCP] - self.payload = ip_pkt.len - (ip_pkt.ihl * 4) - (tcp_pkt.dataofs * 4) + self.flags = tcp_pkt.flags self.window = tcp_pkt.window self.ack = tcp_pkt.ack self.seq = tcp_pkt.seq - self.protocol_name = "IPV4-TCP" - elif self.is_udp(): # udp - udp_pkt = ip_pkt[UDP] - self.payload = ip_pkt.len - (ip_pkt.ihl * 4) - (8 * 4) # UDP header size is always 8 - self.protocol_name = "IPV4-UDP" - - elif self.is_icmp(): # icmp - self.payload = ip_pkt.len - (ip_pkt.ihl * 4) - (8 * 4) # ICMP header size is always 8 - self.protocol_name = "IPV4-ICMP" + self.protocol_length = len(tcp_pkt) + self.payload = len(tcp_pkt) - (tcp_pkt.dataofs * 4) - else: - self.payload = self.payload = ip_pkt.len - (ip_pkt.ihl * 4) - (8 * 4) # default is 8 bytes - self.protocol_name = "IPV4-" + str(self.proto) - Log.log(f'Packet parameter is computing for non TCP and UDP packet type (time = {pkt_time}).', - logging.WARNING) + elif self.protocol == PacketInfo.PROTOCOL_UDP: + udp_pkt = ip_pkt[UDP] - elif self.is_arp(): - self.payload = self.length - 4 - self.protocol_name = "ARP" + self.protocol_length = len(udp_pkt) + self.payload = len(udp_pkt) - (8 * 4) # UDP header size is always 8 - elif self.is_ipv6(): - self.payload = self.length - 4 - self.protocol_name = "IPV6" + elif self.protocol == PacketInfo.PROTOCOL_ICMP or \ + self.protocol == PacketInfo.PROTOCOL_ICMPv6 or \ + self.protocol == PacketInfo.PROTOCOL_IGMP: # icmp + self.payload = 0 - else: - self.payload = self.length - 4 - self.protocol_name = str(self.type) - Log.log(f'Packet parameter is computing for unknown packet type (time = {pkt_time}).', - logging.WARNING) + # elif self.type == PacketInfo.TYPE_IPv6 and ip_pkt.nh == 0 and ip_pkt.haslayer(HBHOptions): + # pass - def is_ip(self): - return self.type == 0x0800 - def is_arp(self): - return self.type == 2054 - def is_ipv6(self): - return self.type == 34525 + else: + self.payload = self.protocol_length - (8 * 4) # default is 8 bytes + Log.log(f'Packet parameter is computing for non TCP and UDP packet type ({self.type_protocol_name} time = {pkt_time} packet = {ip_pkt}).', + logging.WARNING) + if IPv6ExtHdrHopByHop in ip_pkt: + hop_by_hop_header = ip_pkt[IPv6ExtHdrHopByHop] + print("Hop-by-Hop Header:") + print(hop_by_hop_header.show()) # Display the Hop-by-Hop header details - def is_icmp(self): - return self.is_ip() and self.proto == 1 + # Access specific fields + # For example, if you want to access the options in the Hop-by-Hop header + if hop_by_hop_header.options: + for option in hop_by_hop_header.options: + print("Option Type:", option.type) + print("Option Data:", option.data) - def is_tcp(self): - return self.is_ip() and self.proto == 6 - def is_udp(self): - return self.is_ip() and self.proto == 17 + else: + self.payload = self.protocol_length + self.protocol = str(self.type) + Log.log(f'Packet parameter is computing for unknown packet type {hex(self.type)}, time = {pkt_time}).', + logging.WARNING) def get_src(self): - if self.is_ip(): + if self.is_ip_based(): return self.src_ip else: return self.src_mac def get_dst(self): - if self.is_ip(): + if self.is_ip_based(): return self.dst_ip else: return self.dst_mac + def is_ip_based(self): + return self.type == PacketInfo.TYPE_IP or self.type == PacketInfo.TYPE_IPv6 + + def is_tcp(self): + return self.is_ip_based() and self.protocol == PacketInfo.PROTOCOL_TCP + diff --git a/src/ProcessFlowSenderMQTT.py b/src/ProcessFlowSenderMQTT.py index 99c0913..6773208 100644 --- a/src/ProcessFlowSenderMQTT.py +++ b/src/ProcessFlowSenderMQTT.py @@ -4,7 +4,7 @@ class ProcessFlowSenderMQTT: def __init__(self, server_connection_file): - self.client = Connection.build(server_connection_file) if server_connection_file.strip() else False + self.client = Connection.build(server_connection_file) if (not server_connection_file is None) and server_connection_file.strip() else False if self.client: self.client.start() diff --git a/src/ProcessStatusSenderMQTT.py b/src/ProcessStatusSenderMQTT.py index ab31938..57397a4 100644 --- a/src/ProcessStatusSenderMQTT.py +++ b/src/ProcessStatusSenderMQTT.py @@ -10,7 +10,7 @@ class ProcessStatusSenderMQTT: def __init__(self, server_connection_file, voting_interval=0): - self.client = Connection.build(server_connection_file) if server_connection_file.strip() else False + self.client = Connection.build(server_connection_file) if ((not server_connection_file is None) and server_connection_file.strip()) else False self.voting_interval = voting_interval self.link_flows = dict()