diff --git a/Configuration 2 (RAK831)/Config2_LoRadar_Extractor.py b/Configuration 2 (RAK831)/Config2_LoRadar_Extractor.py new file mode 100644 index 0000000..11075da --- /dev/null +++ b/Configuration 2 (RAK831)/Config2_LoRadar_Extractor.py @@ -0,0 +1,1126 @@ +### Libraries to load +import glob, matplotlib, pprint, requests, math, time, os +import pandas as pd +import numpy as np +from collections import Counter +from itertools import repeat +import matplotlib.pyplot as plt + + +### Loading the data +os.chdir = ("/home/pi/lora_gateway/util_pkt_logger") +docs = glob.glob(os.chdir + "/*.csv") + +if len(docs) > 1: + print("1. Reading " + str(len(docs)) + " csv files...") + all_dat = [] + for doc in docs: + all_dat.append(pd.read_csv(doc, encoding = 'unicode_escape')) + + every = pd.concat(all_dat) + every.reset_index(inplace=True) + every = every.dropna(subset=["UTC timestamp"]) + every = every.drop("index", axis=1) + +elif len(docs) == 1: + print("1. Reading one csv file...") + print(docs) + every = pd.read_csv(docs[0], encoding = 'unicode_escape') + every.reset_index(inplace=True) + every = every.dropna(subset=["UTC timestamp"]) + every = every.drop("index", axis=1) + +elif len(docs) == 0: + print("ERROR: No Data Files!") + exit() + + +### Filtering the data for ABP +print("2. Filtering the data for ABP...") +info = every.payload +info = list(map(str, info)) # Convert payload to string. This is needed as some payloads only have integers. +devs = [] +fcnts = [] +mhdrs = [] +fctrls = [] +devaddr_revl = [] +fcnt_revl = [] +devaddr_len = 8 + +for i in info: + devaddr = i[2:11].split("-") + devaddr = "".join(devaddr) + fcnt = i[13:17].split("-") + fcnt = "".join(fcnt) + fctrl = i[11:13].split("-") + fctrl = "".join(fctrl) + mhdr = i[:2].split("-") + mhdr = "".join(mhdr) + devs.append(devaddr) + fcnts.append(fcnt) + fctrls.append(fctrl) + mhdrs.append(mhdr) + +# reversing the little endian form of DevAddr and Fcnt +for row in devs: + fixed = [] + for j in range(int(len(row)/2)): + fixed.append(row[j*2:(j+1)*2]) + fixed.reverse() + devaddr_rev = "".join(fixed) + devaddr_revl.append(devaddr_rev) + +for row in fcnts: + fixed = [] + for j in range(int(len(row)/2)): + fixed.append(row[j*2:(j+1)*2]) + fixed.reverse() + fcnt_rev = "".join(fixed) + fcnt_revl.append(fcnt_rev) + +# Remove payloads shorter than the length of DevAddr, as these will not provide sufficient information to identify ABP devices +ttn_addr = [] +all_addr = [] +other_addr = [] +network = [] +activation = [] +for addr in devaddr_revl: + if len(addr) == devaddr_len: + # Network prefix assignments to the World + if addr[:2] == "26" or addr[:2] == "27": + ttn_addr.append(addr) # potential DevAddr from TTN + all_addr.append(addr) + network.append("World: The Things Network") + activation.append("ABP") + elif addr[:2] == "04" or addr[:2] == "05": + all_addr.append(addr) + network.append("World: Actility") + activation.append("ABP") + elif addr[:2] == "10" or addr[:2] == "11": + all_addr.append(addr) + network.append("World: Orbiwise") + activation.append("ABP") + elif addr[:2] == "1a" or addr[:2] == "1b": + all_addr.append(addr) + network.append("World: SK Telecom") + activation.append("ABP") + elif addr[:2] == "1c" or addr[:2] == "1d": + all_addr.append(addr) + network.append("World: SagemCom") + activation.append("ABP") + elif addr[:2] == "24" or addr[:2] == "25": + all_addr.append(addr) + network.append("World: Kerlink") + activation.append("ABP") + elif addr[:2] == "2a" or addr[:2] == "2b": + all_addr.append(addr) + network.append("World: Cisco Systems") + activation.append("ABP") + elif addr[:2] == "2e" or addr[:2] == "2f": + all_addr.append(addr) + network.append("World: MultiTech Systems") + activation.append("ABP") + elif addr[:2] == "30" or addr[:2] == "31": + all_addr.append(addr) + network.append("World: Loriot") + activation.append("ABP") + elif addr[:2] == "32" or addr[:2] == "33": + all_addr.append(addr) + network.append("World: NNNCo") + activation.append("ABP") + elif addr[:2] == "34" or addr[:2] == "35": + all_addr.append(addr) + network.append("World: Flashnet") + activation.append("ABP") + elif addr[:2] == "36" or addr[:2] == "37": + all_addr.append(addr) + network.append("World: TrackNet") + activation.append("ABP") + elif addr[:2] == "38" or addr[:2] == "39": + all_addr.append(addr) + network.append("World: Lar.Tech") + activation.append("ABP") + elif addr[:2] == "3a" or addr[:2] == "3b": + all_addr.append(addr) + network.append("World: Swiss Led") + activation.append("ABP") + elif addr[:2] == "42" or addr[:2] == "43": + all_addr.append(addr) + network.append("World: Patavina Technologies") + activation.append("ABP") + elif addr[:2] == "48" or addr[:2] == "49": + all_addr.append(addr) + network.append("World: Gimasi") + activation.append("ABP") + elif addr[:2] == "4a" or addr[:2] == "4b": + all_addr.append(addr) + network.append("World: Talkpool") + activation.append("ABP") + elif addr[:2] == "4e" or addr[:2] == "4f": + all_addr.append(addr) + network.append("World: MCF88 SRL") + activation.append("ABP") + elif addr[:2] == "52" or addr[:2] == "53": + all_addr.append(addr) + network.append("World: GIoT") + activation.append("ABP") + elif addr[:2] == "54" or addr[:2] == "55": + all_addr.append(addr) + network.append("World: M2B Communications") + activation.append("ABP") + elif addr[:2] == "5a" or addr[:2] == "5b": + all_addr.append(addr) + network.append("World: Rai Way") + activation.append("ABP") + elif addr[:2] == "5c" or addr[:2] == "5d": + all_addr.append(addr) + network.append("World: Levikom") + activation.append("ABP") + elif addr[:2] == "60" or addr[:2] == "61": + all_addr.append(addr) + network.append("World: SoftBank") + activation.append("ABP") + elif addr[:2] == "62" or addr[:2] == "63": + all_addr.append(addr) + network.append("World: Inmarsat") + activation.append("ABP") + elif addr[:4] == "e006" or addr[:4] == "e007": + all_addr.append(addr) + network.append("World: IOTCAN") + activation.append("ABP") + elif addr[:4] == "e00a" or addr[:4] == "e00b": + all_addr.append(addr) + network.append("World: IoT Network AS") + activation.append("ABP") + elif addr[:4] == "e00e" or addr[:4] == "e00f": + all_addr.append(addr) + network.append("World: EDF") + activation.append("ABP") + elif addr[:4] == "e018" or addr[:4] == "e019": + all_addr.append(addr) + network.append("World: SenSys") + activation.append("ABP") + elif addr[:4] == "e01c" or addr[:4] == "e01d": + all_addr.append(addr) + network.append("World: Spark") + activation.append("ABP") + elif addr[:4] == "e020" or addr[:4] == "e021": + all_addr.append(addr) + network.append("World: Senet") + activation.append("ABP") + elif addr[:4] == "e026" or addr[:4] == "e027": + all_addr.append(addr) + network.append("World: Actility") + activation.append("ABP") + elif addr[:4] == "e02a" or addr[:4] == "e02b": + all_addr.append(addr) + network.append("World: Kerlink") + activation.append("ABP") + elif addr[:4] == "e02c" or addr[:4] == "e02d": + all_addr.append(addr) + network.append("World: Cisco") + activation.append("ABP") + elif addr[:4] == "e02e" or addr[:4] == "e02f": + all_addr.append(addr) + network.append("World: Schneider Electric") + activation.append("ABP") + elif addr[:4] == "e030" or addr[:4] == "e031": + all_addr.append(addr) + network.append("World: ZENNER") + activation.append("ABP") + elif addr[:4] == "e038" or addr[:4] == "e039": + all_addr.append(addr) + network.append("World: MachineQ/Comcast") + activation.append("ABP") + elif addr[:6] == "fc0004" or addr[:6] == "fc0005" or addr[:6] == "fc0006" or addr[:6] == "fc0007": + all_addr.append(addr) + network.append("World: Nordic Automation Systems") + activation.append("ABP") + elif addr[:6] == "fc0008" or addr[:6] == "fc0009" or addr[:6] == "fc000a" or addr[:6] == "fc000b": + all_addr.append(addr) + network.append("World: ResIOT") + activation.append("ABP") + elif addr[:6] == "fc000c" or addr[:6] == "fc000d" or addr[:6] == "fc000e" or addr[:6] == "fc000f": + all_addr.append(addr) + network.append("World: SYSDEV") + activation.append("ABP") + elif addr[:6] == "fc0020" or addr[:6] == "fc0021" or addr[:6] == "fc0022" or addr[:6] == "fc0023": + all_addr.append(addr) + network.append("World: Definium Technologies") + activation.append("ABP") + elif addr[:6] == "fc0030" or addr[:6] == "fc0031" or addr[:6] == "fc0032" or addr[:6] == "fc0033": + all_addr.append(addr) + network.append("World: nFore Technology") + activation.append("ABP") + elif addr[:6] == "fc0044" or addr[:6] == "fc0045" or addr[:6] == "fc0046" or addr[:6] == "fc0047": + all_addr.append(addr) + network.append("World: Digital Nordix AB (DNX)") + activation.append("ABP") + elif addr[:6] == "fc0058" or addr[:6] == "fc0059" or addr[:6] == "fc005a" or addr[:6] == "fc005b": + all_addr.append(addr) + network.append("World: Schneider Electric") + activation.append("ABP") + elif addr[:6] == "fc0060" or addr[:6] == "fc0061" or addr[:6] == "fc0062" or addr[:6] == "fc0063": + all_addr.append(addr) + network.append("World: ZENNER") + activation.append("ABP") + elif addr[:6] == "fc0068" or addr[:6] == "fc0069" or addr[:6] == "fc006a" or addr[:6] == "fc006b": + all_addr.append(addr) + network.append("World: REQUEA") + activation.append("ABP") + elif addr[:6] == "fc007c" or addr[:6] == "fc007d" or addr[:6] == "fc007e" or addr[:6] == "fc007f": + all_addr.append(addr) + network.append("World: mcf88 SRL") + activation.append("ABP") + elif addr[:6] == "fc0084" or addr[:6] == "fc0085" or addr[:6] == "fc0086" or addr[:6] == "fc0087": + all_addr.append(addr) + network.append("World: Hiber") + activation.append("ABP") + elif addr[:6] == "fc0098" or addr[:6] == "fc0099" or addr[:6] == "fc009a" or addr[:6] == "fc009b": + all_addr.append(addr) + network.append("World: Mirakonta") + activation.append("ABP") + + # Network prefix assignments to Australia + elif addr[:2] == "0a" or addr[:2] == "0b": + all_addr.append(addr) + network.append("Singapore Indonesia Australia Africa India: SingTel") + activation.append("ABP") + elif addr[:2] == "46" or addr[:2] == "47": + all_addr.append(addr) + network.append("Australia New Zealand: Ventia") + activation.append("ABP") + elif addr[:2] == "58" or addr[:2] == "59": + all_addr.append(addr) + network.append("Australia: Airlora") + activation.append("ABP") + + # Neither + else: + all_addr.append(addr) + other_addr.append(addr) # potential DevAddr from non-TTN network + network.append("-") + activation.append("-") + else: + all_addr.append("-") + network.append("-") + activation.append("-") + + + +### Filtering the data for OTAA +print("3. Filtering the data for OTAA...") +info = every.payload +info = list(map(str, info)) +deveuis = [] +appeuis = [] +deveui_revl = [] +appeui_revl = [] +deveui_len = 16 +appeui_len = 16 + +for i in range(len(info)): + if list(map(str, mhdrs))[i] == "00": + deveui = info[i][20:38].split("-") + deveui = "".join(deveui) + appeui = info[i][2:20].split("-") + appeui = "".join(appeui) + deveuis.append(deveui) + appeuis.append(appeui) + else: + deveuis.append("-") + appeuis.append("-") + +for row in deveuis: # reversing DevEUI's little endian form + fixed = [] + for j in range(int(len(row)/2)): + fixed.append(row[j*2:(j+1)*2]) + fixed.reverse() + deveui_rev = "".join(fixed) + deveui_revl.append(deveui_rev) + +for row in appeuis: # reversing AppEUI's little endian form + fixed = [] + for j in range(int(len(row)/2)): + fixed.append(row[j*2:(j+1)*2]) + fixed.reverse() + appeui_rev = "".join(fixed) + appeui_revl.append(appeui_rev) + +filt_deveui = [] # Remove payloads shorter than the length of DevAddr +filt_appeui = [] +all_deveui = [] +all_appeui = [] +for deveui_s in deveui_revl: + if len(deveui_s) == deveui_len: + filt_deveui.append(deveui_s) + all_deveui.append(deveui_s) + else: + all_deveui.append("-") + +for appeui_s in appeui_revl: + if len(appeui_s) == appeui_len: + filt_appeui.append(appeui_s) + all_appeui.append(appeui_s) + else: + all_appeui.append("-") +deveui_results = Counter(filt_deveui) +appeui_results = Counter(filt_appeui) + + + +### Using API to obtain device manufacturer +print("4. Looking up device manufacturers...") +MAC_URL = "https://api.macvendors.com/v1/lookup/%s" +deveui_knowns = [] +appeui_knowns = [] +deveui_total = len(list(deveui_results.keys())) +appeui_total = len(list(appeui_results.keys())) + +num = 1 +starttime = time.time() +for eui in list(deveui_results.keys()): + print("Looking up", num, "out of", deveui_total, "DevEUIs...") + r = requests.get(MAC_URL % eui) + if "data" in list(r.json().keys()): + deveui_knowns.append([eui, r.json()]) + num += 1 + time.sleep(0.5) + +num = 1 +starttime = time.time() +for eui in list(appeui_results.keys()): + print("Looking up", num, "out of", appeui_total, "AppEUIs...") + r = requests.get(MAC_URL % eui) + if "data" in list(r.json().keys()): + appeui_knowns.append([eui, r.json()]) + num += 1 + time.sleep(0.5) + + + +### Adding new columns +# Turn DevEUI and AppEUI knowns into dictionary +print("5. Adding new columns...") +deveui_knowns_dict = {} +appeui_knowns_dict = {} +for i in range(len(deveui_knowns)): + deveui_knowns_dict[deveui_knowns[i][0]] = deveui_knowns[i][1] +for i in range(len(appeui_knowns)): + appeui_knowns_dict[appeui_knowns[i][0]] = appeui_knowns[i][1] + +# Get a list of devaddress and mic +devaddress = [] +mic = [] +for i in range(len(activation)): + if activation[i] != "-": + devaddress.append(devaddr_revl[i]) + if len("".join(info[i].split("-"))[-8:]) == 8: + mic.append("".join(info[i].split("-"))[-8:]) + else: + mic.append("-") + else: + devaddress.append(devaddr_revl[i]) + if len("".join(info[i].split("-"))[-8:]) == 8: + mic.append("".join(info[i].split("-"))[-8:]) + else: + mic.append("-") + +# Create a dataframe for OTAA +dev_matcheui = [] +app_matcheui = [] +dev_manufeui = [] +app_manufeui = [] +manumac = [] +for i in deveui_knowns: + dev_matcheui.append(i[0]) + dev_manufeui.append(i[1].get("data").get("organization_name")) + manumac.append(i[1].get("data").get("assignment")) + +for i in appeui_knowns: + app_matcheui.append(i[0]) + app_manufeui.append(i[1].get("data").get("organization_name")) + +#manufdf = pd.DataFrame({"DevEUI": dev_matcheui, "DevManufacturer": dev_manufeui, "MAC_prefix": manumac, +# "AppEUI": app_matcheui, "AppManufacturer": app_manufeui}) + +dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ]) +deveui_res = dictfilt(deveui_results, dev_matcheui) +appeui_res = dictfilt(appeui_results, app_matcheui) + +# Update the new columns' values +dev_manufid = [] +app_manufid = [] +for i in range(len(all_deveui)): + #if int(every["size"][i]) == 23 and str(mhdrs[i]) == "00": + if (all_deveui[i] in dev_matcheui or all_appeui[i] in app_matcheui) and str(mhdrs[i]) == "00": + activation[i] = "OTAA" + network[i] = "-" + devaddress[i] = all_deveui[i] + if all_deveui[i] in dev_matcheui and all_appeui[i] in app_matcheui: + dev_manufid.append(deveui_knowns_dict[all_deveui[i]].get("data").get("organization_name").replace(',', '')) + app_manufid.append(appeui_knowns_dict[all_appeui[i]].get("data").get("organization_name").replace(',', '')) + elif all_deveui[i] in dev_matcheui and all_appeui[i] not in app_matcheui: + dev_manufid.append(deveui_knowns_dict[all_deveui[i]].get("data").get("organization_name").replace(',', '')) + app_manufid.append("-") + elif all_deveui[i] not in dev_matcheui and all_appeui[i] in app_matcheui: + dev_manufid.append("-") + app_manufid.append(appeui_knowns_dict[all_appeui[i]].get("data").get("organization_name").replace(',', '')) + else: + dev_manufid.append("-") + app_manufid.append("-") + elif int(every["size"][i]) == 23 and str(mhdrs[i]) == "00": + activation[i] = "OTAA" + network[i] = "-" + devaddress[i] = all_deveui[i] + if all_deveui[i] in dev_matcheui and all_appeui[i] in app_matcheui: + dev_manufid.append(deveui_knowns_dict[all_deveui[i]].get("data").get("organization_name").replace(',', '')) + app_manufid.append(appeui_knowns_dict[all_appeui[i]].get("data").get("organization_name").replace(',', '')) + elif all_deveui[i] in dev_matcheui and all_appeui[i] not in app_matcheui: + dev_manufid.append(deveui_knowns_dict[all_deveui[i]].get("data").get("organization_name").replace(',', '')) + app_manufid.append("-") + elif all_deveui[i] not in dev_matcheui and all_appeui[i] in app_matcheui: + dev_manufid.append("-") + app_manufid.append(appeui_knowns_dict[all_appeui[i]].get("data").get("organization_name").replace(',', '')) + else: + dev_manufid.append("-") + app_manufid.append("-") + else: + dev_manufid.append("-") + app_manufid.append("-") + +all_fcnt = [] +all_fctrl = [] +all_mhdr = [] +final_appeui = [] +for i in range(len(activation)): + if activation[i] == "ABP": + final_appeui.append("-") + all_mhdr.append(mhdrs[i]) + all_fcnt.append(fcnt_revl[i]) + all_fctrl.append(fctrls[i]) + elif activation[i] == "-": + final_appeui.append("-") + all_fcnt.append(fcnt_revl[i]) + all_fctrl.append(fctrls[i]) + all_mhdr.append(mhdrs[i]) + elif activation[i] == "OTAA": + final_appeui.append(all_appeui[i]) + all_fcnt.append("-") + all_fctrl.append("-") + all_mhdr.append(mhdrs[i]) + +# Converting MHDR (hex) to binary and getting message type +mtype_raw = [] +for hexdec in all_mhdr: + if hexdec == "-": + mtype_raw.append("-") + elif hexdec == "na": + mtype_raw.append("-") + elif hexdec == "9.": + mtype_raw.append("-") + else: + scale = 16 ## equals to hexadecimal + num_of_bits = 8 + mtype_raw.append(str(bin(int(hexdec, scale))[2:].zfill(num_of_bits)[:3])) + +mtype_desc = [] +for mtype in mtype_raw: + if mtype == "000": + mtype_desc.append("Join-request") + elif mtype == "001": + mtype_desc.append("Join-accept") + elif mtype == "010": + mtype_desc.append("Unconfirmed Data Up") + elif mtype == "011": + mtype_desc.append("Unconfirmed Data Down") + elif mtype == "100": + mtype_desc.append("Confirmed Data Up") + elif mtype == "101": + mtype_desc.append("Confirmed Data Down") + elif mtype == "110": + mtype_desc.append("Rejoin-request") + elif mtype == "111": + mtype_desc.append("Proprietary") + else: + mtype_desc.append("-") + +# Converting fcnt (hex) to decimal +int_fcnt = [] +for fcnt_hex in all_fcnt: + if str(fcnt_hex) == "-": + int_fcnt.append("-") + elif str(fcnt_hex) == "": + int_fcnt.append("-") + else: + dec = int(fcnt_hex, 16) + int_fcnt.append(dec) + +# Correcting the time +yr = [] +month = [] +day = [] +hrs = [] +mins = [] +secs = [] +for i in list(every["UTC timestamp"]): + #split for date and time + date_split = i.split(" ") + time_split = date_split[1].split(":") + + if int(date_split[0].split("-")[0]) == 2013: + y = int(date_split[0].split("-")[0]) + 0 + if len(str(y)) == 1: + yr.append("0"+str(y)) + else: + yr.append(str(y)) + m = int(date_split[0].split("-")[1]) - 0 + if len(str(m)) == 1: + month.append("0"+str(m)) + else: + month.append(str(m)) + d = int(date_split[0].split("-")[2]) - 0 + if len(str(d)) == 1: + day.append("0"+str(d)) + else: + day.append(str(d)) + h = int(time_split[0]) + 0 + if len(str(h)) == 1: + hrs.append("0"+str(h)) + else: + hrs.append(str(h)) + m = int(time_split[1]) + 0 + if len(str(m)) == 1: + mins.append("0"+str(m)) + else: + mins.append(str(m)) + secs.append(time_split[2]) + + elif int(date_split[0].split("-")[0]) == 2014: + y = int(date_split[0].split("-")[0]) + 0 + if len(str(y)) == 1: + yr.append("0"+str(y)) + else: + yr.append(str(y)) + m = int(date_split[0].split("-")[1]) + 0 + if len(str(m)) == 1: + month.append("0"+str(m)) + else: + month.append(str(m)) + d = int(date_split[0].split("-")[2]) + 0 + if len(str(d)) == 1: + day.append("0"+str(d)) + else: + day.append(str(d)) + h = int(time_split[0]) + 0 + if len(str(h)) == 1: + hrs.append("0"+str(h)) + else: + hrs.append(str(h)) + m = int(time_split[1]) + 0 + if len(str(m)) == 1: + mins.append("0"+str(m)) + else: + mins.append(str(m)) + secs.append(time_split[2]) + + else: + yr.append(date_split[0].split("-")[0]) + month.append(date_split[0].split("-")[1]) + day.append(date_split[0].split("-")[2]) + h = int(time_split[0]) + 0 + if len(str(h)) == 1: + hrs.append("0"+str(h)) + else: + hrs.append(str(h)) + mins.append(time_split[1]) + secs.append(time_split[2]) + +yr = list(map(str, yr)) +month = list(map(str, month)) +day = list(map(str, day)) +hrs = list(map(str, hrs)) +mins = list(map(str, mins)) +secs = list(map(str, secs)) + +# correcting cases where mins >= 60 and hours >= 24 +for i in range(len(mins)): + if int(mins[i]) >= 60: + h = int(hrs[i]) + h += 1 + if len(str(h)) == 1: + hrs[i] = "0"+str(h) + else: + hrs[i] = str(h) + m = int(mins[i]) + m -= 60 + if len(str(m)) == 1: + mins[i] = "0"+str(m) + else: + mins[i] = str(m) + +for i in range(len(hrs)): + if int(hrs[i]) >= 24: + d = int(day[i]) + d += 1 + if len(str(d)) == 1: + day[i] = "0"+str(d) + else: + day[i] = str(d) + h = int(hrs[i]) + h -= 24 + if len(str(h)) == 1: + hrs[i] = "0"+str(h) + else: + hrs[i] = str(h) + +# correcting cases where day is > possible value within a certain month +for i in range(len(day)): + if int(day[i]) > 30: + if month[i] in ["4", "6", "9", "11"]: + m = int(month[i]) + m += 1 + if len(str(m)) == 1: + month[i] = "0"+str(m) + else: + month[i] = str(m) + d = int(day[i]) + d -= 30 + if len(str(d)) == 1: + day[i] = "0"+str(d) + else: + day[i] = str(d) + elif int(day[i]) > 28: + if month[i] in ["2"]: + if int(yr[i])%4 == 0 and int(day[i]) > 29: + m = int(month[i]) + m += 1 + if len(str(m)) == 1: + month[i] = "0"+str(m) + else: + month[i] = str(m) + d = int(day[i]) + d -= 29 + if len(str(d)) == 1: + day[i] = "0"+str(d) + else: + day[i] = str(d) + elif int(yr[i])%4 != 0: + m = int(month[i]) + m += 1 + if len(str(m)) == 1: + month[i] = "0"+str(m) + else: + month[i] = str(m) + d = int(day[i]) + d -= 28 + if len(str(d)) == 1: + day[i] = "0"+str(d) + else: + day[i] = str(d) + +# correcting cases where year needs to be changed +for i in range(len(month)): + if int(month[i]) == 12 and int(day[i]) > 31: + y = int(yr[i]) + y += 1 + yr[i] = str(y) + m = 1 + if len(str(m)) == 1: + month[i] = "0"+str(m) + else: + month[i] = str(m) + d = 1 + if len(str(d)) == 1: + day[i] = "0"+str(d) + else: + day[i] = str(d) + +# concatenating the fixed date time +date = [] +aus_time = [] + +for i in range(len(yr)): + date.append("-".join([yr[i], month[i], day[i]])) + aus_time.append(":".join([hrs[i], mins[i], secs[i]])) + +every["date"] = date +every["time"] = aus_time +every["activation"] = activation +every["network"] = network +every["DevEUI or DevAddr"] = devaddress +every["AppEUI"] = final_appeui +every["fctrl"] = all_fctrl +every["fcnt"] = int_fcnt +every["mhdr"] = all_mhdr +every["mtype"] = mtype_raw +every["mtype_desc"] = mtype_desc +every["MIC"] = mic +every["deveui_manufacturer"] = dev_manufid +every["appeui_manufacturer"] = app_manufid + +every = every[['UTC timestamp', 'date', 'time', 'us count', 'frequency', 'RF chain', 'RX chain','status', + 'bandwidth', 'datarate', 'coderate', 'RSSI', 'SNR', 'size', 'DevEUI or DevAddr', + 'AppEUI', 'fctrl', 'fcnt', 'mhdr', 'mtype', 'mtype_desc', 'MIC', 'activation', + 'network', 'deveui_manufacturer', 'appeui_manufacturer', 'payload']] + +filtered = every.copy() +no_fcnt = filtered.loc[(filtered.fcnt == "-") & (filtered.activation != "OTAA")].index +filtered = filtered.drop(no_fcnt) + + + +### Calculating the time difference +# Function for truncating decimals without rounding to 3 dp +print("6. Calculating the time difference in seconds...") +time_sec = [] +timeutc = list(pd.to_datetime(filtered["UTC timestamp"])) +for i in range(len(timeutc)): + time_sec.append(pd.Timedelta(timeutc[i]-timeutc[0]).total_seconds()) + +# Add a new column to the dataframe +filtered["sec_diff"] = time_sec + + + +### Algorithm for identifying unique devices +# Split it from the original dataframe by DevAddr or DevEUI +print("7. Identifying unique devices...") +l_freq = [] +l_devaddr915 = [] +l_devaddr923 = [] +all_freq = [] +filtered = filtered.loc[filtered.status == "CRC_OK "] + +# ISM band +fr915 = [916800000, 917000000, 917200000, 917400000, 917600000, 917800000, 918000000, 91820000] +fr923 = [922000000, 922100000, 922200000, 922400000, 922600000, 922800000, 923000000, 923200000, 923400000] + +freq915 = filtered.loc[filtered.frequency.isin(fr915)] +if freq915.shape[0] > 0: + freq915 = filtered.loc[filtered.frequency.isin(fr915)].index + filtered.loc[freq915, "Freq_Plan"] = "915band" + all_freq.append(freq915) + +freq923 = filtered.loc[filtered.frequency.isin(fr923)] +if freq923.shape[0] > 0: + freq923 = filtered.loc[filtered.frequency.isin(fr923)].index + filtered.loc[freq923, "Freq_Plan"] = "923band" + all_freq.append(freq923) + +if len(all_freq) == 0: + print("ERROR: No valid LoRaWAN packets!") + exit() + +if (freq915.shape[0] and freq923.shape[0]) > 0: + for a, b in filtered.groupby("Freq_Plan", sort=False): + l_freq.append(b.sort_values(by=["sec_diff"])) + #915 + for a, b in l_freq[0].groupby("DevEUI or DevAddr", sort=False): + l_devaddr915.append(b.sort_values(by=["sec_diff"])) + #923 + for a, b in l_freq[1].groupby("DevEUI or DevAddr", sort=False): + l_devaddr923.append(b.sort_values(by=["sec_diff"])) + +elif freq915.shape[0] == 0: + #923 + for a, b in filtered.groupby("DevEUI or DevAddr", sort=False): + l_devaddr923.append(b.sort_values(by=["sec_diff"])) + +elif freq923.shape[0] == 0: + #915 + for a, b in filtered.groupby("DevEUI or DevAddr", sort=False): + l_devaddr915.append(b.sort_values(by=["sec_diff"])) + + +### Counting the number of devices +print("8. Counting the number of devices...") +def truncate(f): + if f == "-": + return "-" + else: + return math.floor(f * 10 ** 3) / 10 ** 3 + +def truncate3(f): + if f == "-": + return "-" + else: + return math.floor(f * 10 ** 3) / 10 ** 3 + +def truncate0(f): + if f == "-": + return "-" + else: + return math.floor(f * 10 ** 0) / 10 ** 0 + +dev_num915 = [] +rate915 = [] +dev_num923 = [] +rate923 = [] +num = 1 + +# Identifying unique devices +l_index_915 = [] +if len(l_devaddr915) > 0: + for i in l_devaddr915: + sec_df = list(map(float, i["sec_diff"])) + sec_df = [x - sec_df[0] for x in sec_df] + sec_df = list(map(truncate, sec_df)) + + devaddr_df = list(i["DevEUI or DevAddr"]) + activation_df = list(i["activation"]) + + # OTAA case + if activation_df[0] == "OTAA": + l_index_915.extend(i.index) + for j in range(len(devaddr_df)): + if j >= 1: + rate915.append(truncate3(float(sec_df[j]) - float(sec_df[j-1]))) + else: + rate915.append("-") + dev_num915.extend(repeat(num, len(devaddr_df))) + num += 1 + + # ABP case + # Single packet case + elif i.shape[0] == 1 and activation_df[0] != "OTAA": + l_index_915.extend(i.index) + dev_num915.append(num) + rate915.append("-") + num += 1 + + # 2 packets or more + elif i.shape[0] > 1 and activation_df[0] == "ABP": + dev_num915.extend(repeat(num, i.shape[0])) + #j = i.sort_values(by=["date", "time"]) # sort by date and time + l_index_915.extend(i.index) + fcnt_df = list(map(int, i["fcnt"])) + i_rate = [] + rate915.append("-") + + for k in range(len(fcnt_df)): # +1 to start from 2nd row + if 1 <= k < len(fcnt_df): # make sure it doesn't exceed the row index limit + if fcnt_df[k] != fcnt_df[k-1]: + i_rate.append(truncate3(((sec_df[k] - sec_df[k-1])/(fcnt_df[k] - fcnt_df[k-1])))) + else: + i_rate.append(truncate3((sec_df[k] - sec_df[k-1]))) + trunc_i_rate = [] + i_rate2 = list(map(np.round, i_rate)) + i_rate_set = list(dict.fromkeys(i_rate2)) # list of the set intervals to look up for assigning devnumber + # Remove rates that are within +-1 rate of each other + for m in range(len(i_rate_set)): + if m == 0: + trunc_i_rate.append(i_rate_set[m]) + elif m >= 1: + if (i_rate_set[m] > all([x+1 for x in trunc_i_rate])) or (i_rate_set[m] < all([x-1 for x in trunc_i_rate])): + trunc_i_rate.append(i_rate_set[m]) + + for element in i_rate: + for i_devnum in range(len(trunc_i_rate)): + if (trunc_i_rate[i_devnum] - 1) <= float(element) <= (trunc_i_rate[i_devnum] + 1): + break + else: + continue + rate915.append(element) + num += 1 + + # For neither OTAA and ABP activation and more than 1 packet + elif i.shape[0] > 1 and activation_df[0] == "-": + dev_num915.extend(repeat(num, i.shape[0])) + #j = i.sort_values(by=["date", "time"]) # sort by date and time + l_index_915.extend(i.index) + fcnt_df = list(i["fcnt"]) + if any(i["fcnt"] == "-"): + for k in range(len(fcnt_df)): # +1 to start from 2nd row + if 1 <= k < len(fcnt_df): # make sure it doesn't exceed the row index limit + rate915.append(truncate3((sec_df[k] - sec_df[k-1]))) + else: + fcnt_df = list(map(int, i["fcnt"])) + i_rate = [] + rate915.append("-") + + for k in range(len(fcnt_df)): # +1 to start from 2nd row + if 1 <= k < len(fcnt_df): # make sure it doesn't exceed the row index limit + if fcnt_df[k] != fcnt_df[k-1]: + i_rate.append(truncate3(((sec_df[k] - sec_df[k-1])/(fcnt_df[k] - fcnt_df[k-1])))) + else: + i_rate.append(truncate3((sec_df[k] - sec_df[k-1]))) + trunc_i_rate = [] + i_rate2 = list(map(np.round, i_rate)) + i_rate_set = list(dict.fromkeys(i_rate2)) # list of the set intervals to look up for assigning devnumber + # Remove rates that are within +-1 rate of each other + for m in range(len(i_rate_set)): + if m == 0: + trunc_i_rate.append(i_rate_set[m]) + elif m >= 1: + if (i_rate_set[m] > all([x+1 for x in trunc_i_rate])) or (i_rate_set[m] < all([x-1 for x in trunc_i_rate])): + trunc_i_rate.append(i_rate_set[m]) + + for element in i_rate: + for i_devnum in range(len(trunc_i_rate)): + if (trunc_i_rate[i_devnum] - 1) <= float(element) <= (trunc_i_rate[i_devnum] + 1): + break + else: + continue + rate915.append(element) + num += 1 + +l_index_923 = [] +if len(l_devaddr923) > 0: + for i in l_devaddr923: + sec_df = list(map(float, i["sec_diff"])) + sec_df = [x - sec_df[0] for x in sec_df] + sec_df = list(map(truncate, sec_df)) + + devaddr_df = list(i["DevEUI or DevAddr"]) + activation_df = list(i["activation"]) + + # OTAA case + if activation_df[0] == "OTAA": + l_index_923.extend(i.index) + for j in range(len(devaddr_df)): + if j >= 1: + rate923.append(truncate3(float(sec_df[j]) - float(sec_df[j-1]))) + else: + rate923.append("-") + dev_num923.extend(repeat(num, len(devaddr_df))) + num += 1 + + # ABP case + # Single packet case + elif i.shape[0] == 1 and activation_df[0] != "OTAA": + l_index_923.extend(i.index) + dev_num923.append(num) + rate923.append("-") + num += 1 + + # 2 packets or more + elif i.shape[0] > 1 and activation_df[0] == "ABP": + dev_num923.extend(repeat(num, i.shape[0])) + #j = i.sort_values(by=["date", "time"]) # sort by date and time + l_index_923.extend(i.index) + fcnt_df = list(map(int, i["fcnt"])) + i_rate = [] + rate923.append("-") + + for k in range(len(fcnt_df)): # +1 to start from 2nd row + if 1 <= k < len(fcnt_df): # make sure it doesn't exceed the row index limit + if fcnt_df[k] != fcnt_df[k-1]: + i_rate.append(truncate3(((sec_df[k] - sec_df[k-1])/(fcnt_df[k] - fcnt_df[k-1])))) + else: + i_rate.append(truncate3((sec_df[k] - sec_df[k-1]))) + trunc_i_rate = [] + i_rate2 = list(map(np.round, i_rate)) + i_rate_set = list(dict.fromkeys(i_rate2)) # list of the set intervals to look up for assigning devnumber + # Remove rates that are within +-1 rate of each other + for m in range(len(i_rate_set)): + if m == 0: + trunc_i_rate.append(i_rate_set[m]) + elif m >= 1: + if (i_rate_set[m] > all([x+1 for x in trunc_i_rate])) or (i_rate_set[m] < all([x-1 for x in trunc_i_rate])): + trunc_i_rate.append(i_rate_set[m]) + + for element in i_rate: + for i_devnum in range(len(trunc_i_rate)): + if (trunc_i_rate[i_devnum] - 1) <= float(element) <= (trunc_i_rate[i_devnum] + 1): + break + else: + continue + rate923.append(element) + num += 1 + + # For neither OTAA and ABP activation and more than 1 packet + elif i.shape[0] > 1 and activation_df[0] == "-": + dev_num923.extend(repeat(num, i.shape[0])) + #j = i.sort_values(by=["date", "time"]) # sort by date and time + l_index_923.extend(i.index) + fcnt_df = list(i["fcnt"]) + if any(i["fcnt"] == "-"): + for k in range(len(fcnt_df)): # +1 to start from 2nd row + if 1 <= k < len(fcnt_df): # make sure it doesn't exceed the row index limit + rate923.append(truncate3((sec_df[k] - sec_df[k-1]))) + else: + fcnt_df = list(map(int, i["fcnt"])) + i_rate = [] + rate923.append("-") + + for k in range(len(fcnt_df)): # +1 to start from 2nd row + if 1 <= k < len(fcnt_df): # make sure it doesn't exceed the row index limit + if fcnt_df[k] != fcnt_df[k-1]: + i_rate.append(truncate3(((sec_df[k] - sec_df[k-1])/(fcnt_df[k] - fcnt_df[k-1])))) + else: + i_rate.append(truncate3((sec_df[k] - sec_df[k-1]))) + trunc_i_rate = [] + i_rate2 = list(map(np.round, i_rate)) + i_rate_set = list(dict.fromkeys(i_rate2)) # list of the set intervals to look up for assigning devnumber + # Remove rates that are within +-1 rate of each other + for m in range(len(i_rate_set)): + if m == 0: + trunc_i_rate.append(i_rate_set[m]) + elif m >= 1: + if (i_rate_set[m] > all([x+1 for x in trunc_i_rate])) or (i_rate_set[m] < all([x-1 for x in trunc_i_rate])): + trunc_i_rate.append(i_rate_set[m]) + + for element in i_rate: + for i_devnum in range(len(trunc_i_rate)): + if (trunc_i_rate[i_devnum] - 1) <= float(element) <= (trunc_i_rate[i_devnum] + 1): + break + else: + continue + rate923.append(element) + num += 1 + + + +### Sort the derived device number, interval, rate by their corresponding index (to match it with original dataframe) +print("9. Sorting the device numbers by index...") +def sortFirst(val): + return val[0] + +all_l_indx = [] +rate = [] +dev_num = [] + +if len(l_devaddr915) > 0: + all_list915 = list(zip(l_index_915, rate915, dev_num915)) + all_list915.sort(key = sortFirst) + + for i in all_list915: + all_l_indx.append(i[0]) + rate.append(i[1]) + dev_num.append(i[2]) + +elif len(l_devaddr) > 0: + all_list923 = list(zip(l_index_923, rate923, dev_num923)) + all_list923.sort(key = sortFirst) + + for i in all_list923: + all_l_indx.append(i[0]) + rate.append(i[1]) + dev_num.append(i[2]) + +all_list = list(zip(all_l_indx, rate, dev_num)) +all_list.sort(key = sortFirst) + + +### Adding the corrected ordered tx_interval and dev_number to the dataframe +print("10. Adding the transmission interval column...") +l_rate = [] +l_dev_num = [] +for i in all_list: + if i[1] != "-": + l_rate.append(float(i[1])) + l_dev_num.append(int(i[2])) + else: + l_rate.append(i[1]) + l_dev_num.append(int(i[2])) + +filtered["tx_interval"] = l_rate +filtered["dev_number"] = l_dev_num + + + +### Save as csv +print("11. Saving as csv...") +filtered.to_csv(pd.datetime.today().strftime('%y%m%d_%H%M%S_') + "LoRadar_data.csv", index=False) +print("Data saved!")