From be0340cedc6d1ebe9abdfb204d05dda9b3efebaf Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:40:45 -0400 Subject: [PATCH 1/9] Add pipeline subpackage --- src/pyfaradaycup/__init__.py | 3 +++ src/pyfaradaycup/pipeline/__init__.py | 1 + 2 files changed, 4 insertions(+) create mode 100644 src/pyfaradaycup/pipeline/__init__.py diff --git a/src/pyfaradaycup/__init__.py b/src/pyfaradaycup/__init__.py index 188e0be..6a6e0dc 100644 --- a/src/pyfaradaycup/__init__.py +++ b/src/pyfaradaycup/__init__.py @@ -1,3 +1,6 @@ +from . import pipeline + + def hello() -> str: """Check that docstrings are tested. diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py new file mode 100644 index 0000000..9c00f97 --- /dev/null +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -0,0 +1 @@ +from ccsds_reader_pipeline import From 09d62b327fb189bcda575956b2451df80ab173f7 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:41:46 -0400 Subject: [PATCH 2/9] Add ccsds_reader_pipeline.py --- .../pipeline/ccsds_reader_pipeline.py | 647 ++++++++++++++++++ 1 file changed, 647 insertions(+) create mode 100644 src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py new file mode 100644 index 0000000..bb3833f --- /dev/null +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -0,0 +1,647 @@ +''' +# $URL: file:///psp/psp_swp_spc_code_repository/trunk/ccsds_reader_pipeline.py $ +# $LastChangedRevision: 103 $ +# $LastChangedDate: 2020-08-13 08:42:52 -0400 (Thu, 13 Aug 2020) $ +# $LastChangedBy: acase $ +''' + +import re, sys, struct, copy, os, datetime +import dateutil.parser +import numpy as np +import binascii +import time +#import Tkinter +#import tkFileDialog + +######################################### +def read_stdin(ptp=False, verbose=False): + """Parse binary stream on stdin""" + pass + +######################################### +def file2bytestr(path='', verbose=False, gzip=False): + try: + if gzip: + import gzip + with gzip.open(path, 'rb') as f: + bytestr = f.read() + return(bytestr) + else: + with open(path, 'rb') as f: + bytestr = f.read() + return(bytestr) + + except: + print('***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting') + print(sys.exc_info()) + import pdb; pdb.set_trace() + sys.exit() + + + +######################################### +def choose_file(path='', ptp=False, verbose=False): + # make sure file exists + try: + open(path, 'r').close() + except: + print('***ERROR*** File can not be read...will give option to choose file') + path='' + + # pop up a dialog to choose a file if path=='' + # path = 'C:\\Users\\comra_000\\SWEAP\\SPC\\FEU\\Testing\\20150228_UCB_SPC_FEU_LVPS_PTP_data\\PTP_data.dat' + if path=='': + print('***ERROR*** Must define a file path') + # root = Tkinter.Tk() + # root.withdraw() + # path = tkFileDialog.askopenfilename() + + return(path) + + + +######################################### +def wrapper_status(path='', verbose=False, gzip=False, spconly=False): + + #get a filename if not specified + path = choose_file(path) + + #convert file to a hex string + bytestr = file2bytestr(path, gzip=gzip) + + #define the apids that are ok + wrapper_apids = range(0x348,0x351) + if spconly: + ok_apids = [0x351,0x352,0x353,0x354,0x35e,0x35f] + else: + ok_apids = range(0x351,0x3a0,1) + + #create a dictionary that we can store data in + data = {'wrap_met':[], 'wrap_apid':[], 'data_met':[], 'data_apid':[], 'wrap_seq':[], 'data_seq':[]} + + #Define a pattern that will match a SWEM wrapper header and an SPC instrument header + pattern = struct.pack('1B',0x0B) + pattern += b'[' + for wrap_ap in wrapper_apids: #allowable wrapper apids + pattern += struct.pack('1B',wrap_ap & 255) + pattern += b']' + pattern += b'.'*10 + pattern += struct.pack('1B',0x0B) + pattern += b'[' + for inst_ap in ok_apids: #allowable SPC instrument apids + pattern += struct.pack('1B',inst_ap & 255) + pattern += b']' + + #Find all occurrences of the beginning of a packet + pkt_inds = np.array([(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)]) + try: + pkt_starts = pkt_inds[:,0] + except: + return(data) + + npackets = len(pkt_starts) + + + #Loop through each packet beginning and decommutate it + for i_pointer, pointer in enumerate(pkt_starts): + wrap_cchead = parse_ccsds_head(bytestr[pointer:pointer+10]) + data_cchead = parse_ccsds_head(bytestr[pointer+12:pointer+22]) + data['wrap_met'].append(wrap_cchead['CCSDS_MET'] ) + data['wrap_apid'].append(wrap_cchead['CCSDS_ApID'] ) + data['wrap_seq'].append(wrap_cchead['CCSDS_SeqCnt'] ) + data['data_met'].append(data_cchead['CCSDS_MET'] ) + data['data_apid'].append(data_cchead['CCSDS_ApID'] ) + data['data_seq'].append(data_cchead['CCSDS_SeqCnt'] ) + + return(data) + +######################################### +def read_file(path='', verbose=False, gzip=False): + """Read a CCSDS File and return data structure""" + + #get a filename if not specified + path = choose_file(path) + + #convert file to a hex string + bytestr = file2bytestr(path, gzip=gzip) + + #define the apids that are ok + wrapper_apids = range(0x348,0x351) + ok_apids = [0x351,0x352,0x353,0x354,0x35e,0x35f] + + #create a dictionary that we can store data in + data = {} + + #store the format for each apid in a dictionary + apidformat = {} + for apid in ok_apids: + apidformat[apid] = get_layout(apid, verbose=verbose) + if apidformat[apid]: + data[apid]={} + for name in apidformat[apid].names: + data[apid][name]=[] + + #create a list of two dictionaries that can keep track of + #the count of good packets found and bad packets found + goodcnt = {} + for thisap in data.keys(): + goodcnt[thisap]=0 + errcnt = {} + pktcnt = [goodcnt,errcnt] + + #Define a pattern that will match a SWEM wrapper header and an SPC instrument header + #0x348 through 0x350 is a SWEM wrapper apid, 0x351,0x352,0x353,0x354,0x35e,0x35f are SPC APIDs + pattern = struct.pack('1B',0x0B) + pattern += b'[' + for wrap_ap in wrapper_apids: #allowable wrapper apids + pattern += struct.pack('1B',wrap_ap & 255) + pattern += b']' + pattern += b'.'*10 + pattern += struct.pack('1B',0x0B) + pattern += b'[' + for inst_ap in ok_apids: #allowable SPC instrument apids + pattern += struct.pack('1B',inst_ap & 255) + pattern += b']' + + #Find all occurrences of the beginning of a packet + pkt_inds = np.array([(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)]) + try: + pkt_starts = pkt_inds[:,0] + except: + return(data) + + npackets = len(pkt_starts) + + #Some variables so we can display progress + updatetime = 0. + starttime = time.time() + + #Loop through each packet beginning and decommutate it + for i_pointer, pointer in enumerate(pkt_starts): + foo = read_bytestr(bytestr, pointer+12, data, apidformat, pktcnt, verbose=verbose) + + #Update status + nowtime = time.time() + if (nowtime-updatetime) >0.5: + sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.'.format( ( np.double(i_pointer) ) / npackets * 100. , nowtime-starttime)) + updatetime = nowtime + + #write out a summary of how things went + nowtime = time.time() + sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.\n\n'.format( 100., nowtime-starttime)) + sys.stdout.write('Packet Summary\n') + for thisapid in pktcnt[0].keys(): + sys.stdout.write('\tAPID {:}: found {:7.0f} packets\n'.format(hex(thisapid), pktcnt[0][thisapid] ) ) + sys.stdout.write('\n') + + + return(data) + + +######################################### +def read_file_sc(path='', verbose=False, ptp=False, gzip=False): + """Read a CCSDS File and return data structure""" + + #get a filename if not specified + path = choose_file(path) + + #convert file to a hex string + bytestr = file2bytestr(path, gzip=gzip) + + #We'll need to find which apid dictionary to use, + #based on which version of FSW was running + #Those versions (and respective dates) are listed in the L1 APID257 file + #That file is created via psp_sc_hsk_257_l052l1.py + #Corresponding SC_HK files that we will read in are in ./sc_hk_def/ + with open('/psp/data/sc_hsk/L1/APID257_combined.txt') as f: lines = f.readlines() + vers_dt = np.array([dateutil.parser.isoparse(line.split(',')[0]) for line in lines]) + versions = np.array([line.split(',')[1].strip() for line in lines]) + + #And we have to hardwire how to relate a particular version number to a SC_HK.blk filename + #This will have to be manually updated every time they update FSW + sc_hk_filenames = {'05.01.01':'SPP.SC.HK.05.01.01_G01.blk', + '05.04.00':'SPP.SC.HK.05.04.00_G04.blk', + '05.05.01':'SPP.SC.HK.05.05.01_G02.blk', + '05.06.00':'SPP.SC.HK.05.06.02_G06.blk' + } + + #get the first packet header in the file and see what the date/time is + #and thus which SC_HK.blk file to use + #we'll assume the first bytes in the file are a header + try: + if ptp: + cchead = parse_ccsds_head(bytestr[17:]) + else: + cchead = parse_ccsds_head(bytestr) + if ( (cchead['CCSDS_Version']!=0) | + (cchead['CCSDS_PacketType']!=0) | + (cchead['CCSDS_SecHdrFlag']!=1) ): + raise ValueError('CCSDS header values not as expected') + file_dt = datetime.datetime(2010,1,1)+datetime.timedelta(seconds=cchead['CCSDS_MET']) + try: + good_time = np.where(vers_dt>8, 2048+inst_ap & 0x00ff) + pattern += b'|' + pattern = pattern[:-1] #get rid of that last "|" + pattern += b')' + + offset_bytes = 15 #since we searched before 2 bytes into the PTP header, we need to offset the rest of the PTP header + else: + pattern = b'(' + for inst_ap in ok_apids: #allowable SPC instrument apids + pattern += struct.pack('2B',(2048+inst_ap & 0xff00)>>8, 2048+inst_ap & 0x00ff) + pattern += b'..' + if inst_ap==0x256: + #because the length shown in SPP.SC.HK.XX.YY.ZZ_GWW.blk doesn't correspond to packet length + #we just hard-code the length + #As of 2020/06/08 there were only two different possible sizes of 0x256 packets 0x098d and 0x0a91 + pattern += b'(\x09\x8d|\x0a\x91)' + else: + pattern += struct.pack('2B',(lengths[inst_ap] & 0xff00)>>8, lengths[inst_ap] & 0x00ff) + pattern += b'|' + pattern = pattern[:-1] #get rid of that last "|" + pattern += b')' + + offset_bytes = 0 #we searched for beginning of CCSDS packets, so no offset necessary + #import pdb; pdb.set_trace() + + #Find all occurrences of the beginning of a packet + pkt_inds = np.array([(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)]) + try: + pkt_starts = pkt_inds[:,0] + except: + return(data) + npackets = len(pkt_starts) + + #Some variables so we can display progress + updatetime = 0. + starttime = time.time() + + #Loop through each packet beginning and decommutate it + for i_pointer, pointer in enumerate(pkt_starts): + foo = read_bytestr(bytestr, pointer+offset_bytes, data, apidformat, pktcnt, verbose=verbose) + + #Update status + nowtime = time.time() + if (nowtime-updatetime) >0.5: + sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.'.format( ( np.double(i_pointer) ) / npackets * 100. , nowtime-starttime)) + updatetime = nowtime + + #write out a summary of how things went + nowtime = time.time() + sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.\n\n'.format( 100., nowtime-starttime)) + sys.stdout.write('Packet Summary\n') + for thisapid in pktcnt[0].keys(): + sys.stdout.write('\tAPID {:}: found {:7.0f} packets\n'.format(hex(thisapid), pktcnt[0][thisapid] ) ) + sys.stdout.write('\n') + + return(data) + +######################################### +def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): + """Take a hex string and find packets""" + + #Parse the CCSDS header + try: + ccsds_head = parse_ccsds_head(bytestr[pointer:pointer+10]) + except ValueError: + if verbose: print('Full CCSDS Header Not Present') + return() + apid = ccsds_head['CCSDS_ApID'] + pkt_len = ccsds_head['CCSDS_PacketLen'] + + #Verify that the CCSDS header is valid + if ccsds_head['CCSDS_Version']!=0: + if verbose: print('CCSDS Version is invalid') + return() + + if ccsds_head['CCSDS_PacketType']!=0: + if verbose: print('CCSDS Type is invalid') + return() + + if ccsds_head['CCSDS_SecHdrFlag']!=1: + if verbose: print('CCSDS Secondary Header flag is invalid') + return() + + #Make sure the full packet is here + if pointer+pkt_len+7 > len(bytestr): + if verbose: print('Full CCSDS packet not available at end of bytestr') + return() + + #This packet only (no PTP header and no wrapper header (if they existed)) + thispkt = bytestr[pointer:pointer+pkt_len+7] + + #make sure we know how to decom this packet + if apid in apidformat.keys(): + #count this as a good packet + pktcnt[0][apid]+=1 + + #parse the packet and add decommed values to data variable + parse_pkt(thispkt, data, apidformat, apid, ccsds_head) #could send this off to a parallel task? Might try that if too slow this way + + else: + if apid in pktcnt[1].keys(): + pktcnt[1][apid]+=1 + else: + pktcnt[1][apid]=1 + + return() + + #we shouldn't make it here + import pdb; pdb.set_trace() + + +######################################### +def parse_ccsds_head(bytestr, verbose=False): + bytearr = struct.unpack('B'*len(bytestr), bytestr) + + exp_length = 10 + if len(bytearr) < exp_length: + raise ValueError('CCSDS header is not as long as expected') + + head = {} + head['CCSDS_Version'] = bytearr[0] >> 5 + head['CCSDS_PacketType'] = (bytearr[0] & 0b00010000) >> 4 + head['CCSDS_SecHdrFlag'] = (bytearr[0] & 0b00001000) >> 3 + head['CCSDS_ApID'] = 256*(bytearr[0] & 0b00000111) + bytearr[1] + head['CCSDS_GroupFlags'] = bytearr[2] >> 6 + head['CCSDS_SeqCnt'] = 256*(bytearr[2] & 0b00111111) + bytearr[3] + head['CCSDS_PacketLen'] = 256*bytearr[4] + bytearr[5] + head['CCSDS_MET'] = 2**24*bytearr[6] + 2**16*bytearr[7] + 2**8*bytearr[8] + bytearr[9] + + #return the dictionary + return(head) + +######################################### +def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): + '''Parse one CCSDS packet''' + + #The format for this APIDs packet list + form = apidformat[apid] + thisdat = data[apid] + + #Convert to a bit string + bytearr = struct.unpack('B'*len(bytestr), bytestr) + str_bin = ''.join([bin(i)[2:].zfill(8) for i in bytearr]) + + + #For SWEAP packets, we just have each mnemonic listed and each bit length + #So we have to step through them in order + #Take care of the variables in sw_data (the repeating bit of the packet) separately + pointer = 0 + if hasattr(form, 'sw_data_vars'): + sw_data_vars_len = len(form.sw_data_vars) + else: + sw_data_vars_len = 0 + + """ + #This *might* be a faster way to parse values? + #This will work for the non sw_data_vars variables, + #but I haven't written anything for the sw_data_vars yet + + for i_bit, bit in enumerate(form.bits[0:len(form.bits)-sw_data_vars_len]): + + bytes = bytearr[form.bytestart[i_bit]:form.byteend[i_bit]+1] #the bytes that contain the value for this mnemonic + valint = sum([bytes[len(bytes)-1-i]<> (7-form.bitend[i_bit]) + + #store in our data variable + thisname = form.names[i_bit] + thisdat[thisname].append(thisval) + """ + + #SC packets are defined in a different format than SWEAP packets + #Each mnemonic has a start byte, start bit, and length + #Loop through each name + if apid in [0x081,0x262,0x07B,0x254,0x257,0x256]: + for i_name, thisname in enumerate(form.names): + startbit = 8*form.startbyte[i_name]+(7-form.startbit[i_name]) + endbit = startbit + form.bits[i_name] + thisbin = str_bin[startbit:endbit] + try: + thisval = int(thisbin,2) + except: + #print(sys.exc_info()) + thisval = -999 + thisdat[thisname].append(thisval) + return + + + + #If the full packet isn't here, then don't bother parsing + if len(bytearr)*8. < sum(form.bits): + print('short packet: {:}'.format(hex(apid))) + return + + for i_bit, bit in enumerate(form.bits[0:len(form.bits)-sw_data_vars_len]): + thisbin = str_bin[pointer:pointer+bit] + try: + thisval = int(thisbin,2) + except: + import pdb; pdb.set_trace() + thisval = -999 + thisname = form.names[i_bit] + + #store in our data variable + thisdat[thisname].append(thisval) + + #advance the pointer + pointer+=bit + + #Read in the portion of the packet that repeats over and over (the data) + if hasattr(form, 'sw_data_vars'): + + #A dictionary to store the lists for this packet + #Which will get appended to the lists from previous packets + newdat = {} + for key in form.sw_data_vars: newdat[key]=[] + + n_vars = len(form.sw_data_vars) + total_sw_data_length = np.sum(form.bits[-n_vars:]) + + while (pointer+total_sw_data_length)<=len(str_bin): + for i in range(n_vars): + thisbin = str_bin[pointer:pointer+form.bits[-n_vars+i]] + try: + thisval = int(thisbin, 2) + except ValueError: + import pdb; pdb.set_trace() + thisval = -999 + + thisname = form.sw_data_vars[i] + newdat[thisname].append(thisval) + pointer+=form.bits[-n_vars+i] + + for key in form.sw_data_vars: + thisdat[key].append(newdat[key]) + + +######################################### +class apid_obj(): + def __init__(self): + self.names = [] + self.bits = [] + self.bytestart = [] + self.bitstart = [] + self.byteend = [] + self.bitend = [] + self.data = {} + self.startbyte = [] + self.startbit = [] + + +######################################### +def get_layout(apid, verbose=False): + try: + file = open('sweap_tlm.blk') + except: + if verbose: print("***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py") + try: + thisdir = os.path.realpath(__file__) + thisdir = '\\'.join(thisdir.split('\\')[0:-1]) + file = open(thisdir+'\\sweap_tlm.blk') + except: + print(sys.exc_info()) + import pdb; pdb.set_trace() + lines = file.readlines() + for i, line in enumerate(lines): + + if line[0:8]=='APID_{:}'.format(hex(apid)[2:].zfill(3)).upper(): + if verbose: print('APID {:} Format Found'.format(hex(apid)[2:]).upper()) + thisapid = apid_obj() + thisapid.apid=apid + line='' #so that the while loop will start out ok + while line[0:4]!='APID': + i+=1 + line = lines[i] + try: + if line.strip()[0] not in ['(','{','}',')']: + pieces = re.split(',|;', line.strip()) + thisapid.names.append(pieces[0].strip()) + thisapid.bits.append(int(pieces[3].strip())) + thisapid.data[pieces[0].strip()] = [] + if hasattr(thisapid, 'sw_data_vars'): + thisapid.sw_data_vars.append(thisapid.names[-1]) + elif ((line.strip()[0:9]=='( SW_DATA') | (line.strip()[0:12]=='( SW_SPC_SCI')): + thisapid.sw_data_vars = [] + except IndexError: + break + except: + print(sys.exc_info()) + import pdb; pdb.set_trace() + + + + start = np.array([0] + [sum(thisapid.bits[0:i]) for i in range(1,len(thisapid.bits))]) + length = np.array(thisapid.bits)-1 + thisapid.bytestart = np.floor(start/8.).astype(int) + thisapid.bitstart = start-8*thisapid.bytestart.astype(int) + endbits = start+length + thisapid.byteend = np.floor(endbits/8.).astype(int) + thisapid.bitend = endbits-8*thisapid.byteend.astype(int) + + return(thisapid) + + #if we didn't find that APID + print('***ERROR*** [ccsds_reader_pipeline] Did not find APID {:}'.format(hex(apid)[2:]).upper()) + return(None) + +######################################### +def get_layout_sc(apid, verbose=False, filename=''): + try: + file = open(filename) + print('using sc_hk file: {:}'.format(filename)) + except: + print('could not open SC HK BLK file') + print(sys.exc_info()) + import pdb; pdb.set_trace() + + lines = file.readlines() + for i, line in enumerate(lines): + if line[0:11]=='SC_HK_0x{:}'.format(hex(apid)[2:].zfill(3).upper()): + if verbose: print('APID {:} Format Found'.format(hex(apid)[2:]).upper()) + thisapid = apid_obj() + thisapid.apid=apid + + line='' + while line[0:4]!='SC_H': + i+=1 + line = lines[i].strip() + if line[0:8]=='( Block[': + length = int(line.split('[')[1].split(']')[0]) + try: + if line[0] not in ['(','{','}',')']: + pieces = re.split(',|;', line) + #for some reason packet definitions with many bytes look like "mnemonic[32], x, y, z (for 32 byte long packet), rather than having the bit length actually show 8*32 bits + if pieces[0][-1]==']': + bitlength = 8*int(pieces[0].split('[')[1].split(']')[0]) + pieces[3] = str(bitlength) + pieces[0] = pieces[0].split('[')[0] + thisapid.names.append(pieces[0].strip()) + thisapid.startbyte.append(int(pieces[1].strip())) + thisapid.startbit.append(int(pieces[2].strip())) + thisapid.bits.append(int(pieces[3].strip())) + except IndexError: + break + except: + print(sys.exc_info()) + import pdb; pdb.set_trace() + return(thisapid, length) + #if we didn't find that APID + print('***ERROR*** [ccsds_reader_pipeline] Did not find APID {:}'.format(hex(apid)[2:]).upper()) + return(None) + +if __name__=="__main__": read_file(ptp=False, verbose=True) From aa89acfbf2879273d4f98bc5254f66ba8a0c10ee Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:42:05 -0400 Subject: [PATCH 3/9] Remove import --- src/pyfaradaycup/pipeline/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py index 9c00f97..e69de29 100644 --- a/src/pyfaradaycup/pipeline/__init__.py +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -1 +0,0 @@ -from ccsds_reader_pipeline import From f15718d1f56d4aa1631e2c65d25177c77903a95e Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:46:25 -0400 Subject: [PATCH 4/9] Apply fixes from ruff format and ruff check --- .../pipeline/ccsds_reader_pipeline.py | 1287 +++++++++-------- 1 file changed, 691 insertions(+), 596 deletions(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index bb3833f..28e2d4b 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -1,452 +1,524 @@ -''' +""" # $URL: file:///psp/psp_swp_spc_code_repository/trunk/ccsds_reader_pipeline.py $ # $LastChangedRevision: 103 $ # $LastChangedDate: 2020-08-13 08:42:52 -0400 (Thu, 13 Aug 2020) $ # $LastChangedBy: acase $ -''' +""" + +import datetime +import os +import re +import struct +import sys +import time -import re, sys, struct, copy, os, datetime import dateutil.parser import numpy as np -import binascii -import time -#import Tkinter -#import tkFileDialog + +# import Tkinter +# import tkFileDialog + ######################################### def read_stdin(ptp=False, verbose=False): - """Parse binary stream on stdin""" - pass - + """Parse binary stream on stdin""" + + ######################################### -def file2bytestr(path='', verbose=False, gzip=False): - try: - if gzip: - import gzip - with gzip.open(path, 'rb') as f: - bytestr = f.read() - return(bytestr) - else: - with open(path, 'rb') as f: - bytestr = f.read() - return(bytestr) - - except: - print('***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting') - print(sys.exc_info()) - import pdb; pdb.set_trace() - sys.exit() - +def file2bytestr(path="", verbose=False, gzip=False): + try: + if gzip: + import gzip + + with gzip.open(path, "rb") as f: + bytestr = f.read() + return bytestr + with open(path, "rb") as f: + bytestr = f.read() + return bytestr + + except: + print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") + print(sys.exc_info()) + import pdb + + pdb.set_trace() + sys.exit() ######################################### -def choose_file(path='', ptp=False, verbose=False): - # make sure file exists - try: - open(path, 'r').close() - except: - print('***ERROR*** File can not be read...will give option to choose file') - path='' - - # pop up a dialog to choose a file if path=='' - # path = 'C:\\Users\\comra_000\\SWEAP\\SPC\\FEU\\Testing\\20150228_UCB_SPC_FEU_LVPS_PTP_data\\PTP_data.dat' - if path=='': - print('***ERROR*** Must define a file path') - # root = Tkinter.Tk() - # root.withdraw() - # path = tkFileDialog.askopenfilename() - - return(path) - +def choose_file(path="", ptp=False, verbose=False): + # make sure file exists + try: + open(path).close() + except: + print("***ERROR*** File can not be read...will give option to choose file") + path = "" + + # pop up a dialog to choose a file if path=='' + # path = 'C:\\Users\\comra_000\\SWEAP\\SPC\\FEU\\Testing\\20150228_UCB_SPC_FEU_LVPS_PTP_data\\PTP_data.dat' + if path == "": + print("***ERROR*** Must define a file path") + # root = Tkinter.Tk() + # root.withdraw() + # path = tkFileDialog.askopenfilename() + + return path + - ######################################### -def wrapper_status(path='', verbose=False, gzip=False, spconly=False): - - #get a filename if not specified - path = choose_file(path) - - #convert file to a hex string - bytestr = file2bytestr(path, gzip=gzip) - - #define the apids that are ok - wrapper_apids = range(0x348,0x351) - if spconly: - ok_apids = [0x351,0x352,0x353,0x354,0x35e,0x35f] - else: - ok_apids = range(0x351,0x3a0,1) - - #create a dictionary that we can store data in - data = {'wrap_met':[], 'wrap_apid':[], 'data_met':[], 'data_apid':[], 'wrap_seq':[], 'data_seq':[]} - - #Define a pattern that will match a SWEM wrapper header and an SPC instrument header - pattern = struct.pack('1B',0x0B) - pattern += b'[' - for wrap_ap in wrapper_apids: #allowable wrapper apids - pattern += struct.pack('1B',wrap_ap & 255) - pattern += b']' - pattern += b'.'*10 - pattern += struct.pack('1B',0x0B) - pattern += b'[' - for inst_ap in ok_apids: #allowable SPC instrument apids - pattern += struct.pack('1B',inst_ap & 255) - pattern += b']' - - #Find all occurrences of the beginning of a packet - pkt_inds = np.array([(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)]) - try: - pkt_starts = pkt_inds[:,0] - except: - return(data) - - npackets = len(pkt_starts) - - - #Loop through each packet beginning and decommutate it - for i_pointer, pointer in enumerate(pkt_starts): - wrap_cchead = parse_ccsds_head(bytestr[pointer:pointer+10]) - data_cchead = parse_ccsds_head(bytestr[pointer+12:pointer+22]) - data['wrap_met'].append(wrap_cchead['CCSDS_MET'] ) - data['wrap_apid'].append(wrap_cchead['CCSDS_ApID'] ) - data['wrap_seq'].append(wrap_cchead['CCSDS_SeqCnt'] ) - data['data_met'].append(data_cchead['CCSDS_MET'] ) - data['data_apid'].append(data_cchead['CCSDS_ApID'] ) - data['data_seq'].append(data_cchead['CCSDS_SeqCnt'] ) - - return(data) - +def wrapper_status(path="", verbose=False, gzip=False, spconly=False): + + # get a filename if not specified + path = choose_file(path) + + # convert file to a hex string + bytestr = file2bytestr(path, gzip=gzip) + + # define the apids that are ok + wrapper_apids = range(0x348, 0x351) + if spconly: + ok_apids = [0x351, 0x352, 0x353, 0x354, 0x35E, 0x35F] + else: + ok_apids = range(0x351, 0x3A0, 1) + + # create a dictionary that we can store data in + data = { + "wrap_met": [], + "wrap_apid": [], + "data_met": [], + "data_apid": [], + "wrap_seq": [], + "data_seq": [], + } + + # Define a pattern that will match a SWEM wrapper header and an SPC instrument header + pattern = struct.pack("1B", 0x0B) + pattern += b"[" + for wrap_ap in wrapper_apids: # allowable wrapper apids + pattern += struct.pack("1B", wrap_ap & 255) + pattern += b"]" + pattern += b"." * 10 + pattern += struct.pack("1B", 0x0B) + pattern += b"[" + for inst_ap in ok_apids: # allowable SPC instrument apids + pattern += struct.pack("1B", inst_ap & 255) + pattern += b"]" + + # Find all occurrences of the beginning of a packet + pkt_inds = np.array( + [(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)] + ) + try: + pkt_starts = pkt_inds[:, 0] + except: + return data + + npackets = len(pkt_starts) + + # Loop through each packet beginning and decommutate it + for i_pointer, pointer in enumerate(pkt_starts): + wrap_cchead = parse_ccsds_head(bytestr[pointer : pointer + 10]) + data_cchead = parse_ccsds_head(bytestr[pointer + 12 : pointer + 22]) + data["wrap_met"].append(wrap_cchead["CCSDS_MET"]) + data["wrap_apid"].append(wrap_cchead["CCSDS_ApID"]) + data["wrap_seq"].append(wrap_cchead["CCSDS_SeqCnt"]) + data["data_met"].append(data_cchead["CCSDS_MET"]) + data["data_apid"].append(data_cchead["CCSDS_ApID"]) + data["data_seq"].append(data_cchead["CCSDS_SeqCnt"]) + + return data + + ######################################### -def read_file(path='', verbose=False, gzip=False): - """Read a CCSDS File and return data structure""" - - #get a filename if not specified - path = choose_file(path) - - #convert file to a hex string - bytestr = file2bytestr(path, gzip=gzip) - - #define the apids that are ok - wrapper_apids = range(0x348,0x351) - ok_apids = [0x351,0x352,0x353,0x354,0x35e,0x35f] - - #create a dictionary that we can store data in - data = {} - - #store the format for each apid in a dictionary - apidformat = {} - for apid in ok_apids: - apidformat[apid] = get_layout(apid, verbose=verbose) - if apidformat[apid]: - data[apid]={} - for name in apidformat[apid].names: - data[apid][name]=[] - - #create a list of two dictionaries that can keep track of - #the count of good packets found and bad packets found - goodcnt = {} - for thisap in data.keys(): - goodcnt[thisap]=0 - errcnt = {} - pktcnt = [goodcnt,errcnt] - - #Define a pattern that will match a SWEM wrapper header and an SPC instrument header - #0x348 through 0x350 is a SWEM wrapper apid, 0x351,0x352,0x353,0x354,0x35e,0x35f are SPC APIDs - pattern = struct.pack('1B',0x0B) - pattern += b'[' - for wrap_ap in wrapper_apids: #allowable wrapper apids - pattern += struct.pack('1B',wrap_ap & 255) - pattern += b']' - pattern += b'.'*10 - pattern += struct.pack('1B',0x0B) - pattern += b'[' - for inst_ap in ok_apids: #allowable SPC instrument apids - pattern += struct.pack('1B',inst_ap & 255) - pattern += b']' - - #Find all occurrences of the beginning of a packet - pkt_inds = np.array([(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)]) - try: - pkt_starts = pkt_inds[:,0] - except: - return(data) - - npackets = len(pkt_starts) - - #Some variables so we can display progress - updatetime = 0. - starttime = time.time() - - #Loop through each packet beginning and decommutate it - for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr(bytestr, pointer+12, data, apidformat, pktcnt, verbose=verbose) - - #Update status - nowtime = time.time() - if (nowtime-updatetime) >0.5: - sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.'.format( ( np.double(i_pointer) ) / npackets * 100. , nowtime-starttime)) - updatetime = nowtime - - #write out a summary of how things went - nowtime = time.time() - sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.\n\n'.format( 100., nowtime-starttime)) - sys.stdout.write('Packet Summary\n') - for thisapid in pktcnt[0].keys(): - sys.stdout.write('\tAPID {:}: found {:7.0f} packets\n'.format(hex(thisapid), pktcnt[0][thisapid] ) ) - sys.stdout.write('\n') +def read_file(path="", verbose=False, gzip=False): + """Read a CCSDS File and return data structure""" + # get a filename if not specified + path = choose_file(path) + + # convert file to a hex string + bytestr = file2bytestr(path, gzip=gzip) + + # define the apids that are ok + wrapper_apids = range(0x348, 0x351) + ok_apids = [0x351, 0x352, 0x353, 0x354, 0x35E, 0x35F] + + # create a dictionary that we can store data in + data = {} + + # store the format for each apid in a dictionary + apidformat = {} + for apid in ok_apids: + apidformat[apid] = get_layout(apid, verbose=verbose) + if apidformat[apid]: + data[apid] = {} + for name in apidformat[apid].names: + data[apid][name] = [] + + # create a list of two dictionaries that can keep track of + # the count of good packets found and bad packets found + goodcnt = {} + for thisap in data: + goodcnt[thisap] = 0 + errcnt = {} + pktcnt = [goodcnt, errcnt] + + # Define a pattern that will match a SWEM wrapper header and an SPC instrument header + # 0x348 through 0x350 is a SWEM wrapper apid, 0x351,0x352,0x353,0x354,0x35e,0x35f are SPC APIDs + pattern = struct.pack("1B", 0x0B) + pattern += b"[" + for wrap_ap in wrapper_apids: # allowable wrapper apids + pattern += struct.pack("1B", wrap_ap & 255) + pattern += b"]" + pattern += b"." * 10 + pattern += struct.pack("1B", 0x0B) + pattern += b"[" + for inst_ap in ok_apids: # allowable SPC instrument apids + pattern += struct.pack("1B", inst_ap & 255) + pattern += b"]" + + # Find all occurrences of the beginning of a packet + pkt_inds = np.array( + [(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)] + ) + try: + pkt_starts = pkt_inds[:, 0] + except: + return data + + npackets = len(pkt_starts) + + # Some variables so we can display progress + updatetime = 0.0 + starttime = time.time() + + # Loop through each packet beginning and decommutate it + for i_pointer, pointer in enumerate(pkt_starts): + foo = read_bytestr( + bytestr, pointer + 12, data, apidformat, pktcnt, verbose=verbose + ) + + # Update status + nowtime = time.time() + if (nowtime - updatetime) > 0.5: + sys.stdout.write( + "\b" * 40 + + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." + ) + updatetime = nowtime + + # write out a summary of how things went + nowtime = time.time() + sys.stdout.write( + "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" + ) + sys.stdout.write("Packet Summary\n") + for thisapid in pktcnt[0].keys(): + sys.stdout.write( + f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" + ) + sys.stdout.write("\n") + + return data + - - return(data) - - ######################################### -def read_file_sc(path='', verbose=False, ptp=False, gzip=False): - """Read a CCSDS File and return data structure""" - - #get a filename if not specified - path = choose_file(path) - - #convert file to a hex string - bytestr = file2bytestr(path, gzip=gzip) - - #We'll need to find which apid dictionary to use, - #based on which version of FSW was running - #Those versions (and respective dates) are listed in the L1 APID257 file - #That file is created via psp_sc_hsk_257_l052l1.py - #Corresponding SC_HK files that we will read in are in ./sc_hk_def/ - with open('/psp/data/sc_hsk/L1/APID257_combined.txt') as f: lines = f.readlines() - vers_dt = np.array([dateutil.parser.isoparse(line.split(',')[0]) for line in lines]) - versions = np.array([line.split(',')[1].strip() for line in lines]) - - #And we have to hardwire how to relate a particular version number to a SC_HK.blk filename - #This will have to be manually updated every time they update FSW - sc_hk_filenames = {'05.01.01':'SPP.SC.HK.05.01.01_G01.blk', - '05.04.00':'SPP.SC.HK.05.04.00_G04.blk', - '05.05.01':'SPP.SC.HK.05.05.01_G02.blk', - '05.06.00':'SPP.SC.HK.05.06.02_G06.blk' - } - - #get the first packet header in the file and see what the date/time is - #and thus which SC_HK.blk file to use - #we'll assume the first bytes in the file are a header - try: - if ptp: - cchead = parse_ccsds_head(bytestr[17:]) - else: - cchead = parse_ccsds_head(bytestr) - if ( (cchead['CCSDS_Version']!=0) | - (cchead['CCSDS_PacketType']!=0) | - (cchead['CCSDS_SecHdrFlag']!=1) ): - raise ValueError('CCSDS header values not as expected') - file_dt = datetime.datetime(2010,1,1)+datetime.timedelta(seconds=cchead['CCSDS_MET']) - try: - good_time = np.where(vers_dt> 8, 2048 + inst_ap & 0x00FF + ) + pattern += b"|" + pattern = pattern[:-1] # get rid of that last "|" + pattern += b")" + + offset_bytes = 15 # since we searched before 2 bytes into the PTP header, we need to offset the rest of the PTP header + else: + pattern = b"(" + for inst_ap in ok_apids: # allowable SPC instrument apids + pattern += struct.pack( + "2B", (2048 + inst_ap & 0xFF00) >> 8, 2048 + inst_ap & 0x00FF + ) + pattern += b".." + if inst_ap == 0x256: + # because the length shown in SPP.SC.HK.XX.YY.ZZ_GWW.blk doesn't correspond to packet length + # we just hard-code the length + # As of 2020/06/08 there were only two different possible sizes of 0x256 packets 0x098d and 0x0a91 + pattern += b"(\x09\x8d|\x0a\x91)" + else: + pattern += struct.pack( + "2B", (lengths[inst_ap] & 0xFF00) >> 8, lengths[inst_ap] & 0x00FF + ) + pattern += b"|" + pattern = pattern[:-1] # get rid of that last "|" + pattern += b")" + + offset_bytes = ( + 0 # we searched for beginning of CCSDS packets, so no offset necessary + ) + # import pdb; pdb.set_trace() + + # Find all occurrences of the beginning of a packet + pkt_inds = np.array( + [(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)] + ) + try: + pkt_starts = pkt_inds[:, 0] + except: + return data + npackets = len(pkt_starts) + + # Some variables so we can display progress + updatetime = 0.0 + starttime = time.time() + + # Loop through each packet beginning and decommutate it + for i_pointer, pointer in enumerate(pkt_starts): + foo = read_bytestr( + bytestr, pointer + offset_bytes, data, apidformat, pktcnt, verbose=verbose + ) + + # Update status + nowtime = time.time() + if (nowtime - updatetime) > 0.5: + sys.stdout.write( + "\b" * 40 + + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." + ) + updatetime = nowtime + + # write out a summary of how things went + nowtime = time.time() + sys.stdout.write( + "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" + ) + sys.stdout.write("Packet Summary\n") + for thisapid in pktcnt[0].keys(): + sys.stdout.write( + f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" + ) + sys.stdout.write("\n") + + return data + - - #define the apids that are ok - ok_apids =[0x081, 0x262, 0x07B, 0x254, 0x257, 0x256] - lengths = {} #store the length of each apid that we'll find in the sc_hk file - - #create a dictionary that we can store data in - data = {} - #store the format for each apid in a dictionary - apidformat = {} - for apid in ok_apids: - apidformat[apid], lengths[apid] = get_layout_sc(apid, verbose=verbose, filename=os.path.join('sc_hk_def',sc_hk_filename)) - if apidformat[apid]: - data[apid]={} - for name in apidformat[apid].names: - data[apid][name]=[] - - #since sc_hk file lists total length, but we search for length in apid header (total length - 7) - #Also, packets need to be multiples of 2 bytes, so actual packet length will be rounded up to nearest multiple of 2 - for key,val in lengths.items(): lengths[key] = 2*np.ceil((val)/2.).astype(int) - 7 - - #create a list of two dictionaries that can keep track of - #the count of good packets found and bad packets found - goodcnt = {} - for thisap in data.keys(): - goodcnt[thisap]=0 - errcnt = {} - pktcnt = [goodcnt,errcnt] - - if ptp: - pattern = struct.pack('3B',0x03,0x00,0xBB) #2,3,4,5,6th bytes (start from zero) of PTP header - pattern += b'.'*12 - pattern += b'(' - for inst_ap in ok_apids: #allowable SPC instrument apids - pattern += struct.pack('2B',(2048+inst_ap & 0xff00)>>8, 2048+inst_ap & 0x00ff) - pattern += b'|' - pattern = pattern[:-1] #get rid of that last "|" - pattern += b')' - - offset_bytes = 15 #since we searched before 2 bytes into the PTP header, we need to offset the rest of the PTP header - else: - pattern = b'(' - for inst_ap in ok_apids: #allowable SPC instrument apids - pattern += struct.pack('2B',(2048+inst_ap & 0xff00)>>8, 2048+inst_ap & 0x00ff) - pattern += b'..' - if inst_ap==0x256: - #because the length shown in SPP.SC.HK.XX.YY.ZZ_GWW.blk doesn't correspond to packet length - #we just hard-code the length - #As of 2020/06/08 there were only two different possible sizes of 0x256 packets 0x098d and 0x0a91 - pattern += b'(\x09\x8d|\x0a\x91)' - else: - pattern += struct.pack('2B',(lengths[inst_ap] & 0xff00)>>8, lengths[inst_ap] & 0x00ff) - pattern += b'|' - pattern = pattern[:-1] #get rid of that last "|" - pattern += b')' - - offset_bytes = 0 #we searched for beginning of CCSDS packets, so no offset necessary - #import pdb; pdb.set_trace() - - #Find all occurrences of the beginning of a packet - pkt_inds = np.array([(m.start(0), m.end(0)) for m in re.finditer(pattern, bytestr, re.DOTALL)]) - try: - pkt_starts = pkt_inds[:,0] - except: - return(data) - npackets = len(pkt_starts) - - #Some variables so we can display progress - updatetime = 0. - starttime = time.time() - - #Loop through each packet beginning and decommutate it - for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr(bytestr, pointer+offset_bytes, data, apidformat, pktcnt, verbose=verbose) - - #Update status - nowtime = time.time() - if (nowtime-updatetime) >0.5: - sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.'.format( ( np.double(i_pointer) ) / npackets * 100. , nowtime-starttime)) - updatetime = nowtime - - #write out a summary of how things went - nowtime = time.time() - sys.stdout.write('\b'*40 + '{:5.1f}% Complete. ET={:6.2f} sec.\n\n'.format( 100., nowtime-starttime)) - sys.stdout.write('Packet Summary\n') - for thisapid in pktcnt[0].keys(): - sys.stdout.write('\tAPID {:}: found {:7.0f} packets\n'.format(hex(thisapid), pktcnt[0][thisapid] ) ) - sys.stdout.write('\n') - - return(data) - ######################################### def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): - """Take a hex string and find packets""" - - #Parse the CCSDS header - try: - ccsds_head = parse_ccsds_head(bytestr[pointer:pointer+10]) - except ValueError: - if verbose: print('Full CCSDS Header Not Present') - return() - apid = ccsds_head['CCSDS_ApID'] - pkt_len = ccsds_head['CCSDS_PacketLen'] - - #Verify that the CCSDS header is valid - if ccsds_head['CCSDS_Version']!=0: - if verbose: print('CCSDS Version is invalid') - return() - - if ccsds_head['CCSDS_PacketType']!=0: - if verbose: print('CCSDS Type is invalid') - return() - - if ccsds_head['CCSDS_SecHdrFlag']!=1: - if verbose: print('CCSDS Secondary Header flag is invalid') - return() - - #Make sure the full packet is here - if pointer+pkt_len+7 > len(bytestr): - if verbose: print('Full CCSDS packet not available at end of bytestr') - return() + """Take a hex string and find packets""" + # Parse the CCSDS header + try: + ccsds_head = parse_ccsds_head(bytestr[pointer : pointer + 10]) + except ValueError: + if verbose: + print("Full CCSDS Header Not Present") + return () + apid = ccsds_head["CCSDS_ApID"] + pkt_len = ccsds_head["CCSDS_PacketLen"] - #This packet only (no PTP header and no wrapper header (if they existed)) - thispkt = bytestr[pointer:pointer+pkt_len+7] - - #make sure we know how to decom this packet - if apid in apidformat.keys(): - #count this as a good packet - pktcnt[0][apid]+=1 - - #parse the packet and add decommed values to data variable - parse_pkt(thispkt, data, apidformat, apid, ccsds_head) #could send this off to a parallel task? Might try that if too slow this way - - else: - if apid in pktcnt[1].keys(): - pktcnt[1][apid]+=1 - else: - pktcnt[1][apid]=1 - - return() - - #we shouldn't make it here - import pdb; pdb.set_trace() - - -######################################### + # Verify that the CCSDS header is valid + if ccsds_head["CCSDS_Version"] != 0: + if verbose: + print("CCSDS Version is invalid") + return () + + if ccsds_head["CCSDS_PacketType"] != 0: + if verbose: + print("CCSDS Type is invalid") + return () + + if ccsds_head["CCSDS_SecHdrFlag"] != 1: + if verbose: + print("CCSDS Secondary Header flag is invalid") + return () + + # Make sure the full packet is here + if pointer + pkt_len + 7 > len(bytestr): + if verbose: + print("Full CCSDS packet not available at end of bytestr") + return () + + # This packet only (no PTP header and no wrapper header (if they existed)) + thispkt = bytestr[pointer : pointer + pkt_len + 7] + + # make sure we know how to decom this packet + if apid in apidformat.keys(): + # count this as a good packet + pktcnt[0][apid] += 1 + + # parse the packet and add decommed values to data variable + parse_pkt( + thispkt, data, apidformat, apid, ccsds_head + ) # could send this off to a parallel task? Might try that if too slow this way + + elif apid in pktcnt[1].keys(): + pktcnt[1][apid] += 1 + else: + pktcnt[1][apid] = 1 + + return () + + # we shouldn't make it here + import pdb + + pdb.set_trace() + + +######################################### def parse_ccsds_head(bytestr, verbose=False): - bytearr = struct.unpack('B'*len(bytestr), bytestr) - - exp_length = 10 - if len(bytearr) < exp_length: - raise ValueError('CCSDS header is not as long as expected') - - head = {} - head['CCSDS_Version'] = bytearr[0] >> 5 - head['CCSDS_PacketType'] = (bytearr[0] & 0b00010000) >> 4 - head['CCSDS_SecHdrFlag'] = (bytearr[0] & 0b00001000) >> 3 - head['CCSDS_ApID'] = 256*(bytearr[0] & 0b00000111) + bytearr[1] - head['CCSDS_GroupFlags'] = bytearr[2] >> 6 - head['CCSDS_SeqCnt'] = 256*(bytearr[2] & 0b00111111) + bytearr[3] - head['CCSDS_PacketLen'] = 256*bytearr[4] + bytearr[5] - head['CCSDS_MET'] = 2**24*bytearr[6] + 2**16*bytearr[7] + 2**8*bytearr[8] + bytearr[9] - - #return the dictionary - return(head) + bytearr = struct.unpack("B" * len(bytestr), bytestr) + + exp_length = 10 + if len(bytearr) < exp_length: + raise ValueError("CCSDS header is not as long as expected") + + head = {} + head["CCSDS_Version"] = bytearr[0] >> 5 + head["CCSDS_PacketType"] = (bytearr[0] & 0b00010000) >> 4 + head["CCSDS_SecHdrFlag"] = (bytearr[0] & 0b00001000) >> 3 + head["CCSDS_ApID"] = 256 * (bytearr[0] & 0b00000111) + bytearr[1] + head["CCSDS_GroupFlags"] = bytearr[2] >> 6 + head["CCSDS_SeqCnt"] = 256 * (bytearr[2] & 0b00111111) + bytearr[3] + head["CCSDS_PacketLen"] = 256 * bytearr[4] + bytearr[5] + head["CCSDS_MET"] = ( + 2**24 * bytearr[6] + 2**16 * bytearr[7] + 2**8 * bytearr[8] + bytearr[9] + ) + + # return the dictionary + return head + ######################################### def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): - '''Parse one CCSDS packet''' - - #The format for this APIDs packet list - form = apidformat[apid] - thisdat = data[apid] - - #Convert to a bit string - bytearr = struct.unpack('B'*len(bytestr), bytestr) - str_bin = ''.join([bin(i)[2:].zfill(8) for i in bytearr]) + """Parse one CCSDS packet""" + # The format for this APIDs packet list + form = apidformat[apid] + thisdat = data[apid] - - #For SWEAP packets, we just have each mnemonic listed and each bit length - #So we have to step through them in order - #Take care of the variables in sw_data (the repeating bit of the packet) separately - pointer = 0 - if hasattr(form, 'sw_data_vars'): - sw_data_vars_len = len(form.sw_data_vars) - else: - sw_data_vars_len = 0 - - """ + # Convert to a bit string + bytearr = struct.unpack("B" * len(bytestr), bytestr) + str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) + + # For SWEAP packets, we just have each mnemonic listed and each bit length + # So we have to step through them in order + # Take care of the variables in sw_data (the repeating bit of the packet) separately + pointer = 0 + if hasattr(form, "sw_data_vars"): + sw_data_vars_len = len(form.sw_data_vars) + else: + sw_data_vars_len = 0 + + """ #This *might* be a faster way to parse values? #This will work for the non sw_data_vars variables, #but I haven't written anything for the sw_data_vars yet @@ -463,185 +535,208 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): thisdat[thisname].append(thisval) """ - #SC packets are defined in a different format than SWEAP packets - #Each mnemonic has a start byte, start bit, and length - #Loop through each name - if apid in [0x081,0x262,0x07B,0x254,0x257,0x256]: - for i_name, thisname in enumerate(form.names): - startbit = 8*form.startbyte[i_name]+(7-form.startbit[i_name]) - endbit = startbit + form.bits[i_name] - thisbin = str_bin[startbit:endbit] - try: - thisval = int(thisbin,2) - except: - #print(sys.exc_info()) - thisval = -999 - thisdat[thisname].append(thisval) - return + # SC packets are defined in a different format than SWEAP packets + # Each mnemonic has a start byte, start bit, and length + # Loop through each name + if apid in [0x081, 0x262, 0x07B, 0x254, 0x257, 0x256]: + for i_name, thisname in enumerate(form.names): + startbit = 8 * form.startbyte[i_name] + (7 - form.startbit[i_name]) + endbit = startbit + form.bits[i_name] + thisbin = str_bin[startbit:endbit] + try: + thisval = int(thisbin, 2) + except: + # print(sys.exc_info()) + thisval = -999 + thisdat[thisname].append(thisval) + return + + # If the full packet isn't here, then don't bother parsing + if len(bytearr) * 8.0 < sum(form.bits): + print(f"short packet: {hex(apid)}") + return + + for i_bit, bit in enumerate(form.bits[0 : len(form.bits) - sw_data_vars_len]): + thisbin = str_bin[pointer : pointer + bit] + try: + thisval = int(thisbin, 2) + except: + import pdb + + pdb.set_trace() + thisval = -999 + thisname = form.names[i_bit] + + # store in our data variable + thisdat[thisname].append(thisval) + + # advance the pointer + pointer += bit + + # Read in the portion of the packet that repeats over and over (the data) + if hasattr(form, "sw_data_vars"): + # A dictionary to store the lists for this packet + # Which will get appended to the lists from previous packets + newdat = {} + for key in form.sw_data_vars: + newdat[key] = [] + + n_vars = len(form.sw_data_vars) + total_sw_data_length = np.sum(form.bits[-n_vars:]) + + while (pointer + total_sw_data_length) <= len(str_bin): + for i in range(n_vars): + thisbin = str_bin[pointer : pointer + form.bits[-n_vars + i]] + try: + thisval = int(thisbin, 2) + except ValueError: + import pdb + + pdb.set_trace() + thisval = -999 + + thisname = form.sw_data_vars[i] + newdat[thisname].append(thisval) + pointer += form.bits[-n_vars + i] + + for key in form.sw_data_vars: + thisdat[key].append(newdat[key]) - - - #If the full packet isn't here, then don't bother parsing - if len(bytearr)*8. < sum(form.bits): - print('short packet: {:}'.format(hex(apid))) - return - - for i_bit, bit in enumerate(form.bits[0:len(form.bits)-sw_data_vars_len]): - thisbin = str_bin[pointer:pointer+bit] - try: - thisval = int(thisbin,2) - except: - import pdb; pdb.set_trace() - thisval = -999 - thisname = form.names[i_bit] - - #store in our data variable - thisdat[thisname].append(thisval) - - #advance the pointer - pointer+=bit - - #Read in the portion of the packet that repeats over and over (the data) - if hasattr(form, 'sw_data_vars'): - - #A dictionary to store the lists for this packet - #Which will get appended to the lists from previous packets - newdat = {} - for key in form.sw_data_vars: newdat[key]=[] - - n_vars = len(form.sw_data_vars) - total_sw_data_length = np.sum(form.bits[-n_vars:]) - - while (pointer+total_sw_data_length)<=len(str_bin): - for i in range(n_vars): - thisbin = str_bin[pointer:pointer+form.bits[-n_vars+i]] - try: - thisval = int(thisbin, 2) - except ValueError: - import pdb; pdb.set_trace() - thisval = -999 - - thisname = form.sw_data_vars[i] - newdat[thisname].append(thisval) - pointer+=form.bits[-n_vars+i] - - for key in form.sw_data_vars: - thisdat[key].append(newdat[key]) - ######################################### -class apid_obj(): - def __init__(self): - self.names = [] - self.bits = [] - self.bytestart = [] - self.bitstart = [] - self.byteend = [] - self.bitend = [] - self.data = {} - self.startbyte = [] - self.startbit = [] - - +class apid_obj: + def __init__(self): + self.names = [] + self.bits = [] + self.bytestart = [] + self.bitstart = [] + self.byteend = [] + self.bitend = [] + self.data = {} + self.startbyte = [] + self.startbit = [] + + ######################################### def get_layout(apid, verbose=False): - try: - file = open('sweap_tlm.blk') - except: - if verbose: print("***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py") - try: - thisdir = os.path.realpath(__file__) - thisdir = '\\'.join(thisdir.split('\\')[0:-1]) - file = open(thisdir+'\\sweap_tlm.blk') - except: - print(sys.exc_info()) - import pdb; pdb.set_trace() - lines = file.readlines() - for i, line in enumerate(lines): - - if line[0:8]=='APID_{:}'.format(hex(apid)[2:].zfill(3)).upper(): - if verbose: print('APID {:} Format Found'.format(hex(apid)[2:]).upper()) - thisapid = apid_obj() - thisapid.apid=apid - line='' #so that the while loop will start out ok - while line[0:4]!='APID': - i+=1 - line = lines[i] - try: - if line.strip()[0] not in ['(','{','}',')']: - pieces = re.split(',|;', line.strip()) - thisapid.names.append(pieces[0].strip()) - thisapid.bits.append(int(pieces[3].strip())) - thisapid.data[pieces[0].strip()] = [] - if hasattr(thisapid, 'sw_data_vars'): - thisapid.sw_data_vars.append(thisapid.names[-1]) - elif ((line.strip()[0:9]=='( SW_DATA') | (line.strip()[0:12]=='( SW_SPC_SCI')): - thisapid.sw_data_vars = [] - except IndexError: - break - except: - print(sys.exc_info()) - import pdb; pdb.set_trace() - - - - start = np.array([0] + [sum(thisapid.bits[0:i]) for i in range(1,len(thisapid.bits))]) - length = np.array(thisapid.bits)-1 - thisapid.bytestart = np.floor(start/8.).astype(int) - thisapid.bitstart = start-8*thisapid.bytestart.astype(int) - endbits = start+length - thisapid.byteend = np.floor(endbits/8.).astype(int) - thisapid.bitend = endbits-8*thisapid.byteend.astype(int) - - return(thisapid) - - #if we didn't find that APID - print('***ERROR*** [ccsds_reader_pipeline] Did not find APID {:}'.format(hex(apid)[2:]).upper()) - return(None) + try: + file = open("sweap_tlm.blk") + except: + if verbose: + print( + "***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py" + ) + try: + thisdir = os.path.realpath(__file__) + thisdir = "\\".join(thisdir.split("\\")[0:-1]) + file = open(thisdir + "\\sweap_tlm.blk") + except: + print(sys.exc_info()) + import pdb + + pdb.set_trace() + lines = file.readlines() + for i, line in enumerate(lines): + if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): + if verbose: + print(f"APID {hex(apid)[2:]} Format Found".upper()) + thisapid = apid_obj() + thisapid.apid = apid + line = "" # so that the while loop will start out ok + while line[0:4] != "APID": + i += 1 + line = lines[i] + try: + if line.strip()[0] not in ["(", "{", "}", ")"]: + pieces = re.split(",|;", line.strip()) + thisapid.names.append(pieces[0].strip()) + thisapid.bits.append(int(pieces[3].strip())) + thisapid.data[pieces[0].strip()] = [] + if hasattr(thisapid, "sw_data_vars"): + thisapid.sw_data_vars.append(thisapid.names[-1]) + elif (line.strip()[0:9] == "( SW_DATA") | ( + line.strip()[0:12] == "( SW_SPC_SCI" + ): + thisapid.sw_data_vars = [] + except IndexError: + break + except: + print(sys.exc_info()) + import pdb + + pdb.set_trace() + + start = np.array( + [0] + [sum(thisapid.bits[0:i]) for i in range(1, len(thisapid.bits))] + ) + length = np.array(thisapid.bits) - 1 + thisapid.bytestart = np.floor(start / 8.0).astype(int) + thisapid.bitstart = start - 8 * thisapid.bytestart.astype(int) + endbits = start + length + thisapid.byteend = np.floor(endbits / 8.0).astype(int) + thisapid.bitend = endbits - 8 * thisapid.byteend.astype(int) + + return thisapid + + # if we didn't find that APID + print( + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() + ) + return None + ######################################### -def get_layout_sc(apid, verbose=False, filename=''): - try: - file = open(filename) - print('using sc_hk file: {:}'.format(filename)) - except: - print('could not open SC HK BLK file') - print(sys.exc_info()) - import pdb; pdb.set_trace() - - lines = file.readlines() - for i, line in enumerate(lines): - if line[0:11]=='SC_HK_0x{:}'.format(hex(apid)[2:].zfill(3).upper()): - if verbose: print('APID {:} Format Found'.format(hex(apid)[2:]).upper()) - thisapid = apid_obj() - thisapid.apid=apid - - line='' - while line[0:4]!='SC_H': - i+=1 - line = lines[i].strip() - if line[0:8]=='( Block[': - length = int(line.split('[')[1].split(']')[0]) - try: - if line[0] not in ['(','{','}',')']: - pieces = re.split(',|;', line) - #for some reason packet definitions with many bytes look like "mnemonic[32], x, y, z (for 32 byte long packet), rather than having the bit length actually show 8*32 bits - if pieces[0][-1]==']': - bitlength = 8*int(pieces[0].split('[')[1].split(']')[0]) - pieces[3] = str(bitlength) - pieces[0] = pieces[0].split('[')[0] - thisapid.names.append(pieces[0].strip()) - thisapid.startbyte.append(int(pieces[1].strip())) - thisapid.startbit.append(int(pieces[2].strip())) - thisapid.bits.append(int(pieces[3].strip())) - except IndexError: - break - except: - print(sys.exc_info()) - import pdb; pdb.set_trace() - return(thisapid, length) - #if we didn't find that APID - print('***ERROR*** [ccsds_reader_pipeline] Did not find APID {:}'.format(hex(apid)[2:]).upper()) - return(None) - -if __name__=="__main__": read_file(ptp=False, verbose=True) +def get_layout_sc(apid, verbose=False, filename=""): + try: + file = open(filename) + print(f"using sc_hk file: {filename}") + except: + print("could not open SC HK BLK file") + print(sys.exc_info()) + import pdb + + pdb.set_trace() + + lines = file.readlines() + for i, line in enumerate(lines): + if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": + if verbose: + print(f"APID {hex(apid)[2:]} Format Found".upper()) + thisapid = apid_obj() + thisapid.apid = apid + + line = "" + while line[0:4] != "SC_H": + i += 1 + line = lines[i].strip() + if line[0:8] == "( Block[": + length = int(line.split("[")[1].split("]")[0]) + try: + if line[0] not in ["(", "{", "}", ")"]: + pieces = re.split(",|;", line) + # for some reason packet definitions with many bytes look like "mnemonic[32], x, y, z (for 32 byte long packet), rather than having the bit length actually show 8*32 bits + if pieces[0][-1] == "]": + bitlength = 8 * int(pieces[0].split("[")[1].split("]")[0]) + pieces[3] = str(bitlength) + pieces[0] = pieces[0].split("[")[0] + thisapid.names.append(pieces[0].strip()) + thisapid.startbyte.append(int(pieces[1].strip())) + thisapid.startbit.append(int(pieces[2].strip())) + thisapid.bits.append(int(pieces[3].strip())) + except IndexError: + break + except: + print(sys.exc_info()) + import pdb + + pdb.set_trace() + return (thisapid, length) + # if we didn't find that APID + print( + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() + ) + return None + + +if __name__ == "__main__": + read_file(ptp=False, verbose=True) From f4bdf21861bf49e46607ed919b016418a931c035 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:48:52 -0400 Subject: [PATCH 5/9] Add comments to ignore ruff check errors --- .../pipeline/ccsds_reader_pipeline.py | 228 +++++++++--------- 1 file changed, 114 insertions(+), 114 deletions(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index 28e2d4b..3f15962 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -3,7 +3,7 @@ # $LastChangedRevision: 103 $ # $LastChangedDate: 2020-08-13 08:42:52 -0400 (Thu, 13 Aug 2020) $ # $LastChangedBy: acase $ -""" +""" # noqa: D400 import datetime import os @@ -20,45 +20,45 @@ ######################################### -def read_stdin(ptp=False, verbose=False): - """Parse binary stream on stdin""" +def read_stdin(ptp=False, verbose=False): # noqa: ANN001, ANN201, FBT002 + """Parse binary stream on stdin""" # noqa: D400 ######################################### -def file2bytestr(path="", verbose=False, gzip=False): +def file2bytestr(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 try: if gzip: - import gzip + import gzip # noqa: PLC0415 with gzip.open(path, "rb") as f: bytestr = f.read() - return bytestr - with open(path, "rb") as f: + return bytestr # noqa: RET504 + with open(path, "rb") as f: # noqa: PTH123 bytestr = f.read() - return bytestr + return bytestr # noqa: RET504, TRY300 - except: - print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") - print(sys.exc_info()) - import pdb + except: # noqa: E722 + print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") # noqa: T201 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 sys.exit() ######################################### -def choose_file(path="", ptp=False, verbose=False): +def choose_file(path="", ptp=False, verbose=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 # make sure file exists try: - open(path).close() - except: - print("***ERROR*** File can not be read...will give option to choose file") + open(path).close() # noqa: PTH123 + except: # noqa: E722 + print("***ERROR*** File can not be read...will give option to choose file") # noqa: T201 path = "" # pop up a dialog to choose a file if path=='' # path = 'C:\\Users\\comra_000\\SWEAP\\SPC\\FEU\\Testing\\20150228_UCB_SPC_FEU_LVPS_PTP_data\\PTP_data.dat' if path == "": - print("***ERROR*** Must define a file path") + print("***ERROR*** Must define a file path") # noqa: T201 # root = Tkinter.Tk() # root.withdraw() # path = tkFileDialog.askopenfilename() @@ -67,7 +67,7 @@ def choose_file(path="", ptp=False, verbose=False): ######################################### -def wrapper_status(path="", verbose=False, gzip=False, spconly=False): +def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 # get a filename if not specified path = choose_file(path) @@ -77,7 +77,7 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # define the apids that are ok wrapper_apids = range(0x348, 0x351) - if spconly: + if spconly: # noqa: SIM108 ok_apids = [0x351, 0x352, 0x353, 0x354, 0x35E, 0x35F] else: ok_apids = range(0x351, 0x3A0, 1) @@ -111,13 +111,13 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): ) try: pkt_starts = pkt_inds[:, 0] - except: + except: # noqa: E722 return data - npackets = len(pkt_starts) + npackets = len(pkt_starts) # noqa: F841 # Loop through each packet beginning and decommutate it - for i_pointer, pointer in enumerate(pkt_starts): + for i_pointer, pointer in enumerate(pkt_starts): # noqa: B007 wrap_cchead = parse_ccsds_head(bytestr[pointer : pointer + 10]) data_cchead = parse_ccsds_head(bytestr[pointer + 12 : pointer + 22]) data["wrap_met"].append(wrap_cchead["CCSDS_MET"]) @@ -131,8 +131,8 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): ######################################### -def read_file(path="", verbose=False, gzip=False): - """Read a CCSDS File and return data structure""" +def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901, FBT002 + """Read a CCSDS File and return data structure""" # noqa: D400 # get a filename if not specified path = choose_file(path) @@ -183,7 +183,7 @@ def read_file(path="", verbose=False, gzip=False): ) try: pkt_starts = pkt_inds[:, 0] - except: + except: # noqa: E722 return data npackets = len(pkt_starts) @@ -194,13 +194,13 @@ def read_file(path="", verbose=False, gzip=False): # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr( + foo = read_bytestr( # noqa: F841 bytestr, pointer + 12, data, apidformat, pktcnt, verbose=verbose ) # Update status nowtime = time.time() - if (nowtime - updatetime) > 0.5: + if (nowtime - updatetime) > 0.5: # noqa: PLR2004 sys.stdout.write( "\b" * 40 + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." @@ -213,7 +213,7 @@ def read_file(path="", verbose=False, gzip=False): "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" ) sys.stdout.write("Packet Summary\n") - for thisapid in pktcnt[0].keys(): + for thisapid in pktcnt[0].keys(): # noqa: SIM118 sys.stdout.write( f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" ) @@ -223,8 +223,8 @@ def read_file(path="", verbose=False, gzip=False): ######################################### -def read_file_sc(path="", verbose=False, ptp=False, gzip=False): - """Read a CCSDS File and return data structure""" +def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0915 + """Read a CCSDS File and return data structure""" # noqa: D400 # get a filename if not specified path = choose_file(path) @@ -236,7 +236,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # Those versions (and respective dates) are listed in the L1 APID257 file # That file is created via psp_sc_hsk_257_l052l1.py # Corresponding SC_HK files that we will read in are in ./sc_hk_def/ - with open("/psp/data/sc_hsk/L1/APID257_combined.txt") as f: + with open("/psp/data/sc_hsk/L1/APID257_combined.txt") as f: # noqa: PTH123 lines = f.readlines() vers_dt = np.array([dateutil.parser.isoparse(line.split(",")[0]) for line in lines]) versions = np.array([line.split(",")[1].strip() for line in lines]) @@ -254,7 +254,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # and thus which SC_HK.blk file to use # we'll assume the first bytes in the file are a header try: - if ptp: + if ptp: # noqa: SIM108 cchead = parse_ccsds_head(bytestr[17:]) else: cchead = parse_ccsds_head(bytestr) @@ -263,8 +263,8 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): | (cchead["CCSDS_PacketType"] != 0) | (cchead["CCSDS_SecHdrFlag"] != 1) ): - raise ValueError("CCSDS header values not as expected") - file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( + raise ValueError("CCSDS header values not as expected") # noqa: EM101, TRY003 + file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( # noqa: DTZ001 seconds=cchead["CCSDS_MET"] ) try: @@ -272,10 +272,10 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: - print(sys.exc_info()) - print("Could not find which SC_HK file to use based on packet header") - print("Attempting to find correct date based on filename/path") + except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + print("Could not find which SC_HK file to use based on packet header") # noqa: T201 + print("Attempting to find correct date based on filename/path") # noqa: T201 try: match = re.search( os.path.sep @@ -285,7 +285,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): + os.path.sep, path, ).span() - file_dt = datetime.datetime( + file_dt = datetime.datetime( # noqa: DTZ001 int(path[match[0] + 1 : match[0] + 5]), 1, 1 ) + datetime.timedelta(days=int(path[match[0] + 6 : match[0] + 9]) - 1) try: @@ -293,8 +293,8 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: - print( + except: # noqa: E722 + print( # noqa: T201 "***WARNING*** Could not find date based on filename...using most recent" ) sc_hk_filename = sc_hk_filenames[-1] @@ -309,7 +309,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): apidformat = {} for apid in ok_apids: apidformat[apid], lengths[apid] = get_layout_sc( - apid, verbose=verbose, filename=os.path.join("sc_hk_def", sc_hk_filename) + apid, verbose=verbose, filename=os.path.join("sc_hk_def", sc_hk_filename) # noqa: PTH118 ) if apidformat[apid]: data[apid] = {} @@ -351,7 +351,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): "2B", (2048 + inst_ap & 0xFF00) >> 8, 2048 + inst_ap & 0x00FF ) pattern += b".." - if inst_ap == 0x256: + if inst_ap == 0x256: # noqa: PLR2004 # because the length shown in SPP.SC.HK.XX.YY.ZZ_GWW.blk doesn't correspond to packet length # we just hard-code the length # As of 2020/06/08 there were only two different possible sizes of 0x256 packets 0x098d and 0x0a91 @@ -375,7 +375,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): ) try: pkt_starts = pkt_inds[:, 0] - except: + except: # noqa: E722 return data npackets = len(pkt_starts) @@ -385,13 +385,13 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr( + foo = read_bytestr( # noqa: F841 bytestr, pointer + offset_bytes, data, apidformat, pktcnt, verbose=verbose ) # Update status nowtime = time.time() - if (nowtime - updatetime) > 0.5: + if (nowtime - updatetime) > 0.5: # noqa: PLR2004 sys.stdout.write( "\b" * 40 + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." @@ -404,7 +404,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" ) sys.stdout.write("Packet Summary\n") - for thisapid in pktcnt[0].keys(): + for thisapid in pktcnt[0].keys(): # noqa: SIM118 sys.stdout.write( f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" ) @@ -414,14 +414,14 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): ######################################### -def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): - """Take a hex string and find packets""" +def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0913, RET503 + """Take a hex string and find packets""" # noqa: D400 # Parse the CCSDS header try: ccsds_head = parse_ccsds_head(bytestr[pointer : pointer + 10]) except ValueError: if verbose: - print("Full CCSDS Header Not Present") + print("Full CCSDS Header Not Present") # noqa: T201 return () apid = ccsds_head["CCSDS_ApID"] pkt_len = ccsds_head["CCSDS_PacketLen"] @@ -429,30 +429,30 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # Verify that the CCSDS header is valid if ccsds_head["CCSDS_Version"] != 0: if verbose: - print("CCSDS Version is invalid") + print("CCSDS Version is invalid") # noqa: T201 return () if ccsds_head["CCSDS_PacketType"] != 0: if verbose: - print("CCSDS Type is invalid") + print("CCSDS Type is invalid") # noqa: T201 return () if ccsds_head["CCSDS_SecHdrFlag"] != 1: if verbose: - print("CCSDS Secondary Header flag is invalid") + print("CCSDS Secondary Header flag is invalid") # noqa: T201 return () # Make sure the full packet is here if pointer + pkt_len + 7 > len(bytestr): if verbose: - print("Full CCSDS packet not available at end of bytestr") + print("Full CCSDS packet not available at end of bytestr") # noqa: T201 return () # This packet only (no PTP header and no wrapper header (if they existed)) thispkt = bytestr[pointer : pointer + pkt_len + 7] # make sure we know how to decom this packet - if apid in apidformat.keys(): + if apid in apidformat.keys(): # noqa: SIM118 # count this as a good packet pktcnt[0][apid] += 1 @@ -461,7 +461,7 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): thispkt, data, apidformat, apid, ccsds_head ) # could send this off to a parallel task? Might try that if too slow this way - elif apid in pktcnt[1].keys(): + elif apid in pktcnt[1].keys(): # noqa: SIM118 pktcnt[1][apid] += 1 else: pktcnt[1][apid] = 1 @@ -469,18 +469,18 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): return () # we shouldn't make it here - import pdb + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 ######################################### -def parse_ccsds_head(bytestr, verbose=False): +def parse_ccsds_head(bytestr, verbose=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 bytearr = struct.unpack("B" * len(bytestr), bytestr) exp_length = 10 if len(bytearr) < exp_length: - raise ValueError("CCSDS header is not as long as expected") + raise ValueError("CCSDS header is not as long as expected") # noqa: EM101, TRY003 head = {} head["CCSDS_Version"] = bytearr[0] >> 5 @@ -499,21 +499,21 @@ def parse_ccsds_head(bytestr, verbose=False): ######################################### -def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): - """Parse one CCSDS packet""" +def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # noqa: ANN001, ANN201, ARG001, C901, FBT002, PLR0912, PLR0913 + """Parse one CCSDS packet""" # noqa: D400 # The format for this APIDs packet list form = apidformat[apid] thisdat = data[apid] # Convert to a bit string bytearr = struct.unpack("B" * len(bytestr), bytestr) - str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) + str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) # noqa: FURB116 # For SWEAP packets, we just have each mnemonic listed and each bit length # So we have to step through them in order # Take care of the variables in sw_data (the repeating bit of the packet) separately pointer = 0 - if hasattr(form, "sw_data_vars"): + if hasattr(form, "sw_data_vars"): # noqa: SIM108 sw_data_vars_len = len(form.sw_data_vars) else: sw_data_vars_len = 0 @@ -533,7 +533,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): #store in our data variable thisname = form.names[i_bit] thisdat[thisname].append(thisval) - """ + """ # noqa: W291, W293 # SC packets are defined in a different format than SWEAP packets # Each mnemonic has a start byte, start bit, and length @@ -545,7 +545,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): thisbin = str_bin[startbit:endbit] try: thisval = int(thisbin, 2) - except: + except: # noqa: E722 # print(sys.exc_info()) thisval = -999 thisdat[thisname].append(thisval) @@ -553,17 +553,17 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # If the full packet isn't here, then don't bother parsing if len(bytearr) * 8.0 < sum(form.bits): - print(f"short packet: {hex(apid)}") + print(f"short packet: {hex(apid)}") # noqa: T201 return for i_bit, bit in enumerate(form.bits[0 : len(form.bits) - sw_data_vars_len]): thisbin = str_bin[pointer : pointer + bit] try: thisval = int(thisbin, 2) - except: - import pdb + except: # noqa: E722 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 thisval = -999 thisname = form.names[i_bit] @@ -590,9 +590,9 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): try: thisval = int(thisbin, 2) except ValueError: - import pdb + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 thisval = -999 thisname = form.sw_data_vars[i] @@ -604,8 +604,8 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): ######################################### -class apid_obj: - def __init__(self): +class apid_obj: # noqa: D101, N801 + def __init__(self): # noqa: ANN204 self.names = [] self.bits = [] self.bytestart = [] @@ -618,34 +618,34 @@ def __init__(self): ######################################### -def get_layout(apid, verbose=False): +def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 try: - file = open("sweap_tlm.blk") - except: + file = open("sweap_tlm.blk") # noqa: PTH123, SIM115 + except: # noqa: E722 if verbose: - print( + print( # noqa: T201 "***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py" ) try: thisdir = os.path.realpath(__file__) thisdir = "\\".join(thisdir.split("\\")[0:-1]) - file = open(thisdir + "\\sweap_tlm.blk") - except: - print(sys.exc_info()) - import pdb + file = open(thisdir + "\\sweap_tlm.blk") # noqa: PTH123, SIM115 + except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 lines = file.readlines() for i, line in enumerate(lines): - if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): + if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): # noqa: FURB116 if verbose: - print(f"APID {hex(apid)[2:]} Format Found".upper()) + print(f"APID {hex(apid)[2:]} Format Found".upper()) # noqa: FURB116, T201 thisapid = apid_obj() thisapid.apid = apid - line = "" # so that the while loop will start out ok + line = "" # so that the while loop will start out ok # noqa: PLW2901 while line[0:4] != "APID": - i += 1 - line = lines[i] + i += 1 # noqa: PLW2901 + line = lines[i] # noqa: PLW2901 try: if line.strip()[0] not in ["(", "{", "}", ")"]: pieces = re.split(",|;", line.strip()) @@ -660,11 +660,11 @@ def get_layout(apid, verbose=False): thisapid.sw_data_vars = [] except IndexError: break - except: - print(sys.exc_info()) - import pdb + except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 start = np.array( [0] + [sum(thisapid.bits[0:i]) for i in range(1, len(thisapid.bits))] @@ -679,36 +679,36 @@ def get_layout(apid, verbose=False): return thisapid # if we didn't find that APID - print( - f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() + print( # noqa: T201 + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # noqa: FURB116 ) return None ######################################### -def get_layout_sc(apid, verbose=False, filename=""): +def get_layout_sc(apid, verbose=False, filename=""): # noqa: ANN001, ANN201, C901, D103, FBT002 try: - file = open(filename) - print(f"using sc_hk file: {filename}") - except: - print("could not open SC HK BLK file") - print(sys.exc_info()) - import pdb + file = open(filename) # noqa: PTH123, SIM115 + print(f"using sc_hk file: {filename}") # noqa: T201 + except: # noqa: E722 + print("could not open SC HK BLK file") # noqa: T201 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 lines = file.readlines() for i, line in enumerate(lines): - if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": + if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": # noqa: FURB116 if verbose: - print(f"APID {hex(apid)[2:]} Format Found".upper()) + print(f"APID {hex(apid)[2:]} Format Found".upper()) # noqa: FURB116, T201 thisapid = apid_obj() thisapid.apid = apid - line = "" + line = "" # noqa: PLW2901 while line[0:4] != "SC_H": - i += 1 - line = lines[i].strip() + i += 1 # noqa: PLW2901 + line = lines[i].strip() # noqa: PLW2901 if line[0:8] == "( Block[": length = int(line.split("[")[1].split("]")[0]) try: @@ -725,15 +725,15 @@ def get_layout_sc(apid, verbose=False, filename=""): thisapid.bits.append(int(pieces[3].strip())) except IndexError: break - except: - print(sys.exc_info()) - import pdb + except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 return (thisapid, length) # if we didn't find that APID - print( - f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() + print( # noqa: T201 + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # noqa: FURB116 ) return None From ac9012f7791215710ce6380eecbf7b12f57adb8d Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:49:26 -0400 Subject: [PATCH 6/9] Use absolute rather than relative import --- src/pyfaradaycup/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyfaradaycup/__init__.py b/src/pyfaradaycup/__init__.py index 6a6e0dc..a8b4374 100644 --- a/src/pyfaradaycup/__init__.py +++ b/src/pyfaradaycup/__init__.py @@ -1,4 +1,4 @@ -from . import pipeline +from pyfaradaycup import pipeline def hello() -> str: From 31df0f39b7154e8259ca9419ae945a074c23a9ea Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:52:02 -0400 Subject: [PATCH 7/9] Apply ruff fixes --- src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index 3f15962..a970e5d 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -285,9 +285,12 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 + os.path.sep, path, ).span() - file_dt = datetime.datetime( # noqa: DTZ001 - int(path[match[0] + 1 : match[0] + 5]), 1, 1 - ) + datetime.timedelta(days=int(path[match[0] + 6 : match[0] + 9]) - 1) + file_dt = ( + datetime.datetime( # noqa: DTZ001 + int(path[match[0] + 1 : match[0] + 5]), 1, 1 + ) + + datetime.timedelta(days=int(path[match[0] + 6 : match[0] + 9]) - 1) + ) try: good_time = np.where(vers_dt < file_dt)[0][-1] except IndexError: @@ -309,7 +312,9 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 apidformat = {} for apid in ok_apids: apidformat[apid], lengths[apid] = get_layout_sc( - apid, verbose=verbose, filename=os.path.join("sc_hk_def", sc_hk_filename) # noqa: PTH118 + apid, + verbose=verbose, + filename=os.path.join("sc_hk_def", sc_hk_filename), # noqa: PTH118 ) if apidformat[apid]: data[apid] = {} From 508e5fb616db4cd77b4f75d4e424d8415236a34e Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 14:56:51 -0400 Subject: [PATCH 8/9] Allow more positional arguments --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index be859cd..cebbbda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -274,6 +274,7 @@ lint.flake8-tidy-imports.banned-api."typing.Tuple".msg = "Deprecated alias. Chan lint.flake8-tidy-imports.banned-api."typing.Type".msg = "Deprecated alias. Change to type" lint.isort.known-first-party = [ "pyfaradaycup" ] lint.pydocstyle.convention = "numpy" +lint.pylint.max-positional-args = 6 [tool.pytest] # https://docs.pytest.org/en/stable/reference/customize.html#pyproject-toml addopts = [ From be5cd6db46d25e479d5978774a5f9e6ff1751be6 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Thu, 3 Sep 2026 16:51:07 -0400 Subject: [PATCH 9/9] Add ty:ignore comments --- .../pipeline/ccsds_reader_pipeline.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index a970e5d..38a8697 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -5,6 +5,21 @@ # $LastChangedBy: acase $ """ # noqa: D400 +__all__ = [ + "apid_obj", + "choose_file", + "file2bytestr", + "get_layout", + "get_layout_sc", + "parse_ccsds_head", + "parse_pkt", + "read_bytestr", + "read_file", + "read_file_sc", + "read_stdin", + "wrapper_status", +] + import datetime import os import re @@ -284,7 +299,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 + "[0-9][0-9][0-9]" + os.path.sep, path, - ).span() + ).span() # ty: ignore[unresolved-attribute] file_dt = ( datetime.datetime( # noqa: DTZ001 int(path[match[0] + 1 : match[0] + 5]), 1, 1 @@ -300,7 +315,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 print( # noqa: T201 "***WARNING*** Could not find date based on filename...using most recent" ) - sc_hk_filename = sc_hk_filenames[-1] + sc_hk_filename = sc_hk_filenames[-1] # ty: ignore[invalid-argument-type] # define the apids that are ok ok_apids = [0x081, 0x262, 0x07B, 0x254, 0x257, 0x256] @@ -525,11 +540,11 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no """ #This *might* be a faster way to parse values? - #This will work for the non sw_data_vars variables, + #This will work for the non sw_data_vars variables, #but I haven't written anything for the sw_data_vars yet - + for i_bit, bit in enumerate(form.bits[0:len(form.bits)-sw_data_vars_len]): - + bytes = bytearr[form.bytestart[i_bit]:form.byteend[i_bit]+1] #the bytes that contain the value for this mnemonic valint = sum([bytes[len(bytes)-1-i]<