From ed7678d74ac8b8e8e1eaad4ed62d5ecc110f6e2b Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:19:04 -0400 Subject: [PATCH 1/4] Add swp_spc_l02l1.py --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 971 +++++++++++++++++++++ 1 file changed, 971 insertions(+) create mode 100644 src/pyfaradaycup/pipeline/swp_spc_l02l1.py diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py new file mode 100644 index 0000000..3bacedc --- /dev/null +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -0,0 +1,971 @@ +""" +# $URL: file:///psp/psp_swp_spc_code_repository/trunk/swp_spc_l02l1.py $ +# $LastChangedRevision: 97 $ +# $LastChangedDate: 2020-08-04 09:20:42 -0400 (Tue, 04 Aug 2020) $ +# $LastChangedBy: acase $ +""" + +import numpy as np +import argparse, sys, os, datetime, math, glob + +try: + from spacepy import pycdf +except: + print(sys.exc_info()) + print("***ERRORR*** Could not import pycdf from spacepy") + print( + "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" + ) + sys.exit() +import ccsds_reader_pipeline as cc +import distutils.dir_util +import spiceypy + +# Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft +# into L0.5 or L1 CDF files + +# Requirements: Must have a reference to a skeleton file for the ApID that you wish to convert_one + +# Input: path to a binary L0 file + +# Output: saves a CDF file + +# Revision History +# 2020/02/03 - Fix timing bug in 0x351 that arose when switching to spiceypy. Add capability to produce 0x352 (time series) files. Will require update to 0x352 (time series) skeleton also. Update bug in naming of L1 files coming from gzip (spacecraft files, probably). Remove versioning from skeleton files (since SVN is taking care of that). +# 2020/01/29 - Use spiceypy to adjust time of each measurement to SCET; remove unused command-line argument options +# Change method by which newest skeleton files are found; various other cleanups +# 2019/08/27 - Calling new ccsds_reader that separates out s/c from instr. packets +# - Also, new ccsds_reader will use new packet finder (much faster), and does not scan through packets like before +# 2019/08/23 - Fixed bug that was reading in the oldest rather than newest skeleton file +# - Added capability to read 0x081 and 0x262 packets from s/c hsk files. Requires link to SC_HK.blk block definition file +# - Added revision history + + +def main( + l0file="", + l1dir="", + logdir="", + spacecraft=False, + ptp=False, + gzip=False, + apidreq=0, + overwrite=False, + verbose=False, +): + """Convert a single L0 file to L1""" + + # Try to create a filename for the new CDF that we're going to create + l0dirname = os.path.dirname(l0file) + l0basename = os.path.basename(l0file) + if l1dir == "": + l1dir = ( + l0dirname # use input L0 directory for L1 files, if nothing else specified + ) + + # Get a version of filename with no extension + l0file_noext = os.path.splitext(l0basename)[0] + if l0file_noext[-3:] == "ptp": + l0file_noext = os.path.splitext(l0file_noext)[0] + + # Open a log file to write to + nowdt = datetime.datetime.now() + if logdir == "": + logdir = l1dir # use L1 file output directory for log file, if nothing else specified + distutils.dir_util.mkpath( + logdir + ) # in case the directory doesn't exist, this will create it + logpath = os.path.join( + logdir, + "swp_spc_l02l1_{:04.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}.log".format( + nowdt.year, nowdt.month, nowdt.day, nowdt.hour, nowdt.minute, nowdt.second + ), + ) + try: + global logfile + logfile = open(logpath, "w") + except: + print("\n***ERROR*** Could not open log file!\n") + sys.exit(1) + + # Write some information to the log file + statusmsg("scriptname = swp_spc_l02l1.py", verbose=verbose) + statusmsg("timerun = " + nowdt.isoformat(), verbose=verbose) + statusmsg("l0file = " + l0file, verbose=verbose) + statusmsg("l1dir = " + l1dir, verbose=verbose) + statusmsg("spacecraft = " + repr(spacecraft), verbose=verbose) + statusmsg("ptp = " + repr(ptp), verbose=verbose) + statusmsg("gzip = " + repr(gzip), verbose=verbose) + statusmsg("apid = " + hex(apidreq), verbose=verbose) + statusmsg("overwrite = " + repr(overwrite), verbose=verbose) + + # Make sure the L0 file exists and is readable + try: + foo = open(l0file, "r") + foo.close() + statusmsg("L0 file exists and is readable") + except IOError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Input L0 file could not be read...exiting", + screen=True, + verbose=verbose, + ) + import pdb + + pdb.set_trace() + sys.exit() + + # Load in Leap Second Kernel + statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest leap second kernel...") + tls_path = get_newest_kernel(tls=True) + if not tls_path: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not find leap second kernel...exiting" + ) + sys.exit() + else: + try: + statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(tls_path)) + spiceypy.furnsh(tls_path) + except: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not furnsh leap second kernel...exiting" + ) + sys.exit() + + # Load in S/C Clock Kernel + statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest S/C clock kernel...") + sclk_path = get_newest_kernel(sclk=True) + if not sclk_path: + statusmsg("***ERROR*** [swp_spc_l02l1.py] Could not find SCLK kernel...exiting") + sys.exit() + else: + try: + statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(sclk_path)) + spiceypy.furnsh(sclk_path) + except: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not furnsh SCLK kernel...exiting" + ) + sys.exit() + + # Read in the L0 file into a python SPC data structure + if spacecraft: + statusmsg("Event = Starting reading file: spacecraft") + l0data = cc.read_file_sc(path=l0file, ptp=ptp, verbose=verbose, gzip=gzip) + else: + statusmsg("Event = Starting reading file: non-spacecraft (instrument)") + l0data = cc.read_file(path=l0file, verbose=verbose, gzip=gzip) + statusmsg("Event = Finished reading file") + + # Loop through the APIDs that we got + for apid in l0data.keys(): + statusmsg("Event = Beginning APID: {:}".format(hex(apid))) + + if apid == 0x07B: + statusmsg( + "***WARNING*** [swp_spc_l02l1] APID 0x07B CDFs not yet implemented", + screen=True, + verbose=verbose, + ) + continue + + # Make sure we need to do this apid + if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: + statusmsg("No packets found for this apid.") + continue # skip this apid if there were no packets received + if (apidreq != 0) & (apidreq != apid): + statusmsg("This apid not requested by user") + continue # skip this apid if user only wanted one apid and this isn't it + + # Filename for the L1 file we're about to write for this apid + l1path = os.path.join( + l1dir, + l0file_noext + + "_APID{:}_L1.cdf".format(str(hex(apid)[2:].zfill(3)).upper()), + ) + statusmsg("About to write: " + l1path) + + # Make sure the skeleton file exists and is readable + try: + skeleton_filename = get_newest_skeleton(apid) + foo = open(skeleton_filename, "r") + foo.close() + statusmsg("Skeleton to be used: " + skeleton_filename) + except IOError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Skeleton file could not be read...moving to next apid", + screen=True, + verbose=verbose, + ) + statusmsg("Tried to use skeleton file: " + skeleton_filename) + continue + except TypeError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={:} could not be found...moving to next apid".format( + hex(apid) + ), + screen=True, + verbose=verbose, + ) + continue + + # See if the CDF file already exists + try: + # try to open and close it + statusmsg("Using L1 path: " + l1path, screen=True, verbose=verbose) + foo = open(l1path) + foo.close() + + # if we get here, this file already exists; so delete it, if desired + statusmsg( + "***INFO*** [swp_spc_l02l1] L1 CDF file ({:}) already exists".format( + l1path + ), + screen=True, + verbose=verbose, + ) + if overwrite: + statusmsg( + "***INFO*** [swp_spc_l02l1] Overwriting existing L1 CDF", + screen=True, + verbose=verbose, + ) + os.remove(l1path) + else: + statusmsg( + "***ERROR*** [swp_spc_l02l1] L1 CDF already exists, and overwrite (-o option) was not requested...exiting.", + screen=True, + verbose=verbose, + ) + raise (SystemExit) + except SystemExit: + sys.exit() + except IOError: + pass # Apparently the file did not exist already + except: + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + statusmsg( + "\n***ERROR*** [swp_spc_l02l1] Could not check existence/delete L1 CDF file path. Exiting...\n", + screen=True, + verbose=verbose, + ) + sys.exit() + + # Create a new CDF file from the provided skeleton + try: + cdf = pycdf.CDF(l1path, skeleton_filename) + except "CDFError": + statusmsg( + "\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={:})...continuing to next APID\n).".format( + apid + ), + screen=True, + verbose=verbose, + ) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) + continue + + # Run a different procedure to put data into CDF file depending on APID + cdfproc = { + 0x081: cdf35e_35f, + 0x1DE: cdf35e_35f, + 0x254: cdf35e_35f, + 0x256: cdf35e_35f, + 0x257: cdf35e_35f, + 0x262: cdf35e_35f, + 0x351: cdf351_353_354, + 0x352: cdf352, + 0x353: cdf351_353_354, + 0x354: cdf351_353_354, + 0x35E: cdf35e_35f, + 0x35F: cdf35e_35f, + } + try: + cdfproc[apid](cdf, l0data[apid], verbose=verbose) + except: + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + statusmsg( + "***WARNING*** [swp_spc_l02l1] CDF not processed for APID={:}".format( + hex(apid) + ), + screen=True, + verbose=verbose, + ) + continue + + # Close the CDF + # import pdb; pdb.set_trace() + cdf.close() + + statusmsg( + "***INFO*** [swp_spc_l02l1] Script complete.", screen=True, verbose=verbose + ) + + # Close the log file + logfile.close() + + +##################################################### +## +##################################################### +def cdf35e_35f(cdf, dat, verbose=False): + """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" + + # Calculate MET from the variables in the L0 data + # MET of each NYS + if "CCSDS_MET" in dat.keys(): + scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPC_SUBSEC"]) + elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["FSW_HK_HK_INST_TPSH_MET_SEC"], + dat["FSW_HK_HK_INST_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["PDU_PRIO94_TPSH_MET_SEC"], + dat["PDU_PRIO94_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["HK_HIGH_TPSH_MET_SEC"], dat["HK_HIGH_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["HK_FSWL_TPSH_MET_SEC"], dat["HK_FSWL_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["HK_LOW_TPSH_MET_SEC"], dat["HK_LOW_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["RIU_DERIVED_TPSH_MET_SEC"], + dat["RIU_DERIVED_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + + else: + statusmsg("Failed: could not create Epoch variable") + return + + # Fill in values for each variable + keys = cdf.keys() + dat["Epoch"] = scet + + for key in keys: + try: + cdf[key] = dat[key] # create variable and insert data + except KeyError: + if key not in dat.keys(): + cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] + except: + import pdb + + pdb.set_trace() + statusmsg( + "Failed : Key:{:} failed insert into CDF".format(key), + screen=True, + verbose=verbose, + ) + statusmsg(sys.exc_info()) + + +##################################################### +## +##################################################### +def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): + """Fill up a CDF with SCI, ALL, or RSS data.""" + # Take data sorted by NYS, and produce one long variable with all data + + # APID of this packet + apid = dat["CCSDS_ApID"][0] + + # Each different packet will require a different variable to calculate + # The number of measurements each NYS + if apid == 0x351: + length_var = "A1S" + elif apid == 0x353: + length_var = "ASIN" + elif apid == 0x354: + length_var = "ARSS" + + # Calculate MET from the variables in the L0 data + # MET of each NYS + scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) + + # MET of each measurement (to be filled in in the future) + scet_exp = [] + + # Same keys as original data dictionary, but will hold one variable per key + # instead of one for every NYS for every key + dat_exp = {} + for key in dat.keys(): + dat_exp[key] = [] + + # Create the 'Epoch' variable in our data array + dat_exp["Epoch"] = [] + + # Loop through each NYS + itst_warned = False + for i in range(len(scet)): + this_scet = scet[i] + + # Number of ticks each measurement takes (1024 ticks per NYS) + ticks_per_meas = dat["SW_SPC_INTTIME"][i] + dat["SW_SPC_SERVTIME"][i] + + # Make sure ST and IT are allowed values + if (math.log(ticks_per_meas, 2)) % 1 != 0: + # the SPC FPGA will default to IT=6, ST=2 (the power-on defaults) if a non-integer power of 2 IT+ST is requested + ticks_per_meas = 8 + + if not itst_warned: + statusmsg( + "***WARNING*** [swp_spc_l02l1] The reported IT+ST is not an even power of 2. Using IT=6,ST=2...", + screen=True, + verbose=verbose, + ) + itst_warned = True + + # throw out packets if the IT and ST are different than both previous and next packets + # this is almost surely an improperly identified packet that probably + # isn't even an SPC packet, but got decommutated as such + try: + if ( + (dat["SW_SPC_INTTIME"][i] != dat["SW_SPC_INTTIME"][i - 1]) + & (dat["SW_SPC_INTTIME"][i] != dat["SW_SPC_INTTIME"][i + 1]) + & (dat["SW_SPC_SERVTIME"][i] != dat["SW_SPC_SERVTIME"][i - 1]) + & (dat["SW_SPC_SERVTIME"][i] != dat["SW_SPC_SERVTIME"][i + 1]) + ): + statusmsg( + "***WARNING***IT+ST not 2^n, and not same as prev. and next values...so skipping this packet.", + screen=True, + verbose=verbose, + ) + continue + except IndexError: + statusmsg( + "***WARNING***IT+ST not 2^n, and not same as prev. and next values...so skipping this packet.", + screen=True, + verbose=verbose, + ) + continue + + # Time that each measurement took this NYS + tm_per_meas = (1.0 / 1171.875) * ticks_per_meas + + # Number of measurements this NYS + nmeas = len(dat[length_var][i]) + + # Expected number of measurements in a NYS + exp_nmeas = 1024.0 / 1171.875 / tm_per_meas + + # Calculate the time array for this NYS + add_time = np.arange(0, tm_per_meas * (nmeas - 0.1), tm_per_meas) + + # See if there were any times when we had retraces + win = np.array(dat["WINDOW"][i]) + rtpix = (np.where((win[1:] - win[:-1]) < 0)[0]) + 1 + + # Add on some time for each of the retraces + # We don't need to do this if the expected number of measurements is equal to the number of measurements we received + # This is because of the possibility that HV DAC tables were not loaded (probably only on the ground). + # In that case, the FPGA does not actually take the time to do a retrace since it does not have to slew to a new DAC value + # It doesn't matter that it is slewing to a new 'Window', since every window will have the same DAC value + # There is a slight bug here in that the 'final' NYS after a HALT is sent, will likely be a partial packet + # so we might get tricked on our logical check here for the final packet when we do not have DAC tables loaded + if nmeas != exp_nmeas: + for thisrtpix in rtpix: + add_time[thisrtpix:] += tm_per_meas + + # If we're in an AllGain packet, then the beginning of the packet might not be the beginning of the NYS (which is the time noted in the header) + if apid == 0x351: + pktnum = dat["SW_SPC_PKTNUM"][i] + # if pktnum==0: import pdb; pdb.set_trace() + if pktnum != 0: + if len(dat_exp["Epoch"]) == 0: + continue # if file started on pktnum other than zero, then we can't know precise timing for the first 1-3 packets + + add_time += (dat_exp["Epoch"][-1] / 1e9 - this_scet / 1e9) + tm_per_meas + + # in case retrace was at end of last packet + if (win[0] - dat_exp["WINDOW"][-1]) < 0: + add_time += tm_per_meas + + # Extend the new expanded dt + dscet_extend = [this_scet + 1e9 * thisaddtime for thisaddtime in add_time] + dat_exp["Epoch"].extend(dscet_extend) + + # Extend each of the data arrays + for key in dat.keys(): + try: + dat_exp[key].extend(dat[key][i]) + except TypeError: + expanded = np.ones(nmeas) * dat[key][i] + dat_exp[key].extend(expanded) + + if nocdf: + return dat_exp + else: + # Fill in the CDF + keys = cdf.keys() + + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: + statusmsg( + "Failed : Key:{:} failed insert into CDF".format(key), + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + import pdb + + pdb.set_trace() + + +##################################################### +## +##################################################### +def cdf352(cdf, dat, nocdf=False, verbose=False): + try: + # Calculate SCET from the variables in the L0 data + dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) + + # Same keys as original data dictionary, but will hold one variable per key + # instead of one for every NYS for every key + dat_exp = {} + for key in dat.keys(): + if key[-4:] == "_000": + continue + dat_exp[key] = [] + + dat_exp["VAR0"] = [] + dat_exp["VAR1"] = [] + dat_exp["VAR2"] = [] + dat_exp["VAR3"] = [] + dat_exp["VAR0_NAME"] = [] + dat_exp["VAR1_NAME"] = [] + dat_exp["VAR2_NAME"] = [] + dat_exp["VAR3_NAME"] = [] + + # Var names for each possible channel that might be contained in the packet + avars = ["A0", "A1", "A2", "A3"] + bvars = ["B0", "B1", "B2", "B3"] + cvars = ["C0", "C1", "C2", "C3"] + dvars = ["D0", "D1", "D2", "D3"] + hk1vars = ["HV_DAC_IN", "P3p3_Vmon", "P12_Vmon", "N12_Vmon"] + hk2vars = ["HV_Out", "Rail_Ctrl", "P5_Vmon", "N5_Vmon"] + + # To convert from the ID number in each packet header to the variables that + # the packet actually contains values for + coll2var = {1: avars, 2: bvars, 4: cvars, 8: dvars, 16: hk1vars, 32: hk2vars} + + # Create the 'Epoch' variable in our data array + dat_exp["Epoch"] = [] + + # Time that each measurement took this NYS + tm_per_meas = 1.0 / 32.0 / 1171.875 + + for i in range(len(dt)): + # MET for this NY second + thisdt = dt[i] + + # Collector (or HSK values) that are being used this NYS + # An integer that references which varibles are actually contained in the packet + coll_used = dat["SPC_TIMESERCOLL"][i] + + # Number of measurements this NYS + nmeas = len(dat["G0_000"][i]) # should always be 20*32=640 + + # Packet start time + pkt_start = dat["SPC_TIMESERTICK"][i] / 1171.875 + + # Calculate the time array for this NYS + add_time = pkt_start + np.arange( + 0, tm_per_meas * (nmeas - 0.1), tm_per_meas + ) + + try: + if coll_used not in coll2var.keys(): + raise ValueError( + "Value: {:} not in coll2var.keys()".format(coll_used) + ) # probably a corrupt packet + + dat_exp["VAR0_NAME"].extend( + [coll2var[coll_used][0] for foo in range(nmeas)] + ) + dat_exp["VAR1_NAME"].extend( + [coll2var[coll_used][1] for foo in range(nmeas)] + ) + dat_exp["VAR2_NAME"].extend( + [coll2var[coll_used][2] for foo in range(nmeas)] + ) + dat_exp["VAR3_NAME"].extend( + [coll2var[coll_used][3] for foo in range(nmeas)] + ) + + dat_exp["VAR0"].extend(dat["G0_000"][i]) + dat_exp["VAR1"].extend(dat["G1_000"][i]) + dat_exp["VAR2"].extend(dat["G2_000"][i]) + dat_exp["VAR3"].extend(dat["G3_000"][i]) + + except: + statusmsg( + "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + continue + + # Extend the expanded dt + dt_extend = [thisdt + sec * 1e9 for sec in add_time] + dat_exp["Epoch"].extend(dt_extend) + + # Extend each of the data arrays + for key in dat.keys(): + if key[-4:] == "_000": + continue + try: + dat_exp[key].extend(dat[key][i]) + except TypeError: + expanded = np.ones(nmeas) * dat[key][i] + dat_exp[key].extend(expanded) + + if nocdf: + return dat_exp + else: + # Fill in the CDF + keys = cdf.keys() + + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: + statusmsg( + "Failed : Key:{:} failed insert into CDF".format(key), + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + except: + print(sys.exc_info()) + import pdb + + pdb.set_trace() + + return () + + +##################################################### +## +##################################################### +def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): + """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" + sec_str = ["{:1.0f}".format(i) for i in sec] + subsec_str_base50000 = [ + "{:05.0f}".format(int(i * 50000 / 65536)) for i in subsec + ] # SWEAP has subseconds in 1/65536's of a second + if spacecraft: + subsec_str_base50000 = [ + "{:05.0f}".format(int(i * 50000 / 256)) for i in subsec + ] # S/C has subseconds in 1/256's of a second + + ephem_sec_j2000 = [ + spiceypy.scs2e(-96, sec_str[i] + ":" + subsec_str_base50000[i]) + for i in range(len(sec_str)) + ] + ephem_nanosec_j2000 = [np.round(1e9 * i) for i in ephem_sec_j2000] + + return ephem_nanosec_j2000 + + +##################################################### +## +##################################################### +def statusmsg(string, screen=False, file=True, verbose=False): + """Output status message to screen or logfile (default to file, but not screen)""" + nowdtstr = datetime.datetime.now().isoformat() + if file: + logfile.write(nowdtstr + ", " + string + "\n") + if screen: + if verbose: + print(string) + + +##################################################### +## +##################################################### +def get_newest_kernel(tls=False, sclk=False, verbose=False): + """Find the path to the newest NAIF TLS (leap second) kernel file""" + # Make sure we chose exactly one of the options + if tls + sclk != 1: + return False + + # Search in the MOC data product directory for newest file + if tls: + globdir = "/psp/data/moc_data_products/leap_second_kernel/" + globstr = globdir + "naif00[0-9][0-9].tls" + ndigits = 2 + elif sclk: + globdir = "/psp/data/moc_data_products/operations_sclk_kernel/" + globstr = globdir + "spp_sclk_[0-9][0-9][0-9][0-9].tsc" + ndigits = 4 + + files = glob.glob(globstr) + + # isolate version numbers from the file path and find newest + if tls: + versions = [int(i[-4 - ndigits : -4]) for i in files] + elif sclk: + versions = [int(i[-4 - ndigits : -4]) for i in files] + try: + maxind = np.argmax(versions) + except ValueError: + statusmsg("***ERROR*** Could not find kernel versions") + print(sys.exc_info()) + import pdb + + pdb.set_trace() + return False + + # return path to newest file + path = files[maxind] + return path + + +##################################################### +## +##################################################### +def get_newest_skeleton(apid, verbose=False): + """Find the path to the newest skeleton CDF file""" + + return "cdf_skeletons/psp_swp_spc_l1_{:}_skeleton.cdf".format( + hex(apid)[2:].zfill(3) + ) + + # The remaining code in this function is from when we used skeleton file numbers with a version # in them + # and we had to search for the most recent (highest) version + + # Search for newest file + # globstr = 'cdf_skeletons/spp_apid_{:}_sweap_00000000t000000_v[0-9][0-9].cdf'.format(hex(apid)[2:].zfill(3)) + # ndigits = 2 + + # files = glob.glob(globstr) + + # isolate version numbers from the file path and find newest + # versions = [int(i[-4-ndigits:-4]) for i in files] + + # try: + # maxind = np.argmax(versions) + # except ValueError: + # statusmsg('***ERROR*** Could not find skeleton versions') + # return(False) + + # return path to newest file + # path = files[maxind] + + # return(path) + + +##################################################### +### +##################################################### +def setup(): + """Get user command-line input and set things up""" + + # defaults + l0file_default = "" + l0dir_default = "" + l1dir_default = "" + logdir_default = "" + apid_default = "0" + + # Get User Input + parser = argparse.ArgumentParser(description="") + parser.add_argument( + "-v", + "--verbose", + default=False, + action="store_true", + help="Increase verbosity", + required=False, + ) + parser.add_argument( + "-gz", + "--gzip", + default=False, + action="store_true", + help="Read in L0 file as gzip", + required=False, + ) + parser.add_argument( + "-sc", + "--spacecraft", + default=False, + action="store_true", + help="Look for S/C packets", + required=False, + ) + parser.add_argument( + "-b", + "--batch", + default=False, + action="store_true", + help="Convert all L0 files in same directory as selected", + required=False, + ) + parser.add_argument( + "-r", + "--recursive", + default=False, + action="store_true", + help="Convert all L0 files in given directory and in all subdirectories", + required=False, + ) + parser.add_argument( + "-p", + "--ptp", + default=False, + action="store_true", + help="Indicate that input L0 file is a PTP file", + required=False, + ) + parser.add_argument( + "-o", + "--overwrite", + default=False, + action="store_true", + help="Overwrite existing L1 CDF file, if necessary", + required=False, + ) + parser.add_argument( + "-stc", + "--stcorrect", + default=False, + action="store_true", + help="If ST is wrong (FPGA bug if ST set higher than 2), then try to correct it)", + required=False, + ) + parser.add_argument( + "-a", + "--apid", + help="APID to create L1 file for [0==all] [default={:}]".format(apid_default), + required=False, + default=apid_default, + type=str, + ) + parser.add_argument( + "-l0", + "--l0file", + help="Input L0 File [default={:}]".format(l0file_default), + required=False, + default=l0file_default, + ) + parser.add_argument( + "-d", + "--l0dir", + help="Input L0 Directory (for use with -b or -r [default={:}]".format( + l0dir_default + ), + required=False, + default=l0dir_default, + ) + parser.add_argument( + "-dl1", + "--l1dir", + help="Output L1 Directory [default={:}]".format(l1dir_default), + required=False, + default=l1dir_default, + ) + parser.add_argument( + "-dlog", + "--logdir", + help="Output for Log Files [default={:}]".format(logdir_default), + required=False, + default=logdir_default, + ) + + # Read in the arguments + args = parser.parse_args() + + # Version of the data product + args.version = 2 + + # Make sure we got a good argument set + if (args.batch == 0) & (args.recursive == 0): + if args.l0file == "": + statusmsg( + "***ERROR*** You must provide --l0file, if not using -b or -r", + screen=True, + verbose=verbose, + ) + else: + if args.l0dir == "": + statusmsg( + "***ERROR*** You must provide --l0dir if using -b or -r", + screen=True, + verbose=verbose, + ) + + # Convert APID to an integer (it is read as a string from the command line) + try: + if args.apid[0:2] == "0x": + base = 16 + else: + base = 10 + args.apid = int(args.apid, base) + except TypeError: + statusmsg( + "Trouble parsing desired APID....exiting.", screen=True, verbose=verbose + ) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) + sys.exit() + + # Make sure the environmental variable reference to the data directory is set and readable + try: + datadir = os.environ["PSP_DATA_DIR"] + except: + raise KeyError( + "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" + ) + + if not os.path.exists(datadir): + raise ValueError( + "Directory specified in env. variable PSP_DATA_DIR does not exist" + ) + + # Return to main routine + return args + + +############################################ +#### +############################################ +if __name__ == "__main__": + args = setup() + main( + l0file=args.l0file, + l1dir=args.l1dir, + logdir=args.logdir, + spacecraft=args.spacecraft, + ptp=args.ptp, + gzip=args.gzip, + apidreq=args.apid, + overwrite=args.overwrite, + verbose=args.verbose, + ) From e6692f3e904e3aacc1733164c153784e13d88d83 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:20:46 -0400 Subject: [PATCH 2/4] Make changes through ruff check and ruff format --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 387 ++++++++++----------- 1 file changed, 186 insertions(+), 201 deletions(-) diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index 3bacedc..faabb58 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -3,22 +3,29 @@ # $LastChangedRevision: 97 $ # $LastChangedDate: 2020-08-04 09:20:42 -0400 (Tue, 04 Aug 2020) $ # $LastChangedBy: acase $ -""" +""" # noqa: D400 + +import argparse +import datetime +import glob +import math +import os +import sys import numpy as np -import argparse, sys, os, datetime, math, glob try: from spacepy import pycdf -except: - print(sys.exc_info()) - print("***ERRORR*** Could not import pycdf from spacepy") - print( +except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + print("***ERROR*** Could not import pycdf from spacepy") # noqa: T201 + print( # noqa: T201 "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" ) sys.exit() -import ccsds_reader_pipeline as cc import distutils.dir_util + +import ccsds_reader_pipeline as cc import spiceypy # Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft @@ -41,50 +48,47 @@ # - Added revision history -def main( - l0file="", - l1dir="", - logdir="", - spacecraft=False, - ptp=False, - gzip=False, - apidreq=0, - overwrite=False, - verbose=False, +def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 + l0file="", # noqa: ANN001 + l1dir="", # noqa: ANN001 + logdir="", # noqa: ANN001 + spacecraft=False, # noqa: ANN001, FBT002 + ptp=False, # noqa: ANN001, FBT002 + gzip=False, # noqa: ANN001, FBT002 + apidreq=0, # noqa: ANN001 + overwrite=False, # noqa: ANN001, FBT002 + verbose=False, # noqa: ANN001, FBT002 ): - """Convert a single L0 file to L1""" - + """Convert a single L0 file to L1""" # noqa: D400 # Try to create a filename for the new CDF that we're going to create - l0dirname = os.path.dirname(l0file) - l0basename = os.path.basename(l0file) + l0dirname = os.path.dirname(l0file) # noqa: PTH120 + l0basename = os.path.basename(l0file) # noqa: PTH119 if l1dir == "": l1dir = ( l0dirname # use input L0 directory for L1 files, if nothing else specified ) # Get a version of filename with no extension - l0file_noext = os.path.splitext(l0basename)[0] + l0file_noext = os.path.splitext(l0basename)[0] # noqa: PTH122 if l0file_noext[-3:] == "ptp": - l0file_noext = os.path.splitext(l0file_noext)[0] + l0file_noext = os.path.splitext(l0file_noext)[0] # noqa: PTH122 # Open a log file to write to - nowdt = datetime.datetime.now() + nowdt = datetime.datetime.now() # noqa: DTZ005 if logdir == "": logdir = l1dir # use L1 file output directory for log file, if nothing else specified distutils.dir_util.mkpath( logdir ) # in case the directory doesn't exist, this will create it - logpath = os.path.join( + logpath = os.path.join( # noqa: PTH118 logdir, - "swp_spc_l02l1_{:04.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}.log".format( - nowdt.year, nowdt.month, nowdt.day, nowdt.hour, nowdt.minute, nowdt.second - ), + f"swp_spc_l02l1_{nowdt.year:04.0f}{nowdt.month:02.0f}{nowdt.day:02.0f}{nowdt.hour:02.0f}{nowdt.minute:02.0f}{nowdt.second:02.0f}.log", ) try: - global logfile - logfile = open(logpath, "w") - except: - print("\n***ERROR*** Could not open log file!\n") + global logfile # noqa: PLW0603 + logfile = open(logpath, "w") # noqa: PTH123, SIM115 + except: # noqa: E722 + print("\n***ERROR*** Could not open log file!\n") # noqa: T201 sys.exit(1) # Write some information to the log file @@ -100,18 +104,18 @@ def main( # Make sure the L0 file exists and is readable try: - foo = open(l0file, "r") + foo = open(l0file) # noqa: PTH123, SIM115 foo.close() statusmsg("L0 file exists and is readable") - except IOError: + except OSError: statusmsg( "***ERROR*** [swp_spc_l02l1.py] Input L0 file could not be read...exiting", screen=True, verbose=verbose, ) - import pdb + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 sys.exit() # Load in Leap Second Kernel @@ -124,9 +128,9 @@ def main( sys.exit() else: try: - statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(tls_path)) + statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {tls_path}") spiceypy.furnsh(tls_path) - except: + except: # noqa: E722 statusmsg( "***ERROR*** [swp_spc_l02l1.py] Could not furnsh leap second kernel...exiting" ) @@ -140,9 +144,9 @@ def main( sys.exit() else: try: - statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(sclk_path)) + statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {sclk_path}") spiceypy.furnsh(sclk_path) - except: + except: # noqa: E722 statusmsg( "***ERROR*** [swp_spc_l02l1.py] Could not furnsh SCLK kernel...exiting" ) @@ -158,10 +162,10 @@ def main( statusmsg("Event = Finished reading file") # Loop through the APIDs that we got - for apid in l0data.keys(): - statusmsg("Event = Beginning APID: {:}".format(hex(apid))) + for apid in l0data.keys(): # noqa: SIM118 + statusmsg(f"Event = Beginning APID: {hex(apid)}") - if apid == 0x07B: + if apid == 0x07B: # noqa: PLR2004 statusmsg( "***WARNING*** [swp_spc_l02l1] APID 0x07B CDFs not yet implemented", screen=True, @@ -170,7 +174,7 @@ def main( continue # Make sure we need to do this apid - if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: + if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: # noqa: RUF015 statusmsg("No packets found for this apid.") continue # skip this apid if there were no packets received if (apidreq != 0) & (apidreq != apid): @@ -178,20 +182,19 @@ def main( continue # skip this apid if user only wanted one apid and this isn't it # Filename for the L1 file we're about to write for this apid - l1path = os.path.join( + l1path = os.path.join( # noqa: PTH118 l1dir, - l0file_noext - + "_APID{:}_L1.cdf".format(str(hex(apid)[2:].zfill(3)).upper()), + l0file_noext + f"_APID{str(hex(apid)[2:].zfill(3)).upper()}_L1.cdf", # noqa: FURB116 ) statusmsg("About to write: " + l1path) # Make sure the skeleton file exists and is readable try: skeleton_filename = get_newest_skeleton(apid) - foo = open(skeleton_filename, "r") + foo = open(skeleton_filename) # noqa: PTH123, SIM115 foo.close() statusmsg("Skeleton to be used: " + skeleton_filename) - except IOError: + except OSError: statusmsg( "***ERROR*** [swp_spc_l02l1.py] Skeleton file could not be read...moving to next apid", screen=True, @@ -201,9 +204,7 @@ def main( continue except TypeError: statusmsg( - "***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={:} could not be found...moving to next apid".format( - hex(apid) - ), + f"***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={hex(apid)} could not be found...moving to next apid", screen=True, verbose=verbose, ) @@ -213,14 +214,12 @@ def main( try: # try to open and close it statusmsg("Using L1 path: " + l1path, screen=True, verbose=verbose) - foo = open(l1path) + foo = open(l1path) # noqa: PTH123, SIM115 foo.close() # if we get here, this file already exists; so delete it, if desired statusmsg( - "***INFO*** [swp_spc_l02l1] L1 CDF file ({:}) already exists".format( - l1path - ), + f"***INFO*** [swp_spc_l02l1] L1 CDF file ({l1path}) already exists", screen=True, verbose=verbose, ) @@ -230,19 +229,19 @@ def main( screen=True, verbose=verbose, ) - os.remove(l1path) + os.remove(l1path) # noqa: PTH107 else: statusmsg( "***ERROR*** [swp_spc_l02l1] L1 CDF already exists, and overwrite (-o option) was not requested...exiting.", screen=True, verbose=verbose, ) - raise (SystemExit) + raise (SystemExit) # noqa: TRY301 except SystemExit: sys.exit() - except IOError: + except OSError: pass # Apparently the file did not exist already - except: + except: # noqa: E722 statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) statusmsg( "\n***ERROR*** [swp_spc_l02l1] Could not check existence/delete L1 CDF file path. Exiting...\n", @@ -254,11 +253,9 @@ def main( # Create a new CDF file from the provided skeleton try: cdf = pycdf.CDF(l1path, skeleton_filename) - except "CDFError": + except "CDFError": # noqa: B030 statusmsg( - "\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={:})...continuing to next APID\n).".format( - apid - ), + f"\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={apid})...continuing to next APID\n).", screen=True, verbose=verbose, ) @@ -282,12 +279,10 @@ def main( } try: cdfproc[apid](cdf, l0data[apid], verbose=verbose) - except: + except: # noqa: E722 statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) statusmsg( - "***WARNING*** [swp_spc_l02l1] CDF not processed for APID={:}".format( - hex(apid) - ), + f"***WARNING*** [swp_spc_l02l1] CDF not processed for APID={hex(apid)}", screen=True, verbose=verbose, ) @@ -308,38 +303,37 @@ def main( ##################################################### ## ##################################################### -def cdf35e_35f(cdf, dat, verbose=False): - """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" - +def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, ANN201, C901, FBT002 + """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 # Calculate MET from the variables in the L0 data # MET of each NYS - if "CCSDS_MET" in dat.keys(): + if "CCSDS_MET" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPC_SUBSEC"]) - elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): + elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["FSW_HK_HK_INST_TPSH_MET_SEC"], dat["FSW_HK_HK_INST_TPSH_MET_SUBSEC"], spacecraft=True, ) - elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): + elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["PDU_PRIO94_TPSH_MET_SEC"], dat["PDU_PRIO94_TPSH_MET_SUBSEC"], spacecraft=True, ) - elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): + elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["HK_HIGH_TPSH_MET_SEC"], dat["HK_HIGH_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): + elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["HK_FSWL_TPSH_MET_SEC"], dat["HK_FSWL_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): + elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["HK_LOW_TPSH_MET_SEC"], dat["HK_LOW_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): + elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["RIU_DERIVED_TPSH_MET_SEC"], dat["RIU_DERIVED_TPSH_MET_SUBSEC"], @@ -357,15 +351,15 @@ def cdf35e_35f(cdf, dat, verbose=False): for key in keys: try: cdf[key] = dat[key] # create variable and insert data - except KeyError: - if key not in dat.keys(): + except KeyError: # noqa: PERF203 + if key not in dat.keys(): # noqa: SIM118 cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] - except: - import pdb + except: # noqa: E722 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 statusmsg( - "Failed : Key:{:} failed insert into CDF".format(key), + f"Failed : Key:{key} failed insert into CDF", screen=True, verbose=verbose, ) @@ -375,7 +369,7 @@ def cdf35e_35f(cdf, dat, verbose=False): ##################################################### ## ##################################################### -def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): +def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0915, RET503 """Fill up a CDF with SCI, ALL, or RSS data.""" # Take data sorted by NYS, and produce one long variable with all data @@ -384,11 +378,11 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # Each different packet will require a different variable to calculate # The number of measurements each NYS - if apid == 0x351: + if apid == 0x351: # noqa: PLR2004 length_var = "A1S" - elif apid == 0x353: + elif apid == 0x353: # noqa: PLR2004 length_var = "ASIN" - elif apid == 0x354: + elif apid == 0x354: # noqa: PLR2004 length_var = "ARSS" # Calculate MET from the variables in the L0 data @@ -396,12 +390,12 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) # MET of each measurement (to be filled in in the future) - scet_exp = [] + scet_exp = [] # noqa: F841 # Same keys as original data dictionary, but will hold one variable per key # instead of one for every NYS for every key dat_exp = {} - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 dat_exp[key] = [] # Create the 'Epoch' variable in our data array @@ -416,7 +410,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): ticks_per_meas = dat["SW_SPC_INTTIME"][i] + dat["SW_SPC_SERVTIME"][i] # Make sure ST and IT are allowed values - if (math.log(ticks_per_meas, 2)) % 1 != 0: + if (math.log(ticks_per_meas, 2)) % 1 != 0: # noqa: FURB163 # the SPC FPGA will default to IT=6, ST=2 (the power-on defaults) if a non-integer power of 2 IT+ST is requested ticks_per_meas = 8 @@ -480,7 +474,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): add_time[thisrtpix:] += tm_per_meas # If we're in an AllGain packet, then the beginning of the packet might not be the beginning of the NYS (which is the time noted in the header) - if apid == 0x351: + if apid == 0x351: # noqa: PLR2004 pktnum = dat["SW_SPC_PKTNUM"][i] # if pktnum==0: import pdb; pdb.set_trace() if pktnum != 0: @@ -498,45 +492,44 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): dat_exp["Epoch"].extend(dscet_extend) # Extend each of the data arrays - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 try: dat_exp[key].extend(dat[key][i]) - except TypeError: + except TypeError: # noqa: PERF203 expanded = np.ones(nmeas) * dat[key][i] dat_exp[key].extend(expanded) if nocdf: return dat_exp - else: - # Fill in the CDF - keys = cdf.keys() + # Fill in the CDF + keys = cdf.keys() - # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) - epochloc = np.where(np.array(keys) == "Epoch")[0] - if len(epochloc) != 0: - keys.pop(epochloc[0]) - keys.insert(0, "Epoch") + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") - for key in keys: - try: - # insert data - cdf[key] = dat_exp[key] - except: - statusmsg( - "Failed : Key:{:} failed insert into CDF".format(key), - screen=True, - verbose=verbose, - ) - statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - import pdb + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: # noqa: E722, PERF203 + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 ##################################################### ## ##################################################### -def cdf352(cdf, dat, nocdf=False, verbose=False): +def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915 try: # Calculate SCET from the variables in the L0 data dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) @@ -544,7 +537,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # Same keys as original data dictionary, but will hold one variable per key # instead of one for every NYS for every key dat_exp = {} - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 if key[-4:] == "_000": continue dat_exp[key] = [] @@ -581,7 +574,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): thisdt = dt[i] # Collector (or HSK values) that are being used this NYS - # An integer that references which varibles are actually contained in the packet + # An integer that references which variables are actually contained in the packet coll_used = dat["SPC_TIMESERCOLL"][i] # Number of measurements this NYS @@ -596,9 +589,9 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): ) try: - if coll_used not in coll2var.keys(): - raise ValueError( - "Value: {:} not in coll2var.keys()".format(coll_used) + if coll_used not in coll2var: + raise ValueError( # noqa: TRY003 + f"Value: {coll_used} not in coll2var.keys()" # noqa: EM102 ) # probably a corrupt packet dat_exp["VAR0_NAME"].extend( @@ -619,7 +612,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): dat_exp["VAR2"].extend(dat["G2_000"][i]) dat_exp["VAR3"].extend(dat["G3_000"][i]) - except: + except: # noqa: E722 statusmsg( "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" ) @@ -631,7 +624,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): dat_exp["Epoch"].extend(dt_extend) # Extend each of the data arrays - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 if key[-4:] == "_000": continue try: @@ -642,31 +635,30 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): if nocdf: return dat_exp - else: - # Fill in the CDF - keys = cdf.keys() - - # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) - epochloc = np.where(np.array(keys) == "Epoch")[0] - if len(epochloc) != 0: - keys.pop(epochloc[0]) - keys.insert(0, "Epoch") - for key in keys: - try: - # insert data - cdf[key] = dat_exp[key] - except: - statusmsg( - "Failed : Key:{:} failed insert into CDF".format(key), - screen=True, - verbose=verbose, - ) - statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - except: - print(sys.exc_info()) - import pdb + # Fill in the CDF + keys = cdf.keys() - pdb.set_trace() + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: # noqa: E722, PERF203 + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 + + pdb.set_trace() # noqa: T100 return () @@ -674,15 +666,15 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): ##################################################### ## ##################################################### -def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): - """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" - sec_str = ["{:1.0f}".format(i) for i in sec] +def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 + """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # noqa: D400 + sec_str = [f"{i:1.0f}" for i in sec] subsec_str_base50000 = [ - "{:05.0f}".format(int(i * 50000 / 65536)) for i in subsec + f"{int(i * 50000 / 65536):05.0f}" for i in subsec ] # SWEAP has subseconds in 1/65536's of a second if spacecraft: subsec_str_base50000 = [ - "{:05.0f}".format(int(i * 50000 / 256)) for i in subsec + f"{int(i * 50000 / 256):05.0f}" for i in subsec ] # S/C has subseconds in 1/256's of a second ephem_sec_j2000 = [ @@ -691,27 +683,27 @@ def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): ] ephem_nanosec_j2000 = [np.round(1e9 * i) for i in ephem_sec_j2000] - return ephem_nanosec_j2000 + return ephem_nanosec_j2000 # noqa: RET504 ##################################################### ## ##################################################### -def statusmsg(string, screen=False, file=True, verbose=False): - """Output status message to screen or logfile (default to file, but not screen)""" - nowdtstr = datetime.datetime.now().isoformat() +def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, ANN201, FBT002 + """Output status message to screen or logfile (default to file, but not screen)""" # noqa: D400 + nowdtstr = datetime.datetime.now().isoformat() # noqa: DTZ005 if file: logfile.write(nowdtstr + ", " + string + "\n") - if screen: + if screen: # noqa: SIM102 if verbose: - print(string) + print(string) # noqa: T201 ##################################################### ## ##################################################### -def get_newest_kernel(tls=False, sclk=False, verbose=False): - """Find the path to the newest NAIF TLS (leap second) kernel file""" +def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 + """Find the path to the newest NAIF TLS (leap second) kernel file""" # noqa: D400 # Make sure we chose exactly one of the options if tls + sclk != 1: return False @@ -726,37 +718,32 @@ def get_newest_kernel(tls=False, sclk=False, verbose=False): globstr = globdir + "spp_sclk_[0-9][0-9][0-9][0-9].tsc" ndigits = 4 - files = glob.glob(globstr) + files = glob.glob(globstr) # noqa: PTH207 # isolate version numbers from the file path and find newest - if tls: - versions = [int(i[-4 - ndigits : -4]) for i in files] - elif sclk: + if tls or sclk: versions = [int(i[-4 - ndigits : -4]) for i in files] try: maxind = np.argmax(versions) except ValueError: statusmsg("***ERROR*** Could not find kernel versions") - print(sys.exc_info()) - import pdb + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 return False # return path to newest file path = files[maxind] - return path + return path # noqa: RET504 ##################################################### ## ##################################################### -def get_newest_skeleton(apid, verbose=False): - """Find the path to the newest skeleton CDF file""" - - return "cdf_skeletons/psp_swp_spc_l1_{:}_skeleton.cdf".format( - hex(apid)[2:].zfill(3) - ) +def get_newest_skeleton(apid, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 + """Find the path to the newest skeleton CDF file""" # noqa: D400 + return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # noqa: FURB116 # The remaining code in this function is from when we used skeleton file numbers with a version # in them # and we had to search for the most recent (highest) version @@ -785,9 +772,8 @@ def get_newest_skeleton(apid, verbose=False): ##################################################### ### ##################################################### -def setup(): - """Get user command-line input and set things up""" - +def setup(): # noqa: ANN201 + """Get user command-line input and set things up""" # noqa: D400 # defaults l0file_default = "" l0dir_default = "" @@ -864,7 +850,7 @@ def setup(): parser.add_argument( "-a", "--apid", - help="APID to create L1 file for [0==all] [default={:}]".format(apid_default), + help=f"APID to create L1 file for [0==all] [default={apid_default}]", required=False, default=apid_default, type=str, @@ -872,30 +858,28 @@ def setup(): parser.add_argument( "-l0", "--l0file", - help="Input L0 File [default={:}]".format(l0file_default), + help=f"Input L0 File [default={l0file_default}]", required=False, default=l0file_default, ) parser.add_argument( "-d", "--l0dir", - help="Input L0 Directory (for use with -b or -r [default={:}]".format( - l0dir_default - ), + help=f"Input L0 Directory (for use with -b or -r [default={l0dir_default}]", required=False, default=l0dir_default, ) parser.add_argument( "-dl1", "--l1dir", - help="Output L1 Directory [default={:}]".format(l1dir_default), + help=f"Output L1 Directory [default={l1dir_default}]", required=False, default=l1dir_default, ) parser.add_argument( "-dlog", "--logdir", - help="Output for Log Files [default={:}]".format(logdir_default), + help=f"Output for Log Files [default={logdir_default}]", required=False, default=logdir_default, ) @@ -912,41 +896,42 @@ def setup(): statusmsg( "***ERROR*** You must provide --l0file, if not using -b or -r", screen=True, - verbose=verbose, - ) - else: - if args.l0dir == "": - statusmsg( - "***ERROR*** You must provide --l0dir if using -b or -r", - screen=True, - verbose=verbose, + verbose=verbose, # noqa: F821 ) + elif args.l0dir == "": + statusmsg( + "***ERROR*** You must provide --l0dir if using -b or -r", + screen=True, + verbose=verbose, # noqa: F821 + ) # Convert APID to an integer (it is read as a string from the command line) try: - if args.apid[0:2] == "0x": + if args.apid[0:2] == "0x": # noqa: SIM108 base = 16 else: base = 10 args.apid = int(args.apid, base) except TypeError: statusmsg( - "Trouble parsing desired APID....exiting.", screen=True, verbose=verbose + "Trouble parsing desired APID....exiting.", + screen=True, + verbose=verbose, # noqa: F821 ) - statusmsg(sys.exc_info(), screen=True, verbose=verbose) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) # noqa: F821 sys.exit() # Make sure the environmental variable reference to the data directory is set and readable try: datadir = os.environ["PSP_DATA_DIR"] - except: - raise KeyError( - "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" + except: # noqa: E722 + raise KeyError( # noqa: B904, TRY003 + "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" # noqa: EM101 ) - if not os.path.exists(datadir): - raise ValueError( - "Directory specified in env. variable PSP_DATA_DIR does not exist" + if not os.path.exists(datadir): # noqa: PTH110 + raise ValueError( # noqa: TRY003 + "Directory specified in env. variable PSP_DATA_DIR does not exist" # noqa: EM101 ) # Return to main routine From dad85934ad69367dcdda3d33f0caf82e762ff04a Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:26:25 -0400 Subject: [PATCH 3/4] Minor cleanup of file --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 35 ++++++++++------------ 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index faabb58..41b0b95 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -5,6 +5,18 @@ # $LastChangedBy: acase $ """ # noqa: D400 +__all__ = [ + "cdf35e_35f", + "cdf351_353_354", + "cdf352", + "get_newest_kernel", + "get_newest_skeleton", + "main", + "secsubsec2scet", + "setup", + "statusmsg", +] + import argparse import datetime import glob @@ -300,10 +312,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 logfile.close() -##################################################### -## -##################################################### -def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, ANN201, C901, FBT002 +def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, FBT002 """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 # Calculate MET from the variables in the L0 data # MET of each NYS @@ -526,10 +535,8 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 pdb.set_trace() # noqa: T100 -##################################################### -## -##################################################### -def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915 + +def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, FBT002 try: # Calculate SCET from the variables in the L0 data dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) @@ -663,9 +670,6 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, return () -##################################################### -## -##################################################### def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # noqa: D400 sec_str = [f"{i:1.0f}" for i in sec] @@ -686,9 +690,6 @@ def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN00 return ephem_nanosec_j2000 # noqa: RET504 -##################################################### -## -##################################################### def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, ANN201, FBT002 """Output status message to screen or logfile (default to file, but not screen)""" # noqa: D400 nowdtstr = datetime.datetime.now().isoformat() # noqa: DTZ005 @@ -699,9 +700,6 @@ def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, print(string) # noqa: T201 -##################################################### -## -##################################################### def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 """Find the path to the newest NAIF TLS (leap second) kernel file""" # noqa: D400 # Make sure we chose exactly one of the options @@ -738,9 +736,6 @@ def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, AN return path # noqa: RET504 -##################################################### -## -##################################################### def get_newest_skeleton(apid, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 """Find the path to the newest skeleton CDF file""" # noqa: D400 return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # noqa: FURB116 From 3b91f1b996709f3b0dbcd4336f45753988d3ae04 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:30:58 -0400 Subject: [PATCH 4/4] Update imports --- src/pyfaradaycup/pipeline/__init__.py | 2 ++ src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py index e69de29..6e86e0f 100644 --- a/src/pyfaradaycup/pipeline/__init__.py +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -0,0 +1,2 @@ +from . import swp_spc_l02l1 +from . import ccsds_reader_pipeline diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index 41b0b95..4673164 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -29,17 +29,21 @@ try: from spacepy import pycdf except: # noqa: E722 + # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) + # then we don't need this. print(sys.exc_info()) # noqa: T201 print("***ERROR*** Could not import pycdf from spacepy") # noqa: T201 print( # noqa: T201 "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" ) sys.exit() + import distutils.dir_util -import ccsds_reader_pipeline as cc import spiceypy +import pyfaradaycup.pipeline.ccsds_reader_pipeline as cc + # Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft # into L0.5 or L1 CDF files @@ -535,7 +539,6 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 pdb.set_trace() # noqa: T100 - def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, FBT002 try: # Calculate SCET from the variables in the L0 data