diff --git a/.gitignore b/.gitignore index 0982d9c80..32fc1465d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ Work build +dist + RELEASE-VERSION.txt *.egg* @@ -47,4 +49,6 @@ avaframe/com1DFA/*.so # Byte-compiled / optimized / DLL files # see https://github.com/github/gitignore/blob/main/Python.gitignore -*.py[cod] \ No newline at end of file +*.py[cod] + +*.rcf \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 323c5fec4..4f33aed1c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include avaframe/RELEASE-VERSION.txt include avaframe/version.py include avaframe/in3Utils/logging.conf include avaframe/com1DFA/*.pyx +include avaframe/com9MoTVoellmy/MoT-Voellmy*.exe recursive-include avaframe *Cfg.ini global-exclude local_*Cfg.ini prune benchmarks diff --git a/avaframe/ana4Stats/probAna.py b/avaframe/ana4Stats/probAna.py index f69944fd2..2f7f533d7 100644 --- a/avaframe/ana4Stats/probAna.py +++ b/avaframe/ana4Stats/probAna.py @@ -9,6 +9,9 @@ import logging import pathlib from scipy.stats import qmc +from SALib.sample import morris +import pickle +from deepdiff import grep import avaframe.out3Plot.plotUtils as pU from avaframe.in3Utils import cfgUtils @@ -28,60 +31,60 @@ def createComModConfig(cfgProb, avaDir, modName): - """ create configuration file for performing sims with modName com module - - Parameters - ----------- - cfgProb: configParser object - configuration settings - avaDir: pathlib path - path to avalanche directory - modName: module - computational module - - Returns - ------- - cfgFiles: list - list of paths to newly generated configuration files for com module inlcuding - parameter variations + """create configuration file for performing sims with modName com module + + Parameters + ----------- + cfgProb: configParser object + configuration settings + avaDir: pathlib path + path to avalanche directory + modName: module + computational module + + Returns + ------- + cfgFiles: list + list of paths to newly generated configuration files for com module inlcuding + parameter variations """ # setup where configuration file is saved modNameString = str(pathlib.Path(modName.__file__).stem) - outDir = avaDir / 'Work' / ('%sConfigFiles' % modNameString) + outDir = avaDir / "Work" / ("%sConfigFiles" % modNameString) fU.makeADir(outDir) # check variation settings - variationsDict = makeDictFromVars(cfgProb['PROBRUN']) + variationsDict = makeDictFromVars(cfgProb["PROBRUN"]) - if cfgProb['PROBRUN'].getint('samplingStrategy') == 2: - log.info('Probability run performed by varying one parameter at a time - local approach.') + if cfgProb["PROBRUN"].getint("samplingStrategy") == 2: + log.info("Probability run performed by varying one parameter at a time - local approach.") cfgFiles = cfgFilesLocalApproach(variationsDict, cfgProb, modName, outDir) else: - log.info('Probability run perfromed drawing parameter set from full sample.') + log.info("Probability run perfromed drawing parameter set from full sample.") cfgFiles = cfgFilesGlobalApproach(avaDir, cfgProb, modName, outDir) return cfgFiles, outDir def cfgFilesGlobalApproach(avaDir, cfgProb, modName, outDir): - """ create configuration files with all parameter variations - drawn from full sample - for performing sims with modName comModule - - Parameters - ----------- - cfgProb: configParser object - configuration settings - avaDir: pathlib path - path to avalanche directory - modName: module - computational module - - Returns - ------- - cfgFiles: list - list of paths to newly generated configuration files for com module inlcuding parameter - variations + """create configuration files with all parameter variations - drawn from full sample + for performing sims with modName comModule + + Parameters + ----------- + cfgProb: configParser object + configuration settings + avaDir: pathlib path + path to avalanche directory + modName: module + computational module + + Returns + ------- + cfgFiles: list + list of paths to newly generated configuration files for com module inlcuding parameter + variations """ # create sample of all parameter variations @@ -89,17 +92,17 @@ def cfgFilesGlobalApproach(avaDir, cfgProb, modName, outDir): # create plot of parameter sample if variation of two parameters for paramValuesD in paramValuesDList: - if 'releaseScenario' in paramValuesD.keys(): - releaseScenario = paramValuesD['releaseScenario'] + if "releaseScenario" in paramValuesD.keys(): + releaseScenario = paramValuesD["releaseScenario"] else: - releaseScenario = '' - plotDir = avaDir / 'Outputs' / 'ana4Stats' / 'plots' - if len(paramValuesD['names']) == 2: + releaseScenario = "" + plotDir = avaDir / "Outputs" / "ana4Stats" / "plots" + if len(paramValuesD["names"]) == 2: sP.plotSample(paramValuesD, plotDir, releaseScenario=releaseScenario) - elif len(paramValuesD['varParNamesInitial']) == 2: + elif len(paramValuesD["varParNamesInitial"]) == 2: sP.plotThSampleFromVals(paramValuesD, plotDir) else: - log.debug('More or less than two parameters have been varied - no plot of sample available') + log.debug("More or less than two parameters have been varied - no plot of sample available") # write cfg files one for each parameter set drawn from full sample cfgFiles = createCfgFiles(paramValuesDList, modName, cfgProb, cfgPath=outDir) @@ -108,23 +111,21 @@ def cfgFilesGlobalApproach(avaDir, cfgProb, modName, outDir): def cfgFilesLocalApproach(variationsDict, cfgProb, modName, outDir): - """ create configuration file for performing sims with modName com module - - Parameters - ----------- - variationsDict: dict - dictionary with for each varName, varVariation, varSteps, and type of variation - cfgProb: configParser object - configuration settings - avaDir: pathlib path - path to avalanche directory - modName: module - computational module - - Returns - ------- - cfgFiles: dict - dictionary of paths to newly generated configuration files for com module for all parameters + """create configuration file for performing sims with modName com module + + Parameters + ----------- + variationsDict: dict + dictionary with for each varName, varVariation, varSteps, and type of variation + cfgProb: configParser object + configuration settings + modName: module + computational module + + Returns + ------- + cfgFiles: dict + dictionary of paths to newly generated configuration files for com module for all parameters """ @@ -133,13 +134,13 @@ def cfgFilesLocalApproach(variationsDict, cfgProb, modName, outDir): # define configuration files # get filename of module modNameString = str(pathlib.Path(modName.__file__).stem) - cfgFile = outDir / ('probRun%sCfg%s.ini' % (modNameString, varName)) + cfgFile = outDir / ("probRun%sCfg%s.ini" % (modNameString, varName)) # use cfgFile, local com module settings or default settings if local not available modCfg = fetchStartCfg(modName, cfgProb) modCfg = updateCfgRange(modCfg, cfgProb, varName, variationsDict[varName]) - with open(cfgFile, 'w') as configfile: + with open(cfgFile, "w") as configfile: modCfg.write(configfile) # append cfgFiles to list cfgFiles.append(cfgFile) @@ -148,28 +149,28 @@ def cfgFilesLocalApproach(variationsDict, cfgProb, modName, outDir): def updateCfgRange(cfg, cfgProb, varName, varDict): - """ update cfg with a range for parameters in cfgProb - - Parameters - ----------- - cfg: configparser object - configuration object to update - cfgProb: configParser object - configparser object with info on update - varName: str - name of parameter used for variation - varDict: dict - dictionary with variationValue and numberOfSteps for varName - - Returns - -------- - cfg: configParser - updated configuration object + """update cfg with a range for parameters in cfgProb + + Parameters + ----------- + cfg: configparser object + configuration object to update + cfgProb: configParser object + configparser object with info on update + varName: str + name of parameter used for variation + varDict: dict + dictionary with variationValue and numberOfSteps for varName + + Returns + -------- + cfg: configParser + updated configuration object """ # set reference values of parameters - override values in com module configurations - varParList = cfgProb['PROBRUN']['varParList'].split('|') + varParList = cfgProb["PROBRUN"]["varParList"].split("|") # also for the other parameters that are varied subsequently # first check if no parameter variation in provided for these parameters in the com module ini # if so - error @@ -177,213 +178,322 @@ def updateCfgRange(cfg, cfgProb, varName, varDict): # this is now done for parameter VARNAME from inputs # get range, steps and reference value of parameter to perform variations - valVariation = varDict['variationValue'] - valSteps = varDict['numberOfSteps'] - valVal = cfg['GENERAL'][varName] - variationType = varDict['variationType'] + valVariation = varDict["variationValue"] + valSteps = varDict["numberOfSteps"] + valVal = cfg["GENERAL"][varName] + variationType = varDict["variationType"] - if variationType.lower() == 'normaldistribution': + if variationType.lower() == "normaldistribution": # get computeFromDistribution configuration and apply override - cfgDist = cfgUtils.getModuleConfig(cP, fileOverride='', modInfo=False, toPrint=False, - onlyDefault=cfgProb['in1Data_computeFromDistribution_override'].getboolean('defaultConfig')) + cfgDist = cfgUtils.getModuleConfig( + cP, + fileOverride="", + modInfo=False, + toPrint=False, + onlyDefault=cfgProb["in1Data_computeFromDistribution_override"].getboolean("defaultConfig"), + ) cfgDist, cfgProb = cfgHandling.applyCfgOverride(cfgDist, cfgProb, cP, addModValues=False) # set variation in configuration - if varName in ['relTh', 'entTh', 'secondaryRelTh']: + if varName in ["relTh", "entTh", "secondaryRelTh"]: # if variation using normal distribution - if variationType.lower() == 'normaldistribution': - parName = varName + 'DistVariation' - if valVariation == '': - valVariation = '-' - parValue = (variationType + '$' - + valSteps + '$' + valVariation + '$' - + cfgDist['GENERAL']['minMaxInterval'] + '$' - + cfgDist['GENERAL']['buildType'] + '$' - + cfgDist['GENERAL']['support']) + if variationType.lower() == "normaldistribution": + parName = varName + "DistVariation" + if valVariation == "": + valVariation = "-" + parValue = ( + variationType + + "$" + + valSteps + + "$" + + valVariation + + "$" + + cfgDist["GENERAL"]["minMaxInterval"] + + "$" + + cfgDist["GENERAL"]["buildType"] + + "$" + + cfgDist["GENERAL"]["support"] + ) # if variation using percent - elif variationType.lower() == 'percent': - parName = varName + 'PercentVariation' - parValue = valVariation + '$' + valSteps + elif variationType.lower() == "percent": + parName = varName + "PercentVariation" + parValue = valVariation + "$" + valSteps # if variation using absolute range - elif variationType.lower() == 'range': - parName = varName + 'RangeVariation' - parValue = valVariation + '$' + valSteps - if 'ci' in valVariation: - message = ('Variation Type: range - variationValue is %s not a valid option - only \ - scalar value allowed or consider variationType rangefromci' % - valVariation) + elif variationType.lower() == "range": + parName = varName + "RangeVariation" + parValue = valVariation + "$" + valSteps + if "ci" in valVariation: + message = ( + "Variation Type: range - variationValue is %s not a valid option - only \ + scalar value allowed or consider variationType rangefromci" + % valVariation + ) log.error(message) raise AssertionError(message) - elif variationType.lower() == 'rangefromci': - parName = varName + 'RangeFromCiVariation' - parValue = valVariation + '$' + valSteps + elif variationType.lower() == "rangefromci": + parName = varName + "RangeFromCiVariation" + parValue = valVariation + "$" + valSteps else: - message = ('Variation Type: %s - not a valid option, options are: percent, range, \ - normaldistribution, rangefromci' % variationType) + message = ( + "Variation Type: %s - not a valid option, options are: percent, range, \ + normaldistribution, rangefromci" + % variationType + ) log.error(message) raise AssertionError(message) # write parameter variation for varName in config file - cfg['GENERAL'][parName] = parValue + cfg["GENERAL"][parName] = parValue else: # set variation - if variationType.lower() == 'normaldistribution': - cfgDist = {'sampleSize': valSteps, 'mean': valVal, - 'buildType': cfgProb['in1Data_computeFromDistribution_override']['buildType'], - 'buildValue': valVariation, - 'minMaxInterval': cfgDist['GENERAL']['minMaxInterval'], - 'support': cfgDist['GENERAL']['support']} + if variationType.lower() == "normaldistribution": + cfgDist = { + "sampleSize": valSteps, + "mean": valVal, + "buildType": cfgProb["in1Data_computeFromDistribution_override"]["buildType"], + "buildValue": valVariation, + "minMaxInterval": cfgDist["GENERAL"]["minMaxInterval"], + "support": cfgDist["GENERAL"]["support"], + } _, valValues, _, _ = cP.extractNormalDist(cfgDist) - cfg['GENERAL'][varName] = dP.writeToCfgLine(valValues) - elif variationType.lower() == 'percent': - cfg['GENERAL'][varName] = '%s$%s$%s' % (valVal, valVariation, valSteps) - valValues = fU.splitIniValueToArraySteps(cfg['GENERAL'][varName]) + cfg["GENERAL"][varName] = dP.writeToCfgLine(valValues) + elif variationType.lower() == "percent": + cfg["GENERAL"][varName] = "%s$%s$%s" % (valVal, valVariation, valSteps) + valValues = fU.splitIniValueToArraySteps(cfg["GENERAL"][varName]) - elif variationType.lower() == 'range': - if '-' in valVariation or '+' in valVariation: + elif variationType.lower() == "range": + if "-" in valVariation or "+" in valVariation: valStart = str(float(valVal) + float(valVariation)) valStop = float(valVal) else: valStart = str(float(valVal) - float(valVariation)) valStop = str(float(valVal) + float(valVariation)) - cfg['GENERAL'][varName] = '%s:%s:%s' % (valStart, valStop, valSteps) + cfg["GENERAL"][varName] = "%s:%s:%s" % (valStart, valStop, valSteps) valValues = np.linspace(float(valStart), float(valStop), int(valSteps)) else: - message = ('Variation Type: %s - not a valid option, options are: percent, range, \ - normaldistribution, rangefromci' % variationType) + message = ( + "Variation Type: %s - not a valid option, options are: percent, range, \ + normaldistribution, rangefromci" + % variationType + ) log.error(message) raise AssertionError(message) # add a scenario Name to VISUALISATION - cfg['VISUALISATION']['scenario'] = varName + cfg["VISUALISATION"]["scenario"] = varName return cfg def checkParameterSettings(cfg, varParList): - """ check if parameter settings in comMod configuration do not inlcude variation for parameters to be varied + """check if parameter settings in comMod configuration do not inlcude variation for parameters to be varied - Parameters - ----------- - cfg: configparser object - configuration settings - varParList: list - list of parameters (names) that shall be varied + Parameters + ----------- + cfg: configparser object + configuration settings + varParList: list + list of parameters (names) that shall be varied """ # set a list of all thickness parameters that are set to be read from shp file thReadFromShp = [] - # loop over all parameters and check if no variation is set and if read from shp for varPar in varParList: - if any(chars in cfg['GENERAL'][varPar] for chars in ['|', '$', ':']): - message = ('Only one reference value is allowed for %s: but %s is given' % - (varPar, cfg['GENERAL'][varPar])) + # Check if valid parameter exists in any section and check for duplicates + _ = checkIfParameterInConfig(cfg, varPar) + + # Fetch section where parameter was found + section = fetchParameterSection(cfg, varPar) + + if any(chars in cfg[section][varPar] for chars in ["|", "$", ":"]): + message = "Only one reference value is allowed for %s: but %s is given" % ( + varPar, + cfg[section][varPar], + ) log.error(message) raise AssertionError(message) - elif varPar in ['entTh', 'relTh', 'secondaryRelTh']: - thFromShp = varPar + 'FromShp' + elif varPar in ["entTh", "relTh", "secondaryRelTh"]: + thFromShp = varPar + "FromShp" # check if reference settings have already variation of varPar - _ = checkForNumberOfReferenceValues(cfg['GENERAL'], varPar) + _ = checkForNumberOfReferenceValues(cfg["GENERAL"], varPar) # check if th read from shp file - if cfg['GENERAL'].getboolean(thFromShp): + if cfg["GENERAL"].getboolean(thFromShp): thReadFromShp.append(varPar) return True, thReadFromShp +def checkIfParameterInConfig(cfg, varPar): + """ + Checks the existence and uniqueness of a parameter within a configuration object. + + This function searches for a specified parameter within a configuration object, ensuring the + parameter exists and is not duplicated across sections. If the parameter is found multiple + times or not found at all, an error is logged. If the parameter is found in only one section, + that section is returned. + + Parameters + ---------- + cfg : configparser object + The configuration object in which to search for the specified parameter. + varPar : str + The name of the parameter to locate within the configuration. + + Returns + ------- + True if the parameter is found and unique. + """ + + # search for parameter in cfg object + matches = cfg | grep(varPar, verbose_level=2) + + # Check if matches is empty + if not matches.get("matched_paths"): + message = "'%s' is not a valid parameter" % varPar + log.error(message) + raise AssertionError(message) + + # Exact key match (case-insensitive) + exactKeyMatch = { + path: val + for path, val in matches["matched_paths"].items() + if path.lower().endswith(f"['{varPar.lower()}']") + } + + # Check for duplicates + if len(exactKeyMatch) != 1: + message = "Parameter '%s' does not uniquely match a single configuration parameter." % varPar + log.error(message) + raise AssertionError(message) + + return True + + +def fetchParameterSection(cfg, parameter): + """Fetch the section name that contains the specified parameter in a configuration file. + + Parameters + ---------- + cfg : configparser object + Configuration settings + parameter : str + Name of the parameter to find + + Returns + ------- + str + Name of the section containing the parameter or None if the parameter is not found. + """ + + for section in cfg.sections(): + if parameter in cfg[section]: + return section + + return None + + def checkForNumberOfReferenceValues(cfgGen, varPar): - """ check if in reference configuration no variation option of varPar is set - if set - throw error + """check if in reference configuration no variation option of varPar is set + if set - throw error - Parameters - ----------- - cfgGen: configparser object - reference configuration settings - varPar: str - name of parameter to be checked + Parameters + ----------- + cfgGen: configparser object + reference configuration settings + varPar: str + name of parameter to be checked """ - thPV = varPar + 'PercentVariation' - thRV = varPar + 'RangeVariation' - thDV = varPar + 'DistVariation' - thRCiV = varPar + 'RangeFromCiVariation' + thPV = varPar + "PercentVariation" + thRV = varPar + "RangeVariation" + thDV = varPar + "DistVariation" + thRCiV = varPar + "RangeFromCiVariation" # check if variation is set - if cfgGen[thPV] != '' or cfgGen[thRV] != '' or cfgGen[thDV] != '' or cfgGen[thRCiV] != '': - message = ('Only one reference value is allowed for %s: but %s %s, %s %s, %s %s, %s %s is given' % - (varPar, thPV, cfgGen[thPV], thRV, cfgGen[thRV], thDV, cfgGen[thDV], thRCiV, cfgGen[thRCiV])) + if cfgGen[thPV] != "" or cfgGen[thRV] != "" or cfgGen[thDV] != "" or cfgGen[thRCiV] != "": + message = "Only one reference value is allowed for %s: but %s %s, %s %s, %s %s, %s %s is given" % ( + varPar, + thPV, + cfgGen[thPV], + thRV, + cfgGen[thRV], + thDV, + cfgGen[thDV], + thRCiV, + cfgGen[thRCiV], + ) log.error(message) raise AssertionError(message) return True -def probAnalysis(avaDir, cfg, modName, parametersDict='', inputDir='', probConf='', simDFActual=''): - """ Compute probability map of a given set of simulation result exceeding a particular threshold and save to outDir - - Parameters - ---------- - avaDir: str - path to avalanche directory - cfg : dict - configuration read from ini file of probAna function - modName - name of computational module that was used to run the simulations - to locate results files and filtering options - parametersDict: dict - dictionary with simulation parameters to filter simulations - only available if modName=com1DFA - inputDir : str - optional - path to directory where data that should be analysed can be found in - a subfolder called peakFiles and configurationFiles, required if not in module results - probConf : str - name of probability configuration - simDFActual: pandas dataFrame - dataframe of simulation configurations that shall be used for prob analysis +def probAnalysis(avaDir, cfg, modName, parametersDict="", inputDir="", probConf="", simDFActual=""): + """Compute probability map of a given set of simulation result exceeding a particular threshold and save to outDir + + Parameters + ---------- + avaDir: str + path to avalanche directory + cfg : dict + configuration read from ini file of probAna function + modName + name of computational module that was used to run the simulations - to locate results files and filtering options + parametersDict: dict + dictionary with simulation parameters to filter simulations - only available if modName=com1DFA + inputDir : str + optional - path to directory where data that should be analysed can be found in + a subfolder called peakFiles and configurationFiles, required if not in module results + probConf : str + name of probability configuration + simDFActual: pandas dataFrame + dataframe of simulation configurations that shall be used for prob analysis """ avaDir = pathlib.Path(avaDir) # set output directory - outDir = avaDir / 'Outputs' / 'ana4Stats' + outDir = avaDir / "Outputs" / "ana4Stats" fU.makeADir(outDir) # fetch all result files and filter simulations according to parametersDict - if modName.lower() == 'com1dfa': + if modName.lower() == "com1dfa": simNameList = cfgHandling.filterSims(avaDir, parametersDict, specDir=inputDir, simDF=simDFActual) filtering = True else: simNameList = [] filtering = False - log.info('No filtering available for this comMod: %s' % modName) + log.info("No filtering available for this comMod: %s" % modName) # initialize flag if analysis has been performed or e.g. no matching files found analysisPerformed = False if simNameList == [] and filtering: # no matching sims found for filtering criteria - log.warning('No matching simulations found for filtering criteria') + log.warning("No matching simulations found for filtering criteria") return analysisPerformed # if matching sims found - perform analysis - if inputDir == '': - inputDir = avaDir / 'Outputs' / modName / 'peakFiles' + if inputDir == "": + inputDir = avaDir / "Outputs" / modName / "peakFiles" peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) else: - inputDirPF = inputDir / 'peakFiles' + inputDirPF = inputDir / "peakFiles" peakFilesDF = fU.makeSimDF(inputDirPF, avaDir=avaDir) if len(peakFilesDF) == 0: - message = 'No peak files found in %s' % str(inputDir) + message = "No peak files found in %s" % str(inputDir) log.error(message) raise FileNotFoundError(message) # get header info from peak files - this should be the same for all peakFiles - header = IOf.readRasterHeader(peakFilesDF['files'][0]) - refData = IOf.readRaster(peakFilesDF['files'][0]) - nRows = header['nrows'] - nCols = header['ncols'] + header = IOf.readRasterHeader(peakFilesDF["files"][0]) + refData = IOf.readRaster(peakFilesDF["files"][0]) + nRows = header["nrows"] + nCols = header["ncols"] # Initialise array for computations probSum = np.zeros((nRows, nCols)) @@ -391,131 +501,143 @@ def probAnalysis(avaDir, cfg, modName, parametersDict='', inputDir='', probConf= contourDict = {} # Loop through peakFiles and compute probability - for m in range(len(peakFilesDF['names'])): - + for m in range(len(peakFilesDF["names"])): # only take simulations that match filter criteria from parametersDict - if (peakFilesDF['simName'][m] in simNameList) or filtering == False: + if (peakFilesDF["simName"][m] in simNameList) or filtering == False: # Load peak field for desired peak field parameter - if peakFilesDF['resType'][m] == cfg['GENERAL']['peakVar']: - + if peakFilesDF["resType"][m] == cfg["GENERAL"]["peakVar"]: # Load data - fileName = peakFilesDF['files'][m] + fileName = peakFilesDF["files"][m] dataLim = np.zeros((nRows, nCols)) fileData = IOf.readRaster(fileName) # check if extent is the same as first loaded dataset # if not - remesh and print warning - if fileData['header']['nrows'] != nRows or fileData['header']['ncols'] != nCols: - log.warning('datasets used to create probMap do not match in extent - remeshing: %s to cellSize %s' % - (fileName, header['cellsize'])) + if fileData["header"]["nrows"] != nRows or fileData["header"]["ncols"] != nCols: + log.warning( + "datasets used to create probMap do not match in extent - remeshing: %s to cellSize %s" + % (fileName, header["cellsize"]) + ) dataRead, _ = gT.resizeData(fileData, refData) else: - dataRead = fileData['rasterData'] + dataRead = fileData["rasterData"] data = np.flipud(dataRead) # fetch contourline info - xGrid, yGrid, _, _ = gT.makeCoordGridFromHeader(refData['header']) - contourDictXY = pU.fetchContourCoords(xGrid, yGrid, fileData['rasterData'], float(cfg['GENERAL']['peakLim'])) + xGrid, yGrid, _, _ = gT.makeCoordGridFromHeader(refData["header"]) + contourDictXY = pU.fetchContourCoords( + xGrid, + yGrid, + fileData["rasterData"], + float(cfg["GENERAL"]["peakLim"]), + ) contourDict[fileName.stem] = contourDictXY - log.info('File Name: %s , simulation parameter %s ' % (fileName, cfg['GENERAL']['peakVar'])) + log.info("File Name: %s , simulation parameter %s " % (fileName, cfg["GENERAL"]["peakVar"])) # Check if peak values exceed desired threshold - dataLim[data > float(cfg['GENERAL']['peakLim'])] = 1.0 + dataLim[data > float(cfg["GENERAL"]["peakLim"])] = 1.0 probSum = probSum + dataLim count = count + 1 # Create probability map ranging from 0-1 probMap = probSum / count - unit = pU.cfgPlotUtils['unit%s' % cfg['GENERAL']['peakVar']] - log.info('probability analysis performed for peak parameter: %s and a peak value ' - 'threshold of: %s %s' % (cfg['GENERAL']['peakVar'], cfg['GENERAL']['peakLim'], unit)) - log.info('%s peak fields added to analysis' % count) + unit = pU.cfgPlotUtils["unit%s" % cfg["GENERAL"]["peakVar"]] + log.info( + "probability analysis performed for peak parameter: %s and a peak value " + "threshold of: %s %s" % (cfg["GENERAL"]["peakVar"], cfg["GENERAL"]["peakLim"], unit) + ) + log.info("%s peak fields added to analysis" % count) # Save to raster file avaName = avaDir.name - outFileName = '%s_prob_%s_%s_lim%s' % (avaName, - probConf, - cfg['GENERAL']['peakVar'], - cfg['GENERAL']['peakLim']) + outFileName = "%s_prob_%s_%s_lim%s" % ( + avaName, + probConf, + cfg["GENERAL"]["peakVar"], + cfg["GENERAL"]["peakLim"], + ) outFile = outDir / outFileName IOf.writeResultToRaster(header, probMap, outFile) - log.info('Prob result written to %s' % outFile) + log.info("Prob result written to %s" % outFile) analysisPerformed = True return analysisPerformed, contourDict def makeDictFromVars(cfg): - """ create a dictionary with info on parameter variation for all parameter in - varParList + """create a dictionary with info on parameter variation for all parameter in + varParList - Parameters - ----------- - cfg: configparser object - configuration settings, here varParList, variationValue, numberOfSteps + Parameters + ----------- + cfg: configparser object + configuration settings, here varParList, variationValue, numberOfSteps - Returns - -------- - variationsDict: dict - dictionary with for each varName, varVariation, varSteps, and type of variation + Returns + -------- + variationsDict: dict + dictionary with for each varName, varVariation, varSteps, and type of variation """ - varParList = cfg['varParList'].split('|') - varParTypes = cfg['varParType'].split('|') - varValues = cfg['variationValue'].split('|') - varSteps = cfg['numberOfSteps'].split('|') - varTypes = cfg['variationType'].split('|') + varParList = cfg["varParList"].split("|") + varValues = cfg["variationValue"].split("|") + varSteps = cfg["numberOfSteps"].split("|") + varTypes = cfg["variationType"].split("|") # check if value is provided for each parameter - if cfg.getint('samplingStrategy') == 1: - lengthsPar = 'varParType' - elif cfg.getint('samplingStrategy') == 2: - lengthsPar = 'numberOfSteps' + if cfg.getint("samplingStrategy") == 1: + lengthsPar = "varParType" + elif cfg.getint("samplingStrategy") == 2: + lengthsPar = "numberOfSteps" else: - message = 'Chosen sampling strategy not valid: options are 1 or 2' + message = "Chosen sampling strategy not valid: options are 1 or 2" log.error(message) raise AssertionError(message) - if (len(varParList) == len(varValues) == len(cfg[lengthsPar].split('|')) == len(varTypes)) is False: - message = ('For every parameter in varParList a variationValue, %s and variationType needs to be provided' % lengthsPar) + if (len(varParList) == len(varValues) == len(cfg[lengthsPar].split("|")) == len(varTypes)) is False: + message = ( + "For every parameter in varParList a variationValue, %s and variationType needs to be provided" + % lengthsPar + ) log.error(message) raise AssertionError(message) # check if correct values provided for rangefromci - rangeFromCi = [idx for idx, v in enumerate(varTypes) if v.lower() == 'rangefromci'] + rangeFromCi = [idx for idx, v in enumerate(varTypes) if v.lower() == "rangefromci"] varValuesRCi = np.asarray(varValues)[rangeFromCi] - ciCheck = [False for ci in varValuesRCi if ci != 'ci95'] + ciCheck = [False for ci in varValuesRCi if ci != "ci95"] if len(ciCheck) > 0: - message = 'If rangefromci is chosen as variation type, ci95 is required as variationValue' + message = "If rangefromci is chosen as variation type, ci95 is required as variationValue" log.error(message) raise AssertionError(message) variationsDict = {} - if cfg.getint('samplingStrategy') == 2: + if cfg.getint("samplingStrategy") == 2: for idx, val in enumerate(varParList): - variationsDict[val] = {'variationValue': varValues[idx], 'numberOfSteps': varSteps[idx], - 'variationType': varTypes[idx]} + variationsDict[val] = { + "variationValue": varValues[idx], + "numberOfSteps": varSteps[idx], + "variationType": varTypes[idx], + } return variationsDict def fetchThicknessInfo(avaDir): - """ Fetch input data for avaDir and thickness info - - Parameters - ------------ - cfg: configparser object - configuration settings - avaDir: pathlib path or str - path to avalanche directory - - Returns - ----------- - inputSimFilesAll: dict - dictionary with info on available input data (release areas, entrainment, and thickness info) + """Fetch input data for avaDir and thickness info + + Parameters + ------------ + avaDir: pathlib path or str + path to avalanche directory + + Returns + ----------- + inputSimFilesAll: dict + dictionary with info on available input data (release areas, entrainment, and thickness info) """ # fetch input data - dem, release-, entrainment- and resistance areas (and secondary release areas) @@ -528,44 +650,44 @@ def fetchThicknessInfo(avaDir): def createSampleFromConfig(avaDir, cfgProb, comMod): - """ Create a sample of parameters for a desired parameter variation, - and draw nSample sets of parameter values - if thickness values read from shp for comMod, convert sample values for these + """Create a sample of parameters for a desired parameter variation, + and draw nSample sets of parameter values + if thickness values read from shp for comMod, convert sample values for these - Parameters - ------------ - avaDir: pathlib path - path to avalanche directory - cfgProb: configparser object - configuration settings for parameter variation - comMod: computational module - module to perform then sims for parameter variation + Parameters + ------------ + avaDir: pathlib path + path to avalanche directory + cfgProb: configparser object + configuration settings for parameter variation + comMod: computational module + module to perform then sims for parameter variation - Returns - -------- - paramValuesDList: list - list of paramValuesD (multiple if multiple release area scenarios) + Returns + -------- + paramValuesDList: list + list of paramValuesD (multiple if multiple release area scenarios) - - names: list, list of parameter names (that are varied) - - values: numpy nd array, as many rows as sets of parameter values and as many rows as parameters - - typeList: list, list of types of parameters (float, ...) - - thFromIni: str, str of parameter names where the base value is read from shape + - names: list, list of parameter names (that are varied) + - values: numpy nd array, as many rows as sets of parameter values and as many rows as parameters + - typeList: list, list of types of parameters (float, ...) + - thFromIni: str, str of parameter names where the base value is read from shape - """ + """ # read initial configuration cfgStart = fetchStartCfg(comMod, cfgProb) # fetch parameter names for parameter variation and variation value and variation type - varParList = cfgProb['PROBRUN']['varParList'].split('|') - valVariationValue = cfgProb['PROBRUN']['variationValue'].split('|') - varType = cfgProb['PROBRUN']['variationType'].split('|') + varParList = cfgProb["PROBRUN"]["varParList"].split("|") + valVariationValue = cfgProb["PROBRUN"]["variationValue"].split("|") + varType = cfgProb["PROBRUN"]["variationType"].split("|") # check if thickness parameters are actually read from shp file _, thReadFromShp = checkParameterSettings(cfgStart, varParList) modNameString = str(pathlib.Path(comMod.__file__).stem) - if modNameString.lower() == "com1dfa": + if modNameString.lower() in ["com1dfa", "com8motpsa"]: # check if thickness parameters are actually read from shp file _, thReadFromShp = checkParameterSettings(cfgStart, varParList) else: @@ -573,18 +695,32 @@ def createSampleFromConfig(avaDir, cfgProb, comMod): # create sets of parameters values for parameter variation if len(thReadFromShp) > 0: - paramValuesDList = createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParList, - valVariationValue, varType, thReadFromShp) + paramValuesDList = createSampleWithVariationForThParameters( + avaDir, + cfgProb, + cfgStart, + varParList, + valVariationValue, + varType, + thReadFromShp, + ) else: - paramValuesD = createSampleWithVariationStandardParameters(cfgProb, cfgStart, varParList, valVariationValue, - varType) + paramValuesD = createSampleWithVariationStandardParameters( + cfgProb, cfgStart, varParList, valVariationValue, varType + ) paramValuesDList = [paramValuesD] + # save dictionary to pickle file + outDir = pathlib.Path(avaDir, "Outputs", "ana4Stats") + fU.makeADir(outDir) + with open(outDir / "paramValuesD.pickle", "wb") as fi: + pickle.dump(paramValuesDList[0], fi) + return paramValuesDList def createSampleWithVariationStandardParameters(cfgProb, cfgStart, varParList, valVariationValue, varType): - """ create a sample for a parameter variation using latin hypercube sampling + """create a sample for a parameter variation using latin hypercube sampling Parameters ------------ @@ -615,16 +751,16 @@ def createSampleWithVariationStandardParameters(cfgProb, cfgStart, varParList, v lowerBounds = [] upperBounds = [] for idx, varPar in enumerate(varParList): - # if parameter value directly set in configuration modify the value directly - varVal = cfgStart['GENERAL'].getfloat(varPar) - if varType[idx].lower() == 'percent': - lB = varVal - varVal * (float(valVariationValue[idx]) / 100.) - uB = varVal + varVal * (float(valVariationValue[idx]) / 100.) - elif varType[idx].lower() == 'range': + section = fetchParameterSection(cfgStart, varPar) + varVal = cfgStart[section].getfloat(varPar) + if varType[idx].lower() == "percent": + lB = varVal - varVal * (float(valVariationValue[idx]) / 100.0) + uB = varVal + varVal * (float(valVariationValue[idx]) / 100.0) + elif varType[idx].lower() == "range": lB = varVal - float(valVariationValue[idx]) uB = varVal + float(valVariationValue[idx]) else: - message = ('Variation method: %s not a valid option' % varType[idx]) + message = "Variation method: %s not a valid option" % varType[idx] log.error(message) raise AssertionError(message) # update bounds @@ -636,45 +772,49 @@ def createSampleWithVariationStandardParameters(cfgProb, cfgStart, varParList, v sampleWBounds = qmc.scale(sample, lowerBounds, upperBounds) # create dictionary with all the info - paramValuesD = {'names': varParList, - 'values': sampleWBounds, - 'typeList': cfgProb['PROBRUN']['varParType'].split('|'), - 'thFromIni': ''} + paramValuesD = { + "names": varParList, + "values": sampleWBounds, + "typeList": cfgProb["PROBRUN"]["varParType"].split("|"), + "thFromIni": "", + } return paramValuesD -def createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParList, valVariationValue, varType, thReadFromShp): - """ Create a sample of parameters for a desired parameter variation, - and fetch thickness values from shp file and perform variation for each feature within - shapefile but treating the features of one shapefile as not-independent +def createSampleWithVariationForThParameters( + avaDir, cfgProb, cfgStart, varParList, valVariationValue, varType, thReadFromShp +): + """Create a sample of parameters for a desired parameter variation, + and fetch thickness values from shp file and perform variation for each feature within + shapefile but treating the features of one shapefile as not-independent - paramsValuesD dict in output list contains + paramsValuesD dict in output list contains - Parameters - ------------ - cfgProb: configparser object - configuration settings for parameter variation - cfgStart: configparser object - configuration settings for comMod without variation values - varParList: list - list of parameters that shall be varied - valVariationValue: list - list if value used for variation - varType: list - list of type of variation for each parameter (percent, range, rangefromci) + Parameters + ------------ + cfgProb: configparser object + configuration settings for parameter variation + cfgStart: configparser object + configuration settings for comMod without variation values + varParList: list + list of parameters that shall be varied + valVariationValue: list + list if value used for variation + varType: list + list of type of variation for each parameter (percent, range, rangefromci) - Returns - -------- - paramValuesDList: list - list of paramValuesD (multiple if multiple release area scenarios) + Returns + -------- + paramValuesDList: list + list of paramValuesD (multiple if multiple release area scenarios) - - names: list, list of parameter names (that are varied) - - values: numpy nd array, as many rows as sets of parameter values and as many rows as parameters - - typeList: list, list of types of parameters (float, ...) - - thFromIni: str, str of parameter names where the base value is read from shape + - names: list, list of parameter names (that are varied) + - values: numpy nd array, as many rows as sets of parameter values and as many rows as parameters + - typeList: list, list of types of parameters (float, ...) + - thFromIni: str, str of parameter names where the base value is read from shape """ @@ -682,7 +822,7 @@ def createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParLi inputSimFiles = fetchThicknessInfo(avaDir) paramValuesDList = [] - for iRel, relF in enumerate(inputSimFiles['relFiles']): + for iRel, relF in enumerate(inputSimFiles["relFiles"]): paramValuesD = {} # create lower and upper bounds for all thickness parameters - taking into account all features fullListOfParameters = [] @@ -692,11 +832,13 @@ def createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParLi ciValues = np.asarray([]) for idx1, varPar in enumerate(varParList): if varPar in thReadFromShp: - ciRequired = varType[idx1].lower() == 'rangefromci' - thV, ciV, thFeatureNames = fetchThThicknessLists(varPar, inputSimFiles, relF, ciRequired=ciRequired) + ciRequired = varType[idx1].lower() == "rangefromci" + thV, ciV, thFeatureNames = fetchThThicknessLists( + varPar, inputSimFiles, relF, ciRequired=ciRequired + ) # add to list all the parameter names fullListOfParameters = fullListOfParameters + thFeatureNames - parentParameterId = parentParameterId + [varParList.index(varPar)]*len(thFeatureNames) + parentParameterId = parentParameterId + [varParList.index(varPar)] * len(thFeatureNames) thValues = np.append(thValues, thV) ciValues = np.append(ciValues, ciV) else: @@ -708,38 +850,67 @@ def createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParLi # initialize lower and upper bounds required to get a sample for the parameter values # numpy arrays required to do masking as lists don't work for a list indices - varValList = np.asarray([cfgStart['GENERAL'].getfloat(varPar) if varPar in staParameter else thValues[idx] for idx, varPar in enumerate(fullListOfParameters)]) - fullValVar = np.asarray([float(valVariationValue[i]) if valVariationValue[i] != 'ci95' else np.nan for i in parentParameterId]) + varValList = np.asarray( + [ + ( + cfgStart[fetchParameterSection(cfgStart, varPar)].getfloat(varPar) + if varPar in staParameter + else thValues[idx] + ) + for idx, varPar in enumerate(fullListOfParameters) + ] + ) + fullValVar = np.asarray( + [ + float(valVariationValue[i]) if valVariationValue[i] != "ci95" else np.nan + for i in parentParameterId + ] + ) fullVarType = np.asarray([varType[i].lower() for i in parentParameterId]) - lowerBounds = np.asarray([None]*len(fullListOfParameters)) - upperBounds = np.asarray([None]*len(fullListOfParameters)) + lowerBounds = np.asarray([None] * len(fullListOfParameters)) + upperBounds = np.asarray([None] * len(fullListOfParameters)) # set lower and upper bounds depending on varType (percent, range, rangefromci) - lowerBounds[fullVarType == 'percent'] = (varValList[fullVarType == 'percent'] - - varValList[fullVarType == 'percent'] * (fullValVar[fullVarType == 'percent'] / 100.)) - upperBounds[fullVarType == 'percent'] = (varValList[fullVarType == 'percent'] + - varValList[fullVarType == 'percent'] * (fullValVar[fullVarType == 'percent'] / 100.)) - - lowerBounds[fullVarType == 'range'] = (varValList[fullVarType == 'range'] - - fullValVar[fullVarType == 'range']) - upperBounds[fullVarType == 'range'] = (varValList[fullVarType == 'range'] + - fullValVar[fullVarType == 'range']) - - lowerBounds[fullVarType == 'rangefromci'] = (varValList[fullVarType == 'rangefromci'] - - ciValues[fullVarType == 'rangefromci']) - upperBounds[fullVarType == 'rangefromci'] = (varValList[fullVarType == 'rangefromci'] + - ciValues[fullVarType == 'rangefromci']) - - # create a sample of parameter values using scipy latin hypercube sampling + lowerBounds[fullVarType == "percent"] = varValList[fullVarType == "percent"] - varValList[ + fullVarType == "percent" + ] * (fullValVar[fullVarType == "percent"] / 100.0) + upperBounds[fullVarType == "percent"] = varValList[fullVarType == "percent"] + varValList[ + fullVarType == "percent" + ] * (fullValVar[fullVarType == "percent"] / 100.0) + + lowerBounds[fullVarType == "range"] = ( + varValList[fullVarType == "range"] - fullValVar[fullVarType == "range"] + ) + upperBounds[fullVarType == "range"] = ( + varValList[fullVarType == "range"] + fullValVar[fullVarType == "range"] + ) + + lowerBounds[fullVarType == "rangefromci"] = ( + varValList[fullVarType == "rangefromci"] - ciValues[fullVarType == "rangefromci"] + ) + upperBounds[fullVarType == "rangefromci"] = ( + varValList[fullVarType == "rangefromci"] + ciValues[fullVarType == "rangefromci"] + ) + + # create a sample of parameter values using scipy latin hypercube or morris sampling sample = createSample(cfgProb, varParList) # create a full sample including those thickness values for the potentially multiple features # however, the thickness values for one parameter (relTh or entTh or secondaryRelTh) should not # be independent for the different features within one parameter - fullSample = np.zeros((int(cfgProb['PROBRUN']['nSample']), len(fullListOfParameters))) + if cfgProb["PROBRUN"]["sampleMethod"] == "morris": + fullSample = np.zeros( + ( + int(cfgProb["PROBRUN"]["nSample"]) * (len(varParList) + 1), + len(fullListOfParameters), + ) + ) + else: + fullSample = np.zeros((int(cfgProb["PROBRUN"]["nSample"]), len(fullListOfParameters))) + for idx, varPar in enumerate(fullListOfParameters): - lB = [0]*len(varParList) - uB = [1]*len(varParList) + lB = [0] * len(varParList) + uB = [1] * len(varParList) lB[parentParameterId[idx]] = lowerBounds[idx] uB[parentParameterId[idx]] = upperBounds[idx] parSample = qmc.scale(sample, lB, uB) @@ -747,13 +918,15 @@ def createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParLi # create dictionary with all the info thFromIni = cfgUtils.convertToCfgList(list(set(varParList).symmetric_difference(set(staParameter)))) - paramValuesD = {'names': fullListOfParameters, - 'values': fullSample, - 'typeList': cfgProb['PROBRUN']['varParType'].split('|'), - 'thFromIni': thFromIni, - 'thVariationBasedOnFromShp': thReadFromShp, - 'varParNamesInitial': varParList, - 'releaseScenario': relF.stem} + paramValuesD = { + "names": fullListOfParameters, + "values": fullSample, + "typeList": cfgProb["PROBRUN"]["varParType"].split("|"), + "thFromIni": thFromIni, + "thVariationBasedOnFromShp": thReadFromShp, + "varParNamesInitial": varParList, + "releaseScenario": relF.stem, + } paramValuesDList.append(paramValuesD) @@ -761,31 +934,47 @@ def createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParLi def createSample(cfgProb, varParList): - """ create a sample of parameters - - Parameters - ----------- - cfgProb: configparser object - configuration settings - varParList: list - list of parameters used for creating a sample - - Returns - -------- - sample: scipy object - sample object of given dimension that can be adjusted to desired bounds + """create a sample of parameters + + Parameters + ----------- + cfgProb: configparser object + configuration settings + varParList: list + list of parameters used for creating a sample + + Returns + -------- + sample: scipy object + sample object of given dimension that can be adjusted to desired bounds """ # random generator initialized with seed - randomGen = np.random.default_rng(cfgProb['PROBRUN'].getint('sampleSeed')) + sampleSeed = cfgProb["PROBRUN"].getint("sampleSeed") + randomGen = np.random.default_rng(sampleSeed) + nTrajectories = cfgProb["PROBRUN"].getint("nSample") + + # create a sample of parameter values using salib morris sampling + if cfgProb["PROBRUN"]["sampleMethod"].lower() == "morris": + param_ranges = { + "num_vars": len(varParList), + "names": varParList, + "bounds": [[0, 1]] * len(varParList), + } + sample = morris.sample( + param_ranges, + N=nTrajectories, # number of trajectories + num_levels=6, # how many discrete values per parameter + seed=sampleSeed, + ) # create a sample of parameter values using scipy latin hypercube sampling - if cfgProb['PROBRUN']['sampleMethod'].lower() == 'latin': + elif cfgProb["PROBRUN"]["sampleMethod"].lower() == "latin": sampler = qmc.LatinHypercube(d=len(varParList), seed=randomGen) - sample = sampler.random(n=int(cfgProb['PROBRUN']['nSample'])) - log.info('Parameter sample created using latin hypercube sampling') + sample = sampler.random(n=int(cfgProb["PROBRUN"]["nSample"])) + log.info("Parameter sample created using latin hypercube sampling") else: - message = ('Sampling method: %s not a valid option' % cfgProb['PROBRUN']['sampleMethod']) + message = "Sampling method: %s not a valid option" % cfgProb["PROBRUN"]["sampleMethod"] log.error(message) raise AssertionError(message) @@ -793,68 +982,72 @@ def createSample(cfgProb, varParList): def fetchThThicknessLists(varPar, inputSimFiles, releaseFile, ciRequired=False): - """ fetch the desired thickness shp file info on thickness, id and ci values - of all available features in shp file - - Parameters - ----------- - varPar: str - name of thickness parameter - inputSimFiles: dict - dictionary with info in input data - ciRequired: bool - if True throw error if ci Values not provided - - Returns - -------- - thicknessFeatureNames: list - list of names of thickness features - thValues: list - list of thickness values for all features - ciValues: list - list of ci values for all feature + """fetch the desired thickness shp file info on thickness, id and ci values + of all available features in shp file + + Parameters + ----------- + varPar: str + name of thickness parameter + inputSimFiles: dict + dictionary with info in input data + ciRequired: bool + if True throw error if ci Values not provided + + Returns + -------- + thicknessFeatureNames: list + list of names of thickness features + thValues: list + list of thickness values for all features + ciValues: list + list of ci values for all feature """ - if varPar == 'relTh': - thFile = [inputSimFiles['relFiles'][idx] for idx, relF in enumerate(inputSimFiles['relFiles']) if relF == releaseFile][0] - elif varPar == 'entTh': - thFile = inputSimFiles['entFile'] - elif varPar == 'secondaryRelTh': - thFile = inputSimFiles['secondaryReleaseFile'] + if varPar == "relTh": + thFile = [ + inputSimFiles["relFiles"][idx] + for idx, relF in enumerate(inputSimFiles["relFiles"]) + if relF == releaseFile + ][0] + elif varPar == "entTh": + thFile = inputSimFiles["entFile"] + elif varPar == "secondaryRelTh": + thFile = inputSimFiles["secondaryReleaseFile"] infoDict = inputSimFiles[thFile.stem] - thicknessFeatureNames = [varPar+str(id) for id in infoDict['id']] - thValues = [float(th) for th in infoDict['thickness']] - ciValues = [float(ci) if ci != 'None' else np.nan for ci in infoDict['ci95']] + thicknessFeatureNames = [varPar + str(id) for id in infoDict["id"]] + thValues = [float(th) for th in infoDict["thickness"]] + ciValues = [float(ci) if ci != "None" else np.nan for ci in infoDict["ci95"]] if np.nan in ciValues and ciRequired: - msg = ('ci95 values required in shape file but not provided for %s' % varPar) + msg = "ci95 values required in shape file but not provided for %s" % varPar log.error(msg) raise AssertionError(msg) return thValues, ciValues, thicknessFeatureNames -def createCfgFiles(paramValuesDList, comMod, cfg, cfgPath=''): - """ create all config files required to run com Module from parameter variations using paramValues +def createCfgFiles(paramValuesDList, comMod, cfg, cfgPath=""): + """create all config files required to run com Module from parameter variations using paramValues - Parameters - ----------- - paramValuesDList: list - list of dictionaries with parameter names and values (array of all sets of parameter values, - one row per value set) - multiple dictionaries if multiple release area scenarios and thFromShp - comMod: com module - computational module - cfg: configparser object - configuration settings - cfgPath: str - path where cfg files should be saved to + Parameters + ----------- + paramValuesDList: list + list of dictionaries with parameter names and values (array of all sets of parameter values, + one row per value set) + multiple dictionaries if multiple release area scenarios and thFromShp + comMod: com module + computational module + cfg: configparser object + configuration settings + cfgPath: str + path where cfg files should be saved to - Returns - -------- - cfgFiles: list - list of cfg file paths for comMod including the updated values of the parameters to vary + Returns + -------- + cfgFiles: list + list of cfg file paths for comMod including the updated values of the parameters to vary """ @@ -867,16 +1060,21 @@ def createCfgFiles(paramValuesDList, comMod, cfg, cfgPath=''): for paramValuesD in paramValuesDList: # read initial configuration cfgStart = fetchStartCfg(comMod, cfg) - for count1, pVal in enumerate(paramValuesD['values']): - for index, par in enumerate(paramValuesD['names']): - cfgStart['GENERAL'][par] = str(pVal[index]) - if modName.lower() == 'com1dfa': - cfgStart['VISUALISATION']['scenario'] = str(count1) - cfgStart['INPUT']['thFromIni'] = paramValuesD['thFromIni'] - if 'releaseScenario' in paramValuesD.keys(): - cfgStart['INPUT']['releaseScenario'] = paramValuesD['releaseScenario'] - cfgF = pathlib.Path(cfgPath, ('%d_%sCfg.ini' % (countS, modName))) - with open(cfgF, 'w') as configfile: + for count1, pVal in enumerate(paramValuesD["values"]): + for index, par in enumerate(paramValuesD["names"]): + section = fetchParameterSection(cfgStart, par) + # If parameter not found in any section, add it to 'GENERAL'. + if section is not None: + cfgStart[section][par] = str(pVal[index]) + else: + cfgStart["GENERAL"][par] = str(pVal[index]) + if modName.lower() == "com1dfa": + cfgStart["VISUALISATION"]["scenario"] = str(count1) + cfgStart["INPUT"]["thFromIni"] = paramValuesD["thFromIni"] + if "releaseScenario" in paramValuesD.keys(): + cfgStart["INPUT"]["releaseScenario"] = paramValuesD["releaseScenario"] + cfgF = pathlib.Path(cfgPath, ("%d_%sCfg.ini" % (countS, modName))) + with open(cfgF, "w") as configfile: cfgStart.write(configfile) # append file path to list of cfg files cfgFiles.append(cfgF) @@ -886,28 +1084,32 @@ def createCfgFiles(paramValuesDList, comMod, cfg, cfgPath=''): def fetchStartCfg(comMod, cfgProb): - """ fetch start configuration of comMod - if onlyDefault use default comModCfg.ini and if false check if there is a local_comModCfg.ini - - Parameters - ----------- - comMod: computational module - module where configuration is read from - cfgProb: configparser object - configuration settings of probAna with collection_comMod_override section - - Returns - -------- - cfgStart: configparser object - configuration object of comMod + """fetch start configuration of comMod + if onlyDefault use default comModCfg.ini and if false check if there is a local_comModCfg.ini + + Parameters + ----------- + comMod: computational module + module where configuration is read from + cfgProb: configparser object + configuration settings of probAna with collection_comMod_override section + + Returns + -------- + cfgStart: configparser object + configuration object of comMod """ # get filename of module modName = str(pathlib.Path(comMod.__file__).stem) modP = (pathlib.Path(comMod.__file__).resolve().parent).stem # fetch comMod config - cfgStart = cfgUtils.getModuleConfig(comMod, fileOverride='', toPrint=False, - onlyDefault=cfgProb['%s_%s_override' % (modP, modName)].getboolean('defaultConfig')) + cfgStart = cfgUtils.getModuleConfig( + comMod, + fileOverride="", + toPrint=False, + onlyDefault=cfgProb["%s_%s_override" % (modP, modName)].getboolean("defaultConfig"), + ) # override with parameters set in in the cfgProb comMod_override section cfgStart, cfgProb = cfgHandling.applyCfgOverride(cfgStart, cfgProb, comMod, addModValues=False) @@ -916,28 +1118,30 @@ def fetchStartCfg(comMod, cfgProb): def fetchProbConfigs(cfg): - """ fetch configurations of prob run in order to filter simulations - e.g. to create probability maps for different scenarios - - Parameters - ----------- - cfg: configparser object - configuration setting, here used: samplingStrategy, varParList - - Returns - -------- - probConfigs: dict - dictionary with one key per config and a dict per key with parameter and value + """fetch configurations of prob run in order to filter simulations + e.g. to create probability maps for different scenarios + + Parameters + ----------- + cfg: configparser object + configuration setting, here used: samplingStrategy, varParList + + Returns + -------- + probConfigs: dict + dictionary with one key per config and a dict per key with parameter and value """ - probConfigs = {'includeAll': {}} + probConfigs = {"includeAll": {}} - if cfg.getint('samplingStrategy') == 2: - for par in cfg['varParList'].split('|'): - probConfigs['include' + par] = {'scenario': par} - log.info('Probability maps are created for full parameter variation and for %s separately' % - cfg['varParList']) + if cfg.getint("samplingStrategy") == 2: + for par in cfg["varParList"].split("|"): + probConfigs["include" + par] = {"scenario": par} + log.info( + "Probability maps are created for full parameter variation and for %s separately" + % cfg["varParList"] + ) else: - log.info('Probability map is created for full parameter variation') + log.info("Probability map is created for full parameter variation") return probConfigs diff --git a/avaframe/ana4Stats/probAnaCfg.ini b/avaframe/ana4Stats/probAnaCfg.ini index adb255be1..9f2d67f25 100644 --- a/avaframe/ana4Stats/probAnaCfg.ini +++ b/avaframe/ana4Stats/probAnaCfg.ini @@ -38,9 +38,9 @@ samplingStrategy = 1 # #++++++VARIATION INFO FOR DRAW SAMPLES FROM FULL SET OF VARIATIONS # type of parameters that shall be varied -separated by | (options: float) varParType = float|float -# factor used to create the number of samples +# factor used to create the number of samples, if morris number of samples depends on number of varied variables and number of trajectories, for now use nSample as number of trajectories nSample = 40 -# sample method used to create sample (options: latin) +# sample method used to create sample (options: latin, morris) sampleMethod = latin # seed for random generator sampleSeed = 12345 @@ -74,6 +74,12 @@ frictModel = samosAT defaultConfig = True +[com8MoTPSA_com8MoTPSA_override] +# use default com1DFA config as base configuration (True) and override following parameters +# if False and local is available use local +defaultConfig = True + + [in1Data_computeFromDistribution_override] # use default config as base configuration (True) and override following parameters # if False and local is available use local diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index dbfa58313..8750f7822 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -63,7 +63,7 @@ debugPlot = cfgAVA["FLAGS"].getboolean("debugPlot") -def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo): +def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo, module=com1DFA): """preprocess information from configuration, read input data and gather into inputSimFiles, create one config object for each of all desired simulations, create dataFrame with one line per simulations of already existing sims in avalancheDir @@ -78,6 +78,8 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo): path to configuration file if overwrite is desired - optional if not local (if available) or default configuration will be loaded if cfgInfo is a configparser object take this as initial config + module: module + module to be used for task (optional) Returns -------- @@ -93,17 +95,17 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo): # read initial configuration if typeCfgInfo in ["cfgFromFile", "cfgFromDefault"]: - cfgStart = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgInfo, toPrint=False) + cfgStart = cfgUtils.getModuleConfig(module, fileOverride=cfgInfo, toPrint=False) elif typeCfgInfo == "cfgFromObject": cfgStart = cfgInfo # fetch input data and create work and output directories inputSimFilesAll, outDir, simDFExisting, simNameExisting = com1DFATools.initializeInputs( - avalancheDir, cfgStart["GENERAL"].getboolean("cleanRemeshedRasters") + avalancheDir, cfgStart["GENERAL"].getboolean("cleanRemeshedRasters"), module ) # create dictionary with one key for each simulation that shall be performed - simDict = dP.createSimDict(avalancheDir, com1DFA, cfgStart, inputSimFilesAll, simNameExisting) + simDict = dP.createSimDict(avalancheDir, module, cfgStart, inputSimFilesAll, simNameExisting) return simDict, outDir, inputSimFilesAll, simDFExisting @@ -1329,7 +1331,6 @@ def initializeParticles(cfg, releaseLine, dem, inputSimLines="", logName="", rel particles["idFixed"] = idFixed # initialize enthalpy particles["totalEnthalpy"] = TIni * cpIce + gravAcc * particles["z"] - particles["massPerPart"] = massPerPart particles["mTot"] = np.sum(particles["m"]) particles["tPlot"] = 0 @@ -2739,7 +2740,7 @@ def exportFields( ) -def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting=""): +def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting="", module=com1DFA): """Prepare a dictionary with simulations that shall be run with varying parameters following the variation dict Parameters @@ -2753,6 +2754,8 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting simNameExisting: list list of simulation names that already exist (optional). If provided, only carry on simulations that do not exist + module: module + module to be used for task (optional) Returns ------- @@ -2761,6 +2764,9 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting simType and contains full configuration configparser object for simulation run """ + # extract the name of the module + modName = module.__name__.split(".")[-1] + # get list of simulation types that are desired if "simTypeList" in variationDict: simTypeList = variationDict["simTypeList"] @@ -2843,11 +2849,15 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting # check if DEM in Inputs has desired mesh size pathToDem = dP.checkRasterMeshSize(cfgSim, inputSimFiles["demFile"], "DEM") cfgSim["INPUT"]["DEM"] = pathToDem - if ( - cfgSim["GENERAL"]["relThFromFile"] == "True" - or cfgSim["GENERAL"]["frictModel"].lower() == "spatialvoellmy" - ): - dem = IOf.readRaster(pathlib.Path(cfgSim["GENERAL"]["avalancheDir"], "Inputs", pathToDem)) + if modName == "com1DFA": + if ( + cfgSim["GENERAL"]["relThFromFile"] == "True" + or cfgSim["GENERAL"]["frictModel"].lower() == "spatialvoellmy" + ): + dem = IOf.readRaster(pathlib.Path(cfgSim["GENERAL"]["avalancheDir"], "Inputs", pathToDem)) + elif modName == "com8MoTPSA": + if cfgSim["GENERAL"]["relThFromFile"] == "True": + dem = IOf.readRaster(pathlib.Path(cfgSim["GENERAL"]["avalancheDir"], "Inputs", pathToDem)) # check if RELTH in Inputs has desired mesh size if cfgSim["GENERAL"]["relThFromFile"] == "True": @@ -2856,15 +2866,16 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting else: cfgSim["INPUT"]["relThFile"] = "" - # check if spatialVoellmy is chosen that friction fields have correct extent - if cfgSim["GENERAL"]["frictModel"].lower() == "spatialvoellmy": - for fric in ["mu", "xi"]: - pathToFric = dP.checkExtentAndCellSize(cfgSim, inputSimFiles["%sFile" % fric], dem, fric) - cfgSim["INPUT"]["%sFile" % fric] = pathToFric + if modName == "com1DFA": + # check if spatialVoellmy is chosen that friction fields have correct extent + if cfgSim["GENERAL"]["frictModel"].lower() == "spatialvoellmy": + for fric in ["mu", "xi"]: + pathToFric = dP.checkExtentAndCellSize(cfgSim, inputSimFiles["%sFile" % fric], dem, fric) + cfgSim["INPUT"]["%sFile" % fric] = pathToFric - # add info about dam file path to the cfg - if cfgSim["GENERAL"]["dam"] == "True" and inputSimFiles["damFile"] is not None: - cfgSim["INPUT"]["DAM"] = str(pathlib.Path("DAM", inputSimFiles["damFile"].name)) + # add info about dam file path to the cfg + if cfgSim["GENERAL"]["dam"] == "True" and inputSimFiles["damFile"] is not None: + cfgSim["INPUT"]["DAM"] = str(pathlib.Path("DAM", inputSimFiles["damFile"].name)) # add info about entrainment file path to the cfg if "ent" in row._asdict()["simTypeList"] and inputSimFiles["entFile"] is not None: @@ -2878,23 +2889,35 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting cfgSim = dP.appendShpThickness(cfgSim) # check differences to default and add indicator to name - defID, _ = com1DFATools.compareSimCfgToDefaultCfgCom1DFA(cfgSim) + defID, _ = com1DFATools.compareSimCfgToDefaultCfgCom1DFA(cfgSim, module) - # if frictModel is samosATAuto compute release vol - if cfgSim["GENERAL"]["frictModel"].lower() == "samosatauto": - pathToDemFull = pathlib.Path(cfgSim["GENERAL"]["avalancheDir"], "Inputs", pathToDem) - relVolume = fetchRelVolume(rel, cfgSim, pathToDemFull, inputSimFiles["secondaryReleaseFile"]) - else: - relVolume = "" + # predefine different size classification indices + frictIndi = None + volIndi = None + + pathToDemFull = pathlib.Path(cfgSim["GENERAL"]["avalancheDir"], "Inputs", pathToDem) + + if modName == "com1DFA": + # if frictModel is samosATAuto compute release vol + if cfgSim["GENERAL"]["frictModel"].lower() == "samosatauto": + relVolume = fetchRelVolume(rel, cfgSim, pathToDemFull, inputSimFiles["secondaryReleaseFile"]) + else: + relVolume = "" + + # check sphKernelRadius setting + cfgSim = checkCfg.checkCellSizeKernelRadius(cfgSim) - # check sphKernelRadius setting - cfgSim = checkCfg.checkCellSizeKernelRadius(cfgSim) + # only keep friction model parameters that are used + cfgSim = checkCfg.checkCfgFrictionModel(cfgSim, inputSimFiles, relVolume=relVolume) - # only keep friction model parameters that are used - cfgSim = checkCfg.checkCfgFrictionModel(cfgSim, inputSimFiles, relVolume=relVolume) + # set frictModelIndicator, this needs to happen AFTER checkCfgFrictModel + frictIndi = com1DFATools.setFrictTypeIndicator(cfgSim) + + elif modName == "com8MoTPSA": + relVolume = fetchRelVolume(rel, cfgSim, pathToDemFull, inputSimFiles["secondaryReleaseFile"]) - # set frictModelIndicator, this needs to happen AFTER checkCfgFrictModel - frictIndi = com1DFATools.setFrictTypeIndicator(cfgSim) + # set Volume class identificator + volIndi = setVolumeIndicator(cfgSim, relVolume) # convert back to configParser object cfgSimObject = cfgUtils.convertDictToConfigParser(cfgSim) @@ -2908,7 +2931,7 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting relNameSim, simHash, defID, - frictIndi, + frictIndi or volIndi, row._asdict()["simTypeList"], cfgSim["GENERAL"]["modelType"], ], @@ -2924,18 +2947,21 @@ def prepareVarSimDict(standardCfg, inputSimFiles, variationDict, simNameExisting "relFile": rel, "cfgSim": cfgSimObject, } - # write configuration file - cfgUtils.writeCfgFile( - cfgSimObject["GENERAL"]["avalancheDir"], - com1DFA, - cfgSimObject, - fileName=simName, - ) + if modName == "com1DFA": + # write configuration file, dont need to write cfg file for com8MoTPSA (does this later when creating rcf file) + cfgUtils.writeCfgFile( + cfgSimObject["GENERAL"]["avalancheDir"], + com1DFA, + cfgSimObject, + fileName=simName, + ) else: log.warning("Simulation %s already exists, not repeating it" % simName) log.info("Done preparing variations -----") + # TODO: maybe treat this in some other way, i.e. adding an "finalDEM" or similar inputSimFiles.pop("demFile") + inputSimFiles["demFile"] = pathToDemFull return simDict @@ -3192,6 +3218,32 @@ def initializeRelVol(cfg, demVol, releaseFile, radius, releaseType="primary"): return relVolume +def setVolumeIndicator(simCfg, relVolume): + """Sets the Volume indicator for the simname based on the threshold defined in ini file + + Parameters + ----------- + simCfg: dict + simulation configuration + relVolume: float + Volume of the release area in m^3 + + Returns + -------- + VolIndi: str + S, M or L + + """ + if relVolume < float(simCfg["GENERAL"]["volClassSmall"]): + volIndi = "S" + elif relVolume > float(simCfg["GENERAL"]["volClassMedium"]): + volIndi = "L" + else: + volIndi = "M" + + return volIndi + + def saveContToPickle(contourDictXY, outDir, cuSimName): """save contourline x, y coordinates dictionary to a pickle diff --git a/avaframe/com1DFA/com1DFATools.py b/avaframe/com1DFA/com1DFATools.py index 46c570ee6..125deac61 100644 --- a/avaframe/com1DFA/com1DFATools.py +++ b/avaframe/com1DFA/com1DFATools.py @@ -98,7 +98,7 @@ def setFrictTypeIndicator(simCfg): return frictTypeIdentifier -def compareSimCfgToDefaultCfgCom1DFA(simCfg): +def compareSimCfgToDefaultCfgCom1DFA(simCfg, module=com1DFA): """Compares the given simulation configuration (as dict) to the default com1DFA configuration. Disregards values like avalancheDir that are expected to change. Returns True if it is the default + an identifier string: D = Default and @@ -108,6 +108,8 @@ def compareSimCfgToDefaultCfgCom1DFA(simCfg): ----------- simCfg: dict simulation configuration + module: module + module to be used for task (optional) Returns -------- @@ -116,6 +118,9 @@ def compareSimCfgToDefaultCfgCom1DFA(simCfg): """ + # extract the name of the module + modName = module.__name__.split(".")[-1] + log.info("Comparing simCfg to default cfg") defaultIdentifierString = "D" @@ -152,9 +157,10 @@ def compareSimCfgToDefaultCfgCom1DFA(simCfg): # sphKernelSize is set during runtime, make sure it is not reported # as changed if default is set to meshCellSize - if defCfg["GENERAL"]["sphKernelRadius"] == "meshCellSize": - if simCfg["GENERAL"]["sphKernelRadius"] == simCfg["GENERAL"]["meshCellSize"]: - excludeItems.append("root['GENERAL']['sphKernelRadius']") + if modName == "com1DFA": + if defCfg["GENERAL"]["sphKernelRadius"] == "meshCellSize": + if simCfg["GENERAL"]["sphKernelRadius"] == simCfg["GENERAL"]["meshCellSize"]: + excludeItems.append("root['GENERAL']['sphKernelRadius']") # do the diff and analyse # this is the deepdiff > 8.0 version @@ -243,7 +249,7 @@ def _cleanDiffKey(keyString): return keyStr -def createSimDictFromCfgs(cfgMain, cfgPath): +def createSimDictFromCfgs(cfgMain, cfgPath, module=com1DFA): """From multiple cfg files create a simDict with one item for each simulation to perform within these cfg files still parameter variations are allowed @@ -253,6 +259,8 @@ def createSimDictFromCfgs(cfgMain, cfgPath): main configuration of AvaFrame cfgPath: pathlib Path or str path to directory with cfg files + module: module + module to be used for task (optional) Returns -------- @@ -266,7 +274,7 @@ def createSimDictFromCfgs(cfgMain, cfgPath): # fetch input data and create work and output directories # TODO: so for now remeshed dir is cleaned before a run - inputSimFilesAll, outDir, simDFExisting, simNameExisting = initializeInputs(avalancheDir, True) + inputSimFilesAll, outDir, simDFExisting, simNameExisting = initializeInputs(avalancheDir, True, module) # save dem file path as it is deleted from input sim files dict once it is set in the config demFile = inputSimFilesAll["demFile"] @@ -287,12 +295,12 @@ def createSimDictFromCfgs(cfgMain, cfgPath): # loop over all cfgFiles and create simDict for index, cfgFile in enumerate(cfgFilesAll): # read configuration - cfgFromFile = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgFile, toPrint=False) + cfgFromFile = cfgUtils.getModuleConfig(module, fileOverride=cfgFile, toPrint=False) # create dictionary with one key for each simulation that shall be performed # NOTE: sims that are added don't need to be added to the simNameExisting list as # if new identical sims are added the simDict entry is just updated and not a duplicate one added - simDict = dP.createSimDict(avalancheDir, com1DFA, cfgFromFile, inputSimFilesAll, simNameExisting) + simDict = dP.createSimDict(avalancheDir, module, cfgFromFile, inputSimFilesAll, simNameExisting) simDictAll.update(simDict) # reset dem file @@ -301,7 +309,7 @@ def createSimDictFromCfgs(cfgMain, cfgPath): return simDictAll, inputSimFilesAll, simDFExisting, outDir -def initializeInputs(avalancheDir, cleanRemeshedRasters): +def initializeInputs(avalancheDir, cleanRemeshedRasters, module=com1DFA): """Create work and output directories, fetch input files and thickness info If cleanRemeshedRasters is true, the remesh folder contents will be deleted at the beginning @@ -311,6 +319,8 @@ def initializeInputs(avalancheDir, cleanRemeshedRasters): to avalanche directory cleanRemeshedRasters: bool flag if the remesh directory shall be cleaned + module: module + module to be used for task (optional) Returns -------- @@ -320,8 +330,8 @@ def initializeInputs(avalancheDir, cleanRemeshedRasters): path to store outputs """ - # fetch name of module - modName = str(pathlib.Path(com1DFA.__file__).stem) + # extract the name of the module + modName = module.__name__.split(".")[-1] # Create output and work directories _, outDir = inDirs.initialiseRunDirs(avalancheDir, modName, cleanRemeshedRasters) diff --git a/avaframe/com1DFA/deriveParameterSet.py b/avaframe/com1DFA/deriveParameterSet.py index 2fbaefc53..84a8bd27d 100644 --- a/avaframe/com1DFA/deriveParameterSet.py +++ b/avaframe/com1DFA/deriveParameterSet.py @@ -16,6 +16,7 @@ from avaframe.in1Data import getInput as gI from avaframe.in3Utils import cfgUtils from avaframe.in3Utils import geoTrans +import avaframe.com1DFA.com1DFA as com1DFA log = logging.getLogger(__name__) @@ -1034,14 +1035,14 @@ def writeToCfgLine(values): return valString -def createSimDict(avalancheDir, com1DFA, cfgInitial, inputSimFiles, simNameExisting): +def createSimDict(avalancheDir, module, cfgInitial, inputSimFiles, simNameExisting): """Create a simDict with all the simulations that shall be performed Parameters ----------- avalancheDir: pathlib path path to avalanche directory - com1DFA: module + module: module computational module cfgStart: configparser object configuration settings for com1DFA @@ -1068,19 +1069,23 @@ def createSimDict(avalancheDir, com1DFA, cfgInitial, inputSimFiles, simNameExist # create a dictionary with information on which parameter shall be varied for individual simulations # compare cfgStart to default module config for this - modCfg, variationDict = getParameterVariationInfo(avalancheDir, com1DFA, cfgInitial) + modCfg, variationDict = getParameterVariationInfo(avalancheDir, module, cfgInitial) # create a configuration object per simulation to run (from configuration) gathered in simDict # only new simulations are included in this simDict # key is simName and corresponds to one simulation simDict = {} simDict = com1DFA.prepareVarSimDict( - modCfg, inputSimFiles, variationDict, simNameExisting=simNameExisting + modCfg, + inputSimFiles, + variationDict, + simNameExisting=simNameExisting, + module=module, ) # write full configuration (.ini file) to file date = datetime.today() fileName = "sourceConfiguration_" + "{:%d_%m_%Y_%H_%M_%S}".format(date) - cfgUtils.writeCfgFile(avalancheDir, com1DFA, modCfg, fileName=fileName) + cfgUtils.writeCfgFile(avalancheDir, module, modCfg, fileName=fileName) return simDict diff --git a/avaframe/com8MoTPSA/README_dev.md b/avaframe/com8MoTPSA/README_dev.md new file mode 100644 index 000000000..7f5454141 --- /dev/null +++ b/avaframe/com8MoTPSA/README_dev.md @@ -0,0 +1,7 @@ +# com8MoTPSA + +This is the initial development / prototype stage of including control of MoTPSA in AvaFrame. + +For now you need a compiled binary, available from ??? + +This binary needs to live in the avaframe/com8MoTPSA directory diff --git a/avaframe/com8MoTPSA/__init__.py b/avaframe/com8MoTPSA/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/avaframe/com8MoTPSA/com8MoTPSA.py b/avaframe/com8MoTPSA/com8MoTPSA.py new file mode 100644 index 000000000..b44c5f0c9 --- /dev/null +++ b/avaframe/com8MoTPSA/com8MoTPSA.py @@ -0,0 +1,238 @@ +import os +import sys +import platform +import logging +import numpy as np +import pathlib +import time +import shutil + +from avaframe.in3Utils.cfgUtils import cfgToRcf + +if os.name == "nt": + from multiprocessing.pool import ThreadPool as Pool +elif platform.system() == "Darwin": + from multiprocessing.pool import ThreadPool as Pool +else: + from multiprocessing import Pool + +import avaframe.com1DFA.com1DFA as com1DFA +from avaframe.in3Utils import cfgUtils +from avaframe.in2Trans import rasterUtils as rU +from avaframe.com1DFA import particleInitialisation as pI +from avaframe.in1Data import getInput as gI +import avaframe.in3Utils.geoTrans as geoTrans +import avaframe.in3Utils.fileHandlerUtils as fU +from avaframe.out1Peak import outPlotAllPeak as oP +from avaframe.in3Utils.MoTUtils import rewriteDEMtoZeroValues, runAndCheckMoT, MoTGenerateConfigs + +# create local logger +log = logging.getLogger(__name__) + + +def com8MoTPSAMain(cfgMain, cfgInfo=None): + # Get all necessary information from the configuration files + currentModule = sys.modules[__name__] + simDict, inputSimFiles = MoTGenerateConfigs(cfgMain, cfgInfo, currentModule) + + # convert DEM from nan to 0 values + # TODO: suggest MoT-PSA to handle nan values + rewriteDEMtoZeroValues(inputSimFiles["demFile"]) + + log.info("The following simulations will be performed") + for key in simDict: + log.info("Simulation: %s" % key) + + # Preprocess the simulations, mainly creating the rcf files + rcfFiles = com8MoTPSAPreprocess(simDict, inputSimFiles, cfgMain) + + # And now we run the simulations + startTime = time.time() + + log.info("--- STARTING (potential) PARALLEL PART ----") + + # Get number of CPU Cores wanted + nCPU = cfgUtils.getNumberOfProcesses(cfgMain, len(rcfFiles)) + + # Create parallel pool and run + # with multiprocessing.Pool(processes=nCPU) as pool: + with Pool(processes=nCPU) as pool: + results = pool.map(com8MoTPSATask, rcfFiles) + pool.close() + pool.join() + + timeNeeded = "%.2f" % (time.time() - startTime) + log.info("Overall (parallel) com8MoTPSA computation took: %s s " % timeNeeded) + log.info("--- ENDING (potential) PARALLEL PART ----") + + # Postprocess the simulations + com8MoTPSAPostprocess(simDict, cfgMain, inputSimFiles) + + +def com8MoTPSAPostprocess(simDict, cfgMain, inputSimFiles): + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + # Copy max files to output directory + + outputDir = pathlib.Path(avalancheDir) / "Outputs" / "com8MoTPSA" + outputDirPeakFile = pathlib.Path(avalancheDir) / "Outputs" / "com8MoTPSA" / "peakFiles" + fU.makeADir(outputDirPeakFile) + + for key in simDict: + workDir = pathlib.Path(avalancheDir) / "Work" / "com8MoTPSA" / str(key) + + # Copy DataTime.txt + dataTimeFile = workDir / "DataTime.txt" + shutil.copy2(dataTimeFile, outputDir / (str(key) + "_DataTime.txt")) + + # TODO: functionize it + # Copy ppr files + pprFiles = list(workDir.glob("*p?_max*")) + targetFiles = [ + pathlib.Path(str(f.name).replace("null_psa_p1_max", "null_dfa_ppr")) for f in pprFiles + ] + targetFiles = [pathlib.Path(str(f).replace("null_psa_p2_max", "null_psa_ppr")) for f in targetFiles] + targetFiles = [outputDirPeakFile / f for f in targetFiles] + for source, target in zip(pprFiles, targetFiles): + shutil.copy2(source, target) + + # Copy pfd files + pfdFiles = list(workDir.glob("*h?_max*")) + targetFiles = [ + pathlib.Path(str(f.name).replace("null_psa_h1_max", "null_dfa_pfd")) for f in pfdFiles + ] + targetFiles = [pathlib.Path(str(f).replace("null_psa_h2_max", "null_psa_pfd")) for f in targetFiles] + targetFiles = [outputDirPeakFile / f for f in targetFiles] + for source, target in zip(pfdFiles, targetFiles): + shutil.copy2(source, target) + + # Copy pfv files + pfvFiles = list(workDir.glob("*s?_max*")) + targetFiles = [ + pathlib.Path(str(f.name).replace("null_psa_s1_max", "null_dfa_pfv")) for f in pfvFiles + ] + targetFiles = [pathlib.Path(str(f).replace("null_psa_s2_max", "null_psa_pfv")) for f in targetFiles] + targetFiles = [outputDirPeakFile / f for f in targetFiles] + for source, target in zip(pfvFiles, targetFiles): + shutil.copy2(source, target) + + # create plots and report + modName = __name__.split(".")[-1] + reportDir = pathlib.Path(avalancheDir, "Outputs", modName, "reports") + fU.makeADir(reportDir) + print(inputSimFiles["demFile"]) + + dem = rU.readRaster(inputSimFiles["demFile"]) + # Generate plots for all peakFiles + oP.plotAllPeakFields(avalancheDir, cfgMain["FLAGS"], modName, demData=dem) + + +def com8MoTPSATask(rcfFile): + # TODO: Obvious... + os.chdir(os.path.dirname(os.path.abspath(__file__))) + command = ["./MoT-PSA", rcfFile] + # command = ['/home/felix/Versioning/AvaFrame/avaframe/com8MoTPSA/MoT-PSA', rcfFile] + log.info("Run simulation: %s" % rcfFile) + runAndCheckMoT(command) + return command + + +def com8MoTPSAPreprocess(simDict, inputSimFiles, cfgMain): + # Load avalanche directory from general configuration file + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + + workDir = pathlib.Path(avalancheDir) / "Work" / "com8MoTPSA" + cfgFileDir = pathlib.Path(avalancheDir) / "Outputs" / "com8MoTPSA" / "configurationFiles" + fU.makeADir(cfgFileDir) + rcfFiles = list() + + for key in simDict: + # Generate command and run via subprocess.run + # Configuration that needs adjustment + + # load configuration object for current sim + cfg = simDict[key]["cfgSim"] + + # convert release shape to raster with values for current sim + # select release area input data according to chosen release scenario + inputSimFiles = gI.selectReleaseFile(inputSimFiles, cfg["INPUT"]["releaseScenario"]) + # create required input from input files + demOri, inputSimLines = com1DFA.prepareInputData(inputSimFiles, cfg) + + if cfg["GENERAL"].getboolean("iniStep"): + # append buffered release Area + inputSimLines = pI.createReleaseBuffer(cfg, inputSimLines) + + # set thickness values for the release area, entrainment and secondary release areas + relName, inputSimLines, badName = com1DFA.prepareReleaseEntrainment( + cfg, inputSimFiles["releaseScenario"], inputSimLines + ) + + releaseLine = inputSimLines["releaseLine"] + # check if release features overlap between features + # TODO: split releaseheight -> question NGI + dem = rU.readRaster(inputSimFiles["demFile"]) + dem["originalHeader"] = dem["header"].copy() + # releaseLine = geoTrans.prepareArea(releaseLine, dem, np.sqrt(2), combine=True, checkOverlap=False) + if len(inputSimLines["relThField"]) == 0: + # if no release thickness field or function - set release according to shapefile or ini file + # this is a list of release rasters that we want to combine + releaseLine = geoTrans.prepareArea( + releaseLine, + dem, + np.sqrt(2), + thList=releaseLine["thickness"], + combine=True, + checkOverlap=False, + ) + releaseField = releaseLine["rasterData"] + else: + # if relTh provided - set release thickness with field or function + releaseLine = geoTrans.prepareArea( + releaseLine, dem, np.sqrt(2), combine=True, checkOverlap=False + ) + relRasterPoly = releaseLine["rasterData"].copy() + releaseRelThCombined = np.where(relRasterPoly > 0, inputSimLines["relThField"], 0) + releaseField = releaseRelThCombined + + # Generate the work and data dirs for the current simHash + cuWorkDir = workDir / key + workInputDir = cuWorkDir / "Input" + workOutputDir = cuWorkDir / key + + fU.makeADir(cuWorkDir) + fU.makeADir(workInputDir) + + zeroRaster = np.full_like(releaseLine["rasterData"], 0) + + releaseL1 = workInputDir / "releaseLayer1" + releaseL2 = workInputDir / "releaseLayer2" + bedDepth = workInputDir / "dummyBedDepth" + bedDepo = workInputDir / "dummyBedDepo" + bedShear = workInputDir / "dummyBedShear" + rU.writeResultToRaster(dem["header"], releaseField, releaseL1, flip=True) + rU.writeResultToRaster(dem["header"], zeroRaster, releaseL2, flip=True) + rU.writeResultToRaster(dem["header"], zeroRaster, bedDepth) + rU.writeResultToRaster(dem["header"], zeroRaster, bedDepo) + rU.writeResultToRaster(dem["header"], zeroRaster, bedShear) + + # set configuration for MoT-PSA + cfg["Run information"]["Area of Interest"] = cfgMain["MAIN"]["avalancheDir"] + cfg["Run information"]["UTM zone"] = "32N" + cfg["Run information"]["EPSG geodetic datum code"] = "31287" + cfg["Run information"]["Run name"] = cfgMain["MAIN"]["avalancheDir"] + cfg["File names"]["Grid filename"] = str(inputSimFiles["demFile"]) + cfg["File names"]["Release depth 1 filename"] = str(releaseL1) + ".asc" + cfg["File names"]["Release depth 2 filename"] = str(releaseL2) + ".asc" + cfg["File names"]["Bed depth filename"] = str(bedDepth) + ".asc" + cfg["File names"]["Bed deposition filename"] = str(bedDepo) + ".asc" + cfg["File names"]["Bed shear strength filename"] = str(bedShear) + ".asc" + cfg["File names"]["Output filename root"] = str(workOutputDir) + + rcfFileName = cfgFileDir / (str(key) + ".rcf") + currentModule = sys.modules[__name__] + cfgUtils.writeCfgFile(avalancheDir, currentModule, cfg, str(key)) + cfgToRcf(cfg, rcfFileName) + rcfFiles.append(rcfFileName) + return rcfFiles + + diff --git a/avaframe/com8MoTPSA/com8MoTPSACfg.ini b/avaframe/com8MoTPSA/com8MoTPSACfg.ini new file mode 100644 index 000000000..50fbec7d2 --- /dev/null +++ b/avaframe/com8MoTPSA/com8MoTPSACfg.ini @@ -0,0 +1,217 @@ +### Config File - This file contains the main settings for the simulation run +## Copy to local_com8MoTPSACfg.ini and set you parameters + +[GENERAL] +# model type - only for file naming (psa - powder snow avalanche) +modelType = psa + +# list of simulations that shall be performed (null, ent, res, entres, available (use all available input data)) +simTypeList = null + + +#+++++Release thickness++++ +# True if release thickness should be read from shapefile file; if False - relTh read from ini file +relThFromShp = True +# if a variation on relTh shall be performed add here +- percent and number of steps separated by $ +# for example relThPercentVariation=50$10 [%] +relThPercentVariation = +# if a variation on relTh shall be performed add here +- absolute value and number of steps separated by $ +# for example relThRangeVariation=0.5$10 [m] +relThRangeVariation = +# if a variation on relTh shall be performed add here +- ci% value and number of steps separated by $ +# for example relThRangeFromCiVariation= ci95$10 +relThRangeFromCiVariation = +# if variation on relTh shall be performed using a normal distribution in number of steps, +# value of buildType (ci95 value), min and max of dist in percent, buildType (ci95 only allowed), +# support (e.g. 10000) all separated by $: e.g. normaldistribution$numberOfSteps$0.3$95$ci95$10000 +# if relThFromShp=True ci95 is read from shp file too +relThDistVariation = +# release thickness (only considered if relThFromShp=False) [m] +relTh = +# read release thickness directly from file (relThFromShp needs to be False) +relThFromFile = False + + +#+++++Entrainment thickness++++ +# True if entrainment thickness should be read from shapefile file; if False - entTh read from ini file +entThFromShp = True +# if a thickness value is missing for the entrainment feature in the provided shp file this value is used for all features [m] +entThIfMissingInShp = 0.3 +# if a variation on entTh shall be performed add here +- percent and number of steps separated by $ +# for example entThPercentVariation=50$10 [%] +entThPercentVariation = +# if a variation on entTh shall be performed add here +- absolute value and number of steps separated by $ +# for example entThRangeVariation=0.5$10 [m] +entThRangeVariation = +# if a variation on entTh shall be performed add here +- ci% value and number of steps separated by $ +# for example entThRangeFromCiVariation= ci95$10 +entThRangeFromCiVariation = +# if variation on entTh shall be performed using a normal distribution in number of steps, +# value of buildType (ci95 value), min and max of dist in percent, buildType (ci95 only allowed), +# support (e.g. 10000) all separated by $: e.g. normaldistribution$numberOfSteps$0.3$95$ci95$10000 +# if entFromShp=True ci95 is read from shp file too +entThDistVariation = +# entrainment thickness (only considered if entThFromShp=False) [m] +entTh = + +#+++++Secondary release thickness+++++ +# if secRelArea is True - add secondary release area +secRelArea = True +# True if release thickness should be read from shapefile file; if False - secondaryRelTh read from ini file +secondaryRelThFromShp = True +# if a variation on secondaryRelTh shall be performed add here +- percent and number of steps separated by $ +# for example secondaryRelThPercentVariation=50$10 [%] +secondaryRelThPercentVariation = +# if a variation on secondaryRelTh shall be performed add here +- absolute value and number of steps separated by $ +# for example secondaryRelThRangeVariation=0.5$10 [m] +secondaryRelThRangeVariation = +# if a variation on secondaryRelTh shall be performed add here +- ci% value and number of steps separated by $ +# for example secondaryRelThRangeFromCiVariation= ci95$10 +secondaryRelThRangeFromCiVariation = +# if variation on secondaryRelTh shall be performed using a normal distribution in number of steps, +# value of buildType (ci95 value), min and max of dist in percent, buildType (ci95 only allowed), +# support (e.g. 10000) all separated by $: e.g. normaldistribution$numberOfSteps$0.3$95$ci95$10000 +# if secondaryRelThFromShp=True ci95 is read from shp file too +secondaryRelThDistVariation = +# secondary area release thickness (only considered if secondaryRelThFromShp=False) [m] +secondaryRelTh = + +#+++++++++++++Volume classes [m³] +volClassSmall = 25000. +volClassMedium = 60000. + +#+++++++++++++Mesh and interpolation +# interpolation option +# 3 Options available : -0: nearest neighbour interpolation +# -1: equal weights interpolation +# -2: bilinear interpolation +interpOption = 2 +# minimum flow thickness [m] +hmin = 0.05 +# remesh the input rasters or look for remeshed rasters +# expected mesh size [m] +meshCellSize = 5 +# threshold under which no remeshing is done +meshCellSizeThreshold = 0.001 +# clean DEMremeshed directory to ensure remeshing if chosen meshCellsize is different from rasters in Inputs/ +cleanRemeshedRasters = True +# resize files read from Inputs/RASTERS to be resized to extent of DEM as resizeThreshold x meshCellSize +resizeThreshold = 3 + +# Normal computation on rectangular grid +# 4 triangles method 6 triangles method 8 triangles method +# +----U----UR---+---+--... +----+----+----+---+--... +----+----+----+---+--... +# | /|\ | /| | /| 2 /| /| |\ 2 | 3 /| /| Y +# | / | \ | / | | / | / | / | | \ | / | / | ^ +# | / | \ | / | / | / | / | / | / | \ | / | / | / | +# |/ 1 | 2 \|/ |/ |/ 1 |/ 3 |/ |/ | 1 \|/ 4 |/ |/ | +# +----P----L----+---+--... +----*----+----+---+--... +----*----+----+----+--... +-----> X +# |\ 4 | 3 /| /| | 6 /| 4 /| /| | 8 /|\ 5 | /| +# | \ | / | / | | / | / | / | | / | \ | / | +# | \ | / | / | / | / | / | / | / | / | \ | / | / +# | \|/ |/ |/ |/ 5 |/ |/ |/ |/ 7 | 6 \|/ |/ +# +----+----+----+---+--... +----+----+----+---+--... +----+----+----+---+--... +# | /| /| /| | /| /| /| | /| /| /| +# 4 Options available : -1: simple cross product method (with the diagonals P-UR and U-L) +# -4: 4 triangles method +# -6: 6 triangles method +# -8: 8 triangles method +methodMeshNormal = 1 + +#++++++++++++++ Technical values +++++++++++++ +# when checking if a point is within a polygon, one can decide to add a buffer +# arround the polygon (0 means take strictly the points inside, a very small value +# will inclune the points located on the polygon line) +thresholdPointInPoly = 0.001 + + +[INPUT] +# specify a particular release area scenario, provide name of shapefile with or without extension .shp (optional) +releaseScenario = +# important for parameter variation through probRun +thFromIni = + +# Below are the settings for the MoT-PSA model +[Run information] +MoT-Voellmy input file version = 2023-03-06 +Area of Interest = Ryggfonn +UTM zone = 33N +EPSG geodetic datum code = 25833 +Run name = Test_01 + +[File names] +Grid filename = - +Release depth 1 filename = ./h1in08.asc +Release depth 2 filename = ./h2in.asc +Bed depth filename = - +Bed deposition filename = - +Bed shear strength filename = - +Forest density filename = - +Tree diameter filename = - +Start velocity u filename = - +Start velocity v filename = - +Output filename root = ./Run_01/01 +Output format = ESRI_ASCII_Grid + +[Physical_parameters] +Gravitational acceleration (m/s^2) = 9.81 +Density (kg/m^3) = 250.0 +Rheology = Voellmy +Parameters = constant +Dry-friction coefficient (-) = 0.400 +Turbulent drag coefficient (-) = 0.0010 +Effective drag height (m) = 0.0 +Centrifugal effects = yes +Passive earth-pressure coeff. (-) = 1.0 +Basal drag coeff. 0-2 (-) = 0.008 +Basal drag coeff. 1-2 (-) = 0.04 +Top drag coeff. (-) = 0.0001 +Conc_L2_Prof_Coeff_0 (-) = 1.3333 +Conc_L2_Prof_Coeff_1 (-) = -0.66667 +Conc_L2_Prof_Coeff_2 (-) = 0.0 +Speed_L2_Prof_Coeff_0 (-) = 1.4 +Speed_L2_Prof_Coeff_1 (-) = 0.13333 +Speed_L2_Prof_Coeff_2 (-) = -1.4 + +[FOREST_EFFECTS] +Forest effects = no +Tree drag coefficient (-) = 1.0 +Modulus of rupture (MPa) = 50.0 +Forest decay coefficient (m s) = 0.15 + +[ENTRAINMENT] +Entrainment L1 = none +Entrainment L2 = none +Erosion coefficient L1 (-) = 0.0 +Bed strength profile = constant +Bed friction coefficient (-) = 0.0 +Bed density (kg/m^3) = 140.0 +Deposition = yes +Deposition density L1 (kg/m^3) = 400.0 +Suspension model = TJSM +Avalanche shear strength (Pa) = 0.0 +Avalanche shear strength deposit (Pa) = 0.0 +Decay coeff. snow s. s. suspension (-) = 1.0 +Decay coeff. snow s. s. deposition (-) = 1.0 +Entrainment coeff. m12 (-) = 0.0 +Deposition rate 21 (m/s) = 0.50 +Constant density L1 = yes +Evolving geometry = no + +[Numerical parameters] +Simulation time (s) = 1000.0 +Minimum time step (s) = 0.001 +Maximum time step (s) = 0.1 +Output interval (s) = 1.0 +Write velocity vectors = no +Write maximum pressure = yes +Write instant. pressure = no +Minimum flow depth (m) = 0.01 +Minimum speed (m/s) = 0.01 +Momentum threshold (-) = 0.05 +Initial CFL number (-) = 0.8 + +[EXPORTS] +# peak files and plots are exported, option to turn off exports when exportData is set to False +# this affects export of peak files and also generation of peak file plots +exportData = True diff --git a/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py b/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py new file mode 100644 index 000000000..78eb15ac8 --- /dev/null +++ b/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py @@ -0,0 +1,133 @@ +""" + Run script for performing an com8MoTPSA avalanche simulation with parameter variation and performing a + probability analysis with the simulation results + Define settings in ana4Stats/probAnaCfg.ini or your local copy - local_probAnaCfg.ini +""" + +# Load modules +import pathlib +import argparse + +# Local imports +from avaframe.com1DFA import com1DFA +import avaframe.com8MoTPSA.com8MoTPSA as com8MoTPSA +from avaframe.out3Plot import statsPlots as sP +from avaframe.ana4Stats import probAna +from avaframe.in3Utils import initializeProject as initProj +from avaframe.in3Utils import cfgUtils +from avaframe.in3Utils import logUtils +import avaframe.in3Utils.fileHandlerUtils as fU +from avaframe.out3Plot import outQuickPlot as oP +import configparser + + +def runProbAna(avalancheDir=''): + """ Run a com1DFA probability analysis with parameters and only an + avalanche directory as input + + Parameters + ---------- + avalancheDir: str + path to avalanche directory (setup eg. with init scipts) + + Returns + ------- + """ + + # log file name; leave empty to use default runLog.log + logName = 'runAna4ProbAna' + + # Load general configuration filee + cfgMain = cfgUtils.getGeneralConfig() + + # Load avalanche directory from general configuration file, + # if not provided as input argument + cfgMain = cfgUtils.getGeneralConfig() + if avalancheDir != '': + cfgMain['MAIN']['avalancheDir'] = avalancheDir + else: + avalancheDir = cfgMain['MAIN']['avalancheDir'] + + avalancheDir = pathlib.Path(avalancheDir) + + # Start logging + log = logUtils.initiateLogger(avalancheDir, logName) + log.info('MAIN SCRIPT') + log.info('Current avalanche: %s', avalancheDir) + + # Clean input directory(ies) of old work files + initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) + + # Load configuration file for probabilistic run and analysis + cfgProb = cfgUtils.getModuleConfig(probAna) + + # create configuration files for com8MoTPSA simulations including parameter + # variation - defined in the probabilistic config + # prob4AnaCfg.ini or its local copy + cfgFiles, cfgPath = probAna.createComModConfig(cfgProb, avalancheDir, com8MoTPSA) + + # perform com8MoTPSA simulations + # dem, plotDict, reportDictList, simDF = com8MoTPSA.com8MoTPSAMain(cfgMain, cfgInfo=cfgPath) + com8MoTPSA.com8MoTPSAMain(cfgMain, cfgInfo=cfgPath) + + # fetch simDF of only sims that were created in above call to com8MoTPSAMain - do not include sims + # that were run previously and are still located in outDir + simDFActual, _ = cfgUtils.readAllConfigurationInfo(avalancheDir, specDir='', configCsvName='latestSims') + + # check if sampling strategy is from full sample - then only one configuration is possible + probabilityConfigurations = probAna.fetchProbConfigs(cfgProb['PROBRUN']) + + # perform probability analysis + for probConf in probabilityConfigurations: + + # filter simulations according to probability configurations + cfgProb['FILTER'] = probabilityConfigurations[probConf] + log.info('Perform proba analysis for configuration: %s' % probConf) + # provide optional filter criteria for simulations + parametersDict = fU.getFilterDict(cfgProb, 'FILTER') + + # perform probability analysis + anaPerformed, contourDict = probAna.probAnalysis(avalancheDir, + cfgProb, + 'com8MoTPSA', + parametersDict=parametersDict, + probConf=probConf, + simDFActual=simDFActual + ) + if anaPerformed is False: + log.warning('No files found for configuration: %s' % probConf) + + + # make a plot of the contours + inputDir = pathlib.Path(avalancheDir, 'Outputs', 'ana4Stats') + outName = '%s_prob_%s_%s_lim%s' % (str(avalancheDir.stem), + probConf, + cfgProb['GENERAL']['peakVar'], + cfgProb['GENERAL']['peakLim']) + pathDict = {'pathResult': str(inputDir / 'plots'), + 'avaDir': str(avalancheDir), + 'plotScenario': outName + } + oP.plotContours(contourDict, + cfgProb['GENERAL']['peakVar'], + cfgProb['GENERAL']['peakLim'], pathDict, addLegend=False) + + # plot probability maps + sP.plotProbMap(avalancheDir, inputDir, cfgProb, demPlot=True) + + # # copy outputs to folder called like probability configurations + # outputFiles = avalancheDir / 'Outputs' / 'ana4Stats' + # saveFiles = avalancheDir / 'Outputs' / ('ana4Stats_' + probConf) + # shutil.move(outputFiles, saveFiles) + + return + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Run ana4ProbAna workflow') + parser.add_argument('avadir', metavar='a', type=str, nargs='?', default='', + help='the avalanche directory') + + args = parser.parse_args() + runProbAna(str(args.avadir)) diff --git a/avaframe/com8MoTPSA/runCom8MoTPSA.py b/avaframe/com8MoTPSA/runCom8MoTPSA.py new file mode 100644 index 000000000..b14595b4d --- /dev/null +++ b/avaframe/com8MoTPSA/runCom8MoTPSA.py @@ -0,0 +1,86 @@ +""" + Run script for running python com8MoTPSA kernel +""" +# Load modules +# importing general python modules +import time +import argparse +import pathlib +import sys + +# Local imports +import avaframe.in3Utils.initializeProject as initProj +from avaframe.com8MoTPSA import com8MoTPSA +from avaframe.in3Utils import cfgUtils +from avaframe.in3Utils import logUtils +from avaframe.in3Utils import fileHandlerUtils as fU + + +def runCom8MoTPSA(avalancheDir=''): + """ Run com8MoTPSA in the default configuration with only an + avalanche directory as input + + Parameters + ---------- + avalancheDir: str + path to avalanche directory (setup eg. with init scipts) + + Returns + ------- + peakFilesDF: pandas dataframe + dataframe with info about com8MoTPSA peak file locations + """ + + # Time the whole routine + startTime = time.time() + + # log file name; leave empty to use default runLog.log + logName = 'runCom8MoTPSA' + + # Load avalanche directory from general configuration file + # More information about the configuration can be found here + # on the Configuration page in the documentation + cfgMain = cfgUtils.getGeneralConfig() + if avalancheDir != '': + cfgMain['MAIN']['avalancheDir'] = avalancheDir + else: + avalancheDir = cfgMain['MAIN']['avalancheDir'] + + # Start logging + log = logUtils.initiateLogger(avalancheDir, logName) + log.info('MAIN SCRIPT') + log.info('Current avalanche: %s', avalancheDir) + + # ---------------- + # Clean input directory(ies) of old work and output files + # If you just created the ``avalancheDir`` this one should be clean but if you + # already did some calculations you might want to clean it:: + initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) + + # Get module config + cfgCom8MoTPSA = cfgUtils.getModuleConfig(com8MoTPSA, toPrint=False) + + # ---------------- + # Run psa + com8MoTPSA.com8MoTPSAMain(cfgMain, cfgInfo=cfgCom8MoTPSA) + + # # Get peakfiles to return to QGIS + # avaDir = pathlib.Path(avalancheDir) + # inputDir = avaDir / 'Outputs' / 'com8MoTPSA' / 'peakFiles' + # peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) + + # Print time needed + endTime = time.time() + log.info('Took %6.1f seconds to calculate.' % (endTime - startTime)) + + # return peakFilesDF + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run com8MoTPSA workflow') + parser.add_argument('avadir', metavar='avalancheDir', type=str, nargs='?', default='', + help='the avalanche directory') + + args = parser.parse_args() + runCom8MoTPSA(str(args.avadir)) diff --git a/avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe b/avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe new file mode 100755 index 000000000..c1ba9356b Binary files /dev/null and b/avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe differ diff --git a/avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe b/avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe new file mode 100644 index 000000000..2362024b8 Binary files /dev/null and b/avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe differ diff --git a/avaframe/com9MoTVoellmy/__init__.py b/avaframe/com9MoTVoellmy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/avaframe/com9MoTVoellmy/com9MoTVoellmy.py b/avaframe/com9MoTVoellmy/com9MoTVoellmy.py new file mode 100644 index 000000000..9f8ff7519 --- /dev/null +++ b/avaframe/com9MoTVoellmy/com9MoTVoellmy.py @@ -0,0 +1,319 @@ +import os +import platform +import logging +import numpy as np +import pathlib +import time +import shutil +import sys + + +if os.name == "nt": + from multiprocessing.pool import ThreadPool as Pool +elif platform.system() == "Darwin": + from multiprocessing.pool import ThreadPool as Pool +else: + from multiprocessing import Pool + +import avaframe.com1DFA.com1DFA as com1DFA +from avaframe.in3Utils import cfgUtils +from avaframe.in2Trans import rasterUtils as rU +from avaframe.com1DFA import particleInitialisation as pI +from avaframe.in1Data import getInput as gI +import avaframe.in3Utils.geoTrans as geoTrans +import avaframe.in3Utils.fileHandlerUtils as fU +from avaframe.out1Peak import outPlotAllPeak as oP +from avaframe.in3Utils.cfgUtils import cfgToRcf +from avaframe.in3Utils.MoTUtils import rewriteDEMtoZeroValues, runAndCheckMoT, MoTGenerateConfigs, copyMoTFiles + + +# create a local logger +log = logging.getLogger(__name__) + + +def com9MoTVoellmyMain(cfgMain, cfgInfo=None): + """Run MoT-Voellmy simulations using specified configurations. + + This function executes MoT-Voellmy simulations by handling + preprocessing, parallel execution, and postprocessing steps. + + Parameters + ---------- + cfgMain : configparser.ConfigParser + Main configuration settings for the simulation + cfgInfo : str or pathlib.Path or configparser.ConfigParser, optional + Additional configuration information, by default None + Can be: + - Path to configuration file for overrides + - ConfigParser object with initial configuration + - None to use local/default configuration + + Returns + ------- + None + Results are written to the output directory structure + + Notes + ----- + The function performs several key steps: + - Generates configuration settings for all simulations + - Preprocesses input data and creates RCF configuration files + - Executes simulations in parallel using multiple processes + - Handles DEM preparation by converting NaN values to zeros + """ + + # Get all necessary information from the configuration files + currentModule = sys.modules[__name__] # As if you would to import com9MoTVoellmy + simDict, inputSimFiles = MoTGenerateConfigs(cfgMain, cfgInfo, currentModule) + + # convert DEM from nan to 0 values + # TODO: suggest MoT-PSA to handle nan values + rewriteDEMtoZeroValues(inputSimFiles["demFile"]) + + log.info("The following simulations will be performed") + for key in simDict: + log.info("Simulation: %s" % key) + + # Preprocess the simulations, mainly creating the rcf files + rcfFiles = com9MoTVoellmyPreprocess(simDict, inputSimFiles, cfgMain) + + # And now we run the simulations + startTime = time.time() + + log.info("--- STARTING (potential) PARALLEL PART ----") + + # Get number of CPU Cores wanted + nCPU = cfgUtils.getNumberOfProcesses(cfgMain, len(rcfFiles)) + + # Create parallel pool and run + # with multiprocessing.Pool(processes=nCPU) as pool: + with Pool(processes=nCPU) as pool: + results = pool.map(com9MoTVoellmyTask, rcfFiles) + pool.close() + pool.join() + + timeNeeded = "%.2f" % (time.time() - startTime) + log.info("Overall (parallel) com9MoTVoellmy computation took: %s s " % timeNeeded) + log.info("--- ENDING (potential) PARALLEL PART ----") + + # Postprocess the simulations + com9MoTVoellmyPostprocess(simDict, cfgMain, inputSimFiles) + + +def com9MoTVoellmyPostprocess(simDict, cfgMain, inputSimFiles): + """Post-process MoT-Voellmy simulation results. + + This function handles post-processing tasks after MoT-Voellmy simulations complete, + including copying result files to output directories and generating visualization plots. + + Parameters + ---------- + simDict : dict + Dictionary containing simulation configurations, with one entry per simulation + cfgMain : configparser.ConfigParser + Main configuration settings for the simulation + inputSimFiles : dict + Dictionary containing paths to input files (DEM, release areas, etc.) + + Returns + ------- + None + Results are written to the output directory structure + + Notes + ----- + The function performs several key tasks: + - Creates output directory structure + - Copies simulation result files (DataTime.txt, ppr, pfd, pfv files) + - Renames files according to conventions + - Generates visualization plots of peak fields + """ + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + + # Copy max files to output directory + outputDirPeakFile = pathlib.Path(avalancheDir) / "Outputs" / "com9MoTVoellmy" / "peakFiles" + fU.makeADir(outputDirPeakFile) + + for key in simDict: + workDir = pathlib.Path(avalancheDir) / "Work" / "com9MoTVoellmy" / str(key) + + # Copy ppr files + copyMoTFiles(workDir, outputDirPeakFile, "p_max", "ppr") + + # Copy pfd files + copyMoTFiles(workDir, outputDirPeakFile, "h_max", "pfd") + + # Copy pfv files + copyMoTFiles(workDir, outputDirPeakFile, "s_max", "pfv") + + # create plots and report + modName = __name__.split(".")[-1] + reportDir = pathlib.Path(avalancheDir, "Outputs", modName, "reports") + fU.makeADir(reportDir) + + dem = rU.readRaster(inputSimFiles["demFile"]) + + # Generate plots for all peakFiles + oP.plotAllPeakFields(avalancheDir, cfgMain["FLAGS"], modName, demData=dem) + + +def com9MoTVoellmyTask(rcfFile): + """Execute a single MoT-PSA simulation using the provided configuration file. + + Parameters + ---------- + rcfFile : pathlib.Path + Path to the RCF configuration file for the simulation + + Returns + ------- + list + The command that was executed as a list containing the executable path + and configuration file path + + Notes + ----- + Changes to the directory containing this module before executing the simulation. + Uses runAndCheckMoT to execute and monitor the simulation process. + """ + os.chdir(os.path.dirname(os.path.abspath(__file__))) + + if os.name == "nt": + exeName = "MoT-Voellmy_win.exe" + elif platform.system() == "Darwin": + message = "MoT-Voellmy does not support MacOS at the moment" + log.error(message) + raise OSError(message) + else: + exeName = "./MoT-Voellmy_linux.exe" + + command = [exeName, rcfFile] + log.info("Run simulation: %s" % rcfFile) + runAndCheckMoT(command) + return command + + +def com9MoTVoellmyPreprocess(simDict, inputSimFiles, cfgMain): + """Preprocess data for MoT-PSA simulations. + + This function prepares the input data and configuration files needed to run + MoT-PSA simulations. It processes release areas, creates required directories, + and generates configuration files for each simulation. + + Parameters + ---------- + simDict : dict + Dictionary containing simulation configurations with one entry per simulation + inputSimFiles : dict + Dictionary containing paths to input files (DEM, release areas, etc.) + cfgMain : configparser.ConfigParser + Main configuration settings for the simulation + + Returns + ------- + list + List of pathlib.Path objects pointing to generated RCF configuration files + that will be used to run the simulations + + Notes + ----- + The function performs several key steps: + - Creates working directories for each simulation + - Processes release areas and converts them to raster format + - Generates configuration files in RCF format + - Handles both single and multiple release scenarios + """ + # Load avalanche directory from general configuration file + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + + workDir = pathlib.Path(avalancheDir) / "Work" / "com9MoTVoellmy" + cfgFileDir = pathlib.Path(avalancheDir) / "Outputs" / "com9MoTVoellmy" / "configurationFiles" + fU.makeADir(cfgFileDir) + rcfFiles = list() + + for key in simDict: + # Generate command and run via subprocess.run + # Configuration that needs adjustment + + # load configuration object for current sim + cfg = simDict[key]["cfgSim"] + + # convert release shape to raster with values for current sim + # select release area input data according to a chosen release scenario + inputSimFiles = gI.selectReleaseFile(inputSimFiles, cfg["INPUT"]["releaseScenario"]) + # create the required input from input files + demOri, inputSimLines = com1DFA.prepareInputData(inputSimFiles, cfg) + + if cfg["GENERAL"].getboolean("iniStep"): + # append buffered release Area + inputSimLines = pI.createReleaseBuffer(cfg, inputSimLines) + + # set thickness values for the release area, entrainment and secondary release areas + relName, inputSimLines, badName = com1DFA.prepareReleaseEntrainment( + cfg, inputSimFiles["releaseScenario"], inputSimLines + ) + + releaseLine = inputSimLines["releaseLine"] + # check if release features overlap between features + # TODO: split releaseheight -> question NGI + dem = rU.readRaster(inputSimFiles["demFile"]) + dem["originalHeader"] = dem["header"].copy() + # releaseLine = geoTrans.prepareArea(releaseLine, dem, np.sqrt(2), combine=True, checkOverlap=False) + if len(inputSimLines["relThField"]) == 0: + # if no release thickness field or function - set release according to shapefile or ini file + # this is a list of release rasters that we want to combine + releaseLine = geoTrans.prepareArea( + releaseLine, + dem, + np.sqrt(2), + thList=releaseLine["thickness"], + combine=True, + checkOverlap=False, + ) + releaseField = releaseLine["rasterData"] + else: + # if relTh provided - set release thickness with field or function + releaseLine = geoTrans.prepareArea( + releaseLine, dem, np.sqrt(2), combine=True, checkOverlap=False + ) + relRasterPoly = releaseLine["rasterData"].copy() + releaseRelThCombined = np.where(relRasterPoly > 0, inputSimLines["relThField"], 0) + releaseField = releaseRelThCombined + + # Generate the work and data dirs for the current simHash + cuWorkDir = workDir / key + workInputDir = cuWorkDir / "Input" + workOutputDir = cuWorkDir / key + + fU.makeADir(cuWorkDir) + fU.makeADir(workInputDir) + + zeroRaster = np.full_like(releaseLine["rasterData"], 0) + + release = workInputDir / "release" + bedDepth = workInputDir / "dummyBedDepth" + bedShear = workInputDir / "dummyBedShear" + rU.writeResultToRaster(dem["header"], releaseField, release, flip=True) + rU.writeResultToRaster(dem["header"], zeroRaster, bedDepth) + rU.writeResultToRaster(dem["header"], zeroRaster, bedShear) + + # set configuration for MoT-Voellmy + cfg["Run information"]["Area of Interest"] = cfgMain["MAIN"]["avalancheDir"] + cfg["Run information"]["UTM zone"] = "32N" + cfg["Run information"]["EPSG geodetic datum code"] = "31287" + cfg["Run information"]["Run name"] = cfgMain["MAIN"]["avalancheDir"] + cfg["File names"]["Grid filename"] = str(inputSimFiles["demFile"]) + cfg["File names"]["Release depth filename"] = str(release) + ".asc" + cfg["File names"]["Bed depth filename"] = str(bedDepth) + ".asc" + cfg["File names"]["Bed shear strength filename"] = str(bedShear) + ".asc" + cfg["File names"]["Output filename root"] = str(workOutputDir) + + rcfFileName = cfgFileDir / (str(key) + ".rcf") + + currentModule = sys.modules[__name__] + cfgUtils.writeCfgFile(avalancheDir, currentModule, cfg, str(key)) + cfgToRcf(cfg, rcfFileName) + rcfFiles.append(rcfFileName) + return rcfFiles + + diff --git a/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini b/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini new file mode 100644 index 000000000..f65f1db7f --- /dev/null +++ b/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini @@ -0,0 +1,192 @@ +### Config File - This file contains the main settings for the simulation run +## Copy to local_com8MoTPSACfg.ini and set you parameters + +[GENERAL] +# model type - only for file naming (psa - powder snow avalanche) +modelType = dfa + +# list of simulations that shall be performed (null, ent, res, entres, available (use all available input data)) +simTypeList = null + + +#+++++Release thickness++++ +# True if release thickness should be read from shapefile file; if False - relTh read from ini file +relThFromShp = True +# if a variation on relTh shall be performed add here +- percent and number of steps separated by $ +# for example relThPercentVariation=50$10 [%] +relThPercentVariation = +# if a variation on relTh shall be performed add here +- absolute value and number of steps separated by $ +# for example relThRangeVariation=0.5$10 [m] +relThRangeVariation = +# if a variation on relTh shall be performed add here +- ci% value and number of steps separated by $ +# for example relThRangeFromCiVariation= ci95$10 +relThRangeFromCiVariation = +# if variation on relTh shall be performed using a normal distribution in number of steps, +# value of buildType (ci95 value), min and max of dist in percent, buildType (ci95 only allowed), +# support (e.g. 10000) all separated by $: e.g. normaldistribution$numberOfSteps$0.3$95$ci95$10000 +# if relThFromShp=True ci95 is read from shp file too +relThDistVariation = +# release thickness (only considered if relThFromShp=False) [m] +relTh = +# read release thickness directly from file (relThFromShp needs to be False) +relThFromFile = False + + +#+++++Entrainment thickness++++ +# True if entrainment thickness should be read from shapefile file; if False - entTh read from ini file +entThFromShp = True +# if a thickness value is missing for the entrainment feature in the provided shp file this value is used for all features [m] +entThIfMissingInShp = 0.3 +# if a variation on entTh shall be performed add here +- percent and number of steps separated by $ +# for example entThPercentVariation=50$10 [%] +entThPercentVariation = +# if a variation on entTh shall be performed add here +- absolute value and number of steps separated by $ +# for example entThRangeVariation=0.5$10 [m] +entThRangeVariation = +# if a variation on entTh shall be performed add here +- ci% value and number of steps separated by $ +# for example entThRangeFromCiVariation= ci95$10 +entThRangeFromCiVariation = +# if variation on entTh shall be performed using a normal distribution in number of steps, +# value of buildType (ci95 value), min and max of dist in percent, buildType (ci95 only allowed), +# support (e.g. 10000) all separated by $: e.g. normaldistribution$numberOfSteps$0.3$95$ci95$10000 +# if entFromShp=True ci95 is read from shp file too +entThDistVariation = +# entrainment thickness (only considered if entThFromShp=False) [m] +entTh = + +#+++++Secondary release thickness+++++ +# if secRelArea is True - add secondary release area +secRelArea = True +# True if release thickness should be read from shapefile file; if False - secondaryRelTh read from ini file +secondaryRelThFromShp = True +# if a variation on secondaryRelTh shall be performed add here +- percent and number of steps separated by $ +# for example secondaryRelThPercentVariation=50$10 [%] +secondaryRelThPercentVariation = +# if a variation on secondaryRelTh shall be performed add here +- absolute value and number of steps separated by $ +# for example secondaryRelThRangeVariation=0.5$10 [m] +secondaryRelThRangeVariation = +# if a variation on secondaryRelTh shall be performed add here +- ci% value and number of steps separated by $ +# for example secondaryRelThRangeFromCiVariation= ci95$10 +secondaryRelThRangeFromCiVariation = +# if variation on secondaryRelTh shall be performed using a normal distribution in number of steps, +# value of buildType (ci95 value), min and max of dist in percent, buildType (ci95 only allowed), +# support (e.g. 10000) all separated by $: e.g. normaldistribution$numberOfSteps$0.3$95$ci95$10000 +# if secondaryRelThFromShp=True ci95 is read from shp file too +secondaryRelThDistVariation = +# secondary area release thickness (only considered if secondaryRelThFromShp=False) [m] +secondaryRelTh = + +#+++++++++++++Volume classes [m³] +volClassSmall = 25000. +volClassMedium = 60000. + +#+++++++++++++Mesh and interpolation +# interpolation option +# 3 Options available : -0: nearest neighbour interpolation +# -1: equal weights interpolation +# -2: bilinear interpolation +interpOption = 2 +# minimum flow thickness [m] +hmin = 0.05 +# remesh the input rasters or look for remeshed rasters +# expected mesh size [m] +meshCellSize = 5 +# threshold under which no remeshing is done +meshCellSizeThreshold = 0.001 +# clean DEMremeshed directory to ensure remeshing if chosen meshCellsize is different from rasters in Inputs/ +cleanRemeshedRasters = True +# resize files read from Inputs/RASTERS to be resized to extent of DEM as resizeThreshold x meshCellSize +resizeThreshold = 3 + +# Normal computation on rectangular grid +# 4 triangles method 6 triangles method 8 triangles method +# +----U----UR---+---+--... +----+----+----+---+--... +----+----+----+---+--... +# | /|\ | /| | /| 2 /| /| |\ 2 | 3 /| /| Y +# | / | \ | / | | / | / | / | | \ | / | / | ^ +# | / | \ | / | / | / | / | / | / | \ | / | / | / | +# |/ 1 | 2 \|/ |/ |/ 1 |/ 3 |/ |/ | 1 \|/ 4 |/ |/ | +# +----P----L----+---+--... +----*----+----+---+--... +----*----+----+----+--... +-----> X +# |\ 4 | 3 /| /| | 6 /| 4 /| /| | 8 /|\ 5 | /| +# | \ | / | / | | / | / | / | | / | \ | / | +# | \ | / | / | / | / | / | / | / | / | \ | / | / +# | \|/ |/ |/ |/ 5 |/ |/ |/ |/ 7 | 6 \|/ |/ +# +----+----+----+---+--... +----+----+----+---+--... +----+----+----+---+--... +# | /| /| /| | /| /| /| | /| /| /| +# 4 Options available : -1: simple cross product method (with the diagonals P-UR and U-L) +# -4: 4 triangles method +# -6: 6 triangles method +# -8: 8 triangles method +methodMeshNormal = 1 + +#++++++++++++++ Technical values +++++++++++++ +# when checking if a point is within a polygon, one can decide to add a buffer +# arround the polygon (0 means take strictly the points inside, a very small value +# will inclune the points located on the polygon line) +thresholdPointInPoly = 0.001 + + +[INPUT] +# specify a particular release area scenario, provide name of shapefile with or without extension .shp (optional) +releaseScenario = +# important for parameter variation through probRun +thFromIni = + +# Below are the settings for the MoT-Voellmy model +[Run information] +MoT-Voellmy input file version = 2024-09-10 +Area of Interest = Grasdalen +UTM zone = 33N +EPSG geodetic datum code = 25833 +Run name = Ryggfonn_2021-04-11_02 + +[File names] +Grid filename = - +Release depth filename = ./h0.asc +Bed depth filename = - +Bed shear strength filename = - +Forest density filename = - +Tree diameter filename = - +Start velocity u filename = - +Start velocity v filename = - +Output filename root = ./Run_02/02 +Output format = ESRI_ASCII_Grid + +[Physical_parameters] +Gravitational acceleration (m/s^2) = 9.81 +Flow density (kg/m^3) = 250.0 +Bed density (kg/m^3) = 140.0 +Deposit density (kg/m^3) = 450.0 +Rheology = Voellmy +Parameters = constant +Dry-friction coefficient (-) = 0.40 +Turbulent drag coefficient (-) = 0.001 +Effective drag height (m) = 3.0 +Centrifugal effects = yes +Passive earth-pressure coeff. (-) = 1.0 + +[FOREST_EFFECTS] +Forest effects = no +Tree drag coefficient (-) = 1.0 +Modulus of rupture (MPa) = 50.0 +Forest decay coefficient (m/s) = 0.15 + +[ENTRAINMENT] +Entrainment = none +Erosion coefficient (-) = 0.0 +Bed strength profile = global +Bed friction coefficient (-) = 0.25 +Deposition = no +Evolving geometry = no + +[Numerical parameters] +Simulation time (s) = 100.0 +Minimum time step (s) = 0.001 +Maximum time step (s) = 0.2 +Output interval (s) = 1.0 +Write velocity vectors = no +Write maximum pressure = yes +Write instant. pressure = no +Minimum flow depth (m) = 0.01 +Minimum speed (m/s) = 0.01 +Momentum threshold (kg m/s) = 100.0 +Initial CFL number (-) = 0.8 diff --git a/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py b/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py new file mode 100644 index 000000000..ade91cb76 --- /dev/null +++ b/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py @@ -0,0 +1,83 @@ +""" + Run script for running python com9MoTVoellmy kernel +""" +# Load modules +# importing general python modules +import time +import argparse + +# Local imports +import avaframe.in3Utils.initializeProject as initProj +from avaframe.com9MoTVoellmy import com9MoTVoellmy +from avaframe.in3Utils import cfgUtils +from avaframe.in3Utils import logUtils + + +def runCom9MoTVoellmy(avalancheDir=''): + """ Run com9MoTVoellmy in the default configuration with only an + avalanche directory as input + + Parameters + ---------- + avalancheDir: str + path to avalanche directory (setup eg. with init scipts) + + Returns + ------- + peakFilesDF: pandas dataframe + dataframe with info about com9MoTVoellmy peak file locations + """ + + # Time the whole routine + startTime = time.time() + + # log file name; leave empty to use default runLog.log + logName = 'runCom9MoTVoellmy' + + # Load avalanche directory from general configuration file + # More information about the configuration can be found here + # on the Configuration page in the documentation + cfgMain = cfgUtils.getGeneralConfig() + if avalancheDir != '': + cfgMain['MAIN']['avalancheDir'] = avalancheDir + else: + avalancheDir = cfgMain['MAIN']['avalancheDir'] + + # Start logging + log = logUtils.initiateLogger(avalancheDir, logName) + log.info('MAIN SCRIPT') + log.info('Current avalanche: %s', avalancheDir) + + # ---------------- + # Clean input directory(ies) of old work and output files + # If you just created the ``avalancheDir`` this one should be clean but if you + # already did some calculations you might want to clean it:: + initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) + + # Get module config + cfgCom9MoTVoellmy = cfgUtils.getModuleConfig(com9MoTVoellmy, toPrint=False) + + # ---------------- + # Run psa + com9MoTVoellmy.com9MoTVoellmyMain(cfgMain, cfgInfo=cfgCom9MoTVoellmy) + + # # Get peakfiles to return to QGIS + # avaDir = pathlib.Path(avalancheDir) + # inputDir = avaDir / 'Outputs' / 'com8MoTPSA' / 'peakFiles' + # peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) + + # Print time needed + endTime = time.time() + log.info('Took %6.1f seconds to calculate.' % (endTime - startTime)) + + # return peakFilesDF + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run com9MoTVoellmy workflow') + parser.add_argument('avadir', metavar='avalancheDir', type=str, nargs='?', default='', + help='the avalanche directory') + + args = parser.parse_args() + runCom9MoTVoellmy(str(args.avadir)) diff --git a/avaframe/in1Data/getInput.py b/avaframe/in1Data/getInput.py index cd7b3611d..8450291fc 100644 --- a/avaframe/in1Data/getInput.py +++ b/avaframe/in1Data/getInput.py @@ -1,5 +1,5 @@ """ - Fetch input data for avalanche simulations +Fetch input data for avalanche simulations """ import logging @@ -250,13 +250,14 @@ def getInputDataCom1DFA(avaDir): demFile = getDEMPath(avaDir) # check if frictionParameter file is available - # TODO: enable this with geotiff - muFile, entResInfo["mu"] = getAndCheckInputFiles(inputDir, "RASTERS", "mu parameter data", fileExt="raster", - fileSuffix='_mu') + muFile, entResInfo["mu"] = getAndCheckInputFiles( + inputDir, "RASTERS", "mu parameter data", fileExt="raster", fileSuffix="_mu" + ) # check if frictionParameter file is available - xiFile, entResInfo["xi"] = getAndCheckInputFiles(inputDir, "RASTERS", "xi parameter data", fileExt="raster", - fileSuffix='_xi') + xiFile, entResInfo["xi"] = getAndCheckInputFiles( + inputDir, "RASTERS", "xi parameter data", fileExt="raster", fileSuffix="_xi" + ) # return DEM, first item of release, entrainment and resistance areas inputSimFiles = { @@ -328,7 +329,12 @@ def getAndCheckInputFiles(inputDir, folder, inputType, fileExt="shp", fileSuffix if len(OutputFile) < 1: OutputFile = None elif len(OutputFile) > 1: - message = "More than one %s .%s file in %s/%s/ not allowed" % (inputType, fileExt, inputDir, folder) + message = "More than one %s .%s file in %s/%s/ not allowed" % ( + inputType, + fileExt, + inputDir, + folder, + ) log.error(message) raise AssertionError(message) else: @@ -336,7 +342,9 @@ def getAndCheckInputFiles(inputDir, folder, inputType, fileExt="shp", fileSuffix OutputFile = OutputFile[0] if OutputFile.suffix not in supportedFileFormats: - message = "Unsupported file format found for OutputFile %s; shp, asc, tif are allowed" % OutputFile + message = ( + "Unsupported file format found for OutputFile %s; shp, asc, tif are allowed" % OutputFile + ) log.error(message) raise AssertionError(message) @@ -376,7 +384,11 @@ def getThicknessInputSimFiles(inputSimFiles): for releaseA in inputSimFiles["relFiles"]: # fetch thickness and id info from input data thicknessList, idList, ci95List = shpConv.readThickness(releaseA) - inputSimFiles[releaseA.stem] = {"thickness": thicknessList, "id": idList, "ci95": ci95List} + inputSimFiles[releaseA.stem] = { + "thickness": thicknessList, + "id": idList, + "ci95": ci95List, + } # append release scenario name to list releaseScenarioList.append(releaseA.stem) @@ -444,7 +456,10 @@ def updateThicknessCfg(inputSimFiles, cfgInitial): cfgInitial["INPUT"]["entrainmentScenario"] = inputSimFiles["entFile"].stem if inputSimFiles["secondaryReleaseFile"] != None and "secondaryReleaseFile" in thTypeList: cfgInitial = dP.getThicknessValue( - cfgInitial, inputSimFiles, inputSimFiles["secondaryReleaseFile"].stem, "secondaryRelTh" + cfgInitial, + inputSimFiles, + inputSimFiles["secondaryReleaseFile"].stem, + "secondaryRelTh", ) cfgInitial["INPUT"]["secondaryReleaseScenario"] = inputSimFiles["secondaryReleaseFile"].stem diff --git a/avaframe/in3Utils/MoTUtils.py b/avaframe/in3Utils/MoTUtils.py new file mode 100644 index 000000000..4e1201db9 --- /dev/null +++ b/avaframe/in3Utils/MoTUtils.py @@ -0,0 +1,191 @@ +import os +import platform +import subprocess +import logging +import pathlib +import shutil + +import numpy as np + +from avaframe.com1DFA import com1DFATools as com1DFATools, com1DFA as com1DFA +from avaframe.in2Trans import rasterUtils as rU + +log = logging.getLogger(__name__) + +def rewriteDEMtoZeroValues(demFile): + """Set all NaN values in a DEM raster to zero and update the nodata value. + + This function reads a DEM raster file, replaces all NaN values with 0.0, + updates the nodata value in the header to 0.0, and writes the modified + raster back to a new file. + + Parameters + ---------- + demFile : pathlib.Path + Path to the input DEM raster file + + Returns + ------- + None + Writes a new raster file with zero values instead of NaN values. + The output file is saved in the same directory as the input file, + using the same stem name. + + Notes + ----- + The function uses the rasterUtils module for reading and writing raster data. + The output raster is flipped during writing. + """ + demData = rU.readRaster(demFile) + demData["rasterData"][np.isnan(demData["rasterData"])] = 0.0 + demData["header"]["nodata_value"] = 0.0 + newFileName = demFile.parent / demFile.stem + rU.writeResultToRaster(demData["header"], demData["rasterData"], newFileName, flip=True) + + +def runAndCheckMoT(command): + """Execute MoT command and monitor its output. + + This function runs a MoT command as a subprocess and monitors its output, + filtering and logging specific messages while tracking time steps. + + Parameters + ---------- + command : str or list + The command to execute. Can be a string or list of arguments. + + Returns + ------- + None + Function runs the command and logs output but does not return a value. + + Notes + ----- + - Uses different shell settings based on operating system + - Filters output to reduce noise from common status messages + - Logs time step progress every 100 steps + - Handles UTF-8 encoding with replacement of invalid characters + """ + if os.name == "nt": + useShell = True + elif platform.system() == "Darwin": + useShell = False + else: + useShell = False + + # This starts the subprocess + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=useShell, + encoding="utf-8", + errors="replace", + universal_newlines=True, + ) + + printCounter = 0 + counter = 1 + + while True: + realtimeOutput = process.stdout.readline() + + if realtimeOutput == "" and process.poll() is not None: + break + + if realtimeOutput: + line = realtimeOutput.strip() + + # do not pollute output window with time step prints + # TODO: hacky for now + if "Step" in line: + counter = counter + 1 + printCounter = printCounter + 1 + if printCounter > 100: + # print('\r' + line, flush=True, end='') + msg = "Process is running. Reported time steps: " + str(counter) + log.info(msg) + printCounter = 0 + + elif "find_dt" in line: + continue + elif "h1" in line: + continue + elif "h2" in line: + continue + elif "write_data" in line: + continue + elif "update_boundaries" in line: + continue + else: + log.info(line) + + +def MoTGenerateConfigs(cfgMain, cfgInfo, currentModule): + """ + Creates configuration objects for com8MoTPSA. + + Parameters + ------------ + cfgMain: configparser object + main configuration of AvaFrame + cfgInfo: str or pathlib Path or configparser object + path to configuration file if overwrite is desired - optional + if not local (if available) or default configuration will be loaded + if cfgInfo is a configparser object take this as initial config + currentModule: module object + is being passed to cfgUtils.writeCfgFile to create the correct the cfg + + + Returns + -------- + simDict: dict + dictionary with one key per simulation to perform including its config object + inputSimFiles: dict + dictionary with input files info + """ + + # fetch type of cfgInfo + typeCfgInfo = com1DFATools.checkCfgInfoType(cfgInfo) + + if typeCfgInfo == "cfgFromDir": + # preprocessing to create configuration objects for all simulations to run by reading multiple cfg files + simDict, inputSimFiles, simDFExisting, outDir = com1DFATools.createSimDictFromCfgs( + cfgMain, cfgInfo, module=currentModule + ) + else: + # preprocessing to create configuration objects for all simulations to run + simDict, outDir, inputSimFiles, simDFExisting = com1DFA.com1DFAPreprocess( + cfgMain, typeCfgInfo, cfgInfo, module=currentModule + ) + return simDict, inputSimFiles + +def copyMoTFiles(workDir, outputDir, searchString, replaceString): + """ + Copy and rename MoT result files from work directory to output directory. + + Parameters + ---------- + workDir : pathlib.Path + Source directory containing the original p_max files + outputDir : pathlib.Path + Destination directory where renamed ppr files will be copied to + searchString : str + String pattern to search for in the original filenames + replaceString : str + String to replace the searchString with in the new filenames + + Returns + ------- + None + Files are copied to the destination directory with renamed extensions + """ + varFiles = list(workDir.glob("*"+searchString+"*")) + targetFiles = [ + pathlib.Path(str(f.name).replace(searchString, replaceString)) + for f in varFiles + ] + targetFiles = [outputDir / f for f in targetFiles] + + for source, target in zip(varFiles, targetFiles): + shutil.copy2(source, target) \ No newline at end of file diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index a52c50ff0..8a62f6192 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -1,7 +1,7 @@ -''' - Utilities for handling configuration files +""" +Utilities for handling configuration files -''' +""" import configparser import logging @@ -27,25 +27,25 @@ log = logging.getLogger(__name__) -def getGeneralConfig(nameFile=''): - ''' Returns the general configuration for avaframe +def getGeneralConfig(nameFile=""): + """Returns the general configuration for avaframe returns a configParser object Parameters ---------- nameFile: pathlib path optional full path to file, if empty use avaframeCfg from folder one level up - ''' + """ # get path of module modPath = pathlib.Path(avaf.__file__).resolve().parent if isinstance(nameFile, pathlib.Path): - localFile = nameFile.parents[0] / ('local_' + nameFile.name) + localFile = nameFile.parents[0] / ("local_" + nameFile.name) defaultFile = nameFile else: - localFile = modPath / 'local_avaframeCfg.ini' - defaultFile = modPath / 'avaframeCfg.ini' + localFile = modPath / "local_avaframeCfg.ini" + defaultFile = modPath / "avaframeCfg.ini" if localFile.is_file(): iniFile = localFile @@ -55,16 +55,16 @@ def getGeneralConfig(nameFile=''): iniFile = defaultFile compare = False else: - raise FileNotFoundError('None of the provided cfg files exist ') + raise FileNotFoundError("None of the provided cfg files exist ") # Finally read it - cfg, _ = readCompareConfig(iniFile, 'General', compare) + cfg, _ = readCompareConfig(iniFile, "General", compare) return cfg -def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDefault=False): - ''' Returns the configuration for a given module +def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDefault=False): + """Returns the configuration for a given module returns a configParser object module object: module : the calling function provides the already imported @@ -87,10 +87,10 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe Order is as follows: fileOverride -> local_MODULECfg.ini -> MODULECfg.ini - ''' + """ if isinstance(onlyDefault, bool) == False: - message = 'OnlyDefault parameter is not a boolean but %s' % type(onlyDefault) + message = "OnlyDefault parameter is not a boolean but %s" % type(onlyDefault) log.error(message) raise TypeError(message) @@ -101,11 +101,11 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe else: modPath, modName = getModPathName(module) - localFile = modPath / ('local_'+modName+'Cfg.ini') - defaultFile = modPath / (modName+'Cfg.ini') + localFile = modPath / ("local_" + modName + "Cfg.ini") + defaultFile = modPath / (modName + "Cfg.ini") - log.debug('localFile: %s', localFile) - log.debug('defaultFile: %s', defaultFile) + log.debug("localFile: %s", localFile) + log.debug("defaultFile: %s", defaultFile) # Decide which one to take if fileOverride: @@ -114,8 +114,7 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe iniFile = [defaultFile, fileOverride] compare = True else: - raise FileNotFoundError('Provided fileOverride does not exist: ' + - str(fileOverride)) + raise FileNotFoundError("Provided fileOverride does not exist: " + str(fileOverride)) elif localFile.is_file() and not onlyDefault: iniFile = localFile @@ -125,11 +124,10 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe iniFile = defaultFile compare = False else: - raise FileNotFoundError('None of the provided cfg files exist ') + raise FileNotFoundError("None of the provided cfg files exist ") # Finally read it cfg, modDict = readCompareConfig(iniFile, modName, compare, toPrint) - if modInfo: return cfg, modDict @@ -137,7 +135,7 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe def getDefaultModuleConfig(module, toPrint=True): - ''' Returns the default configuration for a given module + """Returns the default configuration for a given module returns a configParser object module object: module : the calling function provides the already imported @@ -148,15 +146,15 @@ def getDefaultModuleConfig(module, toPrint=True): from avaframe.com2AB import com2AB as c2 leads to getModuleConfig(c2) - ''' + """ # get path to the module and its name modPath, modName = getModPathName(module) - defaultFile = modPath / (modName+'Cfg.ini') + defaultFile = modPath / (modName + "Cfg.ini") - log.info('Getting the default config for %s', modName) - log.debug('defaultFile: %s', defaultFile) + log.info("Getting the default config for %s", modName) + log.debug("defaultFile: %s", defaultFile) # Finally read it cfg, _ = readCompareConfig(defaultFile, modName, compare=False, toPrint=toPrint) @@ -165,7 +163,7 @@ def getDefaultModuleConfig(module, toPrint=True): def readCompareConfig(iniFile, modName, compare, toPrint=True): - ''' Read and optionally compare configuration files (if a local and default are both provided) + """Read and optionally compare configuration files (if a local and default are both provided) and inform user of the eventual differences. Take the default as reference. Parameters @@ -184,10 +182,10 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): contains combined config modDict: dict dictionary containing only differences from default - ''' + """ if compare: - log.info('Reading config from: %s and %s' % (iniFile[0], iniFile[1])) + log.info("Reading config from: %s and %s" % (iniFile[0], iniFile[1])) # initialize configparser object to read defCfg = configparser.ConfigParser() defCfg.optionxform = str @@ -196,13 +194,12 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): # read default and local parser files defCfg.read(iniFile[0]) locCfg.read(iniFile[1]) - - log.debug('Writing cfg for: %s', modName) + log.debug("Writing cfg for: %s", modName) # compare to default config and get modification dictionary and config modDict, modCfg = compareTwoConfigs(defCfg, locCfg, toPrint=toPrint) else: - log.info('Reading config from: %s', iniFile) + log.info("Reading config from: %s", iniFile) # initialize our final configparser object modCfg = configparser.ConfigParser() modCfg.optionxform = str @@ -217,58 +214,58 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): def _splitDeepDiffValuesChangedItem(inKey, inVal): - """ splits one item of a deepdiff result into section, key, old value, new value - - Parameters - ----------- - inputKey: str - key of a deepdiff changed_values item - inputValue: dict - value of a deepdiff changed_values item - - Returns - -------- - section: str - section name of changed item - key: str - key name of changed item - oldVal: str - old value - newVal: str - new value + """splits one item of a deepdiff result into section, key, old value, new value + + Parameters + ----------- + inKey: str + key of a deepdiff changed_values item + inVal: dict + value of a deepdiff changed_values item + + Returns + -------- + section: str + section name of changed item + key: str + key name of changed item + oldVal: str + old value + newVal: str + new value """ - splitKey = re.findall(r"\['?([A-Za-z0-9_]+)'?\]", inKey) + splitKey = re.findall(r"\[\s*['\"]([^'\"]+)['\"]\s*\]", inKey) section = splitKey[0] key = splitKey[1] - return section, key, inVal['old_value'], inVal['new_value'] + return section, key, inVal["old_value"], inVal["new_value"] def compareTwoConfigs(defCfg, locCfg, toPrint=False): - """ compare locCfg to defCfg and return a cfg object and modification dict - Values are merged from locCfg to defCfg: - - parameters already in defCfg get the value from locCfg - - additional values in locCfg get added in the resulting Cfg - - Parameters - ----------- - defCfg: configparser object - default configuration - locCfg: configuration object - configuration that is compared to defCfg - toPrint: bool - flag if config shall be printed to log - - Returns - -------- - modInfo: dict - dictionary containing only differences from default - cfg: configParser object - contains combined config + """compare locCfg to defCfg and return a cfg object and modification dict + Values are merged from locCfg to defCfg: + - parameters already in defCfg get the value from locCfg + - additional values in locCfg get added in the resulting Cfg + + Parameters + ----------- + defCfg: configparser object + default configuration + locCfg: configuration object + configuration that is compared to defCfg + toPrint: bool + flag if config shall be printed to log + + Returns + -------- + modInfo: dict + dictionary containing only differences from default + cfg: configParser object + contains combined config """ - log.info('Comparing two configs') + log.info("Comparing two configs") # initialize modInfo and printOutInfo modInfo = dict() @@ -287,7 +284,6 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): except ValueError: cfgDiff = DeepDiff(defCfgD, locCfgD) - # Combine them, different keys are just added, for the same keys, the # local (right) value is used modCfgD = deepcopy(defCfgD) @@ -301,12 +297,12 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): # If toPrint is set, print full configuration: if toPrint: - for line in pformat(modCfgD, sort_dicts=False).split('\n'): + for line in pformat(modCfgD, sort_dicts=False).split("\n"): log.info(line) # Generate modInfo dictionary for output - if 'values_changed' in cfgDiff: - for key, value in cfgDiff['values_changed'].items(): + if "values_changed" in cfgDiff: + for key, value in cfgDiff["values_changed"].items(): section, itemKey, defValue, locValue = _splitDeepDiffValuesChangedItem(key, value) if section not in modInfo: @@ -316,91 +312,91 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): modInfo[section][itemKey] = modString # Log changes - log.info('COMPARING TO DEFAULT, THESE CHANGES HAPPENED:') - for line in cfgDiff.pretty().split('\n'): - log.info(line.replace('root','')) + log.info("COMPARING TO DEFAULT, THESE CHANGES HAPPENED:") + for line in cfgDiff.pretty().split("\n"): + log.info(line.replace("root", "")) return modInfo, modCfg -def writeCfgFile(avaDir, module, cfg, fileName='', filePath=''): - """ Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini - or optional to filePath and with fileName +def writeCfgFile(avaDir, module, cfg, fileName="", filePath=""): + """Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini + or optional to filePath and with fileName - Parameters - ----------- - avaDir: str - path to avalanche directory - module: - module - cfg: configparser object - configuration settings - fileName: str - name of saved configuration file - optional - filePath: str or pathlib path - path where file should be saved to except file name - optional + Parameters + ----------- + avaDir: str + path to avalanche directory + module: + module + cfg: configparser object + configuration settings + fileName: str + name of saved configuration file - optional + filePath: str or pathlib path + path where file should be saved to except file name - optional """ # get filename of module name = pathlib.Path(module.__file__).name - modName = name.split('.')[0] + modName = name.split(".")[0] # set outputs - if filePath == '': - outDir = pathlib.Path(avaDir, 'Outputs', modName, 'configurationFiles') + if filePath == "": + outDir = pathlib.Path(avaDir, "Outputs", modName, "configurationFiles") fU.makeADir(outDir) else: if filePath.is_dir(): outDir = pathlib.Path(filePath) else: - message = '%s is not a valid location for saving cfg file' % str(filePath) + message = "%s is not a valid location for saving cfg file" % str(filePath) log.error(message) raise NotADirectoryError(message) # set path to file - if fileName == '': + if fileName == "": fileName = modName - pathToFile = pathlib.Path(outDir, '%s.ini' % (fileName)) + pathToFile = pathlib.Path(outDir, "%s.ini" % (fileName)) # write file - with open(pathToFile, 'w') as conf: + with open(pathToFile, "w") as conf: cfg.write(conf) return pathToFile -def readCfgFile(avaDir, module='', fileName=''): - """ Read configuration from ini file, if module is provided, module configuration is read from Ouputs, - if fileName is provided configuration is read from fileName +def readCfgFile(avaDir, module="", fileName=""): + """Read configuration from ini file, if module is provided, module configuration is read from Ouputs, + if fileName is provided configuration is read from fileName - Parameters - ----------- - avaDir: str - path to avalanche directory - module: - module - fileName: str - path to file that should be read - optional + Parameters + ----------- + avaDir: str + path to avalanche directory + module: + module + fileName: str + path to file that should be read - optional - Returns - -------- - cfg: configParser object - configuration that is from file + Returns + -------- + cfg: configParser object + configuration that is from file """ # define file that should be read - if fileName != '': + if fileName != "": inFile = fileName - elif module != '': + elif module != "": # get module name name = pathlib.Path(module.__file__).name - modName = name.split('.')[0] + modName = name.split(".")[0] # set input file - inFile = pathlib.Path(avaDir, 'Outputs', '%s_settings.ini' % (modName)) + inFile = pathlib.Path(avaDir, "Outputs", "%s_settings.ini" % (modName)) else: - log.error('Please provide either a module or a fileName to read configuration from file') + log.error("Please provide either a module or a fileName to read configuration from file") raise NameError # read configParser object from input file, case sensitive @@ -413,7 +409,7 @@ def readCfgFile(avaDir, module='', fileName=''): def cfgHash(cfg, typeDict=False): - """ UID hash of a config. Given a configParser object cfg, + """UID hash of a config. Given a configParser object cfg, or a dictionary - then typeDict=True, returns a uid hash Parameters @@ -445,7 +441,7 @@ def cfgHash(cfg, typeDict=False): def convertConfigParserToDict(cfg): - """ create dictionary from configparser object """ + """create dictionary from configparser object""" cfgDict = {} for section in cfg.sections(): @@ -457,7 +453,7 @@ def convertConfigParserToDict(cfg): def convertDictToConfigParser(cfgDict): - """ create configParser object from dict """ + """create configParser object from dict""" cfg = configparser.ConfigParser() cfg.optionxform = str @@ -468,7 +464,7 @@ def convertDictToConfigParser(cfgDict): def writeDictToJson(inDict, outFilePath): - """ write a dictionary to a json file """ + """write a dictionary to a json file""" jsonDict = json.dumps(inDict, sort_keys=True, ensure_ascii=True) f = open(outFilePath, "w") @@ -476,41 +472,48 @@ def writeDictToJson(inDict, outFilePath): f.close() -def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir='', simNameList=[]): - """ Read configurations from all simulations configuration ini files from directory - - Parameters - ----------- - avaDir: str - path to avalanche directory - standardCfg: dict - standard configuration for module - option - writeCSV: bool - True if configuration dataFrame shall be written to csv file - specDir: str - path to a directory where simulation configuration files can be found - optional - simNameList: list - if non-empty list only use cfgFiles that are included within simNameList - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations +def createConfigurationInfo( + avaDir, + comModule="com1DFA", + standardCfg="", + writeCSV=False, + specDir="", + simNameList=[], +): + """Read configurations from all simulations configuration ini files from directory + + Parameters + ----------- + avaDir: str + path to avalanche directory + standardCfg: dict + standard configuration for module - option + writeCSV: bool + True if configuration dataFrame shall be written to csv file + specDir: str + path to a directory where simulation configuration files can be found - optional + simNameList: list + if non-empty list only use cfgFiles that are included within simNameList + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', comModule, 'configurationFiles') - configFiles = list(inDir.glob('*.ini')) + inDir = pathlib.Path(avaDir, "Outputs", comModule, "configurationFiles") + configFiles = list(inDir.glob("*.ini")) if not inDir.is_dir(): - message = 'configuration file directory not found: %s' % (inDir) + message = "configuration file directory not found: %s" % (inDir) log.error(message) raise NotADirectoryError(message) elif configFiles == []: - message = 'No configuration file found in: %s' % (inDir) + message = "No configuration file found in: %s" % (inDir) log.error(message) raise FileNotFoundError(message) @@ -523,16 +526,16 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS else: # create configparser object, convert to json object, write to dataFrame # append all dataFrames - simDF = '' + simDF = "" for cFile in configFiles: - if 'sourceConfiguration' not in str(cFile): + if "sourceConfiguration" not in str(cFile): simName = pathlib.Path(cFile).stem - if '_AF_' in simName: - nameParts = simName.split('_AF_') - infoParts = nameParts[1].split('_') + if "_AF_" in simName: + nameParts = simName.split("_AF_") + infoParts = nameParts[1].split("_") else: - nameParts = simName.split('_') + nameParts = simName.split("_") infoParts = nameParts[1:] simHash = infoParts[0] cfgObject = readCfgFile(avaDir, fileName=cFile) @@ -542,9 +545,9 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS simDF = convertDF2numerics(simDF) # add default configuration - if standardCfg != '': + if standardCfg != "": # read default configuration of this module - simDF = appendCgf2DF('current standard', 'current standard', standardCfg, simDF) + simDF = appendCgf2DF("current standard", "current standard", standardCfg, simDF) # if writeCSV, write dataFrame to csv file if writeCSV: @@ -554,35 +557,38 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS def appendCgf2DF(simHash, simName, cfgObject, simDF): - """ append simulation configuration to the simulation dataframe - only account for sections GENERAL and INPUT - - Parameters - ----------- - simHash: str - hash of the simulation to append - simName: str - name of the simulation - cfgObject: configParser - configuration coresponding to the simulation - simDF: pandas dataFrame - configuration dataframe - - Returns - -------- - simDF: pandas DataFrame - DFappended with the new simulation configuration + """append simulation configuration to the simulation dataframe + append all sections to the dataframe + + Parameters + ----------- + simHash: str + hash of the simulation to append + simName: str + name of the simulation + cfgObject: configParser + configuration coresponding to the simulation + simDF: pandas dataFrame + configuration dataframe + + Returns + -------- + simDF: pandas DataFrame + DFappended with the new simulation configuration """ indexItem = [simHash] cfgDict = convertConfigParserToDict(cfgObject) - simItemDFGeneral = pd.DataFrame(data=cfgDict['GENERAL'], index=indexItem) - simItemDFInput = pd.DataFrame(data=cfgDict['INPUT'], index=indexItem) - if 'VISUALISATION' in cfgDict: - simItemDFVisualisation = pd.DataFrame(data=cfgDict['VISUALISATION'], index=indexItem) - simItemDF = pd.concat([simItemDFGeneral, simItemDFInput, simItemDFVisualisation], axis=1) - else: - simItemDF = pd.concat([simItemDFGeneral, simItemDFInput], axis=1) + simItemDFList = [] + for section in cfgDict: + simItemDFSection = pd.DataFrame(data=cfgDict[section], index=indexItem) + simItemDFList.append(simItemDFSection) + simItemDF = pd.concat(simItemDFList, axis=1) simItemDF = simItemDF.assign(simName=simName) + + # check for duplicates: if yes, rename them by adding Dupl1 to the duplicate name + if simItemDF.columns.duplicated().any(): + renameDuplicates(simItemDF) + if isinstance(simDF, str): simDF = simItemDF else: @@ -590,22 +596,53 @@ def appendCgf2DF(simHash, simName, cfgObject, simDF): return simDF +def renameDuplicates(df): + """ + Rename duplicate column names in the given DataFrame. This ensures all column names in the DataFrame + are unique by adding a suffix 'DuplX' where X is the occurrence number, starting + from 1 for the first duplicate. + + Parameters + ---------- + df : pandas.DataFrame + The input DataFrame whose column names need to be checked for duplicates. + + Returns + ------- + bool + Returns True to indicate the renaming of duplicate column names was successful. + """ + seen = {} + new_cols = [] + + for col in df.columns: + if col not in seen: + seen[col] = 0 + new_cols.append(col) + else: + seen[col] += 1 + new_cols.append(f"{col}_{seen[col]}") + + df.columns = new_cols + return True + + def appendTcpu2DF(simHash, tCPU, tCPUDF): - """ append Tcpu dictionary to the dataframe - - Parameters - ----------- - simHash: str - hash of the simulation corresponding to the tCPU dict to append - tCPU: dict - cpu time dict of the simulation - tCPUDF: pandas dataFrame - tCPU dataframe - - Returns - -------- - simDF: pandas DataFrame - DFappended with the new simulation configuration + """append Tcpu dictionary to the dataframe + + Parameters + ----------- + simHash: str + hash of the simulation corresponding to the tCPU dict to append + tCPU: dict + cpu time dict of the simulation + tCPUDF: pandas dataFrame + tCPU dataframe + + Returns + -------- + simDF: pandas DataFrame + DFappended with the new simulation configuration """ indexItem = [simHash] tCPUItemDF = pd.DataFrame(data=tCPU, index=indexItem) @@ -617,163 +654,188 @@ def appendTcpu2DF(simHash, tCPU, tCPUDF): def convertDF2numerics(simDF): - """ convert a string DF to a numerical one + """convert a string DF to a numerical one - Parameters - ----------- - simDF: pandas dataFrame - dataframe + Parameters + ----------- + simDF: pandas dataFrame + dataframe - Returns - -------- - simDF: pandas DataFrame + Returns + -------- + simDF: pandas DataFrame """ for name, values in simDF.items(): - simDFTest = simDF[name].str.replace('.', '', regex=False) + simDFTest = simDF[name].str.replace(".", "", regex=False) # allow for - sign too - simDFTest = simDFTest.replace('-', '', regex=False) + simDFTest = simDFTest.replace("-", "", regex=False) # check for str(np.nan) as these cannot be converted to numerics by pd.to_numeric # but as friction model parameters are set to nans this is required here - if simDFTest.str.match('nan').any(): + if simDFTest.str.match("nan").any(): simDF = setStrnanToNan(simDF, simDFTest, name) # also include columns where nan is in first row - so check for any row - if simDFTest.str.isdigit().any() and (name != 'tSteps'): + if simDFTest.str.isdigit().any() and (name != "tSteps"): # problem here is that it finds even if not present in | although not in ini - simDFTest = simDF[name].str.replace('|', '§', regex=False) - if simDFTest.str.contains('§').any() == False: + simDFTest = simDF[name].str.replace("|", "§", regex=False) + if simDFTest.str.contains("§").any() == False: simDF[name] = pd.to_numeric(simDF[name]) - log.debug('Converted to numeric %s' % name) + log.debug("Converted to numeric %s" % name) else: - log.debug('Not converted to numeric: %s' % name) + log.debug("Not converted to numeric: %s" % name) return simDF def setStrnanToNan(simDF, simDFTest, name): - """ set pandas element to np.nan if it is a string nan - - Parameters - ----------- - simDF: pandas dataFrame - dataframe - simDFTest: pandas series - series of sim DF column named name - replaced "." with " " - name: str - name of pandas dataframe column - - Returns - -------- - simDF: pandas dataframe - updated pandas dataframe with np.nan values where string nan was + """set pandas element to np.nan if it is a string nan + + Parameters + ----------- + simDF: pandas dataFrame + dataframe + simDFTest: pandas series + series of sim DF column named name + replaced "." with " " + name: str + name of pandas dataframe column + + Returns + -------- + simDF: pandas dataframe + updated pandas dataframe with np.nan values where string nan was """ - nanIndex = simDFTest.str.match('nan', flags=re.IGNORECASE) + nanIndex = simDFTest.str.match("nan", flags=re.IGNORECASE) simIndex = simDF.index.values # loop over each row and use simDF.at to avoid copy vs view warning for index, nanInd in enumerate(nanIndex): if nanInd: simDF.at[simIndex[index], name] = np.nan - log.info('%s for index: %s set to numpy nan' % (name, index)) + log.info("%s for index: %s set to numpy nan" % (name, index)) return simDF -def readConfigurationInfoFromDone(avaDir, specDir='', latest=False): - """ Check avaName/Outputs/com1DFA/configurationFilesDone and pass - names of all files found in this directory and create corresponding simDF - this is useful if e.g. no allConfigurations.csv has - been written but already some simulations have been performed as a txt file is saved in - avaName/Outputs/com1DFA/configurationFiles after the respective simulation has been run - whereas the allConfigurations file is written at the end of a call to com1DFAMain that can - include several individual sims - if latest=True only look for latest simulations in avaName/Outputs/com1DFA/configurationFilesLatest - - Parameters - ----------- - avaDir: str - path to avalanche directory - specDir: str - path to a directory where simulation configuration files directory called configurationFiles can be found - optional - latest: bool - if True check for files found in avaName/Outputs/com1DFA/configurationFilesLatest - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations - simDFName: array - simName column of the dataframe + +def readConfigurationInfoFromDone(avaDir, specDir="", latest=False): + """Check avaName/Outputs/com1DFA/configurationFilesDone and pass + names of all files found in this directory and create corresponding simDF + this is useful if e.g. no allConfigurations.csv has + been written but already some simulations have been performed as a txt file is saved in + avaName/Outputs/com1DFA/configurationFiles after the respective simulation has been run + whereas the allConfigurations file is written at the end of a call to com1DFAMain that can + include several individual sims + if latest=True only look for latest simulations in avaName/Outputs/com1DFA/configurationFilesLatest + + Parameters + ----------- + avaDir: str + path to avalanche directory + specDir: str + path to a directory where simulation configuration files directory called configurationFiles can be found - optional + latest: bool + if True check for files found in avaName/Outputs/com1DFA/configurationFilesLatest + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations + simDFName: array + simName column of the dataframe """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") # search inDir/configurationFilesDone or inDir/configurationFilesLatest (depending on latest flag) for already existing sims if latest: - configDir = inDir / 'configurationFilesLatest' + configDir = inDir / "configurationFilesLatest" else: - configDir = inDir / 'configurationFilesDone' + configDir = inDir / "configurationFilesDone" - existingSims = list(configDir.glob('*.ini')) + existingSims = list(configDir.glob("*.ini")) simNameExisting = [] for fName in existingSims: simNameExisting.append(fName.stem) - if list((inDir / 'configurationFilesDone').glob('*.ini')) == []: - log.info('No existing simulations in Outputs found') + if list((inDir / "configurationFilesDone").glob("*.ini")) == []: + log.info("No existing simulations in Outputs found") simDF = None else: # create simDF (dataFrame with one row per simulation of configuration files found in configDir) - simDF = createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir=specDir, - simNameList=simNameExisting) - - + simDF = createConfigurationInfo( + avaDir, + comModule="com1DFA", + standardCfg="", + writeCSV=False, + specDir=specDir, + simNameList=simNameExisting, + ) # check for allConfigurationsInfo to find computation info and add to info fetched from ini files if latest == False and isinstance(simDF, pd.DataFrame): # check if in allConfigurationsInfo also info for existing sims - simDFALL, _ = readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfigurations') + simDFALL, _ = readAllConfigurationInfo(avaDir, specDir="", configCsvName="allConfigurations") if isinstance(simDFALL, pd.DataFrame): - simDF = simDF.reset_index().merge(simDFALL[['nPart', 'timeLoop', 'timeForce', 'timeForceSPH', 'timePos', 'timeNeigh', - 'timeField', 'nSave', 'nIter', 'simName']], how='left', on='simName').set_index('index') + simDF = ( + simDF.reset_index() + .merge( + simDFALL[ + [ + "nPart", + "timeLoop", + "timeForce", + "timeForceSPH", + "timePos", + "timeNeigh", + "timeField", + "nSave", + "nIter", + "simName", + ] + ], + how="left", + on="simName", + ) + .set_index("index") + ) return simDF, simNameExisting -def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfigurations'): - """ Read allConfigurations.csv file as dataFrame from directory - - Parameters - ----------- - avaDir: str - path to avalanche directory - specDir: str - path to a directory where simulation configuration files can be found - optional - configCsvName: str - name of configuration csv file - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations - simDFName: array - simName column of the dataframe +def readAllConfigurationInfo(avaDir, specDir="", configCsvName="allConfigurations"): + """Read allConfigurations.csv file as dataFrame from directory + + Parameters + ----------- + avaDir: str + path to avalanche directory + specDir: str + path to a directory where simulation configuration files can be found - optional + configCsvName: str + name of configuration csv file + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations + simDFName: array + simName column of the dataframe """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') - configFiles = inDir / ('%s.csv' % configCsvName) + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") + configFiles = inDir / ("%s.csv" % configCsvName) if configFiles.is_file(): - with open(configFiles, 'rb') as file: + with open(configFiles, "rb") as file: simDF = pd.read_csv(file, index_col=0, keep_default_na=False) - simDFName = simDF['simName'].to_numpy() + simDFName = simDF["simName"].to_numpy() else: simDF = None simDFName = [] @@ -781,31 +843,31 @@ def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfiguration return simDF, simDFName -def writeAllConfigurationInfo(avaDir, simDF, specDir='', csvName='allConfigurations.csv'): - """ Write cfg configuration to allConfigurations.csv - - Parameters - ----------- - avaDir: str - path to avalanche directory - simDF: pandas dataFrame - daaframe of the configuration - specDir: str - path to a directory where simulation configuration shal be saved - optional - csvName: str - name of csv file in which to save to - optional - - Returns - -------- - configFiles: pathlib Path - path where the configuration dataframe was saved +def writeAllConfigurationInfo(avaDir, simDF, specDir="", csvName="allConfigurations.csv"): + """Write cfg configuration to allConfigurations.csv + + Parameters + ----------- + avaDir: str + path to avalanche directory + simDF: pandas dataFrame + daaframe of the configuration + specDir: str + path to a directory where simulation configuration shal be saved - optional + csvName: str + name of csv file in which to save to - optional + + Returns + -------- + configFiles: pathlib Path + path where the configuration dataframe was saved """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") configFiles = inDir / csvName simDF.to_csv(configFiles) @@ -814,53 +876,53 @@ def writeAllConfigurationInfo(avaDir, simDF, specDir='', csvName='allConfigurati def convertToCfgList(parameterList): - """ convert a list into a string where individual list items are separated by | + """convert a list into a string where individual list items are separated by | - Parameters - ----------- - parameterList: list - list of parameter values + Parameters + ----------- + parameterList: list + list of parameter values - Returns - --------- - parameterString: str - str with parameter values separated by | + Returns + --------- + parameterString: str + str with parameter values separated by | """ if len(parameterList) == 0: - parameterString = '' + parameterString = "" else: parameterString = parameterList[0] for item in parameterList[1:]: - parameterString = parameterString + '|' + item + parameterString = parameterString + "|" + item return parameterString def getNumberOfProcesses(cfgMain, nSims): - """ Determine how many CPU cores to take for parallel tasks + """Determine how many CPU cores to take for parallel tasks - Parameters - ----------- - cfgMain: configuration object - the main avaframe configuration - nSims: integer - number of simulations that need to be calculated + Parameters + ----------- + cfgMain: configuration object + the main avaframe configuration + nSims: integer + number of simulations that need to be calculated - Returns - --------- - nCPU: int - number of cores to take + Returns + --------- + nCPU: int + number of cores to take """ maxCPU = multiprocessing.cpu_count() - if cfgMain["MAIN"]["nCPU"] == 'auto': - cpuPerc = float(cfgMain["MAIN"]["CPUPercent"]) / 100. + if cfgMain["MAIN"]["nCPU"] == "auto": + cpuPerc = float(cfgMain["MAIN"]["CPUPercent"]) / 100.0 nCPU = math.floor(maxCPU * cpuPerc) else: - nCPU = cfgMain['MAIN'].getint('nCPU') + nCPU = cfgMain["MAIN"].getint("nCPU") # if number of sims is lower than nCPU nCPU = min(nCPU, nSims) @@ -872,7 +934,7 @@ def getNumberOfProcesses(cfgMain, nSims): def getModPathName(module): - """ get the path and name of a module from imported module + """get the path and name of a module from imported module Parameters ------------ @@ -893,4 +955,34 @@ def getModPathName(module): # get filename of module modName = str(pathlib.Path(module.__file__).stem) - return modPath, modName \ No newline at end of file + return modPath, modName + + +def cfgToRcf(cfg, fileName): + """Convert configuration object to RCF format file (used by NGI MoT). + + Takes a ConfigParser object and writes its contents to a file in rcf format, + excluding certain sections and formatting others according to RCF requirements. + + Parameters + ---------- + cfg : configparser.ConfigParser + Configuration object containing sections and their key-value pairs + fileName : str or pathlib.Path + Path to the output file where the RCF format will be written + """ + with open(fileName, "w") as f: + for section in cfg.sections(): + if section in ("FOREST_EFFECTS", "ENTRAINMENT"): + pass + elif section in ("GENERAL", "INPUT"): + continue + else: + f.write(f"# {section.replace('_', ' ')}\n") + f.write("#\n") + for key, value in cfg.items(section): + # key = key.replace('_', ' ') + key = key.strip() + f.write(f"{key:<40}{value}\n") + f.write("#\n") + diff --git a/avaframe/out3Plot/plotUtils.py b/avaframe/out3Plot/plotUtils.py index 740149d50..7673ba8e3 100644 --- a/avaframe/out3Plot/plotUtils.py +++ b/avaframe/out3Plot/plotUtils.py @@ -235,6 +235,7 @@ "ppr": cmapPres, "pfv": cmapSpeed, "pft": cmapThickness, + "pfd": cmapThickness, "P": cmapPres, "FV": cmapSpeed, "FM": cmapThickness, diff --git a/avaframe/out3Plot/plotUtilsCfg.ini b/avaframe/out3Plot/plotUtilsCfg.ini index ad0585e15..6cc8cd51b 100644 --- a/avaframe/out3Plot/plotUtilsCfg.ini +++ b/avaframe/out3Plot/plotUtilsCfg.ini @@ -66,6 +66,7 @@ gravityAcc = 9.81 # units for output variables unitppr = kPa unitpft = m +unitpfd = m unitpfv = $ms^{-1}$ unitpta = ° unitpke = $kJm^{-2}$ @@ -86,6 +87,7 @@ unitdmDet = kg # threshold levels elevMaxppr = 100 elevMaxpft = 1 +elevMaxpfd = 1 elevMaxpfv = 10 elevMaxpta = 40 elevMaxP = 100 @@ -109,6 +111,7 @@ probaColorLevels = 0|0.25|0.50|0.75|1. # contour levels (when adding contour lines on a plot) contourLevelsppr = 1|3|5|10|25|50|100|250 contourLevelspft = 0.1|0.25|0.5|0.75|1 +contourLevelspfd = 0.1|0.25|0.5|0.75|1 contourLevelspfv = 0.5|1|5|10|25|50 contourLevelspta = 10|15|20|25|30|35 contourLevelsP = 1|3|5|10|25|50|100|250 @@ -120,6 +123,7 @@ contourLevelsFTDet = -0.1|-0.25|-0.5|-0.75|-1 # name for result parameters nameppr = peak pressure namepft = peak flow thickness +namepfd = peak flow depth namepfv = peak flow velocity namepta = peak travel angle namepke = peak kinetic energy diff --git a/avaframe/runAna4ProbAna.py b/avaframe/runAna4ProbAna.py index 3c6627329..fba76fe38 100644 --- a/avaframe/runAna4ProbAna.py +++ b/avaframe/runAna4ProbAna.py @@ -127,4 +127,4 @@ def runProbAna(avalancheDir=''): help='the avalanche directory') args = parser.parse_args() - runProbAna(str(args.avadir)) + runProbAna(str(args.avadir)) \ No newline at end of file diff --git a/avaframe/runScripts/runAna3AIMEC.py b/avaframe/runScripts/runAna3AIMEC.py index 96565ecc1..49de10665 100644 --- a/avaframe/runScripts/runAna3AIMEC.py +++ b/avaframe/runScripts/runAna3AIMEC.py @@ -8,6 +8,8 @@ from avaframe.in3Utils import initializeProject as iP from avaframe.in3Utils import cfgUtils from avaframe.in3Utils import logUtils + + # create local logger log = logging.getLogger(__name__) diff --git a/avaframe/runStandardTestsCom1DFA.py b/avaframe/runStandardTestsCom1DFA.py index a8ad4c680..75bc7b214 100644 --- a/avaframe/runStandardTestsCom1DFA.py +++ b/avaframe/runStandardTestsCom1DFA.py @@ -47,8 +47,8 @@ # valuesList = ['resistance'] filterType = 'TAGS' valuesList = ['standardTest', 'standardTestSnowGlide'] -# filterType = 'AVANAME' -# valuesList = ['avaAlr'] +# filterType = 'NAME' +# valuesList = ['avaPyramidNullTest'] testList = tU.filterBenchmarks(testDictList, filterType, valuesList, condition='or') @@ -80,6 +80,7 @@ avaDir = test['AVADIR'] cfgMain['MAIN']['avalancheDir'] = avaDir + print("\n", 40 * "*", "\n", test["NAME"], 40 * "*") # Fetch benchmark test info benchDict = test simNameRef = test['simNameRef'] diff --git a/avaframe/tests/test_MoTUtils.py b/avaframe/tests/test_MoTUtils.py new file mode 100644 index 000000000..ecae4bcb1 --- /dev/null +++ b/avaframe/tests/test_MoTUtils.py @@ -0,0 +1,208 @@ +import pytest +import numpy as np +import pathlib +from unittest.mock import patch, MagicMock, Mock, call +from avaframe.in3Utils import MoTUtils +import os +import platform + + +# Note 3.7.25: these tests are mostly AI-generated + +def test_rewriteDEMtoZeroValues(tmp_path): + # Create mock input data + mockDemData = { + "rasterData": np.array([[1.0, np.nan, 3.0], + [np.nan, 5.0, 6.0]]), + "header": { + "nodata_value": np.nan + } + } + + expectedRasterData = np.array([[1.0, 0.0, 3.0], + [0.0, 5.0, 6.0]]) + + # Create a mock Path object avaTestDir = "avaHockeyChannelPytest" + # mockPath = pathlib.Path(tmp_path) + # outFile = pathlib.Path(tmp_path, 'testDir', 'testDem.asc') + # print(outFile) + mockPath = MagicMock( + spec=pathlib.Path, + parent=MagicMock( + spec=pathlib.Path, + name="testDir" + ), + name="testDem.asc" + ) + # mockPath = MagicMock(spec=pathlib.Path) + # print(mockPath) + # mockPath.stem # = "testDem" + + # Mock the rasterUtils functions + with patch('avaframe.in2Trans.rasterUtils.readRaster') as mockRead: + with patch('avaframe.in2Trans.rasterUtils.writeResultToRaster') as mockWrite: + # Set up the mock return value for readRaster + mockRead.return_value = mockDemData + + # Call the function + MoTUtils.rewriteDEMtoZeroValues(mockPath) + # MoTUtils.rewriteDEMtoZeroValues(outFile) + + # Verify readRaster was called with correct argument + mockRead.assert_called_once_with(mockPath) + # mockRead.assert_called_once_with(outFile) + + # Verify the data was modified correctly + np.testing.assert_array_equal( + mockDemData["rasterData"], + expectedRasterData + ) + + # Verify nodata value was updated + assert mockDemData["header"]["nodata_value"] == 0.0 + + # Verify writeResultToRaster was called with correct arguments + mockWrite.assert_called_once() + call_args = mockWrite.call_args[0] + assert call_args[0] == mockDemData["header"] + + +def test_CopyMoTFiles(tmp_path): + # Create temporary source and destination directories + workDir = tmp_path / "work" + outputDir = tmp_path / "output" + workDir.mkdir() + outputDir.mkdir() + + # Create some test files in work directory + testFiles = [ + "test_p_max_001.txt", + "test_p_max_002.txt", + "another_p_max_file.txt" + ] + + for fileName in testFiles: + (workDir / fileName).touch() + + # Test parameters + searchString = "p_max" + replaceString = "ppr" + + # Create expected target file names + expectedTargets = [ + outputDir / name.replace(searchString, replaceString) + for name in testFiles + ] + + # Mock shutil.copy2 to avoid actual file copying + with patch('shutil.copy2') as mockCopy: + # Run the function + MoTUtils.copyMoTFiles(workDir, outputDir, searchString, replaceString) + + # Check that copy2 was called the correct number of times + assert mockCopy.call_count == len(testFiles) + + # Verify each copy operation + # Get actual calls and sort them by source path + actualCopies = sorted(mockCopy.call_args_list, + key=lambda x: str(x[0][0])) + sourceFilePaths = sorted(workDir.glob(f"*{searchString}*")) + expectedTargets = sorted(expectedTargets) + + for i in range(len(testFiles)): + callArgs = actualCopies[i][0] + assert callArgs[0] == sourceFilePaths[i] + assert callArgs[1] == expectedTargets[i] + + +def test_CopyMoTFiles_EmptyDirectory(tmp_path): + # Test behavior with empty source directory + workDir = tmp_path / "empty_work" + outputDir = tmp_path / "empty_output" + workDir.mkdir() + outputDir.mkdir() + + with patch('shutil.copy2') as mockCopy: + MoTUtils.copyMoTFiles(workDir, outputDir, "p_max", "ppr") + mockCopy.assert_not_called() + +def test_CopyMoTFiles_NoMatchingFiles(tmp_path): + # Test behavior when no files match the search string + workDir = tmp_path / "work" + outputDir = tmp_path / "output" + workDir.mkdir() + outputDir.mkdir() + + # Create files that don't match the search pattern + (workDir / "test1.txt").touch() + (workDir / "test2.txt").touch() + + with patch('shutil.copy2') as mockCopy: + MoTUtils.copyMoTFiles(workDir, outputDir, "p_max", "ppr") + mockCopy.assert_not_called() + +class MockProcess: + def __init__(self, outputLines): + self.outputLines = outputLines + self.currentLine = 0 + self.stdout = self + + def readline(self): + if self.currentLine >= len(self.outputLines): + return "" + line = self.outputLines[self.currentLine] + self.currentLine += 1 + return line + + def poll(self): + return None if self.currentLine < len(self.outputLines) else 0 + +@pytest.mark.parametrize("osName,platformSystem,expectedShell", [ + ("nt", "Windows", True), + ("posix", "Darwin", False), + ("posix", "Linux", False) +]) +def test_RunAndCheckMoT_OSSpecific(osName, platformSystem, expectedShell): + with patch('os.name', osName), \ + patch('platform.system', return_value=platformSystem), \ + patch('subprocess.Popen') as mockPopen: + + mockPopen.return_value = MockProcess([]) + MoTUtils.runAndCheckMoT("testCommand") + + mockPopen.assert_called_once() + assert mockPopen.call_args[1]['shell'] == expectedShell + + + +def test_RunAndCheckMoT_CommandTypes(): + testCases = [ + "singleCommand", + ["command", "with", "arguments"], + "command with spaces" + ] + + for command in testCases: + with patch('subprocess.Popen') as mockPopen: + mockPopen.return_value = MockProcess([]) + MoTUtils.runAndCheckMoT(command) + + mockPopen.assert_called_once() + assert mockPopen.call_args[0][0] == command + +def test_RunAndCheckMoT_ProcessExit(): + # Test proper handling of process termination + testOutput = ["Line 1\n", "Line 2\n"] + + with patch('subprocess.Popen') as mockPopen, \ + patch('avaframe.in3Utils.MoTUtils.log.info') as mockLog: + + mockPopen.return_value = MockProcess(testOutput) + MoTUtils.runAndCheckMoT("testCommand") + + # Verify all output was processed + assert mockLog.call_count == 2 + mockLog.assert_has_calls([ + call("Line 1"), + call("Line 2") + ]) \ No newline at end of file diff --git a/avaframe/tests/test_cfgUtils.py b/avaframe/tests/test_cfgUtils.py index 48168f483..96920697d 100644 --- a/avaframe/tests/test_cfgUtils.py +++ b/avaframe/tests/test_cfgUtils.py @@ -6,6 +6,13 @@ import pytest import configparser import sys +from pathlib import Path +import os +import multiprocessing +from types import ModuleType +import pandas as pd +import numpy as np + def test_getModuleConfig(): @@ -203,4 +210,562 @@ def test_readConfigurationInfoFromDone(tmp_path): simDF, simNameExisting = cfgUtils.readConfigurationInfoFromDone(configDir2, specDir="", latest=True) - assert len(simDF) == 3 \ No newline at end of file + assert len(simDF) == 3 + +def test_cfgToRcf(tmp_path): + """Test the conversion of ConfigParser object to RCF format""" + + # Create a test ConfigParser object with various sections + testCfg = configparser.ConfigParser() + testCfg['GENERAL'] = { + 'parameter1': 'value1', + 'parameter2': 'value2' + } + testCfg['INPUT'] = { + 'input1': 'value1', + 'input2': 'value2' + } + testCfg['FOREST_EFFECTS'] = { + 'forest1': 'value1', + 'forest2': 'value2' + } + testCfg['ENTRAINMENT'] = { + 'entrainment1': 'value1', + 'entrainment2': 'value2' + } + testCfg['TEST_SECTION'] = { + 'test1': 'value1', + 'test2': 'value2' + } + + # Create temporary output file + outputFile = tmp_path / "test.rcf" + + print(outputFile) + # Convert configuration to RCF format + from avaframe.in3Utils.cfgUtils import cfgToRcf + cfgToRcf(testCfg, outputFile) + + # Read the generated file + with open(outputFile, 'r') as f: + fileContent = f.read().splitlines() + + # Verify the content + expectedContent = [ + 'forest1 value1', + 'forest2 value2', + '#', + 'entrainment1 value1', + 'entrainment2 value2', + '#', + '# TEST SECTION', + '#', + 'test1 value1', + 'test2 value2', + '#' + ] + + # Check that GENERAL and INPUT sections are not included + assert '# GENERAL' not in fileContent + assert '# INPUT' not in fileContent + + # Check that FOREST_EFFECTS and ENTRAINMENT sections are handled correctly + assert '# FOREST EFFECTS' not in fileContent + assert '# ENTRAINMENT' not in fileContent + + # Check the formatting of the included section + assert fileContent == expectedContent + +def test_cfgToRcf_empty_config(tmp_path): + """Test conversion of empty ConfigParser object""" + emptyCfg = configparser.ConfigParser() + outputFile = tmp_path / "empty.rcf" + + from avaframe.in3Utils.cfgUtils import cfgToRcf + cfgToRcf(emptyCfg, outputFile) + + with open(outputFile, 'r') as f: + fileContent = f.read() + + assert fileContent == '' + + +def test_cfgToRcf_special_characters(tmp_path): + """Test handling of special characters in keys and values""" + testCfg = configparser.ConfigParser() + testCfg['TEST_SECTION'] = { + 'key with spaces': 'value with spaces', + 'key_with_underscore': 'value_with_underscore', + 'key.with.dots': 'value.with.dots' + } + + outputFile = tmp_path / "special.rcf" + + from avaframe.in3Utils.cfgUtils import cfgToRcf + cfgToRcf(testCfg, outputFile) + + with open(outputFile, 'r') as f: + fileContent = f.read().splitlines() + + # Verify handling of special characters + assert any('key with spaces' in line for line in fileContent) + assert any('key_with_underscore' in line for line in fileContent) + assert any('key.with.dots' in line for line in fileContent) + + +def test_cfgToRcf_file_handling(tmp_path): + """Test file handling edge cases""" + testCfg = configparser.ConfigParser() + testCfg['TEST'] = {'key': 'value'} + + # Test with Path object + pathObj = tmp_path / "test_path.rcf" + from avaframe.in3Utils.cfgUtils import cfgToRcf + cfgToRcf(testCfg, pathObj) + assert pathObj.exists() + + # Test with string path + strPath = str(tmp_path / "test_str.rcf") + cfgToRcf(testCfg, strPath) + assert os.path.exists(strPath) + +def test_getNumberOfProcesses(): + """Test getting the number of CPU cores for parallel processing""" + + # Create test configuration + testCfg = configparser.ConfigParser() + testCfg['MAIN'] = {} + maxCoresAvailable = multiprocessing.cpu_count() + + # Test case 1: Auto CPU with percentage + testCfg['MAIN']['nCPU'] = 'auto' + testCfg['MAIN']['CPUPercent'] = '50' + numSims = 100 + + from avaframe.in3Utils.cfgUtils import getNumberOfProcesses + resultCores = getNumberOfProcesses(testCfg, numSims) + expectedCores = int(maxCoresAvailable * 0.5) + assert resultCores == expectedCores + + # Test case 2: Explicit CPU number + testCfg['MAIN']['nCPU'] = '4' + numSims = 100 + resultCores = getNumberOfProcesses(testCfg, numSims) + assert resultCores == 4 + + # Test case 3: Number of sims less than CPU cores + testCfg['MAIN']['nCPU'] = '8' + numSims = 3 + resultCores = getNumberOfProcesses(testCfg, numSims) + assert resultCores == 3 + + # Test case 4: Auto CPU with 100% + testCfg['MAIN']['nCPU'] = 'auto' + testCfg['MAIN']['CPUPercent'] = '100' + numSims = 100 + resultCores = getNumberOfProcesses(testCfg, numSims) + assert resultCores == maxCoresAvailable + + # Test case 5: Auto CPU with low percentage + testCfg['MAIN']['nCPU'] = 'auto' + testCfg['MAIN']['CPUPercent'] = '25' + numSims = 100 + resultCores = getNumberOfProcesses(testCfg, numSims) + expectedCores = int(maxCoresAvailable * 0.25) + assert resultCores == expectedCores + + +def test_getNumberOfProcesses_edge_cases(): + """Test edge cases for getNumberOfProcesses""" + + testCfg = configparser.ConfigParser() + testCfg['MAIN'] = {} + + # Test case 1: Zero simulations + testCfg['MAIN']['nCPU'] = '4' + numSims = 0 + + from avaframe.in3Utils.cfgUtils import getNumberOfProcesses + resultCores = getNumberOfProcesses(testCfg, numSims) + assert resultCores == 0 + + # Test case 2: Very high CPU number + testCfg['MAIN']['nCPU'] = '999' + numSims = 5 + resultCores = getNumberOfProcesses(testCfg, numSims) + assert resultCores == 5 + + # Test case 3: Auto with very high percentage + testCfg['MAIN']['nCPU'] = 'auto' + testCfg['MAIN']['CPUPercent'] = '200' + numSims = 100 + resultCores = getNumberOfProcesses(testCfg, numSims) + assert resultCores == multiprocessing.cpu_count() * 2 + + +def test_getNumberOfProcesses_invalid_input(): + """Test invalid inputs for getNumberOfProcesses""" + + testCfg = configparser.ConfigParser() + testCfg['MAIN'] = {} + + # Test case: Invalid nCPU value + testCfg['MAIN']['nCPU'] = 'invalid' + numSims = 10 + + from avaframe.in3Utils.cfgUtils import getNumberOfProcesses + with pytest.raises(ValueError): + getNumberOfProcesses(testCfg, numSims) + +import pytest + + +def test_convertToCfgList(): + """Test converting lists to configuration string format""" + + # Test case 1: Basic list with strings + inputList = ['value1', 'value2', 'value3'] + + from avaframe.in3Utils.cfgUtils import convertToCfgList + resultString = convertToCfgList(inputList) + assert resultString == 'value1|value2|value3' + + # Test case 2: Empty list + emptyList = [] + resultString = convertToCfgList(emptyList) + assert resultString == '' + + # Test case 3: Single item list + singleList = ['value1'] + resultString = convertToCfgList(singleList) + assert resultString == 'value1' + + # Test case 4: List with numeric values as strings + numericList = ['1.0', '2.5', '3.7'] + resultString = convertToCfgList(numericList) + assert resultString == '1.0|2.5|3.7' + + # Test case 5: List with mixed content + mixedList = ['text', '123', 'special_chars!@#'] + resultString = convertToCfgList(mixedList) + assert resultString == 'text|123|special_chars!@#' + + +def test_convertToCfgList_edge_cases(): + """Test edge cases for convertToCfgList""" + + from avaframe.in3Utils.cfgUtils import convertToCfgList + + # Test case 1: List with empty strings + emptyStrList = ['', '', ''] + resultString = convertToCfgList(emptyStrList) + assert resultString == '||' + + # Test case 2: List with spaces + spacesList = ['item 1', 'item 2', 'item 3'] + resultString = convertToCfgList(spacesList) + assert resultString == 'item 1|item 2|item 3' + + # Test case 3: List with pipe characters in items + pipeList = ['value|1', 'value|2'] + resultString = convertToCfgList(pipeList) + assert resultString == 'value|1|value|2' + + +def test_getDefaultModuleConfig(tmp_path): + """Test getting default configuration for a module""" + + # Create a mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create default config file + defaultConfigPath = tmp_path / 'testModuleCfg.ini' + testConfig = configparser.ConfigParser() + testConfig['GENERAL'] = { + 'parameter1': 'value1', + 'parameter2': 'value2' + } + testConfig['TEST'] = { + 'setting1': 'test1', + 'setting2': 'test2' + } + + with open(defaultConfigPath, 'w') as configFile: + testConfig.write(configFile) + + # Test getting default config + from avaframe.in3Utils.cfgUtils import getDefaultModuleConfig + resultConfig = getDefaultModuleConfig(testModule, toPrint=False) + + # Verify the configuration + assert 'GENERAL' in resultConfig.sections() + assert 'TEST' in resultConfig.sections() + assert resultConfig['GENERAL']['parameter1'] == 'value1' + assert resultConfig['GENERAL']['parameter2'] == 'value2' + assert resultConfig['TEST']['setting1'] == 'test1' + assert resultConfig['TEST']['setting2'] == 'test2' + + +def test_getDefaultModuleConfig_empty_file(tmp_path): + """Test behavior with empty config file""" + + # Create a mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create empty config file + defaultConfigPath = tmp_path / 'testModuleCfg.ini' + with open(defaultConfigPath, 'w') as configFile: + configFile.write('') + + from avaframe.in3Utils.cfgUtils import getDefaultModuleConfig + resultConfig = getDefaultModuleConfig(testModule, toPrint=False) + + # Verify empty configuration + assert len(resultConfig.sections()) == 0 + + +def test_getDefaultModuleConfig_invalid_input(): + """Test handling of invalid inputs""" + + from avaframe.in3Utils.cfgUtils import getDefaultModuleConfig + + # Test with None + with pytest.raises(AttributeError): + getDefaultModuleConfig(None) + + # Test with non-module object + with pytest.raises(AttributeError): + getDefaultModuleConfig("not a module") + + # Test with module without __file__ attribute + invalidModule = ModuleType('invalidModule') + with pytest.raises(AttributeError): + getDefaultModuleConfig(invalidModule) + + + +def test_writeCfgFile(tmp_path): + """Test writing configuration to a file with default settings""" + + # Create test avalanche directory + avaDir = tmp_path / "avaTest" + avaDir.mkdir() + + # Create mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig['GENERAL'] = { + 'parameter1': 'value1', + 'parameter2': 'value2' + } + + # Write configuration file + from avaframe.in3Utils.cfgUtils import writeCfgFile + resultPath = writeCfgFile(avaDir, testModule, testConfig) + + # Check if file exists in correct location + expectedPath = avaDir / "Outputs" / "testModule" / "configurationFiles" / "testModule.ini" + assert resultPath == expectedPath + assert resultPath.exists() + + # Verify file contents + writtenConfig = configparser.ConfigParser() + writtenConfig.read(resultPath) + assert writtenConfig['GENERAL']['parameter1'] == 'value1' + assert writtenConfig['GENERAL']['parameter2'] == 'value2' + + +def test_writeCfgFile_custom_path(tmp_path): + """Test writing configuration to a custom path""" + + # Create custom directory + customPath = tmp_path / "custom" / "path" + customPath.mkdir(parents=True) + + # Create mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig['TEST'] = {'key': 'value'} + + # Write configuration file with custom path + from avaframe.in3Utils.cfgUtils import writeCfgFile + resultPath = writeCfgFile(tmp_path, testModule, testConfig, filePath=customPath) + + # Verify file location and contents + assert resultPath == customPath / "testModule.ini" + assert resultPath.exists() + + +def test_writeCfgFile_custom_filename(tmp_path): + """Test writing configuration with custom filename""" + + # Create mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig['TEST'] = {'key': 'value'} + + # Write configuration file with custom filename + customName = "customConfig" + from avaframe.in3Utils.cfgUtils import writeCfgFile + resultPath = writeCfgFile(tmp_path, testModule, testConfig, fileName=customName) + + # Verify custom filename + expectedPath = tmp_path / "Outputs" / "testModule" / "configurationFiles" / "customConfig.ini" + assert resultPath == expectedPath + assert resultPath.exists() + + +def test_writeCfgFile_invalid_path(tmp_path): + """Test writing configuration with invalid path""" + + # Create mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig['TEST'] = {'key': 'value'} + + # Create a file instead of a directory + invalidPath = tmp_path / "invalid" + invalidPath.touch() + + # Test with invalid path + from avaframe.in3Utils.cfgUtils import writeCfgFile + with pytest.raises(NotADirectoryError): + writeCfgFile(tmp_path, testModule, testConfig, filePath=invalidPath) + + +def test_writeCfgFile_empty_config(tmp_path): + """Test writing empty configuration""" + + # Create mock module + testModule = ModuleType('testModule') + testModule.__file__ = str(tmp_path / 'testModule.py') + + # Create empty configuration + emptyConfig = configparser.ConfigParser() + + # Write empty configuration + from avaframe.in3Utils.cfgUtils import writeCfgFile + resultPath = writeCfgFile(tmp_path, testModule, emptyConfig) + + # Verify empty file was created + assert resultPath.exists() + assert resultPath.stat().st_size == 0 + + + +def test_setStrnanToNan_basic(): + """Test basic functionality of converting string 'nan' to np.nan""" + + # Create test dataframe + testData = { + 'column1': ['nan', '1.0', 'NaN', '2.0', 'NAN'], + 'column2': ['value1', 'value2', 'value3', 'value4', 'value5'] + } + testDF = pd.DataFrame(testData) + testDFColumn = testDF['column1'] + + # Convert string 'nan' to np.nan + from avaframe.in3Utils.cfgUtils import setStrnanToNan + resultDF = setStrnanToNan(testDF, testDFColumn, 'column1') + + # Check results + assert pd.isna(resultDF.at[0, 'column1']) # 'nan' + assert resultDF.at[1, 'column1'] == '1.0' # number + assert pd.isna(resultDF.at[2, 'column1']) # 'NaN' + assert resultDF.at[3, 'column1'] == '2.0' # number + assert pd.isna(resultDF.at[4, 'column1']) # 'NAN' + + # Check other column remains unchanged + assert resultDF['column2'].equals(testDF['column2']) + + +def test_setStrnanToNan_no_nans(): + """Test behavior when no 'nan' strings are present""" + + # Create test dataframe without 'nan' values + testData = { + 'column1': ['1.0', '2.0', '3.0', '4.0', '5.0'], + 'column2': ['a', 'b', 'c', 'd', 'e'] + } + testDF = pd.DataFrame(testData) + testDFColumn = testDF['column1'] + + # Process dataframe + from avaframe.in3Utils.cfgUtils import setStrnanToNan + resultDF = setStrnanToNan(testDF, testDFColumn, 'column1') + + # Verify no changes were made + assert resultDF.equals(testDF) + + +def test_setStrnanToNan_empty_df(): + """Test handling of empty dataframe""" + + # Create empty dataframe + testDF = pd.DataFrame(columns=['column1', 'column2']) + testDFColumn = pd.Series(dtype='object') + + # Process empty dataframe + from avaframe.in3Utils.cfgUtils import setStrnanToNan + resultDF = setStrnanToNan(testDF, testDFColumn, 'column1') + + # Verify result is still empty + assert resultDF.empty + assert len(resultDF.columns) == 2 + + +def test_setStrnanToNan_mixed_types(): + """Test handling of mixed data types""" + + # Create test dataframe with mixed types + testData = { + 'column1': ['nan', 1.0, 'NaN', True, None], + 'column2': ['a', 'b', 'c', 'd', 'e'] + } + testDF = pd.DataFrame(testData) + testDFColumn = testDF['column1'].astype(str) + + # Process dataframe + from avaframe.in3Utils.cfgUtils import setStrnanToNan + resultDF = setStrnanToNan(testDF, testDFColumn, 'column1') + + # Check results + assert pd.isna(resultDF.at[0, 'column1']) # 'nan' + assert resultDF.at[1, 'column1'] == 1.0 # number + assert pd.isna(resultDF.at[2, 'column1']) # 'NaN' + assert resultDF.at[3, 'column1'] == True # boolean + assert pd.isna(resultDF.at[4, 'column1']) # None + + +def test_setStrnanToNan_case_sensitivity(): + """Test case-insensitive matching of 'nan' strings""" + + # Create test dataframe with different cases of 'nan' + testData = { + 'column1': ['nan', 'NaN', 'NAN', 'nAn', 'Nan'], + 'column2': ['a', 'b', 'c', 'd', 'e'] + } + testDF = pd.DataFrame(testData) + testDFColumn = testDF['column1'] + + # Process dataframe + from avaframe.in3Utils.cfgUtils import setStrnanToNan + resultDF = setStrnanToNan(testDF, testDFColumn, 'column1') + + # Verify all variations of 'nan' were converted + for i in range(5): + assert pd.isna(resultDF.at[i, 'column1']) \ No newline at end of file diff --git a/avaframe/tests/test_probAna.py b/avaframe/tests/test_probAna.py index 0a6e4aab7..df60c694f 100644 --- a/avaframe/tests/test_probAna.py +++ b/avaframe/tests/test_probAna.py @@ -1,9 +1,9 @@ """ - Pytest for module ana4Stats +Pytest for module ana4Stats - This file is part of Avaframe. +This file is part of Avaframe. - """ +""" # Load modules import numpy as np @@ -16,104 +16,133 @@ import configparser import shutil import pathlib +from scipy.stats import qmc + + def test_probAnalysis(tmp_path): - """ test probAna function to compute mask for parameter exceeding threshold """ + """test probAna function to compute mask for parameter exceeding threshold""" # set input directory - avaName = 'avaParabola' - avaTestDir = 'avaParabolaStatsTest' + avaName = "avaParabola" + avaTestDir = "avaParabolaStatsTest" dirPath = pathlib.Path(__file__).parents[0] - avaDir = dirPath / '..' / '..' / 'benchmarks' / avaTestDir + avaDir = dirPath / ".." / ".." / "benchmarks" / avaTestDir avaDirtmp = pathlib.Path(tmp_path, avaName) inputDir = pathlib.Path(tmp_path, avaName) inputDir1 = avaDir shutil.copytree(inputDir1, inputDir) # set configurations - testCfg = os.path.join(inputDir, '%sProbAna_com1DFACfg.ini' % avaName) + testCfg = os.path.join(inputDir, "%sProbAna_com1DFACfg.ini" % avaName) cfgMain = cfgUtils.getModuleConfig(com1DFA, testCfg) # Initialise input in correct format cfg = configparser.ConfigParser() - cfg['GENERAL'] = {'peakLim': 1.0, 'peakVar': 'ppr'} - cfg['FILTER'] = {} + cfg["GENERAL"] = {"peakLim": 1.0, "peakVar": "ppr"} + cfg["FILTER"] = {} # provide optional filter criteria for simulations - parametersDict = fU.getFilterDict(cfg, 'FILTER') + parametersDict = fU.getFilterDict(cfg, "FILTER") # call function to test - pA.probAnalysis(avaDirtmp, cfg, 'com1DFA', parametersDict=parametersDict, inputDir='') - outputPath = os.path.join(avaDirtmp, 'Outputs', 'ana4Stats', 'avaParabola_prob__ppr_lim1.0.asc') + pA.probAnalysis(avaDirtmp, cfg, "com1DFA", parametersDict=parametersDict, inputDir="") + outputPath = os.path.join(avaDirtmp, "Outputs", "ana4Stats", "avaParabola_prob__ppr_lim1.0.asc") print(outputPath) probTest = np.loadtxt(outputPath, skiprows=6) # Load reference solution - probSol = np.loadtxt(os.path.join(inputDir1, 'avaParabola_prob__ppr_lim1.0.txt'), skiprows=6) + probSol = np.loadtxt(os.path.join(inputDir1, "avaParabola_prob__ppr_lim1.0.txt"), skiprows=6) # Compare result to reference solution - testRes = np.allclose(probTest, probSol, atol=1.e-6) + testRes = np.allclose(probTest, probSol, atol=1.0e-6) # Test - assert (testRes is True) + assert testRes is True # call function to test - testInputDir = avaDir / 'Outputs' / 'com1DFA' - avaDirtmp2 = pathlib.Path(tmp_path, 'avaTest') + testInputDir = avaDir / "Outputs" / "com1DFA" + avaDirtmp2 = pathlib.Path(tmp_path, "avaTest") avaDirtmp2.mkdir() - pA.probAnalysis(avaDirtmp2, cfg, 'com1DFA', parametersDict='', inputDir=testInputDir) - probTest2 = np.loadtxt(os.path.join(avaDirtmp2, 'Outputs', 'ana4Stats', 'avaTest_prob__ppr_lim1.0.asc'), skiprows=6) + pA.probAnalysis(avaDirtmp2, cfg, "com1DFA", parametersDict="", inputDir=testInputDir) + probTest2 = np.loadtxt( + os.path.join(avaDirtmp2, "Outputs", "ana4Stats", "avaTest_prob__ppr_lim1.0.asc"), + skiprows=6, + ) # Compare result to reference solution - testRes2 = np.allclose(probTest2, probSol, atol=1.e-6) + testRes2 = np.allclose(probTest2, probSol, atol=1.0e-6) # Test - assert (testRes2 is True) - + assert testRes2 is True # set input directory - avaName2 = 'avaTest2' - avaTestDir2 = 'avaProbTest' + avaName2 = "avaTest2" + avaTestDir2 = "avaProbTest" dirPath2 = pathlib.Path(__file__).parents[0] - avaDir2 = dirPath / 'data' / avaTestDir2 + avaDir2 = dirPath / "data" / avaTestDir2 inputDir2 = pathlib.Path(tmp_path, avaName2) shutil.copytree(avaDir2, inputDir2) cfg2 = configparser.ConfigParser() cfg2.optionxform = str - cfg2['GENERAL'] = {'peakVar': 'ppr', 'unit': 'kPa','peakLim': '1.0'} - cfg2['PLOT'] = {'name': 'probability map', 'cmapType': 'prob', 'levels': '0.95', 'unit': 'fraction', - 'zoomBuffer': 250., 'contrainBuffer': 10., 'meshCellSizeThreshold': 0.001} - modName = 'com1DFA' + cfg2["GENERAL"] = {"peakVar": "ppr", "unit": "kPa", "peakLim": "1.0"} + cfg2["PLOT"] = { + "name": "probability map", + "cmapType": "prob", + "levels": "0.95", + "unit": "fraction", + "zoomBuffer": 250.0, + "contrainBuffer": 10.0, + "meshCellSizeThreshold": 0.001, + } + modName = "com1DFA" # call function to be tested - analysisPerformed, contourDict = pA.probAnalysis(inputDir2, cfg2, modName, parametersDict='', inputDir='', probConf='', simDFActual='') + analysisPerformed, contourDict = pA.probAnalysis( + inputDir2, + cfg2, + modName, + parametersDict="", + inputDir="", + probConf="", + simDFActual="", + ) assert analysisPerformed is True def test_createComModConfig(tmp_path): - """ test creatig a config file """ + """test creatig a config file""" # set input directory - avaName = 'testCom1DFA2' + avaName = "testCom1DFA2" dirPath = pathlib.Path(__file__).parents[0] - inputDir = dirPath / 'data' / avaName + inputDir = dirPath / "data" / avaName avaDir = pathlib.Path(tmp_path, avaName) shutil.copytree(inputDir, avaDir) cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|3', - 'defaultSetup': 'True', 'samplingStrategy': '2', 'varParType': 'float|float'} - cfgProb['sampling_override'] = {'defaultConfig': 'True'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|3", + "defaultSetup": "True", + "samplingStrategy": "2", + "varParType": "float|float", + } + cfgProb["sampling_override"] = {"defaultConfig": "True"} + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "frictModel": "samosAT", + } # call function to be tested cfgFiles, outDir = pA.createComModConfig(cfgProb, avaDir, com1DFA) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) # load cfg from file cfgMu = configparser.ConfigParser() @@ -122,21 +151,26 @@ def test_createComModConfig(tmp_path): cfgMu.read(cfgFiles[0]) cfgRelTh.read(cfgFiles[1]) -# print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], + # print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], # cfgRelTh['GENERAL']['relTh']) - assert cfgMu['GENERAL']['musamosat'] == '0.155$60$2' - assert cfgMu['GENERAL']['relTh'] == '' - assert cfgRelTh['GENERAL']['musamosat'] == '0.155' - assert cfgRelTh['GENERAL']['relTh'] == '' - assert cfgRelTh['GENERAL']['relThFromShp'] == 'True' - assert cfgRelTh['GENERAL']['relThPercentVariation'] == '50$3' - assert cfgRelTh['GENERAL']['relThFromShp'] == 'True' - assert cfgMu['GENERAL']['relThFromShp'] == 'True' - assert cfgRelTh.has_option('GENERAL', 'addStandardConfig') == False - - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'relThFromShp': False, 'relTh': 2., - 'musamosat': 0.155, 'frictModel': 'samosAT'} + assert cfgMu["GENERAL"]["musamosat"] == "0.155$60$2" + assert cfgMu["GENERAL"]["relTh"] == "" + assert cfgRelTh["GENERAL"]["musamosat"] == "0.155" + assert cfgRelTh["GENERAL"]["relTh"] == "" + assert cfgRelTh["GENERAL"]["relThFromShp"] == "True" + assert cfgRelTh["GENERAL"]["relThPercentVariation"] == "50$3" + assert cfgRelTh["GENERAL"]["relThFromShp"] == "True" + assert cfgMu["GENERAL"]["relThFromShp"] == "True" + assert cfgRelTh.has_option("GENERAL", "addStandardConfig") is False + + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "relThFromShp": False, + "relTh": 2.0, + "musamosat": 0.155, + "frictModel": "samosAT", + } # call function to be tested cfgFiles, outDir = pA.createComModConfig(cfgProb, avaDir, com1DFA) @@ -147,491 +181,604 @@ def test_createComModConfig(tmp_path): cfgMu.read(cfgFiles[0]) cfgRelTh.read(cfgFiles[1]) -# print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], + # print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], # cfgRelTh['GENERAL']['relTh']) - assert cfgMu['GENERAL']['musamosat'] == '0.155$60$2' - assert np.isclose(cfgMu['GENERAL'].getfloat('relTh'), 2.) - assert cfgRelTh['GENERAL']['musamosat'] == '0.155' - assert np.isclose(cfgRelTh['GENERAL'].getfloat('relTh'), 2.) - assert cfgRelTh['GENERAL']['relThFromShp'] == 'False' - assert cfgRelTh['GENERAL']['relThPercentVariation'] == '50$3' - assert cfgRelTh['GENERAL']['relThFromShp'] == 'False' - assert cfgMu['GENERAL']['relThFromShp'] == 'False' + assert cfgMu["GENERAL"]["musamosat"] == "0.155$60$2" + assert np.isclose(cfgMu["GENERAL"].getfloat("relTh"), 2.0) + assert cfgRelTh["GENERAL"]["musamosat"] == "0.155" + assert np.isclose(cfgRelTh["GENERAL"].getfloat("relTh"), 2.0) + assert cfgRelTh["GENERAL"]["relThFromShp"] == "False" + assert cfgRelTh["GENERAL"]["relThPercentVariation"] == "50$3" + assert cfgRelTh["GENERAL"]["relThFromShp"] == "False" + assert cfgMu["GENERAL"]["relThFromShp"] == "False" cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|3', - 'defaultSetup': 'True', 'samplingStrategy': '1', - 'varParType': 'float|float', 'nSample': '40', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'relThFromShp': False, 'relTh': 2., - 'musamosat': 0.155, 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|3", + "defaultSetup": "True", + "samplingStrategy": "1", + "varParType": "float|float", + "nSample": "40", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "relThFromShp": False, + "relTh": 2.0, + "musamosat": 0.155, + "frictModel": "samosAT", + } # call function to be tested cfgFiles, outDir = pA.createComModConfig(cfgProb, avaDir, com1DFA) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) for cfgF in cfgFiles: cfgTest = configparser.ConfigParser() cfgTest.read(cfgF) -# print('cfgTest', cfgTest['GENERAL']['relThFromShp'], cfgTest['GENERAL']['relTh'], + # print('cfgTest', cfgTest['GENERAL']['relThFromShp'], cfgTest['GENERAL']['relTh'], # cfgTest['GENERAL']['relThPercentVariation'], cfgTest['GENERAL']['musamosat']) - assert cfgTest['GENERAL']['relThFromShp'] == 'False' - assert cfgTest['GENERAL'].getfloat('relTh') <= 3. - assert cfgTest['GENERAL'].getfloat('relTh') >= 1. - assert cfgTest['GENERAL'].getfloat('musamosat') <= 0.248 - assert cfgTest['GENERAL'].getfloat('musamosat') >= 0.062 + assert cfgTest["GENERAL"]["relThFromShp"] == "False" + assert cfgTest["GENERAL"].getfloat("relTh") <= 3.0 + assert cfgTest["GENERAL"].getfloat("relTh") >= 1.0 + assert cfgTest["GENERAL"].getfloat("musamosat") <= 0.248 + assert cfgTest["GENERAL"].getfloat("musamosat") >= 0.062 cfgTest = configparser.ConfigParser() cfgTest.read(cfgFiles[0]) - assert cfgTest['GENERAL']['relTh'] == '2.2719559079879' + assert cfgTest["GENERAL"]["relTh"] == "2.2719559079879" assert len(cfgFiles) == 40 cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|3', - 'defaultSetup': 'True', 'samplingStrategy': '1', - 'varParType': 'float|float', 'nSample': '40', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|3", + "defaultSetup": "True", + "samplingStrategy": "1", + "varParType": "float|float", + "nSample": "40", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "frictModel": "samosAT", + } # call function to be tested cfgFiles2, outDir = pA.createComModConfig(cfgProb, avaDir, com1DFA) -# print('cfgFiles', cfgFiles2) + # print('cfgFiles', cfgFiles2) cfgTest2 = configparser.ConfigParser() cfgTest2.read(cfgFiles2[0]) cfgTest21 = configparser.ConfigParser() cfgTest21.read(cfgFiles2[40]) -# print('cfgTest', cfgTest['GENERAL']['relThFromShp'], cfgTest['GENERAL']['relTh'], + # print('cfgTest', cfgTest['GENERAL']['relThFromShp'], cfgTest['GENERAL']['relTh'], # cfgTest['GENERAL']['relThPercentVariation'], cfgTest['GENERAL']['musamosat']) - assert cfgTest2['GENERAL']['relThFromShp'] == 'True' - assert cfgTest2['GENERAL']['relTh'] == '' - assert cfgTest2['GENERAL']['relThPercentVariation'] == '' - assert cfgTest2['INPUT']['releaseScenario'] == 'relParabola' - assert cfgTest2.has_option('GENERAL', 'relTh0') + assert cfgTest2["GENERAL"]["relThFromShp"] == "True" + assert cfgTest2["GENERAL"]["relTh"] == "" + assert cfgTest2["GENERAL"]["relThPercentVariation"] == "" + assert cfgTest2["INPUT"]["releaseScenario"] == "relParabola" + assert cfgTest2.has_option("GENERAL", "relTh0") assert len(cfgFiles2) == 80 - assert cfgTest21['GENERAL']['relThFromShp'] == 'True' - assert cfgTest21['GENERAL']['relTh'] == '' - assert cfgTest21['GENERAL']['relThPercentVariation'] == '' - assert cfgTest21['GENERAL']['musamosat'] == cfgTest2['GENERAL']['musamosat'] - assert cfgTest21['INPUT']['releaseScenario'] == 'relParabolaTwo' - assert cfgTest2.has_option('GENERAL', 'relTh0') + assert cfgTest21["GENERAL"]["relThFromShp"] == "True" + assert cfgTest21["GENERAL"]["relTh"] == "" + assert cfgTest21["GENERAL"]["relThPercentVariation"] == "" + assert cfgTest21["GENERAL"]["musamosat"] == cfgTest2["GENERAL"]["musamosat"] + assert cfgTest21["INPUT"]["releaseScenario"] == "relParabolaTwo" + assert cfgTest21.has_option("GENERAL", "relTh0") assert len(cfgFiles2) == 80 cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'range|range', - 'variationValue': '0.2|1.2', 'numberOfSteps': '2|3', - 'samplingStrategy': '2', 'varParType': 'float|float|int'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "range|range", + "variationValue": "0.2|1.2", + "numberOfSteps": "2|3", + "samplingStrategy": "2", + "varParType": "float|float|int", + } + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "frictModel": "samosAT", + } # call function to be tested cfgFiles3, outDir = pA.createComModConfig(cfgProb, avaDir, com1DFA) -# print('cfgFiles', cfgFiles3) + # print('cfgFiles', cfgFiles3) cfgTest3 = configparser.ConfigParser() cfgTest3.read(cfgFiles3[1]) - assert cfgTest3['GENERAL']['relThFromShp'] == 'True' - assert cfgTest3['GENERAL']['relTh'] == '' - assert cfgTest3['GENERAL']['relThRangeVariation'] == '1.2$3' + assert cfgTest3["GENERAL"]["relThFromShp"] == "True" + assert cfgTest3["GENERAL"]["relTh"] == "" + assert cfgTest3["GENERAL"]["relThRangeVariation"] == "1.2$3" assert len(cfgFiles3) == 2 cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh|entTh', 'variationType': 'percent|rangefromci|range', - 'variationValue': '60|ci95|0.15', 'numberOfSteps': '2|3|4', - 'samplingStrategy': '1', - 'varParType': 'float|float|float', 'nSample': '30', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh|entTh", + "variationType": "percent|rangefromci|range", + "variationValue": "60|ci95|0.15", + "numberOfSteps": "2|3|4", + "samplingStrategy": "1", + "varParType": "float|float|float", + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "frictModel": "samosAT", + } # call function to be tested cfgFiles4, outDir = pA.createComModConfig(cfgProb, avaDir, com1DFA) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) cfgTest4 = configparser.ConfigParser() cfgTest4.read(cfgFiles4[0]) -# print('cfgTest', cfgTest4['GENERAL']['relThFromShp'], cfgTest4['GENERAL']['relTh'], + # print('cfgTest', cfgTest4['GENERAL']['relThFromShp'], cfgTest4['GENERAL']['relTh'], # cfgTest4['GENERAL']['relThPercentVariation'], cfgTest4['GENERAL']['musamosat']) - assert cfgTest4['GENERAL']['relThFromShp'] == 'True' - assert cfgTest4['GENERAL']['relTh'] == '' - assert cfgTest4['GENERAL']['relThRangeVariation'] == '' - assert cfgTest4['GENERAL']['relTh0'] != '' - assert cfgTest4['GENERAL']['relTh1'] != '' - assert cfgTest4['GENERAL']['entTh0'] != '' - assert cfgTest4['INPUT']['releaseScenario'] == 'relParabola' + assert cfgTest4["GENERAL"]["relThFromShp"] == "True" + assert cfgTest4["GENERAL"]["relTh"] == "" + assert cfgTest4["GENERAL"]["relThRangeVariation"] == "" + assert cfgTest4["GENERAL"]["relTh0"] != "" + assert cfgTest4["GENERAL"]["relTh1"] != "" + assert cfgTest4["GENERAL"]["entTh0"] != "" + assert cfgTest4["INPUT"]["releaseScenario"] == "relParabola" assert len(cfgFiles4) == 60 def test_updateCfgRange(): - """ test updating cfg values """ + """test updating cfg values""" # setup inputs cfg = configparser.ConfigParser() - cfg['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|2', - 'defaultSetup': 'True', 'varParType': 'float', 'samplingStrategy': '2'} + cfg["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|2", + "defaultSetup": "True", + "varParType": "float", + "samplingStrategy": "2", + } com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'musamosat' + varName = "musamosat" - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert cfgNew['GENERAL']['musamosat'] == '0.155$60$2' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '' + assert cfgNew["GENERAL"]["musamosat"] == "0.155$60$2" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "" com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'relTh' + varName = "relTh" # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert cfgNew['GENERAL']['musamosat'] == '0.155' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '50$2' + assert cfgNew["GENERAL"]["musamosat"] == "0.155" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "50$2" com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'musamosat' + varName = "musamosat" varDict = {} - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert cfgNew['GENERAL']['musamosat'] == '0.155$60$2' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '' + assert cfgNew["GENERAL"]["musamosat"] == "0.155$60$2" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "" com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'relTh' + varName = "relTh" # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert cfgNew['GENERAL']['musamosat'] == '0.155' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '50$2' + assert cfgNew["GENERAL"]["musamosat"] == "0.155" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "50$2" # setup inputs cfg = configparser.ConfigParser() - cfg['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'range|range', - 'variationValue': '0.1|0.5', 'numberOfSteps': '5|12', 'varParType': 'float|float', - 'defaultSetup': 'True', 'samplingStrategy': '2'} + cfg["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "range|range", + "variationValue": "0.1|0.5", + "numberOfSteps": "5|12", + "varParType": "float|float", + "defaultSetup": "True", + "samplingStrategy": "2", + } com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'musamosat' + varName = "musamosat" - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert np.isclose(float(cfgNew['GENERAL']['musamosat'].split(':')[0]), 0.055, rtol=1.e-5) - assert np.isclose(float(cfgNew['GENERAL']['musamosat'].split(':')[1]), 0.255, rtol=1.e-5) - assert cfgNew['GENERAL']['musamosat'].split(':')[2] == '5' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '' + assert np.isclose(float(cfgNew["GENERAL"]["musamosat"].split(":")[0]), 0.055, rtol=1.0e-5) + assert np.isclose(float(cfgNew["GENERAL"]["musamosat"].split(":")[1]), 0.255, rtol=1.0e-5) + assert cfgNew["GENERAL"]["musamosat"].split(":")[2] == "5" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "" com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'relTh' + varName = "relTh" # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert cfgNew['GENERAL']['musamosat'] == '0.155' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThRangeVariation'] == '0.5$12' + assert cfgNew["GENERAL"]["musamosat"] == "0.155" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThRangeVariation"] == "0.5$12" # setup inputs cfg = configparser.ConfigParser() cfg.optionxform = str - cfg['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'normaldistribution|normaldistribution', - 'variationValue': '0.1|0.3', 'numberOfSteps': '3|12', - 'defaultSetup': 'True', 'varParType': 'float|float', 'samplingStrategy': '2'} - cfg['in1Data_computeFromDistribution_override'] = {'buildType': 'ci95', 'minMaxInterval': '95', - 'defaultConfig': 'True'} + cfg["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "normaldistribution|normaldistribution", + "variationValue": "0.1|0.3", + "numberOfSteps": "3|12", + "defaultSetup": "True", + "varParType": "float|float", + "samplingStrategy": "2", + } + cfg["in1Data_computeFromDistribution_override"] = { + "buildType": "ci95", + "minMaxInterval": "95", + "defaultConfig": "True", + } com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'musamosat' + varName = "musamosat" - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert np.isclose(float(cfgNew['GENERAL']['musamosat'].split('|')[0]), 0.055, rtol=1.e-3) - assert np.isclose(float(cfgNew['GENERAL']['musamosat'].split('|')[1]), 0.155, rtol=1.e-3) - assert np.isclose(float(cfgNew['GENERAL']['musamosat'].split('|')[2]), 0.255, rtol=1.e-3) - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThDistVariation'] == '' + assert np.isclose(float(cfgNew["GENERAL"]["musamosat"].split("|")[0]), 0.055, rtol=1.0e-3) + assert np.isclose(float(cfgNew["GENERAL"]["musamosat"].split("|")[1]), 0.155, rtol=1.0e-3) + assert np.isclose(float(cfgNew["GENERAL"]["musamosat"].split("|")[2]), 0.255, rtol=1.0e-3) + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThDistVariation"] == "" com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'relTh' + varName = "relTh" - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) -# print('value', cfgNew['GENERAL']['relThPercentVariation']) + # print('value', cfgNew['GENERAL']['relThPercentVariation']) - assert cfgNew['GENERAL']['musamosat'] == '0.155' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThDistVariation'] == 'normaldistribution$12$0.3$95$ci95$10000' + assert cfgNew["GENERAL"]["musamosat"] == "0.155" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThDistVariation"] == "normaldistribution$12$0.3$95$ci95$10000" # setup inputs cfg = configparser.ConfigParser() - cfg['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|rangefromci', - 'variationValue': '60|ci95', 'numberOfSteps': '2|2', - 'defaultSetup': 'True', 'varParType': 'float|float', 'samplingStrategy': '2'} + cfg["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|rangefromci", + "variationValue": "60|ci95", + "numberOfSteps": "2|2", + "defaultSetup": "True", + "varParType": "float|float", + "samplingStrategy": "2", + } com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'relTh' + varName = "relTh" - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) - assert cfgNew['GENERAL']['musamosat'] == '0.155' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '' - assert cfgNew['GENERAL']['relThRangeFromCiVariation'] == 'ci95$2' + assert cfgNew["GENERAL"]["musamosat"] == "0.155" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "" + assert cfgNew["GENERAL"]["relThRangeFromCiVariation"] == "ci95$2" # setup inputs cfg = configparser.ConfigParser() - cfg['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|range', - 'variationValue': '60|0.25', 'numberOfSteps': '2|8', - 'defaultSetup': 'True', 'varParType': 'float|float', 'samplingStrategy': '2'} + cfg["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|range", + "variationValue": "60|0.25", + "numberOfSteps": "2|8", + "defaultSetup": "True", + "varParType": "float|float", + "samplingStrategy": "2", + } com1DFACfg = cfgUtils.getDefaultModuleConfig(com1DFA) - varName = 'relTh' + varName = "relTh" - varDict = pA.makeDictFromVars(cfg['PROBRUN']) + varDict = pA.makeDictFromVars(cfg["PROBRUN"]) # call function cfgNew = pA.updateCfgRange(com1DFACfg, cfg, varName, varDict[varName]) -# print('cfgNEW', cfgNew['GENERAL']['relThRangeVariation']) + # print('cfgNEW', cfgNew['GENERAL']['relThRangeVariation']) - assert cfgNew['GENERAL']['musamosat'] == '0.155' - assert cfgNew['GENERAL']['relTh'] == '' - assert cfgNew['GENERAL']['relThFromShp'] == 'True' - assert cfgNew['GENERAL']['relThPercentVariation'] == '' - assert cfgNew['GENERAL']['relThRangeVariation'] == '0.25$8' + assert cfgNew["GENERAL"]["musamosat"] == "0.155" + assert cfgNew["GENERAL"]["relTh"] == "" + assert cfgNew["GENERAL"]["relThFromShp"] == "True" + assert cfgNew["GENERAL"]["relThPercentVariation"] == "" + assert cfgNew["GENERAL"]["relThRangeVariation"] == "0.25$8" def test_makeDictFromVars(): - """ test creating a dict from parameter variation info """ + """test creating a dict from parameter variation info""" cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh|entTh', 'variationType': 'percent|rangeFromCi|range', - 'variationValue': '60|ci95|0.15', 'numberOfSteps': '2|3|4', - 'samplingStrategy': '2', - 'varParType': 'float|float', - 'varParType': 'float|float|float', 'nSample': '30', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh|entTh", + "variationType": "percent|rangeFromCi|range", + "variationValue": "60|ci95|0.15", + "numberOfSteps": "2|3|4", + "samplingStrategy": "2", + "varParType": "float|float|float", + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin", + } # call function to be tested - variationD = pA.makeDictFromVars(cfgProb['PROBRUN']) + variationD = pA.makeDictFromVars(cfgProb["PROBRUN"]) assert len(variationD) == 3 - assert len(variationD['relTh']) == 3 - assert variationD['relTh']['numberOfSteps'] == '3' + assert len(variationD["relTh"]) == 3 + assert variationD["relTh"]["numberOfSteps"] == "3" - cfgProb['PROBRUN']['variationType'] = 'percent|range' + cfgProb["PROBRUN"]["variationType"] = "percent|range" - message = ('For every parameter in varParList a variationValue, numberOfSteps and variationType needs to be provided') + message = "For every parameter in varParList a variationValue, numberOfSteps and variationType needs to be provided" with pytest.raises(AssertionError) as e: - assert pA.makeDictFromVars(cfgProb['PROBRUN']) + assert pA.makeDictFromVars(cfgProb["PROBRUN"]) assert message in str(e.value) - cfgProb['PROBRUN']['variationType'] = 'percent|rangefromci|range' - cfgProb['PROBRUN']['variationValue'] = '60|50|0.15' + cfgProb["PROBRUN"]["variationType"] = "percent|rangefromci|range" + cfgProb["PROBRUN"]["variationValue"] = "60|50|0.15" - message = 'If rangefromci is chosen as variation type, ci95 is required as variationValue' + message = "If rangefromci is chosen as variation type, ci95 is required as variationValue" with pytest.raises(AssertionError) as e: - assert pA.makeDictFromVars(cfgProb['PROBRUN']) + assert pA.makeDictFromVars(cfgProb["PROBRUN"]) assert message in str(e.value) def test_createSampleWithVariationStandardParameters(): - """ test creation of sample for standard parameters """ + """test creation of sample for standard parameters""" cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh|entTh', 'variationType': 'percent|range|range', - 'variationValue': '60|0.25|0.15', 'numberOfSteps': '2|3|4', - 'samplingStrategy': '1', 'varParType': 'float|float|float', - 'varParType': 'float|float|float', 'nSample': '30', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - - varParList = cfgProb['PROBRUN']['varParList'].split('|') - valVariationValue = cfgProb['PROBRUN']['variationValue'].split('|') - varType = cfgProb['PROBRUN']['variationType'].split('|') + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh|entTh", + "variationType": "percent|range|range", + "variationValue": "60|0.25|0.15", + "numberOfSteps": "2|3|4", + "samplingStrategy": "1", + "varParType": "float|float|float", + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + + varParList = cfgProb["PROBRUN"]["varParList"].split("|") + valVariationValue = cfgProb["PROBRUN"]["variationValue"].split("|") + varType = cfgProb["PROBRUN"]["variationType"].split("|") # read initial configuration cfgStart = cfgUtils.getDefaultModuleConfig(com1DFA, toPrint=False) - cfgStart['GENERAL']['relThFromShp'] = 'False' - cfgStart['GENERAL']['relTh'] = '1.5' - cfgStart['GENERAL']['entTh'] = '0.4' - paramValuesD = pA.createSampleWithVariationStandardParameters(cfgProb, cfgStart, varParList, valVariationValue, - varType) - - assert len(paramValuesD['values']) == 30 - assert paramValuesD['names'] == ['musamosat', 'relTh', 'entTh'] - assert paramValuesD['thFromIni'] == '' - assert np.amax(paramValuesD['values'][:, 1]) <= 1.75 - assert np.amin(paramValuesD['values'][:, 1]) >= 1.25 - assert np.amax(paramValuesD['values'][:, 0]) <= 0.248 - assert np.amin(paramValuesD['values'][:, 0]) >= 0.062 - assert np.amax(paramValuesD['values'][:, 2]) <= 0.55 - assert np.amin(paramValuesD['values'][:, 2]) >= 0.25 + cfgStart["GENERAL"]["relThFromShp"] = "False" + cfgStart["GENERAL"]["relTh"] = "1.5" + cfgStart["GENERAL"]["entTh"] = "0.4" + paramValuesD = pA.createSampleWithVariationStandardParameters( + cfgProb, cfgStart, varParList, valVariationValue, varType + ) + + assert len(paramValuesD["values"]) == 30 + assert paramValuesD["names"] == ["musamosat", "relTh", "entTh"] + assert paramValuesD["thFromIni"] == "" + assert np.amax(paramValuesD["values"][:, 1]) <= 1.75 + assert np.amin(paramValuesD["values"][:, 1]) >= 1.25 + assert np.amax(paramValuesD["values"][:, 0]) <= 0.248 + assert np.amin(paramValuesD["values"][:, 0]) >= 0.062 + assert np.amax(paramValuesD["values"][:, 2]) <= 0.55 + assert np.amin(paramValuesD["values"][:, 2]) >= 0.25 def test_createSampleWithVariationForThParameters(tmp_path): - """ test if thickness parameters are also included """ + """test if thickness parameters are also included""" # set input directory - avaName = 'testCom1DFA2' + avaName = "testCom1DFA2" dirPath = pathlib.Path(__file__).parents[0] - inputDir = dirPath / 'data' / avaName + inputDir = dirPath / "data" / avaName avaDir = pathlib.Path(tmp_path, avaName) shutil.copytree(inputDir, avaDir) cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh|entTh', 'variationType': 'percent|rangefromci|range', - 'variationValue': '60|ci95|0.15', 'numberOfSteps': '2|3|4', - 'samplingStrategy': '1', - 'varParType': 'float|float|float', 'nSample': '30', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - - varParList = cfgProb['PROBRUN']['varParList'].split('|') - valVariationValue = cfgProb['PROBRUN']['variationValue'].split('|') - varType = cfgProb['PROBRUN']['variationType'].split('|') + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh|entTh", + "variationType": "percent|rangefromci|range", + "variationValue": "60|ci95|0.15", + "numberOfSteps": "2|3|4", + "samplingStrategy": "1", + "varParType": "float|float|float", + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + + varParList = cfgProb["PROBRUN"]["varParList"].split("|") + valVariationValue = cfgProb["PROBRUN"]["variationValue"].split("|") + varType = cfgProb["PROBRUN"]["variationType"].split("|") # read initial configuration cfgStart = cfgUtils.getDefaultModuleConfig(com1DFA, toPrint=False) - thReadFromShp = ['relTh', 'entTh'] + thReadFromShp = ["relTh", "entTh"] - paramValuesDList = pA.createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParList, - valVariationValue, varType, thReadFromShp) + paramValuesDList = pA.createSampleWithVariationForThParameters( + avaDir, cfgProb, cfgStart, varParList, valVariationValue, varType, thReadFromShp + ) paramValuesD = paramValuesDList[0] - assert len(paramValuesD['values']) == 30 - assert paramValuesD['names'] == ['musamosat', 'relTh0', 'relTh1', 'entTh0'] - assert len(paramValuesD['thFromIni'].split('|')) == 2 - assert 'relTh' in paramValuesD['thFromIni'] - assert 'entTh' in paramValuesD['thFromIni'] - assert np.amax(paramValuesD['values'][:, 1]) <= 1.75 - assert np.amin(paramValuesD['values'][:, 1]) >= 1.25 - assert np.amax(paramValuesD['values'][:, 2]) <= 1.4 - assert np.amin(paramValuesD['values'][:, 2]) >= 1.0 - assert np.amax(paramValuesD['values'][:, 0]) <= 0.248 - assert np.amin(paramValuesD['values'][:, 0]) >= 0.062 - assert np.amax(paramValuesD['values'][:, 3]) <= 0.45 - assert np.amin(paramValuesD['values'][:, 3]) >= 0.15 - - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh|entTh', 'variationType': 'percent|rangefromci|rangefromci', - 'variationValue': '60|ci95|ci95', 'numberOfSteps': '2|3|4', - 'samplingStrategy': '1', - 'varParType': 'float|float|float', 'nSample': '30', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - - varParList = cfgProb['PROBRUN']['varParList'].split('|') - valVariationValue = cfgProb['PROBRUN']['variationValue'].split('|') - varType = cfgProb['PROBRUN']['variationType'].split('|') + assert len(paramValuesD["values"]) == 30 + assert paramValuesD["names"] == ["musamosat", "relTh0", "relTh1", "entTh0"] + assert len(paramValuesD["thFromIni"].split("|")) == 2 + assert "relTh" in paramValuesD["thFromIni"] + assert "entTh" in paramValuesD["thFromIni"] + assert np.amax(paramValuesD["values"][:, 1]) <= 1.75 + assert np.amin(paramValuesD["values"][:, 1]) >= 1.25 + assert np.amax(paramValuesD["values"][:, 2]) <= 1.4 + assert np.amin(paramValuesD["values"][:, 2]) >= 1.0 + assert np.amax(paramValuesD["values"][:, 0]) <= 0.248 + assert np.amin(paramValuesD["values"][:, 0]) >= 0.062 + assert np.amax(paramValuesD["values"][:, 3]) <= 0.45 + assert np.amin(paramValuesD["values"][:, 3]) >= 0.15 + + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh|entTh", + "variationType": "percent|rangefromci|rangefromci", + "variationValue": "60|ci95|ci95", + "numberOfSteps": "2|3|4", + "samplingStrategy": "1", + "varParType": "float|float|float", + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + + varParList = cfgProb["PROBRUN"]["varParList"].split("|") + valVariationValue = cfgProb["PROBRUN"]["variationValue"].split("|") + varType = cfgProb["PROBRUN"]["variationType"].split("|") # read initial configuration cfgStart = cfgUtils.getDefaultModuleConfig(com1DFA, toPrint=False) - thReadFromShp = ['relTh', 'entTh'] + thReadFromShp = ["relTh", "entTh"] - paramValuesDList = pA.createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParList, - valVariationValue, varType, thReadFromShp) + paramValuesDList = pA.createSampleWithVariationForThParameters( + avaDir, cfgProb, cfgStart, varParList, valVariationValue, varType, thReadFromShp + ) paramValuesD = paramValuesDList[0] - assert len(paramValuesD['values']) == 30 - assert paramValuesD['names'] == ['musamosat', 'relTh0', 'relTh1', 'entTh0'] - assert len(paramValuesD['thFromIni'].split('|')) == 2 - assert 'relTh' in paramValuesD['thFromIni'] - assert 'entTh' in paramValuesD['thFromIni'] - assert np.amax(paramValuesD['values'][:, 1]) <= 1.75 - assert np.amin(paramValuesD['values'][:, 1]) >= 1.25 - assert np.amax(paramValuesD['values'][:, 2]) <= 1.4 - assert np.amin(paramValuesD['values'][:, 2]) >= 1.0 - assert np.amax(paramValuesD['values'][:, 0]) <= 0.248 - assert np.amin(paramValuesD['values'][:, 0]) >= 0.062 - assert np.amax(paramValuesD['values'][:, 3]) <= 0.4 - assert np.amin(paramValuesD['values'][:, 3]) >= 0.1 - - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh|entTh', 'variationType': 'percent|rangefromci|percent', - 'variationValue': '60|ci95|5', 'numberOfSteps': '2|3|4', - 'samplingStrategy': '1', - 'varParType': 'float|float|float', 'nSample': '30', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - - varParList = cfgProb['PROBRUN']['varParList'].split('|') - valVariationValue = cfgProb['PROBRUN']['variationValue'].split('|') - varType = cfgProb['PROBRUN']['variationType'].split('|') + assert len(paramValuesD["values"]) == 30 + assert paramValuesD["names"] == ["musamosat", "relTh0", "relTh1", "entTh0"] + assert len(paramValuesD["thFromIni"].split("|")) == 2 + assert "relTh" in paramValuesD["thFromIni"] + assert "entTh" in paramValuesD["thFromIni"] + assert np.amax(paramValuesD["values"][:, 1]) <= 1.75 + assert np.amin(paramValuesD["values"][:, 1]) >= 1.25 + assert np.amax(paramValuesD["values"][:, 2]) <= 1.4 + assert np.amin(paramValuesD["values"][:, 2]) >= 1.0 + assert np.amax(paramValuesD["values"][:, 0]) <= 0.248 + assert np.amin(paramValuesD["values"][:, 0]) >= 0.062 + assert np.amax(paramValuesD["values"][:, 3]) <= 0.4 + assert np.amin(paramValuesD["values"][:, 3]) >= 0.1 + + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh|entTh", + "variationType": "percent|rangefromci|percent", + "variationValue": "60|ci95|5", + "numberOfSteps": "2|3|4", + "samplingStrategy": "1", + "varParType": "float|float|float", + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + + varParList = cfgProb["PROBRUN"]["varParList"].split("|") + valVariationValue = cfgProb["PROBRUN"]["variationValue"].split("|") + varType = cfgProb["PROBRUN"]["variationType"].split("|") # read initial configuration cfgStart = cfgUtils.getDefaultModuleConfig(com1DFA, toPrint=False) - cfgStart['GENERAL']['entTh'] = '0.5' - cfgStart['GENERAL']['entThFromShp'] = 'False' - thReadFromShp = ['relTh'] + cfgStart["GENERAL"]["entTh"] = "0.5" + cfgStart["GENERAL"]["entThFromShp"] = "False" + thReadFromShp = ["relTh"] - paramValuesDList = pA.createSampleWithVariationForThParameters(avaDir, cfgProb, cfgStart, varParList, - valVariationValue, varType, thReadFromShp) + paramValuesDList = pA.createSampleWithVariationForThParameters( + avaDir, cfgProb, cfgStart, varParList, valVariationValue, varType, thReadFromShp + ) paramValuesD = paramValuesDList[0] - assert len(paramValuesD['values']) == 30 - assert paramValuesD['names'] == ['musamosat', 'relTh0', 'relTh1', 'entTh'] - assert len(paramValuesD['thFromIni'].split('|')) == 1 - assert 'relTh' in paramValuesD['thFromIni'] - assert np.amax(paramValuesD['values'][:, 1]) <= 1.75 - assert np.amin(paramValuesD['values'][:, 1]) >= 1.25 - assert np.amax(paramValuesD['values'][:, 2]) <= 1.4 - assert np.amin(paramValuesD['values'][:, 2]) >= 1.0 - assert np.amax(paramValuesD['values'][:, 0]) <= 0.248 - assert np.amin(paramValuesD['values'][:, 0]) >= 0.062 - assert np.amax(paramValuesD['values'][:, 3]) <= 0.525 - assert np.amin(paramValuesD['values'][:, 3]) >= 0.475 + assert len(paramValuesD["values"]) == 30 + assert paramValuesD["names"] == ["musamosat", "relTh0", "relTh1", "entTh"] + assert len(paramValuesD["thFromIni"].split("|")) == 1 + assert "relTh" in paramValuesD["thFromIni"] + assert np.amax(paramValuesD["values"][:, 1]) <= 1.75 + assert np.amin(paramValuesD["values"][:, 1]) >= 1.25 + assert np.amax(paramValuesD["values"][:, 2]) <= 1.4 + assert np.amin(paramValuesD["values"][:, 2]) >= 1.0 + assert np.amax(paramValuesD["values"][:, 0]) <= 0.248 + assert np.amin(paramValuesD["values"][:, 0]) >= 0.062 + assert np.amax(paramValuesD["values"][:, 3]) <= 0.525 + assert np.amin(paramValuesD["values"][:, 3]) >= 0.475 def test_createCfgFiles(tmp_path): - """ test writing of cfg files """ + """test writing of cfg files""" - paramValuesD = {'names': ['relTh', 'musamosat'], 'values': np.asarray([[1.2, 0.1], [1.4, 0.12], [1.6, 0.14]]), - 'thFromIni': ''} + paramValuesD = { + "names": ["relTh", "musamosat"], + "values": np.asarray([[1.2, 0.1], [1.4, 0.12], [1.6, 0.14]]), + "thFromIni": "", + } cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': True, 'relTh': '', 'musamosat': 0.2, 'thFromIni': False} + cfgProb["PROBRUN"] = {} + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": True, + "relTh": "", + "musamosat": 0.2, + "thFromIni": False, + } - cfgFiles = pA.createCfgFiles([paramValuesD], com1DFA, cfgProb, cfgPath='') + cfgFiles = pA.createCfgFiles([paramValuesD], com1DFA, cfgProb, cfgPath="") -# print(cfgFiles) + # print(cfgFiles) cfgTest1 = configparser.ConfigParser() cfgTest1.read(cfgFiles[0]) @@ -641,194 +788,218 @@ def test_createCfgFiles(tmp_path): cfgTest3.read(cfgFiles[2]) assert len(cfgFiles) == 3 - assert cfgTest1['GENERAL'].getfloat('relTh') == 1.2 - assert cfgTest1['GENERAL'].getfloat('musamosat') == 0.1 - assert cfgTest1['INPUT']['thFromIni'] == '' - assert cfgTest1['VISUALISATION']['scenario'] == '0' - assert cfgTest2['GENERAL'].getfloat('relTh') == 1.4 - assert cfgTest2['GENERAL'].getfloat('musamosat') == 0.12 - assert cfgTest2['INPUT']['thFromIni'] == '' - assert cfgTest2['VISUALISATION']['scenario'] == '1' - assert cfgTest3['GENERAL'].getfloat('relTh') == 1.6 - assert cfgTest3['GENERAL'].getfloat('musamosat') == 0.14 - assert cfgTest3['INPUT']['thFromIni'] == '' - assert cfgTest3['VISUALISATION']['scenario'] == '2' - - cfgProb['PROBRUN'] = {} - cfgProb['com1DFA_override'] = {'defaultConfig': True} + assert cfgTest1["GENERAL"].getfloat("relTh") == 1.2 + assert cfgTest1["GENERAL"].getfloat("musamosat") == 0.1 + assert cfgTest1["INPUT"]["thFromIni"] == "" + assert cfgTest1["VISUALISATION"]["scenario"] == "0" + assert cfgTest2["GENERAL"].getfloat("relTh") == 1.4 + assert cfgTest2["GENERAL"].getfloat("musamosat") == 0.12 + assert cfgTest2["INPUT"]["thFromIni"] == "" + assert cfgTest2["VISUALISATION"]["scenario"] == "1" + assert cfgTest3["GENERAL"].getfloat("relTh") == 1.6 + assert cfgTest3["GENERAL"].getfloat("musamosat") == 0.14 + assert cfgTest3["INPUT"]["thFromIni"] == "" + assert cfgTest3["VISUALISATION"]["scenario"] == "2" + + cfgProb["PROBRUN"] = {} + cfgProb["com1DFA_override"] = {"defaultConfig": True} def test_fetchStartCfg(tmp_path): - """ test fetching starting cfg """ + """test fetching starting cfg""" cfg = configparser.ConfigParser() cfg.optionxform = str - cfg['com1DFA_com1DFA_override'] = {'defaultConfig': True, 'musamosat': 0.2} + cfg["com1DFA_com1DFA_override"] = {"defaultConfig": True, "musamosat": 0.2} cfgStart = pA.fetchStartCfg(com1DFA, cfg) - assert cfgStart.has_option('GENERAL', 'entTh') is True - assert cfgStart['GENERAL'].getfloat('musamosat') == 0.2 + assert cfgStart.has_option("GENERAL", "entTh") is True + assert cfgStart["GENERAL"].getfloat("musamosat") == 0.2 - cfg['com1DFA_com1DFA_override'] = {'defaultConfig': True} + cfg["com1DFA_com1DFA_override"] = {"defaultConfig": True} cfgStart = pA.fetchStartCfg(com1DFA, cfg) - assert cfgStart.has_option('GENERAL', 'entTh') is True - assert cfgStart['GENERAL'].getfloat('musamosat') == 0.155 + assert cfgStart.has_option("GENERAL", "entTh") is True + assert cfgStart["GENERAL"].getfloat("musamosat") == 0.155 def test_fetchProbConfigs(): - """ test creation of probability configurations """ + """test creation of probability configurations""" cfg = configparser.ConfigParser() cfg.optionxform = str - cfg['PROBRUN'] = {'samplingStrategy': '2', 'varParList': 'relTh|musamosat'} + cfg["PROBRUN"] = {"samplingStrategy": "2", "varParList": "relTh|musamosat"} - probConfigs = pA.fetchProbConfigs(cfg['PROBRUN']) + probConfigs = pA.fetchProbConfigs(cfg["PROBRUN"]) assert len(probConfigs) == 3 - assert 'includemusamosat' in probConfigs.keys() - assert 'includerelTh' in probConfigs.keys() + assert "includemusamosat" in probConfigs.keys() + assert "includerelTh" in probConfigs.keys() - cfg['PROBRUN'] = {'samplingStrategy': '1', 'varParList': 'relTh|musamosat'} + cfg["PROBRUN"] = {"samplingStrategy": "1", "varParList": "relTh|musamosat"} - probConfigs = pA.fetchProbConfigs(cfg['PROBRUN']) + probConfigs = pA.fetchProbConfigs(cfg["PROBRUN"]) assert len(probConfigs) == 1 - assert 'includeAll' in probConfigs.keys() + assert "includeAll" in probConfigs.keys() def test_fetchThicknessInfo(tmp_path): - """ test fetching info on thickness settings if readFromShp """ - avaName = 'testCom1DFA2' + """test fetching info on thickness settings if readFromShp""" + avaName = "testCom1DFA2" dirPath = pathlib.Path(__file__).parents[0] - inputDir = dirPath / 'data' / avaName + inputDir = dirPath / "data" / avaName avaDir = pathlib.Path(tmp_path, avaName) shutil.copytree(inputDir, avaDir) inputSimFiles = pA.fetchThicknessInfo(avaDir) - assert inputSimFiles['releaseScenarioList'] == ['relParabola', 'relParabolaTwo'] - assert inputSimFiles['relParabola']['thickness'][0] == '1.5' - assert inputSimFiles['relParabola']['thickness'][1] == '1.2' - assert inputSimFiles['relParabola']['id'][0] == '0' - assert inputSimFiles['relParabola']['id'][1] == '1' - assert inputSimFiles['relParabola']['ci95'][0] == '0.25' - assert inputSimFiles['relParabola']['ci95'][1] == '0.2' + assert inputSimFiles["releaseScenarioList"] == ["relParabola", "relParabolaTwo"] + assert inputSimFiles["relParabola"]["thickness"][0] == "1.5" + assert inputSimFiles["relParabola"]["thickness"][1] == "1.2" + assert inputSimFiles["relParabola"]["id"][0] == "0" + assert inputSimFiles["relParabola"]["id"][1] == "1" + assert inputSimFiles["relParabola"]["ci95"][0] == "0.25" + assert inputSimFiles["relParabola"]["ci95"][1] == "0.2" def test_fetchThicknessList(tmp_path): - """ test fetching the thickness info """ + """test fetching the thickness info""" - avaName = 'testCom1DFA2' + avaName = "testCom1DFA2" dirPath = pathlib.Path(__file__).parents[0] - inputDir = dirPath / 'data' / avaName + inputDir = dirPath / "data" / avaName avaDir = pathlib.Path(tmp_path, avaName) shutil.copytree(inputDir, avaDir) # fetch input files and corresponding thickness info inputSimFiles = pA.fetchThicknessInfo(avaDir) - thValues, ciValues, thicknessFeatureNames = pA.fetchThThicknessLists('relTh', inputSimFiles, - inputSimFiles['relFiles'][0], ciRequired=True) + thValues, ciValues, thicknessFeatureNames = pA.fetchThThicknessLists( + "relTh", inputSimFiles, inputSimFiles["relFiles"][0], ciRequired=True + ) assert thValues[0] == 1.5 assert thValues[1] == 1.2 assert ciValues[0] == 0.25 assert ciValues[1] == 0.2 - assert thicknessFeatureNames[0] == 'relTh0' - assert thicknessFeatureNames[1] == 'relTh1' + assert thicknessFeatureNames[0] == "relTh0" + assert thicknessFeatureNames[1] == "relTh1" def test_cfgFilesGlobalApproach(tmp_path): - """ test global approach to fetch sample and create cfg files """ + """test global approach to fetch sample and create cfg files""" # set input directory - avaName = 'testCom1DFA2' + avaName = "testCom1DFA2" dirPath = pathlib.Path(__file__).parents[0] - inputDir = dirPath / 'data' / avaName + inputDir = dirPath / "data" / avaName avaDir = pathlib.Path(tmp_path, avaName) shutil.copytree(inputDir, avaDir) - cfgFile = dirPath / 'data' / 'testCom1DFA' / 'probA_com1DFACfg.ini' - outDir = avaDir / 'Outputs' + outDir = avaDir / "Outputs" cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|3', - 'defaultSetup': 'True', 'samplingStrategy': '1', - 'varParType': 'float|float', 'nSample': '40', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'relThFromShp': False, 'relTh': 2., - 'musamosat': 0.155, 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|3", + "defaultSetup": "True", + "samplingStrategy": "1", + "varParType": "float|float", + "nSample": "40", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "relThFromShp": False, + "relTh": 2.0, + "musamosat": 0.155, + "frictModel": "samosAT", + } # call function to be tested cfgFiles = pA.cfgFilesGlobalApproach(avaDir, cfgProb, com1DFA, outDir) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) cfgTest = configparser.ConfigParser() cfgTest.read(cfgFiles[0]) -# print('cfgTest', cfgTest['GENERAL']['relThFromShp'], cfgTest['GENERAL']['relTh'], + # print('cfgTest', cfgTest['GENERAL']['relThFromShp'], cfgTest['GENERAL']['relTh'], # cfgTest['GENERAL']['relThPercentVariation'], cfgTest['GENERAL']['musamosat']) - assert cfgTest['GENERAL']['relThFromShp'] == 'False' - assert cfgTest['GENERAL']['relTh'] == '2.2719559079879' + assert cfgTest["GENERAL"]["relThFromShp"] == "False" + assert cfgTest["GENERAL"]["relTh"] == "2.2719559079879" assert len(cfgFiles) == 40 cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|3', - 'defaultSetup': 'True', 'samplingStrategy': '1', - 'varParType': 'float|float', 'nSample': '40', 'sampleSeed': '12345', - 'sampleMethod': 'latin'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|3", + "defaultSetup": "True", + "samplingStrategy": "1", + "varParType": "float|float", + "nSample": "40", + "sampleSeed": "12345", + "sampleMethod": "latin", + } + cfgProb["com1DFA_com1DFA_override"] = {"defaultConfig": "True"} # call function to be tested cfgFiles2 = pA.cfgFilesGlobalApproach(avaDir, cfgProb, com1DFA, outDir) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) cfgTest1 = configparser.ConfigParser() cfgTest1.read(cfgFiles2[0]) -# print('cfgTest', cfgTest1['GENERAL']['relThFromShp'], cfgTest1['GENERAL']['relTh'], + # print('cfgTest', cfgTest1['GENERAL']['relThFromShp'], cfgTest1['GENERAL']['relTh'], # cfgTest1['GENERAL']['relThPercentVariation'], cfgTest1['GENERAL']['musamosat']) - assert cfgTest1['GENERAL']['relThFromShp'] == 'True' - assert cfgTest1['GENERAL']['relTh'] == '' - assert cfgTest1['GENERAL']['relTh0'] != '' - assert cfgTest1['INPUT']['releaseScenario'] == 'relParabola' + assert cfgTest1["GENERAL"]["relThFromShp"] == "True" + assert cfgTest1["GENERAL"]["relTh"] == "" + assert cfgTest1["GENERAL"]["relTh0"] != "" + assert cfgTest1["INPUT"]["releaseScenario"] == "relParabola" assert len(cfgFiles2) == 80 def test_cfgFilesLocalApproach(tmp_path): - """ test creating cfg files from one at a time var """ + """test creating cfg files from one at a time var""" # set input directory - avaName = 'testCom1DFA2' + avaName = "testCom1DFA2" dirPath = pathlib.Path(__file__).parents[0] - inputDir = dirPath / 'data' / avaName + inputDir = dirPath / "data" / avaName avaDir = pathlib.Path(tmp_path, avaName) shutil.copytree(inputDir, avaDir) - cfgFile = dirPath / 'data' / 'testCom1DFA' / 'probA_com1DFACfg.ini' outDir = avaDir cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|percent', - 'variationValue': '60|50', 'numberOfSteps': '2|3', - 'defaultSetup': 'True', 'samplingStrategy': '2', 'varParType': 'float|float'} - cfgProb['sampling_override'] = {'defaultConfig': 'True'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|percent", + "variationValue": "60|50", + "numberOfSteps": "2|3", + "defaultSetup": "True", + "samplingStrategy": "2", + "varParType": "float|float", + } + cfgProb["sampling_override"] = {"defaultConfig": "True"} + cfgProb["com1DFA_com1DFA_override"] = {"defaultConfig": "True"} # check variation settings - variationsDict = pA.makeDictFromVars(cfgProb['PROBRUN']) + variationsDict = pA.makeDictFromVars(cfgProb["PROBRUN"]) # call function to be tested cfgFiles = pA.cfgFilesLocalApproach(variationsDict, cfgProb, com1DFA, outDir) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) # load cfg from file cfgMu = configparser.ConfigParser() @@ -837,34 +1008,44 @@ def test_cfgFilesLocalApproach(tmp_path): cfgMu.read(cfgFiles[0]) cfgRelTh.read(cfgFiles[1]) -# print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], + # print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], # cfgRelTh['GENERAL']['relTh']) - assert cfgMu['GENERAL']['musamosat'] == '0.155$60$2' - assert cfgMu['GENERAL']['relTh'] == '' - assert cfgRelTh['GENERAL']['musamosat'] == '0.155' - assert cfgRelTh['GENERAL']['relTh'] == '' - assert cfgRelTh['GENERAL']['relThFromShp'] == 'True' - assert cfgRelTh['GENERAL']['relThPercentVariation'] == '50$3' - assert cfgRelTh['GENERAL']['relThFromShp'] == 'True' - assert cfgMu['GENERAL']['relThFromShp'] == 'True' + assert cfgMu["GENERAL"]["musamosat"] == "0.155$60$2" + assert cfgMu["GENERAL"]["relTh"] == "" + assert cfgRelTh["GENERAL"]["musamosat"] == "0.155" + assert cfgRelTh["GENERAL"]["relTh"] == "" + assert cfgRelTh["GENERAL"]["relThFromShp"] == "True" + assert cfgRelTh["GENERAL"]["relThPercentVariation"] == "50$3" + assert cfgRelTh["GENERAL"]["relThFromShp"] == "True" + assert cfgMu["GENERAL"]["relThFromShp"] == "True" cfgProb = configparser.ConfigParser() cfgProb.optionxform = str - cfgProb['PROBRUN'] = {'varParList': 'musamosat|relTh', 'variationType': 'percent|range', - 'variationValue': '60|0.5', 'numberOfSteps': '2|3', - 'samplingStrategy': '2', 'varParType': 'float|float'} - cfgProb['sampling_override'] = {'defaultConfig': 'True'} - cfgProb['com1DFA_com1DFA_override'] = {'defaultConfig': 'True', 'relThFromShp': False, 'relTh': 2., - 'musamosat': 0.155, 'frictModel': 'samosAT'} + cfgProb["PROBRUN"] = { + "varParList": "musamosat|relTh", + "variationType": "percent|range", + "variationValue": "60|0.5", + "numberOfSteps": "2|3", + "samplingStrategy": "2", + "varParType": "float|float", + } + cfgProb["sampling_override"] = {"defaultConfig": "True"} + cfgProb["com1DFA_com1DFA_override"] = { + "defaultConfig": "True", + "relThFromShp": False, + "relTh": 2.0, + "musamosat": 0.155, + "frictModel": "samosAT", + } # check variation settings - variationsDict = pA.makeDictFromVars(cfgProb['PROBRUN']) + variationsDict = pA.makeDictFromVars(cfgProb["PROBRUN"]) # call function to be tested cfgFiles = pA.cfgFilesLocalApproach(variationsDict, cfgProb, com1DFA, outDir) -# print('cfgFiles', cfgFiles) + # print('cfgFiles', cfgFiles) # load cfg from file cfgMu = configparser.ConfigParser() @@ -873,68 +1054,214 @@ def test_cfgFilesLocalApproach(tmp_path): cfgMu.read(cfgFiles[0]) cfgRelTh.read(cfgFiles[1]) -# print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], + # print(cfgMu['GENERAL']['musamosat'], cfgMu['GENERAL']['relTh'], cfgRelTh['GENERAL']['musamosat'], # cfgRelTh['GENERAL']['relTh']) - assert cfgMu['GENERAL']['musamosat'] == '0.155$60$2' - assert np.isclose(cfgMu['GENERAL'].getfloat('relTh'), 2.) - assert cfgRelTh['GENERAL']['musamosat'] == '0.155' - assert np.isclose(cfgRelTh['GENERAL'].getfloat('relTh'), 2.) - assert cfgRelTh['GENERAL']['relThFromShp'] == 'False' - assert cfgRelTh['GENERAL']['relThRangeVariation'] == '0.5$3' - assert cfgRelTh['GENERAL']['relThFromShp'] == 'False' - assert cfgMu['GENERAL']['relThFromShp'] == 'False' + assert cfgMu["GENERAL"]["musamosat"] == "0.155$60$2" + assert np.isclose(cfgMu["GENERAL"].getfloat("relTh"), 2.0) + assert cfgRelTh["GENERAL"]["musamosat"] == "0.155" + assert np.isclose(cfgRelTh["GENERAL"].getfloat("relTh"), 2.0) + assert cfgRelTh["GENERAL"]["relThFromShp"] == "False" + assert cfgRelTh["GENERAL"]["relThRangeVariation"] == "0.5$3" + assert cfgRelTh["GENERAL"]["relThFromShp"] == "False" + assert cfgMu["GENERAL"]["relThFromShp"] == "False" def test_checkParameterSettings(): - """ test if parameter settings are valid """ + """test if parameter settings are valid""" cfg = configparser.ConfigParser() cfg.optionxform = str - cfg['GENERAL'] = {'relTh': '', 'musamosat': '0.155', 'relThFromShp': 'True', 'relThPercentVariation': '', - 'relThRangeVariation': '', 'relThRangeFromCiVariation': '', - 'relThDistVariation': ''} - varParList = ['relTh', 'musamosat'] + cfg["GENERAL"] = { + "relTh": "", + "musamosat": "0.155", + "relThFromShp": "True", + "relThPercentVariation": "", + "relThRangeVariation": "", + "relThRangeFromCiVariation": "", + "relThDistVariation": "", + } + varParList = ["relTh", "musamosat"] check, thReadFromShp = pA.checkParameterSettings(cfg, varParList) assert check assert len(thReadFromShp) == 1 - assert thReadFromShp[0] == 'relTh' + assert thReadFromShp[0] == "relTh" - cfg['GENERAL']['musamosat'] = '0.1:0.2:10' - message = ('Only one reference value is allowed for %s: but %s is given' % - ('musamosat', '0.1:0.2:10')) + cfg["GENERAL"]["musamosat"] = "0.1:0.2:10" + message = "Only one reference value is allowed for %s: but %s is given" % ( + "musamosat", + "0.1:0.2:10", + ) with pytest.raises(AssertionError) as e: assert pA.checkParameterSettings(cfg, varParList) assert message in str(e.value) - cfg['GENERAL'] = {'relTh': '', 'musamosat': '0.155', 'relThFromShp': 'True', 'relThPercentVariation': '', - 'relThRangeVariation': '5$4', 'relThRangeFromCiVariation': '', - 'relThDistVariation': ''} + cfg["GENERAL"] = { + "relTh": "", + "musamosat": "0.155", + "relThFromShp": "True", + "relThPercentVariation": "", + "relThRangeVariation": "5$4", + "relThRangeFromCiVariation": "", + "relThDistVariation": "", + } with pytest.raises(AssertionError) as e: assert pA.checkParameterSettings(cfg, varParList) - assert 'Only one reference value is allowed for relTh' in str(e.value) - assert 'relThRangeVariation' in str(e.value) + assert "Only one reference value is allowed for relTh" in str(e.value) + assert "relThRangeVariation" in str(e.value) def test_checkForNumberOfReferenceValues(): - """ check if reference (base) value already has a variation set for thickness parameters""" + """check if reference (base) value already has a variation set for thickness parameters""" cfg = configparser.ConfigParser() cfg.optionxform = str - cfg['GENERAL'] = {'relTh': '', 'musamosat': '0.155', 'relThFromShp': 'True', 'relThPercentVariation': '', - 'relThRangeVariation': '', 'relThRangeFromCiVariation': '', - 'relThDistVariation': ''} + cfg["GENERAL"] = { + "relTh": "", + "musamosat": "0.155", + "relThFromShp": "True", + "relThPercentVariation": "", + "relThRangeVariation": "", + "relThRangeFromCiVariation": "", + "relThDistVariation": "", + } # call function to be tested - checkIs = pA.checkForNumberOfReferenceValues(cfg['GENERAL'], 'relTh') + checkIs = pA.checkForNumberOfReferenceValues(cfg["GENERAL"], "relTh") assert checkIs is True - cfg['GENERAL']['relThPercentVariation'] = '50$3' + cfg["GENERAL"]["relThPercentVariation"] = "50$3" with pytest.raises(AssertionError) as e: - assert pA.checkForNumberOfReferenceValues(cfg['GENERAL'], 'relTh') - assert 'Only one reference value is allowed for relTh' in str(e.value) + assert pA.checkForNumberOfReferenceValues(cfg["GENERAL"], "relTh") + assert "Only one reference value is allowed for relTh" in str(e.value) + + + + +def test_createSample_latin(): + """Test Latin Hypercube sampling method""" + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig["PROBRUN"] = { + "nSample": "50", + "sampleSeed": "12345", + "sampleMethod": "latin" + } + + testParList = ["param1", "param2", "param3"] + + from avaframe.ana4Stats.probAna import createSample + resultSample = createSample(testConfig, testParList) + + # Check sample properties + assert isinstance(resultSample, np.ndarray) + assert resultSample.shape == (50, 3) # nSample x number of parameters + assert np.all(resultSample >= 0) and np.all(resultSample <= 1) # Values should be in [0,1] + + # Check Latin Hypercube properties + for colIdx in range(resultSample.shape[1]): + # Check if values are well distributed (no clustering) + binCounts = np.histogram(resultSample[:, colIdx], bins=10)[0] + assert np.all(binCounts > 0) # Each bin should have at least one sample + + +def test_createSample_morris(): + """Test Morris sampling method""" + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig["PROBRUN"] = { + "nSample": "4", + "sampleSeed": "12345", + "sampleMethod": "morris" + } + + testParList = ["param1", "param2"] + + from avaframe.ana4Stats.probAna import createSample + resultSample = createSample(testConfig, testParList) + + # Check sample properties for Morris method + expectedSampleSize = 4 * (len(testParList) + 1) # Morris method formula + assert resultSample.shape == (expectedSampleSize, 2) + assert np.all(resultSample >= 0) and np.all(resultSample <= 1) + + +def test_createSample_reproducibility(): + """Test reproducibility with same seed""" + + # Create test configuration + testConfig = configparser.ConfigParser() + testConfig["PROBRUN"] = { + "nSample": "30", + "sampleSeed": "54321", + "sampleMethod": "latin" + } + + testParList = ["param1", "param2"] + + from avaframe.ana4Stats.probAna import createSample + + # Generate two samples with same seed + firstSample = createSample(testConfig, testParList) + secondSample = createSample(testConfig, testParList) + + # Check if samples are identical + np.testing.assert_array_equal(firstSample, secondSample) + + +def test_createSample_different_seeds(): + """Test different seeds produce different samples""" + + # Create test configurations with different seeds + testConfig1 = configparser.ConfigParser() + testConfig1["PROBRUN"] = { + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "latin" + } + + testConfig2 = configparser.ConfigParser() + testConfig2["PROBRUN"] = { + "nSample": "30", + "sampleSeed": "54321", + "sampleMethod": "latin" + } + + testParList = ["param1", "param2"] + + from avaframe.ana4Stats.probAna import createSample + + # Generate samples with different seeds + firstSample = createSample(testConfig1, testParList) + secondSample = createSample(testConfig2, testParList) + + # Check if samples are different + with pytest.raises(AssertionError): + np.testing.assert_array_equal(firstSample, secondSample) + + +def test_createSample_invalid_method(): + """Test handling of invalid sampling method""" + + # Create test configuration with invalid method + testConfig = configparser.ConfigParser() + testConfig["PROBRUN"] = { + "nSample": "30", + "sampleSeed": "12345", + "sampleMethod": "invalid_method" + } + + testParList = ["param1", "param2"] + + from avaframe.ana4Stats.probAna import createSample + + # Check if appropriate error is raised + with pytest.raises(AssertionError): + createSample(testConfig, testParList) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e028890b0..f555060fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,11 +63,15 @@ version_file = "avaframe/RELEASE-VERSION.txt" # Cibuildwhell [tool.cibuildwheel] # Only build on CPython 3.12 -#build = "cp312-*" +#build = ["cp312-*"] skip = ["*musllinux*","*-win32"] build-verbosity = 1 before-build = "pip install cython numpy" + +[tool.cibuildwheel.linux] +repair-wheel-command = "auditwheel --verbose repair -w {dest_dir} {wheel} --plat manylinux_2_34_x86_64" + #Flake8 [tool.flake8] max-line-length = 109 @@ -86,6 +90,7 @@ platforms = ["linux-64", "win-64", "osx-64"] [tool.pixi.dependencies] setuptools = "*" setuptools-scm = "*" +cibuildwheel = ">=3.0.0,<4" # Feature dev [tool.pixi.feature.dev.pypi-dependencies]