diff --git a/avaframe/ana1Tests/simiSolTest.py b/avaframe/ana1Tests/simiSolTest.py index c0ab7a3c7..70905726c 100644 --- a/avaframe/ana1Tests/simiSolTest.py +++ b/avaframe/ana1Tests/simiSolTest.py @@ -57,7 +57,7 @@ def mainSimilaritySol(simiSolCfg): """ # Load configuration - cfg = cfgUtils.getModuleConfig(com1DFA, simiSolCfg) + cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=simiSolCfg) cfgGen = cfg["GENERAL"] cfgSimi = cfg["SIMISOL"] bedFrictionAngleDeg = cfgSimi.getfloat("bedFrictionAngle") diff --git a/avaframe/ana5Utils/DFAPathGeneration.py b/avaframe/ana5Utils/DFAPathGeneration.py index 25d46ec24..625ac29cb 100644 --- a/avaframe/ana5Utils/DFAPathGeneration.py +++ b/avaframe/ana5Utils/DFAPathGeneration.py @@ -51,7 +51,7 @@ def generatePathAndSplitpoint(avalancheDir, cfgDFAPath, cfgMain, runDFAModule): # Clean avalanche directory of old work and output files from module initProj.cleanModuleFiles(avalancheDir, com1DFA, deleteOutput=True) # create and read the default com1DFA config (no local is read) - com1DFACfg = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, + com1DFACfg = cfgUtils.getModuleConfig(com1DFA, avalancheDir, toPrint=False, onlyDefault=cfgDFAPath['com1DFA_com1DFA_override'].getboolean( 'defaultConfig')) # and override with settings from DFAPath config @@ -59,12 +59,12 @@ def generatePathAndSplitpoint(avalancheDir, cfgDFAPath, cfgMain, runDFAModule): addModValues=False) outDir = pathlib.Path(avalancheDir, 'Outputs', 'ana5Utils', 'DFAPath') fU.makeADir(outDir) - # write configuration to file + # write configuration to file for documentation com1DFACfgFile = outDir / 'com1DFAPathGenerationCfg.ini' with open(com1DFACfgFile, 'w') as configfile: com1DFACfg.write(configfile) # call com1DFA and perform simulations - dem, plotDict, reportDictList, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=com1DFACfgFile) + dem, plotDict, reportDictList, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=com1DFACfg) else: # read existing simulation results # read simulation dem demOri = gI.readDEM(avalancheDir) diff --git a/avaframe/ana5Utils/distanceTimeAnalysis.py b/avaframe/ana5Utils/distanceTimeAnalysis.py index 1a9cceed4..70ec496bd 100644 --- a/avaframe/ana5Utils/distanceTimeAnalysis.py +++ b/avaframe/ana5Utils/distanceTimeAnalysis.py @@ -583,10 +583,11 @@ def initializeRangeTime(modName, cfg, dem, simHash): """ # fetch configuration and add info - cfgRangeTime = cfgUtils.getModuleConfig(modName) + avalancheDir = cfg['GENERAL']['avalancheDir'] + cfgRangeTime = cfgUtils.getModuleConfig(modName, avalancheDir) cfgRangeTime['GENERAL']['tEnd'] = cfg['GENERAL']['tEnd'] - cfgRangeTime['GENERAL']['avalancheDir'] = cfg['GENERAL']['avalancheDir'] + cfgRangeTime['GENERAL']['avalancheDir'] = avalancheDir cfgRangeTime['GENERAL']['simHash'] = simHash # fetch time steps for creating range time diagram dtRangeTime = fU.splitTimeValueToArrayInterval(cfgRangeTime['GENERAL']['distanceTimeSteps'], diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index 20fd1b99d..90c4f535d 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -2,6 +2,7 @@ Main functions for python DFA kernel """ +import configparser import copy import inspect import logging @@ -65,7 +66,7 @@ debugPlot = cfgAVA["FLAGS"].getboolean("debugPlot") -def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo, module=com1DFA): +def com1DFAPreprocess(cfgMain, cfgInfo, module=com1DFA): """preprocess information from configuration, read input data and gather into inputSimFiles, create one config object for each of all desired simulations, create dataFrame with one line per simulations of already existing sims in avalancheDir @@ -74,12 +75,9 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo, module=com1DFA): ------------ cfgMain: configparser object main configuration of AvaFrame - typeCfgInfo: str - name of type of cfgInfo (cfgFromFile or cfgFromObject) cfgInfo: str or pathlib Path or configparser object - path to configuration file if overwrite is desired - optional - if not local (if available) or default configuration will be loaded - if cfgInfo is a configparser object take this as initial config + if ConfigParser object: use directly as initial config + if str/Path: path to configuration file override (empty string uses default/local config) module: module module to be used for task (optional) @@ -96,10 +94,11 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo, module=com1DFA): avalancheDir = cfgMain["MAIN"]["avalancheDir"] # read initial configuration - if typeCfgInfo in ["cfgFromFile", "cfgFromDefault"]: - cfgStart = cfgUtils.getModuleConfig(module, fileOverride=cfgInfo, toPrint=False) - elif typeCfgInfo == "cfgFromObject": + if isinstance(cfgInfo, configparser.ConfigParser): cfgStart = cfgInfo + else: + # cfgInfo is file path (str or Path) or empty string + cfgStart = cfgUtils.getModuleConfig(module, avalancheDir, fileOverride=cfgInfo, toPrint=False) # fetch input data and create work and output directories inputSimFilesAll, outDir, simDFExisting, simNameExisting = com1DFATools.initializeInputs( @@ -140,14 +139,13 @@ def com1DFAMain(cfgMain, cfgInfo=""): avalancheDir = cfgMain["MAIN"]["avalancheDir"] - # fetch type of cfgInfo - typeCfgInfo = com1DFATools.checkCfgInfoType(cfgInfo) - if typeCfgInfo == "cfgFromDir": - # preprocessing to create configuration objects for all simulations to run by reading multiple cfg files + # Route based on cfgInfo type: directory Path = batch mode, otherwise single config mode + if isinstance(cfgInfo, pathlib.Path) and cfgInfo.is_dir(): + # Batch mode - cfgInfo is directory path (from getModuleConfig with batchCfgDir) simDict, inputSimFiles, simDFExisting, outDir = com1DFATools.createSimDictFromCfgs(cfgMain, cfgInfo) else: - # preprocessing to create configuration objects for all simulations to run - simDict, outDir, inputSimFiles, simDFExisting = com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo) + # Single config mode - cfgInfo is ConfigParser, file path string, or empty string + simDict, outDir, inputSimFiles, simDFExisting = com1DFAPreprocess(cfgMain, cfgInfo) log.info("The following simulations will be performed") for key in simDict: @@ -3111,7 +3109,8 @@ def _findWrapperModuleInStack(): # Extract the last component (the actual module name) moduleName = frameModule.split(".")[-1] # Check if it matches the comN pattern (starts with "com" followed by a digit) - if re.match(r"^com\d+", moduleName) and not frameModule.endswith("com1DFA.com1DFA"): + # Exclude all modules in the com1DFA package (com1DFA.com1DFA, com1DFA.com1DFATools, etc.) + if re.match(r"^com\d+", moduleName) and not frameModule.startswith("avaframe.com1DFA"): return moduleName return None diff --git a/avaframe/com1DFA/com1DFATools.py b/avaframe/com1DFA/com1DFATools.py index 7c940f61b..5ba11a87a 100644 --- a/avaframe/com1DFA/com1DFATools.py +++ b/avaframe/com1DFA/com1DFATools.py @@ -299,7 +299,7 @@ def createSimDictFromCfgs(cfgMain, cfgPath, module=com1DFA): # loop over all cfgFiles and create simDict for index, cfgFile in enumerate(cfgFilesAll): # read configuration - cfgFromFile = cfgUtils.getModuleConfig(module, fileOverride=cfgFile, toPrint=False) + cfgFromFile = cfgUtils.getModuleConfig(module, avalancheDir, fileOverride=cfgFile, toPrint=False) # create dictionary with one key for each simulation that shall be performed # NOTE: sims that are added don't need to be added to the simNameExisting list as @@ -357,48 +357,6 @@ def initializeInputs(avalancheDir, cleanRemeshedRasters, module=com1DFA): return inputSimFilesAll, outDir, simDFExisting, simNameExisting -def checkCfgInfoType(cfgInfo): - """check if cfgInfo is a configparser object, a file or a directory - - Parameters - ------------ - cfgInfo: configparser object, str or pathlib path - - Returns - --------- - typeCfgInfo: str - name of type of cfgInfo - """ - - if cfgInfo == "": - typeCfgInfo = "cfgFromDefault" - - elif isinstance(cfgInfo, (pathlib.Path, str)): - # if path is provided check if file or directory - cfgInfoPath = pathlib.Path(cfgInfo) - if cfgInfoPath.is_dir(): - typeCfgInfo = "cfgFromDir" - log.info("----- CFG override from directory is used -----") - elif cfgInfo.is_file(): - typeCfgInfo = "cfgFromFile" - log.info("----- CFG override from file is used ----") - - elif isinstance(cfgInfo, configparser.ConfigParser): - typeCfgInfo = "cfgFromObject" - log.info("---- CFG override object is used ----") - - else: - message = ( - "cfgInfo is not of valid format, needs to be a path to a cfg file, \ - directory, configparser object or an empty str, cfgInfo is: %s" - % cfgInfo - ) - log.error(message) - raise AssertionError(message) - - return typeCfgInfo - - def updateResCoeffFields(fields, cfg): """update fields of cRes and detK, coefficients of resistance parameter and detrainment parameter according to the thresholds of FV and FT diff --git a/avaframe/com3Hybrid/com3Hybrid.py b/avaframe/com3Hybrid/com3Hybrid.py index 0c942050a..014e3fb67 100644 --- a/avaframe/com3Hybrid/com3Hybrid.py +++ b/avaframe/com3Hybrid/com3Hybrid.py @@ -6,7 +6,6 @@ """ import pathlib import logging -from configupdater import ConfigUpdater import numpy as np import copy @@ -54,20 +53,18 @@ def maincom3Hybrid(cfgMain, cfgHybrid): # ++++++++++ set configurations for all the used modules and override ++++++++++++ # get comDFA configuration and save to file - com1DFACfg = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, + com1DFACfg = cfgUtils.getModuleConfig(com1DFA, avalancheDir, toPrint=False, onlyDefault=cfgHybrid['com1DFA_com1DFA_override'].getboolean('defaultConfig')) com1DFACfg, cfgHybrid = cfgHandling.applyCfgOverride(com1DFACfg, cfgHybrid, com1DFA, addModValues=False) - com1DFACfgFile = cfgUtils.writeCfgFile(avalancheDir, com1DFA, com1DFACfg, fileName='com1DFA_settings', - filePath=workPath) # fetch configuration for DFAPathGeneration - DFAPathGenerationCfg = cfgUtils.getModuleConfig(DFAPath, fileOverride='', modInfo=False, toPrint=False, + DFAPathGenerationCfg = cfgUtils.getModuleConfig(DFAPath, avalancheDir, toPrint=False, onlyDefault=cfgHybrid['ana5Utils_DFAPathGeneration_override'].getboolean('defaultConfig')) DFAPathGenerationCfg, cfgHybrid = cfgHandling.applyCfgOverride(DFAPathGenerationCfg, cfgHybrid, DFAPath, addModValues=False) # first create configuration object for com2AB - com2ABCfg = cfgUtils.getModuleConfig(com2AB, fileOverride='', modInfo=False, toPrint=False, + com2ABCfg = cfgUtils.getModuleConfig(com2AB, avalancheDir, toPrint=False, onlyDefault=cfgHybrid['com1DFA_com1DFA_override'].getboolean('defaultConfig')) com2ABCfg, cfgHybrid = cfgHandling.applyCfgOverride(com2ABCfg, cfgHybrid, com2AB, addModValues=False) @@ -81,18 +78,16 @@ def maincom3Hybrid(cfgMain, cfgHybrid): iterate = True resultsHybrid = {} while iteration < nIterMax and iterate: - # update the com1DFA mu value in configuration file - updater = ConfigUpdater() - updater.read(com1DFACfgFile) - updater['GENERAL']['mucoulomb'].value = ('%.4f' % muArray[-1]) - updater.update_file() + # create a copy to avoid com1DFAMain modifying the original config + com1DFACfgCopy = copy.deepcopy(com1DFACfg) + com1DFACfgCopy['GENERAL']['mucoulomb'] = '%.4f' % muArray[-1] log.info('Mu is set to: %f' % muArray[-1]) # ++++++++++ RUN COM1DFA +++++++++++ # Run dense flow with coulomb friction # Clean input directory of old work and output files from module initProj.cleanModuleFiles(avalancheDir, com1DFA, deleteOutput=False) - dem, _, _, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=com1DFACfgFile) + dem, _, _, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=com1DFACfgCopy) simID = simDF.index[0] particlesList, timeStepInfo = particleTools.readPartFromPickle(avalancheDir, simName=simID, flagAvaDir=True, comModule='com1DFA') @@ -140,6 +135,10 @@ def maincom3Hybrid(cfgMain, cfgHybrid): iterate = keepIterating(cfgHybrid, alphaArray) iteration = iteration + 1 + # write final com1DFA configuration to file for documentation + cfgUtils.writeCfgFile(avalancheDir, com1DFA, com1DFACfgCopy, fileName='com1DFA_settings', + filePath=workPath) + # fetch fields for desired time step fields, fieldHeader, timeList = com1DFA.readFields(avalancheDir, ['pta'], simName=simID, flagAvaDir=True, comModule='com1DFA', timeStep=timeStepInfo[-1]) diff --git a/avaframe/com5SnowSlide/com5SnowSlide.py b/avaframe/com5SnowSlide/com5SnowSlide.py index 461979a2b..ef3a23dbe 100644 --- a/avaframe/com5SnowSlide/com5SnowSlide.py +++ b/avaframe/com5SnowSlide/com5SnowSlide.py @@ -35,6 +35,7 @@ def com5SnowSlideMain(cfgMain, snowSlideCfg): # get comDFA configuration and update with snow slide parameter set com1DFACfg = cfgUtils.getModuleConfig( com1DFA, + avalancheDir, fileOverride="", modInfo=False, toPrint=False, diff --git a/avaframe/com6RockAvalanche/com6RockAvalanche.py b/avaframe/com6RockAvalanche/com6RockAvalanche.py index df4acce18..4655fcec4 100644 --- a/avaframe/com6RockAvalanche/com6RockAvalanche.py +++ b/avaframe/com6RockAvalanche/com6RockAvalanche.py @@ -29,10 +29,13 @@ def com6RockAvalancheMain(cfgMain, rockAvalancheCfg): """ + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + # ++++++++++ set configurations for com1DFA and override ++++++++++++ # get comDFA configuration and update with snow slide parameter set com1DFACfg = cfgUtils.getModuleConfig( com1DFA, + avalancheDir, fileOverride="", modInfo=False, toPrint=False, diff --git a/avaframe/com7Regional/com7Regional.py b/avaframe/com7Regional/com7Regional.py index 72e4a8936..f861b4388 100644 --- a/avaframe/com7Regional/com7Regional.py +++ b/avaframe/com7Regional/com7Regional.py @@ -141,6 +141,7 @@ def getTotalNumberOfSims(avaDirs, cfgMain, cfgCom7): # Get com1DFA config with regional overrides (same as in processAvaDirCom1Regional) cfgCom1DFA = cfgUtils.getModuleConfig( com1DFA, + str(avaDir), fileOverride="", toPrint=False, onlyDefault=cfgCom7["com1DFA_com1DFA_override"].getboolean("defaultConfig"), @@ -149,7 +150,7 @@ def getTotalNumberOfSims(avaDirs, cfgMain, cfgCom7): # Get simulations for this directory try: - simDict, _, _, _ = com1DFA.com1DFAPreprocess(cfgMainCopy, "cfgFromObject", cfgCom1DFA) + simDict, _, _, _ = com1DFA.com1DFAPreprocess(cfgMainCopy, cfgCom1DFA) totalSims += len(simDict) except Exception as e: log.warning(f"Could not get simulations for {avaDir}: {e}") @@ -194,6 +195,7 @@ def processAvaDirCom1Regional(cfgMain, cfgCom7, avalancheDir): # Create com1DFA configuration for the current avalanche directory and override with regional settings cfgCom1DFA = cfgUtils.getModuleConfig( com1DFA, + str(avalancheDir), fileOverride="", toPrint=False, onlyDefault=cfgCom7["com1DFA_com1DFA_override"].getboolean("defaultConfig"), diff --git a/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py b/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py index 78eb15ac8..bc8c874ad 100644 --- a/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py +++ b/avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py @@ -59,7 +59,7 @@ def runProbAna(avalancheDir=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Load configuration file for probabilistic run and analysis - cfgProb = cfgUtils.getModuleConfig(probAna) + cfgProb = cfgUtils.getModuleConfig(probAna, avalancheDir) # create configuration files for com8MoTPSA simulations including parameter # variation - defined in the probabilistic config diff --git a/avaframe/com8MoTPSA/runCom8MoTPSA.py b/avaframe/com8MoTPSA/runCom8MoTPSA.py index b14595b4d..7a4744e86 100644 --- a/avaframe/com8MoTPSA/runCom8MoTPSA.py +++ b/avaframe/com8MoTPSA/runCom8MoTPSA.py @@ -58,7 +58,7 @@ def runCom8MoTPSA(avalancheDir=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Get module config - cfgCom8MoTPSA = cfgUtils.getModuleConfig(com8MoTPSA, toPrint=False) + cfgCom8MoTPSA = cfgUtils.getModuleConfig(com8MoTPSA, avalancheDir, toPrint=False) # ---------------- # Run psa diff --git a/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py b/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py index d7d3f1750..81ec57c2c 100644 --- a/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py +++ b/avaframe/com9MoTVoellmy/runCom9MoTVoellmy.py @@ -58,7 +58,7 @@ def runCom9MoTVoellmy(avalancheDir="", simType=""): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Get module config - cfgCom9MoTVoellmy = cfgUtils.getModuleConfig(com9MoTVoellmy, toPrint=False) + cfgCom9MoTVoellmy = cfgUtils.getModuleConfig(com9MoTVoellmy, avalancheDir, toPrint=False) # Override simTypeList if provided via command line if simType != "": diff --git a/avaframe/in3Utils/MoTUtils.py b/avaframe/in3Utils/MoTUtils.py index 1a8656771..89b7278c8 100644 --- a/avaframe/in3Utils/MoTUtils.py +++ b/avaframe/in3Utils/MoTUtils.py @@ -7,7 +7,7 @@ import numpy as np -from avaframe.com1DFA import com1DFATools as com1DFATools, com1DFA as com1DFA +from avaframe.com1DFA import com1DFA as com1DFA from avaframe.in2Trans import rasterUtils as rU log = logging.getLogger(__name__) @@ -148,12 +148,9 @@ def MoTGenerateConfigs(cfgMain, cfgInfo, currentModule): dictionary with input files info """ - # fetch type of cfgInfo - typeCfgInfo = com1DFATools.checkCfgInfoType(cfgInfo) - # preprocessing to create configuration objects for all simulations to run simDict, outDir, inputSimFiles, simDFExisting = com1DFA.com1DFAPreprocess( - cfgMain, typeCfgInfo, cfgInfo, module=currentModule + cfgMain, cfgInfo, module=currentModule ) return simDict, inputSimFiles diff --git a/avaframe/in3Utils/cfgHandling.py b/avaframe/in3Utils/cfgHandling.py index 86fef5c4a..730b95dd3 100644 --- a/avaframe/in3Utils/cfgHandling.py +++ b/avaframe/in3Utils/cfgHandling.py @@ -522,111 +522,3 @@ def errorDuplicateListEntry(listKeys, message): raise AssertionError -def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): - """fetch all override sections in cfgFull and write a local_NAMEOVERRIDE.ini configuration file for the - available sections - naming is collection_module_override - if no localCfgPath is provided, default saved to avalancheDir/Inputs/configurationOverrides - where package refers to e.g. ana1Tests, ana3AIMEC, etc- - and module to e.g. energyLineTest.py, ana3AIMEC.py so all python files inside the packages that - have a nameCfg.ini file too - - Parameters - ----------- - cfgFull: configparser - configuration with override sections for modules - avalancheDir: pathlib path or str - path to avalanche directory - localCfgPath: pathlib Path - optional - path to directory to store local_ cfg ini file to - if not provided - local_ cfg ini file is saved to avalanche directory - - """ - - # if a path is provided - save local cfg ini file there - pathProvided = False - if localCfgPath != '': - if pathlib.Path(localCfgPath).is_dir() is False: - message1 = 'Provided path for local cfg files is not a directory: %s' % localCfgPath - log.error(message1) - raise NotADirectoryError(message1) - else: - pathProvided = True - - # Get all override sections - cfgSections = cfgFull.sections() - overrideSections = [sec for sec in cfgSections if "_override" in sec] - - # Check if all override sections have 3 parts separated by underscore - if any(len((match := sec).split("_")) != 3 for sec in overrideSections): - message = ( - "Override section needs to provide moduleName_fileName_override; provided: %s invalid format" - % match - ) - log.error(message) - raise AssertionError(message) - - # Go through sections - for section in overrideSections: - modName = section.split("_")[0] - cfgName = section.split("_")[1] - thisFilePath = pathlib.Path(cfgUtils.__file__).resolve().parents[1] - modPath = thisFilePath / modName - cfgNamePath = modPath / cfgName - locFilePath = modPath - - cfgModule = cfgUtils.getModuleConfig( - cfgNamePath, - fileOverride="", - modInfo=False, - toPrint=False, - onlyDefault=cfgFull[section].getboolean("defaultConfig"), - ) - cfgModule, cfgFull = applyCfgOverride(cfgModule, cfgFull, cfgNamePath, addModValues=False) - - overrideParameters = cfgFull["%s_%s_override" % (modName, cfgName)] - overrideKeys = [item for item in overrideParameters] - - # remove items that are not in Override - cfgModule = _removeCfgItemsNotInOverride(cfgModule, overrideKeys) - - # fetch directory to save local cfg ini file - if pathProvided: - locFilePath = pathlib.Path(localCfgPath) - else: - # if not provided save to default location - locFilePath = pathlib.Path(avalancheDir, 'Inputs', 'configurationOverrides') - fU.makeADir(locFilePath) - - cfgF = pathlib.Path(locFilePath, ("local_%sCfg.ini" % (cfgName))) - if cfgF.is_file(): - warningText = "%s already exists - overwriting file here %s!" % (cfgF.name, cfgF) - log.warning(warningText) - with open(cfgF, "w") as configfile: - cfgModule.write(configfile) - - log.info("%s CONFIGURATION wrote to %s" % (cfgName, str(cfgF))) - - -def _removeCfgItemsNotInOverride(cfgModule, overrideKeys): - """ remove options of cfgModule if not part of overrideKeys - in order to just have override parameters in new local cfg ini file - - Parameters - ------------ - cfgModule: configparser object - configuration of module - overrideKeys: list - list of options of configparser object that have been in override section and should be kept in cfgModule - - Returns - --------- - cfgModule: configparser object - updated configuration - only override parameters left - """ - - for sec in cfgModule.sections(): - for item in cfgModule[sec]: - if item not in overrideKeys: - cfgModule.remove_option(sec, item) - - return cfgModule diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index 6a2035b8d..5bba71ad9 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -95,32 +95,51 @@ def getGeneralConfig(nameFile=""): return cfg -def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDefault=False): +def getModuleConfig( + module, avalancheDir="", fileOverride="", batchCfgDir="", modInfo=False, toPrint=True, onlyDefault=False +): """Returns the configuration for a given module - returns a configParser object + returns a configParser object OR pathlib.Path (when batchCfgDir is used) - module object: module : the calling function provides the already imported - module eg.: - from avaframe.com2AB import com2AB - leads to getModuleConfig(com2AB) - whereas - from avaframe.com2AB import com2AB as c2 - leads to getModuleConfig(c2) - OR: pathlib Path to module (python file) + Priority order: + batchCfgDir (returns Path) -> onlyDefault -> fileOverride -> expert config (CFGs/) -> + local_MODULECfg.ini -> MODULECfg.ini - Str: fileOverride : allows for a completely different file location. However note: - missing values from the default cfg will always be added! + Parameters + ---------- + module : module object or pathlib.Path + The calling function provides the already imported module eg.: + from avaframe.com2AB import com2AB leads to getModuleConfig(com2AB) + OR: pathlib Path to module (python file) + avalancheDir : str or pathlib.Path + Path to avalanche directory. If provided and {avalancheDir}/Inputs/CFGs/{moduleName}Cfg.ini + exists, that config is taken and only filled up with missing values from the default config. + The local_{moduleName}Cfg.ini file is ignored for moduleName then. Default "" skips this check. + fileOverride : str or pathlib.Path + Allows for a completely different file location. Missing values from the + default cfg will always be added. Takes highest priority UNLESS onlyDefault is true. + batchCfgDir : str or pathlib.Path + Path to directory containing multiple .ini config files for batch processing. + When provided, validates the directory exists and contains .ini files, + then returns the path as pathlib.Path (not a ConfigParser). + Takes highest priority - all other config resolution is skipped. + modInfo : bool + If True, return tuple (cfg, modDict) with info on differences to standard config. + Ignored when batchCfgDir is provided. + toPrint : bool + If True, print configuration info + onlyDefault : bool + If True, only use the default configuration (skip all overrides). + Ignored when batchCfgDir is provided. - modInfo: bool - true if dictionary with info on differences to standard config - onlyDefault: bool - if True, only use the default configuration + Returns + ------- + configparser.ConfigParser or pathlib.Path + ConfigParser object with merged configuration, OR + pathlib.Path when batchCfgDir is provided - Order is as follows: - fileOverride -> local_MODULECfg.ini -> MODULECfg.ini """ - if isinstance(onlyDefault, bool) == False: message = "OnlyDefault parameter is not a boolean but %s" % type(onlyDefault) log.error(message) @@ -128,7 +147,6 @@ def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDe if isinstance(module, pathlib.Path): modPath = module.parent - # get filename of module modName = module.stem else: modPath, modName = getModPathName(module) @@ -139,7 +157,28 @@ def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDe log.debug("localFile: %s", localFile) log.debug("defaultFile: %s", defaultFile) - # Decide which one to take + # Handle batchCfgDir - return Path for batch processing (highest priority) + if batchCfgDir: + batchPath = pathlib.Path(batchCfgDir) + if not batchPath.is_dir(): + raise FileNotFoundError("batchCfgDir does not exist: %s" % batchPath) + iniFiles = list(batchPath.glob("*.ini")) + if len(iniFiles) == 0: + raise FileNotFoundError("batchCfgDir contains no .ini files: %s" % batchPath) + log.info("Using batch config directory with %d .ini files: %s", len(iniFiles), batchPath) + return batchPath + + # Handle onlyDefault escape hatch - skip all overrides + if onlyDefault: + if defaultFile.is_file(): + cfg, modDict = readCompareConfig(defaultFile, modName, compare=False, toPrint=toPrint) + if modInfo: + return cfg, modDict + return cfg + else: + raise FileNotFoundError("Default config file does not exist: " + str(defaultFile)) + + # Handle fileOverride (ultimate override) if fileOverride: fileOverride = fU.checkPathlib(fileOverride) if fileOverride.is_file(): @@ -147,18 +186,37 @@ def getModuleConfig(module, fileOverride="", modInfo=False, toPrint=True, onlyDe compare = True else: raise FileNotFoundError("Provided fileOverride does not exist: " + str(fileOverride)) - - elif localFile.is_file() and not onlyDefault: - iniFile = localFile + cfg, modDict = readCompareConfig(iniFile, modName, compare, toPrint) + if modInfo: + return cfg, modDict + return cfg + + # Check for expert config in avalancheDir/Inputs/CFGs/ + expertFile = None + if avalancheDir: + avalanchePath = pathlib.Path(avalancheDir) + expertPath = avalanchePath / "Inputs" / "CFGs" / (modName + "Cfg.ini") + if expertPath.is_file(): + expertFile = expertPath + log.info("Using expert config from %s", expertFile) + + # Determine config source based on priority + if expertFile: + # Expert config exists - ignore local_*, merge with default only + iniFile = [defaultFile, expertFile] + compare = True + elif localFile.is_file(): + # No expert config - use normal local_* behavior iniFile = [defaultFile, localFile] compare = True elif defaultFile.is_file(): + # Only default iniFile = defaultFile compare = False else: raise FileNotFoundError("None of the provided cfg files exist ") - # Finally read it + # Read and merge configs cfg, modDict = readCompareConfig(iniFile, modName, compare, toPrint) if modInfo: return cfg, modDict diff --git a/avaframe/runAna4ProbAna.py b/avaframe/runAna4ProbAna.py index d41a3c42d..a75672158 100644 --- a/avaframe/runAna4ProbAna.py +++ b/avaframe/runAna4ProbAna.py @@ -56,7 +56,7 @@ def runProbAna(avalancheDir=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Load configuration file for probabilistic run and analysis - cfgProb = cfgUtils.getModuleConfig(probAna) + cfgProb = cfgUtils.getModuleConfig(probAna, avalancheDir) # create configuration files for com1DFA simulations including parameter # variation - defined in the probabilistic config diff --git a/avaframe/runAna5DFAPathGeneration.py b/avaframe/runAna5DFAPathGeneration.py index 39f3a11c3..ed99ad7ff 100644 --- a/avaframe/runAna5DFAPathGeneration.py +++ b/avaframe/runAna5DFAPathGeneration.py @@ -59,7 +59,7 @@ def runAna5DFAPathGeneration(avalancheDir='', runDFAModule=''): log.info('Current avalanche: %s', avalancheDir) # load config for path generation (from DFAPathGenerationCfg.ini or its local) - cfgDFAPath = cfgUtils.getModuleConfig(DFAPathGeneration) + cfgDFAPath = cfgUtils.getModuleConfig(DFAPathGeneration, avalancheDir) # Clean DFAPath output in avalanche directory # If you just created the directory this one should be clean but if you diff --git a/avaframe/runCom1DFA.py b/avaframe/runCom1DFA.py index 3d295b088..02fb72b1a 100644 --- a/avaframe/runCom1DFA.py +++ b/avaframe/runCom1DFA.py @@ -1,6 +1,7 @@ """ - Run script for running python com1DFA kernel +Run script for running python com1DFA kernel """ + # Load modules # importing general python modules import time @@ -16,8 +17,9 @@ from avaframe.in3Utils import logUtils from avaframe.in3Utils import fileHandlerUtils as fU -def runCom1DFA(avalancheDir='', calibration=''): - """ Run com1DFA in the default configuration with only an + +def runCom1DFA(avalancheDir="", calibration=""): + """Run com1DFA in the default configuration with only an avalanche directory as input and the (optional) friction calibration size @@ -38,21 +40,21 @@ def runCom1DFA(avalancheDir='', calibration=''): startTime = time.time() # log file name; leave empty to use default runLog.log - logName = 'runCom1DFA' + logName = "runCom1DFA" # Load avalanche directory from general configuration file # More information about the configuration can be found here # on the Configuration page in the documentation cfgMain = cfgUtils.getGeneralConfig() - if avalancheDir != '': - cfgMain['MAIN']['avalancheDir'] = avalancheDir + if avalancheDir != "": + cfgMain["MAIN"]["avalancheDir"] = avalancheDir else: - avalancheDir = cfgMain['MAIN']['avalancheDir'] + avalancheDir = cfgMain["MAIN"]["avalancheDir"] # Start logging log = logUtils.initiateLogger(avalancheDir, logName) - log.info('MAIN SCRIPT') - log.info('Current avalanche: %s', avalancheDir) + log.info("MAIN SCRIPT") + log.info("Current avalanche: %s", avalancheDir) # ---------------- # Clean input directory(ies) of old work and output files @@ -61,18 +63,20 @@ def runCom1DFA(avalancheDir='', calibration=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Set friction model according to cmd argument - cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, toPrint=False) - - if calibration.lower() == 'small': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATSmall' - elif calibration.lower() == 'medium': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATMedium' - elif calibration.lower() == 'large': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosAT' - elif calibration.lower() == 'auto': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATAuto' + cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, avalancheDir, toPrint=False) + + if isinstance(cfgCom1DFA, pathlib.Path): + log.warning("Expert mode detected - friction calibration override is ignored") + elif calibration.lower() == "small": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATSmall" + elif calibration.lower() == "medium": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATMedium" + elif calibration.lower() == "large": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosAT" + elif calibration.lower() == "auto": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATAuto" else: - log.info('no friction calibration override given - using ini') + log.info("no friction calibration override given - using ini") # ---------------- # Run dense flow @@ -80,25 +84,31 @@ def runCom1DFA(avalancheDir='', calibration=''): # Get peakfiles to return to QGIS avaDir = pathlib.Path(avalancheDir) - inputDir = avaDir / 'Outputs' / 'com1DFA' / 'peakFiles' + inputDir = avaDir / "Outputs" / "com1DFA" / "peakFiles" peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) # Print time needed endTime = time.time() - log.info('Took %6.1f seconds to calculate.' % (endTime - startTime)) + log.info("Took %6.1f seconds to calculate." % (endTime - startTime)) return peakFilesDF -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description='Run com1DFA workflow') - parser.add_argument('avadir', metavar='avalancheDir', type=str, nargs='?', default='', - help='the avalanche directory') - parser.add_argument('-fc', '--friction_calibration', choices=['auto', 'large', 'medium', 'small', 'ini'], - type=str, default='ini', - help='friction calibration override, possible values are large, medium , small, auto and ini.' + - 'Overrides default AND local configs') +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="Run com1DFA workflow") + parser.add_argument( + "avadir", metavar="avalancheDir", type=str, nargs="?", default="", help="the avalanche directory" + ) + parser.add_argument( + "-fc", + "--friction_calibration", + choices=["auto", "large", "medium", "small", "ini"], + type=str, + default="ini", + help="friction calibration override, possible values are large, medium , small, auto and ini." + + "Overrides default AND local configs", + ) args = parser.parse_args() runCom1DFA(str(args.avadir), str(args.friction_calibration)) diff --git a/avaframe/runCom2AB.py b/avaframe/runCom2AB.py index 0b8e47602..6a6e8a8b8 100644 --- a/avaframe/runCom2AB.py +++ b/avaframe/runCom2AB.py @@ -66,7 +66,7 @@ def runCom2AB(avalancheDir='', smallAva=False): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Call the main com2AB functions - cfgAB = cfgUtils.getModuleConfig(com2AB) + cfgAB = cfgUtils.getModuleConfig(com2AB, avalancheDir) # Override smallAva from ini file if needed if smallAva: diff --git a/avaframe/runCom5SnowSlide.py b/avaframe/runCom5SnowSlide.py index 63f46a424..81d87de7b 100644 --- a/avaframe/runCom5SnowSlide.py +++ b/avaframe/runCom5SnowSlide.py @@ -56,7 +56,7 @@ def runCom5SnowSlide(avalancheDir=""): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # load snow slide tool config - snowSlideCfg = cfgUtils.getModuleConfig(com5SnowSlide) + snowSlideCfg = cfgUtils.getModuleConfig(com5SnowSlide, avalancheDir) # perform com1DFA simulation with snow slide settings _, plotDict, reportDictList, _ = com5SnowSlide.com5SnowSlideMain(cfgMain, snowSlideCfg) diff --git a/avaframe/runCom6RockAvalanche.py b/avaframe/runCom6RockAvalanche.py index a415a8e2b..0d670beea 100644 --- a/avaframe/runCom6RockAvalanche.py +++ b/avaframe/runCom6RockAvalanche.py @@ -55,7 +55,7 @@ def runCom6RockAvalanche(avalancheDir=""): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # load rock avalanche config - rockAvalancheCfg = cfgUtils.getModuleConfig(com6RockAvalanche) + rockAvalancheCfg = cfgUtils.getModuleConfig(com6RockAvalanche, avalancheDir) # perform com1DFA simulation with rock avalanche settings _, plotDict, reportDictList, _ = com6RockAvalanche.com6RockAvalancheMain(cfgMain, rockAvalancheCfg) diff --git a/avaframe/runCom6Scarp.py b/avaframe/runCom6Scarp.py index adf2d4e17..8a42827d9 100644 --- a/avaframe/runCom6Scarp.py +++ b/avaframe/runCom6Scarp.py @@ -49,7 +49,7 @@ def runScarpAnalysisWorkflow(inputDir="", method=""): initProj.cleanSingleAvaDir(inputDir, deleteOutput=False) # load scarp config - scarpCfg = cfgUtils.getModuleConfig(scarp) + scarpCfg = cfgUtils.getModuleConfig(scarp, inputDir) # Set method according to cmd argument if method.lower() == "plane": diff --git a/avaframe/runCom7Regional.py b/avaframe/runCom7Regional.py index bd806e2e0..8103ad08b 100644 --- a/avaframe/runCom7Regional.py +++ b/avaframe/runCom7Regional.py @@ -58,7 +58,7 @@ def runCom7Regional(avalancheDir="", splitInputs=False, runComputations=False): outputDir = pathlib.Path(avalancheDir) / "com7Regional" # Load split inputs module configuration - splitCfg = cfgUtils.getModuleConfig(sI) + splitCfg = cfgUtils.getModuleConfig(sI, avalancheDir) # Run splitting process sI.splitInputsMain(pathlib.Path(avalancheDir), outputDir, splitCfg, cfgMain) @@ -72,7 +72,7 @@ def runCom7Regional(avalancheDir="", splitInputs=False, runComputations=False): # Run main computations if requested if runComputations: # Load module configuration - cfg = cfgUtils.getModuleConfig(com7, fileOverride="", toPrint=False, onlyDefault=False) + cfg = cfgUtils.getModuleConfig(com7, avalancheDir, toPrint=False) # Call main function allPeakFilesDir, mergedRastersDir = com7.com7RegionalMain(cfgMain, cfg) diff --git a/avaframe/runIn1RelInfo.py b/avaframe/runIn1RelInfo.py index 80a2319af..fb46e0348 100644 --- a/avaframe/runIn1RelInfo.py +++ b/avaframe/runIn1RelInfo.py @@ -48,7 +48,7 @@ def runRelInfo(avalancheDir=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Load configuration file for com1DFA module - cfg = cfgUtils.getModuleConfig(com1DFA) + cfg = cfgUtils.getModuleConfig(com1DFA, avalancheDir) # fetch input data and create release info dataframe and csv file relDFDict = gI.createReleaseStats(avalancheDir, cfg) diff --git a/avaframe/runOperational.py b/avaframe/runOperational.py index 45b5566eb..06cbcfd63 100644 --- a/avaframe/runOperational.py +++ b/avaframe/runOperational.py @@ -84,7 +84,7 @@ def runOperational(avalancheDir=''): com2AB.readABinputs(avalancheDir) # if not: Run Alpha Beta - cfgAB = cfgUtils.getModuleConfig(com2AB) + cfgAB = cfgUtils.getModuleConfig(com2AB, avalancheDir) pathDict, dem, splitPoint, eqParams, resAB = com2AB.com2ABMain(cfgAB, avalancheDir) abShpFile = outAB.writeABtoSHP(pathDict, resAB) diff --git a/avaframe/runPlotCom1DFA.py b/avaframe/runPlotCom1DFA.py index bc335dd8f..fda7da358 100644 --- a/avaframe/runPlotCom1DFA.py +++ b/avaframe/runPlotCom1DFA.py @@ -30,7 +30,7 @@ log.info('MAIN SCRIPT') log.info('Current avalanche: %s', avalancheDir) modName = 'com1DFA' -cfg = cfgUtils.getModuleConfig(com1DFA) +cfg = cfgUtils.getModuleConfig(com1DFA, avalancheDir) # Create output and work directories outputDir = pathlib.Path(avalancheDir, 'Outputs', modName) inDirPart = outputDir / 'particles' diff --git a/avaframe/runProbAnalysisOnly.py b/avaframe/runProbAnalysisOnly.py index 86e4520a4..5c2da082a 100644 --- a/avaframe/runProbAnalysisOnly.py +++ b/avaframe/runProbAnalysisOnly.py @@ -59,7 +59,7 @@ def runProbAna(avalancheDir="", modName=""): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Load configuration file for probabilistic run and analysis - cfgProb = cfgUtils.getModuleConfig(probAna) + cfgProb = cfgUtils.getModuleConfig(probAna, avalancheDir) if modName == "": modName = cfgProb["GENERAL"]["modName"] diff --git a/avaframe/runScripts/runAna3AIMEC.py b/avaframe/runScripts/runAna3AIMEC.py index 49de10665..76a037cff 100644 --- a/avaframe/runScripts/runAna3AIMEC.py +++ b/avaframe/runScripts/runAna3AIMEC.py @@ -44,7 +44,7 @@ def runAna3AIMEC(avalancheDir, cfg, inputDir='', demFileName=''): # Load all input Parameters from config file # get the configuration of an already imported module # write config to log file - cfg = cfgUtils.getModuleConfig(ana3AIMEC) + cfg = cfgUtils.getModuleConfig(ana3AIMEC, avalancheDir) iP.cleanModuleFiles(avalancheDir, ana3AIMEC) diff --git a/avaframe/runScripts/runAna3AIMECCompMods.py b/avaframe/runScripts/runAna3AIMECCompMods.py index 7bc0fe15b..4684b0f20 100644 --- a/avaframe/runScripts/runAna3AIMECCompMods.py +++ b/avaframe/runScripts/runAna3AIMECCompMods.py @@ -39,7 +39,7 @@ def runAna3AIMECCompMods(avalancheDir=''): # Load all input Parameters from config file # get the configuration of an already imported module # write config to log file - cfg = cfgUtils.getModuleConfig(ana3AIMEC) + cfg = cfgUtils.getModuleConfig(ana3AIMEC, avalancheDir) iP.cleanModuleFiles(avalancheDir, ana3AIMEC) diff --git a/avaframe/runScripts/runAnalyzeDamBreak.py b/avaframe/runScripts/runAnalyzeDamBreak.py index 0d630a550..a05f41609 100644 --- a/avaframe/runScripts/runAnalyzeDamBreak.py +++ b/avaframe/runScripts/runAnalyzeDamBreak.py @@ -36,7 +36,7 @@ # Load configuration damBreakCfg = pathlib.Path(avalancheDir, 'Inputs', 'damBreak_com1DFACfg.ini') -cfg = cfgUtils.getModuleConfig(com1DFA, damBreakCfg) +cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=damBreakCfg) cfgGen = cfg['GENERAL'] # Load flow thickness from analytical solution diff --git a/avaframe/runScripts/runAnalyzeSimilaritySol.py b/avaframe/runScripts/runAnalyzeSimilaritySol.py index 8c3c86d08..505f0f284 100644 --- a/avaframe/runScripts/runAnalyzeSimilaritySol.py +++ b/avaframe/runScripts/runAnalyzeSimilaritySol.py @@ -42,7 +42,7 @@ # Load configuration for similarity solution test simiSolCfg = pathlib.Path(avalancheDir, 'Inputs', 'simiSol_com1DFACfg.ini') -cfg = cfgUtils.getModuleConfig(com1DFA, simiSolCfg) +cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=simiSolCfg) # create output directory for test result plots outDirTest = pathlib.Path(avalancheDir, 'Outputs', 'ana1Tests') diff --git a/avaframe/runScripts/runCom3Hybrid.py b/avaframe/runScripts/runCom3Hybrid.py index 9ed57d38f..27e506d3d 100644 --- a/avaframe/runScripts/runCom3Hybrid.py +++ b/avaframe/runScripts/runCom3Hybrid.py @@ -33,5 +33,5 @@ initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=True) # Load configuration for hybrid model -cfgHybrid = cfgUtils.getModuleConfig(com3Hybrid) +cfgHybrid = cfgUtils.getModuleConfig(com3Hybrid, avalancheDir) com3Hybrid.maincom3Hybrid(cfgMain, cfgHybrid) diff --git a/avaframe/runScripts/runComputeDist.py b/avaframe/runScripts/runComputeDist.py index b1f7f9846..b5cfe108e 100644 --- a/avaframe/runScripts/runComputeDist.py +++ b/avaframe/runScripts/runComputeDist.py @@ -26,7 +26,7 @@ # Load input parameters from configuration file cfgMain = cfgUtils.getGeneralConfig() -cfg = cfgUtils.getModuleConfig(cF) +cfg = cfgUtils.getModuleConfig(cF, avalancheDir) cfgGen = cfg['GENERAL'] # log file name; leave empty to use default runLog.log diff --git a/avaframe/runScripts/runCreateProbCfgsFlowPy.py b/avaframe/runScripts/runCreateProbCfgsFlowPy.py index 8988863dd..2aa8feb84 100644 --- a/avaframe/runScripts/runCreateProbCfgsFlowPy.py +++ b/avaframe/runScripts/runCreateProbCfgsFlowPy.py @@ -30,7 +30,7 @@ initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Load configuration file for probabilistic run and analysis -cfgProb = cfgUtils.getModuleConfig(probAna) +cfgProb = cfgUtils.getModuleConfig(probAna, avalancheDir) # create configuration files for com1DFA simulations including parameter # variation - defined in the probabilistic config @@ -42,6 +42,6 @@ # loop over all cfgFiles for index, cfgF in enumerate(cfgFiles): # read configuration - cfgFromFile = cfgUtils.getModuleConfig(com4FlowPy, fileOverride=cfgF, toPrint=False) + cfgFromFile = cfgUtils.getModuleConfig(com4FlowPy, avalancheDir, fileOverride=cfgF, toPrint=False) log.info('exp: %s, alpha: %s' % (cfgFromFile['GENERAL']['exp'], cfgFromFile['GENERAL']['alpha'])) \ No newline at end of file diff --git a/avaframe/runScripts/runEnergyLineTest.py b/avaframe/runScripts/runEnergyLineTest.py index 70e4bbcc1..e62171f00 100644 --- a/avaframe/runScripts/runEnergyLineTest.py +++ b/avaframe/runScripts/runEnergyLineTest.py @@ -38,11 +38,11 @@ iP.cleanSingleAvaDir(avalancheDir, deleteOutput=False) workPath = pathlib.Path(avalancheDir, 'Work', 'energyLineTest') fU.makeADir(workPath) -energyLineTestCfg = cfgUtils.getModuleConfig(energyLineTest) +energyLineTestCfg = cfgUtils.getModuleConfig(energyLineTest, avalancheDir) # ++++++++++ set configurations for all the used modules and override ++++++++++++ # get comDFA configuration and save to file -com1DFACfg = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, +com1DFACfg = cfgUtils.getModuleConfig(com1DFA, avalancheDir, toPrint=False, onlyDefault=energyLineTestCfg['com1DFA_com1DFA_override'].getboolean('defaultConfig')) com1DFACfg, energyLineTestCfg = cfgHandling.applyCfgOverride(com1DFACfg, energyLineTestCfg, com1DFA, addModValues=False) com1DFACfgFile = cfgUtils.writeCfgFile(avalancheDir, com1DFA, com1DFACfg, fileName='com1DFA_settings', diff --git a/avaframe/runScripts/runGenProjTopoRelease.py b/avaframe/runScripts/runGenProjTopoRelease.py index 9df778e38..83a572982 100644 --- a/avaframe/runScripts/runGenProjTopoRelease.py +++ b/avaframe/runScripts/runGenProjTopoRelease.py @@ -46,8 +46,8 @@ log.info('MAIN SCRIPT') # Load input parameters from configuration file -cfgT = cfgUtils.getModuleConfig(gT) -cfgR = cfgUtils.getModuleConfig(gR) +cfgT = cfgUtils.getModuleConfig(gT, avalancheDir) +cfgR = cfgUtils.getModuleConfig(gR, avalancheDir) # Call main function to generate DEMs [z, name_ext, outDir] = gT.generateTopo(cfgT, avalancheDir) diff --git a/avaframe/runScripts/runGenerateTopo.py b/avaframe/runScripts/runGenerateTopo.py index b83712a05..98234caaf 100755 --- a/avaframe/runScripts/runGenerateTopo.py +++ b/avaframe/runScripts/runGenerateTopo.py @@ -22,7 +22,7 @@ log.info('MAIN SCRIPT') # Load input parameters from configuration file -cfg = cfgUtils.getModuleConfig(gT) +cfg = cfgUtils.getModuleConfig(gT, avalancheDir) # Call main function to generate DEMs [z, name_ext, outDir] = gT.generateTopo(cfg, avalancheDir) diff --git a/avaframe/runScripts/runParticleAnalysisPlots.py b/avaframe/runScripts/runParticleAnalysisPlots.py index 080689b75..2306e0d28 100644 --- a/avaframe/runScripts/runParticleAnalysisPlots.py +++ b/avaframe/runScripts/runParticleAnalysisPlots.py @@ -48,13 +48,13 @@ modName = 'com1DFA' # ----------------------- -# load configuration of particle analysis using com1DFA as computational module -cfgPartAna = cfgUtils.getModuleConfig(oPartAna) -resTypePlots = fU.splitIniValueToArraySteps(cfgPartAna['GENERAL']['resTypePlots']) # Load avalanche directory from general configuration file cfgMain = cfgUtils.getGeneralConfig() avalancheDir = cfgMain['MAIN']['avalancheDir'] avaDir = pathlib.Path(avalancheDir) +# load configuration of particle analysis using com1DFA as computational module +cfgPartAna = cfgUtils.getModuleConfig(oPartAna, avalancheDir) +resTypePlots = fU.splitIniValueToArraySteps(cfgPartAna['GENERAL']['resTypePlots']) # create outDir outDir = avaDir / 'Outputs' / 'out3Plot' / 'particleAnalysis' fU.makeADir(outDir) @@ -68,7 +68,7 @@ if cfgPartAna['GENERAL'].getboolean('runCom1DFA'): log.info('Perform com1DFA runs using the override section in the outParticlesAnalysis ini file') # get the configuration of com1DFA using overrides - cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, + cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, avalancheDir, toPrint=False, onlyDefault=cfgPartAna['com1DFA_com1DFA_override'].getboolean('defaultConfig')) cfgCom1DFA, cfgPartAna = cfgHandling.applyCfgOverride(cfgCom1DFA, cfgPartAna, com1DFA, addModValues=False) @@ -80,7 +80,7 @@ SimDF, _ = cfgUtils.readAllConfigurationInfo(avalancheDir) # get the configuration of aimec using overrides -cfgAimec = cfgUtils.getModuleConfig(ana3AIMEC, fileOverride='', modInfo=False, toPrint=False, +cfgAimec = cfgUtils.getModuleConfig(ana3AIMEC, avalancheDir, toPrint=False, onlyDefault=cfgPartAna['ana3AIMEC_ana3AIMEC_override'].getboolean('defaultConfig')) cfgAimec, cfgPartAna = cfgHandling.applyCfgOverride(cfgAimec, cfgPartAna, ana3AIMEC, addModValues=False) # fetch anaMod from aimec settings @@ -172,7 +172,7 @@ _, resAnalysisDF, _, newRasters, _ = ana3AIMEC.fullAimecAnalysis(avalancheDir, cfgAimec) # create mtiInfo dicts for tt-diagram - cfgRangeTime = cfgUtils.getModuleConfig(dtAna, fileOverride='', modInfo=False, toPrint=False, + cfgRangeTime = cfgUtils.getModuleConfig(dtAna, avalancheDir, toPrint=False, onlyDefault=cfgPartAna['ana5Utils_distanceTimeAnalysis_override'].getboolean('defaultConfig')) cfgRangeTime, cfgPartAna = cfgHandling.applyCfgOverride(cfgRangeTime, cfgPartAna, dtAna, addModValues=False) cfgRangeTime['GENERAL']['avalancheDir'] = avalancheDir diff --git a/avaframe/runScripts/runPlotContoursFromAsc.py b/avaframe/runScripts/runPlotContoursFromAsc.py index 04dac701f..db9f82f52 100644 --- a/avaframe/runScripts/runPlotContoursFromAsc.py +++ b/avaframe/runScripts/runPlotContoursFromAsc.py @@ -9,7 +9,6 @@ from avaframe.in3Utils import cfgUtils from avaframe.out3Plot import outQuickPlot as oQ import avaframe.runScripts.runPlotContoursFromAsc as rCon -from avaframe.in3Utils import cfgHandling def plotContoursFromAsc(cfg, avalancheDir): @@ -25,13 +24,12 @@ def plotContoursFromAsc(cfg, avalancheDir): if __name__ == "__main__": - # Load configuration for runPlotContour - cfg = cfgUtils.getModuleConfig(rCon, fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) - # fetch input directory cfgMain = cfgUtils.getGeneralConfig() avaDir = cfgMain["MAIN"]["avalancheDir"] - cfgHandling.rewriteLocalCfgs(cfg, avaDir) + # Load configuration for runPlotContour + cfg = cfgUtils.getModuleConfig(rCon, avaDir, toPrint=False) + plotContoursFromAsc(cfg, avaDir) diff --git a/avaframe/runScripts/runProbAna.py b/avaframe/runScripts/runProbAna.py index 81a13e484..7998549a3 100644 --- a/avaframe/runScripts/runProbAna.py +++ b/avaframe/runScripts/runProbAna.py @@ -45,7 +45,7 @@ initProj.cleanSingleAvaDir(avaDir) # Load configuration file for probabilistic run and analysis - cfgProb = cfgUtils.getModuleConfig(probAna, fileOverride=probAnaCfg) + cfgProb = cfgUtils.getModuleConfig(probAna, avaDir, fileOverride=probAnaCfg) # create configuration files for com1DFA simulations including parameter # variation - defined in the probabilistic config diff --git a/avaframe/runScripts/runQuickPlotOne.py b/avaframe/runScripts/runQuickPlotOne.py index 32589a334..1de64dc84 100644 --- a/avaframe/runScripts/runQuickPlotOne.py +++ b/avaframe/runScripts/runQuickPlotOne.py @@ -19,7 +19,7 @@ avalancheDir = cfgMain['MAIN']['avalancheDir'] # load configuration for plot generation -cfg = cfgUtils.getModuleConfig(outQuickPlot) +cfg = cfgUtils.getModuleConfig(outQuickPlot, avalancheDir) cfgPlot = cfg['ONEPLOT'] # Start logging diff --git a/avaframe/runScripts/runRangeTimeDiagram.py b/avaframe/runScripts/runRangeTimeDiagram.py index 7e7526d23..2f5f53d67 100644 --- a/avaframe/runScripts/runRangeTimeDiagram.py +++ b/avaframe/runScripts/runRangeTimeDiagram.py @@ -49,7 +49,7 @@ # Load all input Parameters from config file # get the configuration of an already imported module # write config to log file -cfgRangeTime = cfgUtils.getModuleConfig(dtAna) +cfgRangeTime = cfgUtils.getModuleConfig(dtAna, avalancheDir) cfgRangeTime['GENERAL']['avalancheDir'] = avalancheDir if preProcessedData: diff --git a/avaframe/runScripts/runRotationTest.py b/avaframe/runScripts/runRotationTest.py index 7be51c77a..c9c4f76af 100644 --- a/avaframe/runScripts/runRotationTest.py +++ b/avaframe/runScripts/runRotationTest.py @@ -41,22 +41,22 @@ iP.cleanSingleAvaDir(avalancheDir, deleteOutput=False) workPath = pathlib.Path(avalancheDir, 'Work', 'energyLineTest') fU.makeADir(workPath) -rotationTestCfg = cfgUtils.getModuleConfig(rotationTest) +rotationTestCfg = cfgUtils.getModuleConfig(rotationTest, avalancheDir) # ++++++++++ set configurations for all the used modules and override ++++++++++++ # get comDFA configuration and save to file -com1DFACfg = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, +com1DFACfg = cfgUtils.getModuleConfig(com1DFA, avalancheDir, toPrint=False, onlyDefault=rotationTestCfg['com1DFA_com1DFA_override'].getboolean('defaultConfig')) com1DFACfg, rotationTestCfg = cfgHandling.applyCfgOverride(com1DFACfg, rotationTestCfg, com1DFA, addModValues=False) com1DFACfgFile = cfgUtils.writeCfgFile(avalancheDir, com1DFA, com1DFACfg, fileName='com1DFA_settings', filePath=workPath) # get energyLine configuration and save to file -energyLineTestCfg = cfgUtils.getModuleConfig(energyLineTest, fileOverride='', modInfo=False, toPrint=False, +energyLineTestCfg = cfgUtils.getModuleConfig(energyLineTest, avalancheDir, toPrint=False, onlyDefault=rotationTestCfg['ana1Tests_energyLineTest_override'].getboolean('defaultConfig')) energyLineTestCfg, rotationTestCfg = cfgHandling.applyCfgOverride(energyLineTestCfg, rotationTestCfg, energyLineTest, addModValues=False) # get ana3AIMEC configuration -AIMECCfg = cfgUtils.getModuleConfig(ana3AIMEC, fileOverride='', modInfo=False, toPrint=False, +AIMECCfg = cfgUtils.getModuleConfig(ana3AIMEC, avalancheDir, toPrint=False, onlyDefault=rotationTestCfg['ana3AIMEC_ana3AIMEC_override'].getboolean('defaultConfig')) AIMECCfg, rotationTestCfg = cfgHandling.applyCfgOverride(AIMECCfg, rotationTestCfg, ana3AIMEC, addModValues=False) AIMECCfgFile = cfgUtils.writeCfgFile(avalancheDir, ana3AIMEC, AIMECCfg, fileName='ana3AIMEC_settings', diff --git a/avaframe/runScripts/runSimilaritySol.py b/avaframe/runScripts/runSimilaritySol.py index db3f3791a..10603619c 100644 --- a/avaframe/runScripts/runSimilaritySol.py +++ b/avaframe/runScripts/runSimilaritySol.py @@ -53,7 +53,7 @@ updater['GENERAL']['meshCellSize'].value = sphKernelRadius updater.update_file() - cfg = cfgUtils.getModuleConfig(com1DFA, simiSolCfg) + cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=simiSolCfg) # Define release thickness distribution demFile = gI.getDEMPath(avalancheDir) relDict = simiSolTest.getReleaseThickness(avalancheDir, cfg, demFile) diff --git a/avaframe/runScripts/runStatsExample.py b/avaframe/runScripts/runStatsExample.py index edb042523..c3137468c 100644 --- a/avaframe/runScripts/runStatsExample.py +++ b/avaframe/runScripts/runStatsExample.py @@ -36,7 +36,7 @@ avaDir = 'data/avaHockeyChannel' cfgMain['MAIN']['avalancheDir'] = avaDir -cfgStats = cfgUtils.getModuleConfig(getStats) +cfgStats = cfgUtils.getModuleConfig(getStats, avaDir) cfg = cfgStats['GENERAL'] # Clean input directory(ies) of old work and output files initProj.cleanSingleAvaDir(avaDir) @@ -65,7 +65,7 @@ initProj.cleanModuleFiles(avaDir, ana3AIMEC) # run aimec statsAimecCfg = pathlib.Path('..', 'benchmarks', avaNameTest, '%sStats_ana3AIMECCfg.ini' % (avaName)) - cfgAIMEC = cfgUtils.getModuleConfig(ana3AIMEC, statsAimecCfg) + cfgAIMEC = cfgUtils.getModuleConfig(ana3AIMEC, fileOverride=statsAimecCfg) cfgAimecSetup = cfgAIMEC['AIMECSETUP'] # Setup input from com1DFA diff --git a/avaframe/runScripts/runStatsPlots.py b/avaframe/runScripts/runStatsPlots.py index e1a355312..4aa7eefcd 100644 --- a/avaframe/runScripts/runStatsPlots.py +++ b/avaframe/runScripts/runStatsPlots.py @@ -37,7 +37,7 @@ flagShow = cfgMain['FLAGS'].getboolean('showPlot') avaDir = cfgMain['MAIN']['avalancheDir'] -cfgStats = cfgUtils.getModuleConfig(getStats) +cfgStats = cfgUtils.getModuleConfig(getStats, avaDir) cfg = cfgStats['GENERAL'] # set output directory, first ava in list @@ -56,7 +56,7 @@ # clean all existing aimec files first initProj.cleanModuleFiles(avaDir, ana3AIMEC) # fetch config for aimec - cfgAIMEC = cfgUtils.getModuleConfig(ana3AIMEC) + cfgAIMEC = cfgUtils.getModuleConfig(ana3AIMEC, avaDir) # run aimec pathDict, rasterTransfo, resAnalysisDF, plotDict = rAimec.runAna3AIMEC(avaDir, cfgAIMEC) diff --git a/avaframe/runScripts/runTestFP.py b/avaframe/runScripts/runTestFP.py index 15548b535..e3700c23a 100644 --- a/avaframe/runScripts/runTestFP.py +++ b/avaframe/runScripts/runTestFP.py @@ -35,7 +35,7 @@ # Load configuration FPCfg = pathlib.Path(avalancheDir, 'Inputs', 'FlatPlane_com1DFACfg.ini') -cfg = cfgUtils.getModuleConfig(com1DFA, FPCfg) +cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=FPCfg) cfgGen = cfg['GENERAL'] cfgFP = cfg['FPSOL'] diff --git a/avaframe/runScripts/runThalwegTimeDiagram.py b/avaframe/runScripts/runThalwegTimeDiagram.py index e01fe544d..daac84a58 100644 --- a/avaframe/runScripts/runThalwegTimeDiagram.py +++ b/avaframe/runScripts/runThalwegTimeDiagram.py @@ -39,7 +39,7 @@ # Load all input Parameters from config file # get the configuration of an already imported module # write config to log file -cfgRangeTime = cfgUtils.getModuleConfig(dtAna) +cfgRangeTime = cfgUtils.getModuleConfig(dtAna, avalancheDir) cfgRangeTime['GENERAL']['avalancheDir'] = avalancheDir if preProcessedData: diff --git a/avaframe/runScripts/runVariationsTestsCom1DFA.py b/avaframe/runScripts/runVariationsTestsCom1DFA.py index fbb225b67..63ee0293f 100644 --- a/avaframe/runScripts/runVariationsTestsCom1DFA.py +++ b/avaframe/runScripts/runVariationsTestsCom1DFA.py @@ -127,7 +127,7 @@ # load configuration aimecCfg = refDir / ('%s_AIMECCfg.ini' % test['AVANAME']) if aimecCfg.is_file(): - cfgAimec = cfgUtils.getModuleConfig(ana3AIMEC, aimecCfg) + cfgAimec = cfgUtils.getModuleConfig(ana3AIMEC,fileOverride=aimecCfg) else: cfgAimec = cfgUtils.getDefaultModuleConfig(ana3AIMEC) cfgAimec['AIMECSETUP']['resType'] = aimecResType @@ -158,7 +158,7 @@ # Create plots for report # Load input parameters from configuration file - cfgRep = cfgUtils.getModuleConfig(generateCompareReport) + cfgRep = cfgUtils.getModuleConfig(generateCompareReport, avaDir) plotListRep = {} reportD['Simulation Difference'] = {} diff --git a/avaframe/runStandardTestsCom1DFA.py b/avaframe/runStandardTestsCom1DFA.py index 5c29c7c1e..2d96d9126 100644 --- a/avaframe/runStandardTestsCom1DFA.py +++ b/avaframe/runStandardTestsCom1DFA.py @@ -125,8 +125,7 @@ def runSingleTest( # get comDFA configuration and update with snow slide parameter set standardCfg = cfgUtils.getModuleConfig( com1DFA, - fileOverride="", - modInfo=False, + avaDir, toPrint=False, onlyDefault=snowSlideCfg["com1DFA_com1DFA_override"].getboolean("defaultConfig"), ) @@ -172,7 +171,7 @@ def runSingleTest( # load configuration aimecCfg = refDir / ("%s_AIMECCfg.ini" % test["AVANAME"]) if aimecCfg.is_file(): - cfgAimec = cfgUtils.getModuleConfig(ana3AIMEC, aimecCfg) + cfgAimec = cfgUtils.getModuleConfig(ana3AIMEC, fileOverride=aimecCfg) else: cfgAimec = cfgUtils.getDefaultModuleConfig(ana3AIMEC) @@ -204,7 +203,7 @@ def runSingleTest( # Create plots for report # Load input parameters from configuration file - cfgRep = cfgUtils.getModuleConfig(generateCompareReport) + cfgRep = cfgUtils.getModuleConfig(generateCompareReport, avaDir) plotListRep = {} reportD["Simulation Difference"] = {} diff --git a/avaframe/runTmp1Ex.py b/avaframe/runTmp1Ex.py index 67026282f..910c49c19 100644 --- a/avaframe/runTmp1Ex.py +++ b/avaframe/runTmp1Ex.py @@ -20,7 +20,7 @@ # Load all input Parameters from config file # get the configuration of an already imported module # Write config to log file -cfg = cfgUtils.getModuleConfig(tmp1Ex) +cfg = cfgUtils.getModuleConfig(tmp1Ex, avalancheDir) # Different ways to call functions tmp1Ex.tmp1ExMain(cfg) diff --git a/avaframe/runUpdateBenchmarkTestsCom1DFA.py b/avaframe/runUpdateBenchmarkTestsCom1DFA.py index e4af99917..2c1cd9b2c 100644 --- a/avaframe/runUpdateBenchmarkTestsCom1DFA.py +++ b/avaframe/runUpdateBenchmarkTestsCom1DFA.py @@ -84,8 +84,7 @@ # get comDFA configuration and update with snow slide parameter set standardCfg = cfgUtils.getModuleConfig( com1DFA, - fileOverride="", - modInfo=False, + avaDir, toPrint=False, onlyDefault=snowSlideCfg["com1DFA_com1DFA_override"].getboolean("defaultConfig"), ) diff --git a/avaframe/tests/data/avaExpertCfgTest/Inputs/CFGs/test_logUtilsCfg.ini b/avaframe/tests/data/avaExpertCfgTest/Inputs/CFGs/test_logUtilsCfg.ini new file mode 100644 index 000000000..efbcd2a00 --- /dev/null +++ b/avaframe/tests/data/avaExpertCfgTest/Inputs/CFGs/test_logUtilsCfg.ini @@ -0,0 +1,3 @@ +[GENERAL] +inputDir = expert/override/path +fullLog = False diff --git a/avaframe/tests/data/avaMalformedCfgTest/Inputs/CFGs/test_logUtilsCfg.ini b/avaframe/tests/data/avaMalformedCfgTest/Inputs/CFGs/test_logUtilsCfg.ini new file mode 100644 index 000000000..20fda997f --- /dev/null +++ b/avaframe/tests/data/avaMalformedCfgTest/Inputs/CFGs/test_logUtilsCfg.ini @@ -0,0 +1,2 @@ +[GENERAL +inputDir = missing bracket diff --git a/avaframe/tests/test_MoTUtils.py b/avaframe/tests/test_MoTUtils.py index a4c5fb446..373a0743c 100644 --- a/avaframe/tests/test_MoTUtils.py +++ b/avaframe/tests/test_MoTUtils.py @@ -605,10 +605,8 @@ def test_MoTGenerateConfigs(tmp_path): "relFiles": [] } - with patch('avaframe.in3Utils.MoTUtils.com1DFATools.checkCfgInfoType') as mockCheckType, \ - patch('avaframe.in3Utils.MoTUtils.com1DFA.com1DFAPreprocess') as mockPreprocess: + with patch('avaframe.in3Utils.MoTUtils.com1DFA.com1DFAPreprocess') as mockPreprocess: - mockCheckType.return_value = "None" mockPreprocess.return_value = (mockSimDict, tmp_path / "out", mockInputSimFiles, None) # Call function @@ -617,7 +615,6 @@ def test_MoTGenerateConfigs(tmp_path): # Verify results assert simDict == mockSimDict assert inputSimFiles == mockInputSimFiles - mockCheckType.assert_called_once_with(cfgInfo) mockPreprocess.assert_called_once() diff --git a/avaframe/tests/test_ana1Tests.py b/avaframe/tests/test_ana1Tests.py index 03f4c24ab..d4de25976 100644 --- a/avaframe/tests/test_ana1Tests.py +++ b/avaframe/tests/test_ana1Tests.py @@ -214,7 +214,7 @@ def test_mainEnergyLineTest(tmp_path): # ---------------- # ++++++++++ set configurations for all the used modules and override ++++++++++++ # get comDFA configuration and save to file - com1DFACfg = cfgUtils.getModuleConfig(com1DFA, fileOverride='', modInfo=False, toPrint=False, + com1DFACfg = cfgUtils.getModuleConfig(com1DFA, toPrint=False, onlyDefault=energyLineTestCfg['com1DFA_com1DFA_override'].getboolean('defaultConfig')) com1DFACfg, energyLineTestCfg = cfgHandling.applyCfgOverride(com1DFACfg, energyLineTestCfg, com1DFA, addModValues=False) diff --git a/avaframe/tests/test_cfgHandling.py b/avaframe/tests/test_cfgHandling.py index b01492381..595656b81 100644 --- a/avaframe/tests/test_cfgHandling.py +++ b/avaframe/tests/test_cfgHandling.py @@ -5,7 +5,6 @@ import pytest import configparser import logging -import os from avaframe.in3Utils import cfgUtils from avaframe.in3Utils import cfgHandling @@ -240,82 +239,3 @@ def test_applyCfgOverride(caplog): assert len(cfgToOverride.items("GENERAL")) == 3 -def test_rewriteLocalCfg(tmp_path): - """test rewriting local cfg from override section""" - - cfgFull = configparser.ConfigParser() - cfgFull.optionxform = str - avalancheDir = pathlib.Path(tmp_path, "testAvaDir") - os.makedirs(avalancheDir, exist_ok=True) - - cfgFull["GENERAL"] = {"testP": 1.0, "name": True} - cfgFull["com1DFA_override"] = {"defaultConfig": True, "test1": 1.0} - - with pytest.raises(AssertionError) as e: - cfgHandling.rewriteLocalCfgs(cfgFull, avalancheDir) - assert ( - "Override section needs to provide moduleName_fileName_override; provided: com1DFA_override invalid format" - in str(e.value) - ) - - localCfgPath = pathlib.Path(tmp_path, "testCfg") - os.makedirs(localCfgPath, exist_ok=True) - - cfgFull = configparser.ConfigParser() - cfgFull.optionxform = str - - cfgFull["GENERAL"] = {"testP": 1.0, "name": True} - cfgFull["com1DFA_com1DFA_override"] = {"defaultConfig": True, "meshCellSize": 1.0} - - cfgHandling.rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=localCfgPath) - - cfgNew = localCfgPath / "local_com1DFACfg.ini" - - readCfg = configparser.ConfigParser() - readCfg.read(cfgNew) - - assert cfgNew.is_file() - assert readCfg.has_option("GENERAL", "tEnd") is False - assert readCfg["GENERAL"].getfloat("meshCellSize") == 1.0 - - cfgFull = configparser.ConfigParser() - cfgFull.optionxform = str - - cfgFull["GENERAL"] = {"testP": 1.0, "name": True} - cfgFull["com1DFA_com1DFA_override"] = {"defaultConfig": True, "meshCellSize": 10.0} - - cfgHandling.rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath='') - - cfgNew2 = avalancheDir / 'Inputs' / 'configurationOverrides' / "local_com1DFACfg.ini" - - readCfg2 = configparser.ConfigParser() - readCfg2.read(cfgNew2) - - assert cfgNew2.is_file() - assert readCfg2.has_option("GENERAL", "tEnd") is False - assert readCfg2["GENERAL"].getfloat("meshCellSize") == 10.0 - - localCfgPath = pathlib.Path(tmp_path, "test2") - - with pytest.raises(NotADirectoryError) as e: - cfgHandling.rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=localCfgPath) - assert "Provided path for local cfg files is not a directory" in str(e.value) - - -def test_removeCfgItemsNotInOverride(): - """test removing keys of cfg that are not in overrideKeys""" - - cfgModule = configparser.ConfigParser() - cfgModule.optionxform = str - - cfgModule["GENERAL"] = {"name1": 1.0, "name2": 2} - cfgModule["VISU"] = {"test1": 1.0, "test2": 2} - - overrideKeys = ["test2", "name1"] - - cfgModule = cfgHandling._removeCfgItemsNotInOverride(cfgModule, overrideKeys) - - assert cfgModule.has_option("GENERAL", "name1") - assert cfgModule.has_option("VISU", "test2") - assert cfgModule.has_option("GENERAL", "name2") is False - assert cfgModule.has_option("VISU", "test1") is False diff --git a/avaframe/tests/test_cfgUtils.py b/avaframe/tests/test_cfgUtils.py index 0dfef3056..8c5e232a2 100644 --- a/avaframe/tests/test_cfgUtils.py +++ b/avaframe/tests/test_cfgUtils.py @@ -46,6 +46,189 @@ def test_getModuleConfig(): assert cfg['GOODSECTION2']['goodKey4'] == 'False' +def test_getModuleConfigExpertDir(): + """Test expert config from avalancheDir/Inputs/CFGs/ takes priority over local_*""" + from avaframe.tests import test_logUtils + + # Setup: avalanche directory with expert config + testDir = pathlib.Path(__file__).parent + avalancheDir = testDir / "data" / "avaExpertCfgTest" + + # Load config with avalancheDir - should use expert config + cfg = cfgUtils.getModuleConfig(test_logUtils, avalancheDir=avalancheDir) + + # Expert config sets inputDir = "expert/override/path" and fullLog = False + # This should override both default AND local_ values + assert cfg["GENERAL"]["inputDir"] == "expert/override/path" + assert cfg["GENERAL"]["fullLog"] == "False" + + # Other values should come from default config (not local_*) + # The default test_logUtilsCfg.ini has goodKey1 = 1 + assert cfg["GOODSECTION1"]["goodKey1"] == "1" + + +def test_getModuleConfigEmptyAvalancheDir(): + """Test that empty avalancheDir uses normal local_* behavior""" + from avaframe.tests import test_logUtils + + # Load config without avalancheDir - should use local_* as before + cfg = cfgUtils.getModuleConfig(test_logUtils, avalancheDir="") + + # Should match original behavior - local_ overrides default + # local_test_logUtilsCfg.ini has inputDir = "path/to/avalanche" + assert cfg["GENERAL"]["inputDir"] == "path/to/avalanche" + assert cfg["GENERAL"]["fullLog"] == "True" + + +def test_getModuleConfigMissingExpertConfig(): + """Test that missing expert config falls back to local_* behavior""" + from avaframe.tests import test_logUtils + + # Use avalanche dir that exists but has no expert config for this module + testDir = pathlib.Path(__file__).parent + # avaTestInputs exists but has no CFGs/ directory + avalancheDir = testDir / "data" / "avaTestInputs" + + cfg = cfgUtils.getModuleConfig(test_logUtils, avalancheDir=avalancheDir) + + # Should fall back to local_* behavior + assert cfg["GENERAL"]["inputDir"] == "path/to/avalanche" + assert cfg["GENERAL"]["fullLog"] == "True" + + +def test_getModuleConfigOnlyDefaultSkipsExpert(): + """Test that onlyDefault=True skips expert config""" + from avaframe.tests import test_logUtils + + testDir = pathlib.Path(__file__).parent + avalancheDir = testDir / "data" / "avaExpertCfgTest" + + # With onlyDefault=True, should ignore expert config + cfg = cfgUtils.getModuleConfig(test_logUtils, avalancheDir=avalancheDir, onlyDefault=True) + + # Should use default values, not expert config values + # Default test_logUtilsCfg.ini has inputDir = "path/to/avalanche" + assert cfg["GENERAL"]["inputDir"] == "path/to/avalanche" + + +def test_getModuleConfigFileOverrideBeatsExpert(): + """Test that fileOverride takes priority over expert config""" + from avaframe.tests import test_logUtils + + testDir = pathlib.Path(__file__).parent + avalancheDir = testDir / "data" / "avaExpertCfgTest" + fileOverride = testDir / "local_test_logUtilsCfg.ini" + + # fileOverride should win over expert config + cfg = cfgUtils.getModuleConfig(test_logUtils, avalancheDir=avalancheDir, fileOverride=fileOverride) + + # Should use fileOverride values, not expert config + # local_test_logUtilsCfg.ini has inputDir = "path/to/avalanche" + assert cfg["GENERAL"]["inputDir"] == "path/to/avalanche" + assert cfg["GENERAL"]["fullLog"] == "True" + + +def test_getModuleConfigMalformedExpertConfig(): + """Test that malformed expert config raises ConfigParser error""" + from avaframe.tests import test_logUtils + + testDir = pathlib.Path(__file__).parent + avalancheDir = testDir / "data" / "avaMalformedCfgTest" + + with pytest.raises(configparser.Error): + cfgUtils.getModuleConfig(test_logUtils, avalancheDir=avalancheDir) + + +def test_getModuleConfigExpertPartialOverride(): + """Test that expert config with partial parameters merges with default""" + from avaframe.tests import test_logUtils + + testDir = pathlib.Path(__file__).parent + avalancheDir = testDir / "data" / "avaExpertCfgTest" + + cfg = cfgUtils.getModuleConfig(test_logUtils, avalancheDir=avalancheDir) + + # Expert config only sets [GENERAL] inputDir and fullLog + # All other sections/values should come from default + assert cfg["GENERAL"]["inputDir"] == "expert/override/path" # from expert + assert cfg["GENERAL"]["fullLog"] == "False" # from expert + + # These should come from default config + assert "GOODSECTION1" in cfg.sections() + assert "GOODSECTION2" in cfg.sections() + assert cfg["GOODSECTION1"]["goodKey1"] == "1" + + +def test_getModuleConfigBatchCfgDir(tmp_path): + """Test that batchCfgDir returns pathlib.Path when valid directory with .ini files""" + from avaframe.tests import test_logUtils + + # Setup: create temp dir with .ini files + cfgDir = tmp_path / "cfgs" + cfgDir.mkdir() + (cfgDir / "test1.ini").write_text("[GENERAL]\nkey = value1\n") + (cfgDir / "test2.ini").write_text("[GENERAL]\nkey = value2\n") + + result = cfgUtils.getModuleConfig(test_logUtils, batchCfgDir=cfgDir) + + assert isinstance(result, pathlib.Path) + assert result == cfgDir + + +def test_getModuleConfigBatchCfgDirNotExists(tmp_path): + """Test that batchCfgDir raises FileNotFoundError for non-existent directory""" + from avaframe.tests import test_logUtils + + nonExistentDir = tmp_path / "does_not_exist" + + with pytest.raises(FileNotFoundError, match="batchCfgDir does not exist"): + cfgUtils.getModuleConfig(test_logUtils, batchCfgDir=nonExistentDir) + + +def test_getModuleConfigBatchCfgDirEmpty(tmp_path): + """Test that batchCfgDir raises FileNotFoundError for directory without .ini files""" + from avaframe.tests import test_logUtils + + emptyDir = tmp_path / "empty_cfgs" + emptyDir.mkdir() + # Create a non-ini file to ensure it's not just checking for "any file" + (emptyDir / "readme.txt").write_text("not a config") + + with pytest.raises(FileNotFoundError, match="batchCfgDir contains no .ini files"): + cfgUtils.getModuleConfig(test_logUtils, batchCfgDir=emptyDir) + + +def test_getModuleConfigBatchCfgDirPriority(tmp_path): + """Test that batchCfgDir takes priority and ignores fileOverride and expert config""" + from avaframe.tests import test_logUtils + + # Setup: batchCfgDir with .ini files + cfgDir = tmp_path / "batch_cfgs" + cfgDir.mkdir() + (cfgDir / "batch.ini").write_text("[GENERAL]\nkey = batch\n") + + # Setup: fileOverride that would normally be used + overrideFile = tmp_path / "override.ini" + overrideFile.write_text("[GENERAL]\nkey = override\n") + + # Setup: avalancheDir with expert config that would normally be used + avalancheDir = tmp_path / "avaTest" + expertDir = avalancheDir / "Inputs" / "CFGs" + expertDir.mkdir(parents=True) + (expertDir / "test_logUtilsCfg.ini").write_text("[GENERAL]\nkey = expert\n") + + # batchCfgDir should win - returns Path, not ConfigParser + result = cfgUtils.getModuleConfig( + test_logUtils, + avalancheDir=avalancheDir, + fileOverride=overrideFile, + batchCfgDir=cfgDir + ) + + assert isinstance(result, pathlib.Path) + assert result == cfgDir + + def test_getGeneralConfig(): '''Test for module getGeneralConfig''' diff --git a/avaframe/tests/test_com1DFA.py b/avaframe/tests/test_com1DFA.py index 35870fbc7..34219c595 100644 --- a/avaframe/tests/test_com1DFA.py +++ b/avaframe/tests/test_com1DFA.py @@ -2697,7 +2697,7 @@ def test_runCom1DFA(tmp_path, caplog): com1DFA, fileOverride=cfgFile, modInfo=True ) - dem, plotDict, reportDictList, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=cfgFile) + dem, plotDict, reportDictList, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=modCfg) print("DONE") @@ -3137,7 +3137,7 @@ def test_tSteps_output_behavior(tmp_path, caplog): cfgMain = cfgUtils.getGeneralConfig() cfgMain["MAIN"]["avalancheDir"] = str(avaDir1) # Modify config to have empty tSteps and NO parameter variations - cfg = cfgUtils.getModuleConfig(com1DFA, cfgFile1) + cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgFile1) cfg["GENERAL"]["tSteps"] = "" cfg["GENERAL"]["tEnd"] = "10" # Short simulation cfg["GENERAL"]["dt"] = "0.1" # Single value, no variations @@ -3170,7 +3170,7 @@ def test_tSteps_output_behavior(tmp_path, caplog): cfgMain["MAIN"]["avalancheDir"] = str(avaDir2) # Modify config to have explicit tSteps including t=0 and NO parameter variations - cfg2 = cfgUtils.getModuleConfig(com1DFA, cfgFile2) + cfg2 = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgFile2) cfg2["GENERAL"]["tSteps"] = "0|5" cfg2["GENERAL"]["tEnd"] = "10" # Short simulation cfg2["GENERAL"]["dt"] = "0.1" # Single value, no variations @@ -3200,7 +3200,7 @@ def test_tSteps_output_behavior(tmp_path, caplog): cfgMain["MAIN"]["avalancheDir"] = str(avaDir3) # Modify config to have exportData = False - cfg3 = cfgUtils.getModuleConfig(com1DFA, cfgFile3) + cfg3 = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgFile3) cfg3["GENERAL"]["tSteps"] = "" cfg3["GENERAL"]["tEnd"] = "5" # Very short simulation cfg3["GENERAL"]["dt"] = "0.1" @@ -3369,3 +3369,47 @@ def test_getModuleNames(): assert result == ("com5SnowSlide", "com5"), ( f"Expected ('com5SnowSlide', 'com5'), got {result}" ) + + +def test_com1DFAMainWithPathCfgInfo(tmp_path, caplog): + """Test that com1DFAMain handles pathlib.Path directory cfgInfo for batch mode + + When cfgInfo is a pathlib.Path pointing to a directory, com1DFAMain should route + directly to batch mode (createSimDictFromCfgs). + This is the code path used when getModuleConfig is called with batchCfgDir parameter. + """ + from unittest.mock import patch + + # Setup avalanche directory structure + avaDir = tmp_path / "avaTest" + avaDir.mkdir() + (avaDir / "Inputs").mkdir() + (avaDir / "Outputs").mkdir() + (avaDir / "Work").mkdir() + + # Setup batch config directory with .ini files + cfgDir = tmp_path / "batch_cfgs" + cfgDir.mkdir() + (cfgDir / "sim1.ini").write_text("[GENERAL]\nrelThPercentile = 50\n") + + # Create cfgMain + cfgMain = configparser.ConfigParser() + cfgMain["MAIN"] = {"avalancheDir": str(avaDir)} + + # Pass pathlib.Path directly as cfgInfo + cfgInfoPath = pathlib.Path(cfgDir) + + with patch("avaframe.com1DFA.com1DFA.com1DFATools.createSimDictFromCfgs") as mockCreateSimDict: + + mockCreateSimDict.return_value = ({}, {}, None, tmp_path / "out") + + # Call with directory Path - should route to createSimDictFromCfgs + try: + com1DFA.com1DFAMain(cfgMain, cfgInfo=cfgInfoPath) + except Exception: + pass # We expect it to fail due to missing simulations, but that's OK + + # createSimDictFromCfgs should be called with the Path + mockCreateSimDict.assert_called_once() + callArgs = mockCreateSimDict.call_args + assert callArgs[0][1] == cfgInfoPath diff --git a/avaframe/tests/test_damBreak.py b/avaframe/tests/test_damBreak.py index c8982058e..ac7c8bcb0 100644 --- a/avaframe/tests/test_damBreak.py +++ b/avaframe/tests/test_damBreak.py @@ -32,11 +32,10 @@ def test_mainCompareSimSolCom1DFA(tmp_path): cfgMain = cfgUtils.getGeneralConfig() cfgMain["MAIN"]["avalancheDir"] = str(avalancheDir) - cfg = cfgUtils.getModuleConfig(com1DFA, damBreakCfg) - # call com1DFA to perform simulations - provide configuration file and release thickness function - # (may be multiple sims) + cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=damBreakCfg) + # call com1DFA to perform simulations try: - _, _, _, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=damBreakCfg) + _, _, _, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=cfg) except NotImplementedError as e: if "iniStep=True is currently not supported" in str(e): pytest.skip( diff --git a/avaframe/tests/test_probAna.py b/avaframe/tests/test_probAna.py index 79b2f6462..272cc16ae 100644 --- a/avaframe/tests/test_probAna.py +++ b/avaframe/tests/test_probAna.py @@ -34,7 +34,7 @@ def test_probAnalysis(tmp_path): # set configurations testCfg = os.path.join(inputDir, "%sProbAna_com1DFACfg.ini" % avaName) - cfgMain = cfgUtils.getModuleConfig(com1DFA, testCfg) + cfgMain = cfgUtils.getModuleConfig(com1DFA, fileOverride=testCfg) # Initialise input in correct format cfg = configparser.ConfigParser() diff --git a/avaframe/tests/test_simiSol.py b/avaframe/tests/test_simiSol.py index aeed221ec..6d2ce1afa 100644 --- a/avaframe/tests/test_simiSol.py +++ b/avaframe/tests/test_simiSol.py @@ -30,7 +30,7 @@ def test_mainCompareSimSolCom1DFA(tmp_path): cfgMain = cfgUtils.getGeneralConfig() cfgMain['MAIN']['avalancheDir'] = str(avalancheDir) - cfg = cfgUtils.getModuleConfig(com1DFA, simiSolCfg) + cfg = cfgUtils.getModuleConfig(com1DFA, fileOverride=simiSolCfg) # adjust settings for faster computation times cfg["GENERAL"]["cMax"] = "0.04" cfg["GENERAL"]["sphKernelRadius"] = "8" @@ -38,7 +38,7 @@ def test_mainCompareSimSolCom1DFA(tmp_path): cfg["GENERAL"]["aPPK"] = "-0.5|-1" cfg["GENERAL"]["meshCellSize"] = "8" - # write updated cfg to file + # write updated cfg to file (needed for mainSimilaritySol later) cfgFile = pathlib.Path(destDir, "%s.ini" % ("simiSolUpdated_com1DFACfg")) with open(cfgFile, "w") as conf: cfg.write(conf) @@ -46,9 +46,8 @@ def test_mainCompareSimSolCom1DFA(tmp_path): # Define release thickness distribution demFile = gI.getDEMPath(avalancheDir) relDict = simiSolTest.getReleaseThickness(avalancheDir, cfg, demFile) - # call com1DFA to perform simulations - provide configuration file and release thickness function - # (may be multiple sims) - _, _, _, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=cfgFile) + # call com1DFA to perform simulations + _, _, _, simDF = com1DFA.com1DFAMain(cfgMain, cfgInfo=cfg) simDF, _ = cfgUtils.readAllConfigurationInfo(avalancheDir) solSimi = simiSolTest.mainSimilaritySol(cfgFile)