Copy input files (grid, logfile-definitions) to output container + defect repairs (#603)

* Enhancement and Defect Repairs:
- Added code to copy any grid file and/or logfile-definitions file specified to output container.
- Copying a large grid file could take time, and take up much space, so added new program option '--store-input-files' which is TRUE by default.  If FALSE, neither the grid file (if specified) nor the logfile-definitions file (if specified) will be copied to the output container (if TRUE, both will be copied (if specified)).
- Fixed issue #600: changed pythonSubmit.py to treat fully-qualified grid filenames and fully-qualified logfile-definitions filenames correctly (i.e. don't add CWD if the filename is already fully-qualified).
- Fixed issue #601: changed pythonSubmit.py to put all boolean parameters on the commandline, with True or False value.

* Update Log.h

Removed extraneous include

Author: jeffriley

docs/COMPAS_LaTeX/sections/Appendices/ProgramOptions.tex

@@ -307,6 +307,8 @@ \section{Program Options}\label{sec:ProgramOptionsAppendix}
 
 \programOption{stellar-zeta-prescription}{}{Prescription for stellar zeta. \\ Options: \lcb\ STARTRACK, SOBERMAN, HURLEY, ARBITRARY \rcb}{SOBERMAN}
 
+\programOption{store-input-files}{}{Enables copying of any specified grid file and/or logfile-definitios file to the COMPAS output container}{TRUE}
+
 \programOption{switch-log}{}{Enables printing of the Switch Log logfile}{FALSE}
 
 \programOption{timestep-multiplier}{}{Multiplicative factor for timestep duration}{1.0}

docs/COMPAS_LaTeX/sections/RevisionHistory.tex

@@ -73,6 +73,19 @@
 
 \revisionHistoryRow{18 December 2020}{2.12}{Removed references to Mass\_Transfer\_Case\_Initial parameter}{Reinhold Willcox}
 
+\end{tabularx}  % why is this too wide?
+
+
+\begin{tabularx}{\linewidth}{
+    |>{\hsize=0.8\hsize}X
+    |>{\hsize=0.3\hsize}X
+    |>{\hsize=2.1\hsize}X
+    |>{\hsize=0.8\hsize}X
+    |
+}
+
+\hline                                                                      % 
+
 \revisionHistoryRow{08 January 2021}{2.13}{Added (brief) description of HDF5 logfile support.}{}
 \revisionHistoryRow{}{}{Added description of option \mbox{hdf5-buffer-size}.}{}
 \revisionHistoryRow{}{}{Added description of option \mbox{hdf5-chunk-size}.}{}
@@ -83,6 +96,7 @@
 \revisionHistoryRow{06 April 2021}{2.15}{Added FARMER prescription for option pulsational-pair-instability-prescription}{Lieke van Son}
 \revisionHistoryRow{20 April 2021}{2.16}{Added option add-options-to-sysparms}{Jeff Riley}
 \revisionHistoryRow{18 May 2021}{2.17}{Changed default LBV prescription}{Tom Wagg}
+\revisionHistoryRow{28 July 2021}{2.18}{Added option store-input-files}{Jeff Riley}
 
 \end{tabularx}  % why is this too wide?
 \normalsize

preProcessing/pythonSubmit.py

@@ -3,11 +3,13 @@
 import os
 from subprocess import call
 import re
+import ntpath
 
 # Check if we are using python 3
 python_version = sys.version_info[0]
 print("python_version =", python_version)
 
+
 class pythonProgramOptions:
     """
     A class to store and access COMPAS program options in python
@@ -24,7 +26,6 @@ class pythonProgramOptions:
     # docker container (src, obj, bin), and the COMPAS executable resides
     # in the bin directory (rather than the src directory)
     compas_executable_override = os.environ.get('COMPAS_EXECUTABLE_PATH')
-    print('compas_executable_override', compas_executable_override)
 
     if (compas_executable_override is None):
         git_directory = os.environ.get('COMPAS_ROOT_DIR')
@@ -53,7 +54,7 @@ class pythonProgramOptions:
 
     if (compas_logs_output_override is None):
         output = os.getcwd()
-        output_container = None                 # names the directory to be created and in which log files are created.  Default in COMPAS is "COMPAS_Output"
+        output_container = None                                 # names the directory to be created and in which log files are created.  Default in COMPAS is "COMPAS_Output"
     else:
         output = compas_logs_output_override
         output_container = None
@@ -79,18 +80,28 @@ class pythonProgramOptions:
     grid_filename = None                                        # grid file name (e.g. 'mygrid.txt')
 
     if grid_filename != None:
-        if compas_input_path_override == None:
-            grid_filename = os.getcwd() + '/' + grid_filename.strip("'\"")
-        else:
-            grid_filename = compas_input_path_override + '/' + grid_filename.strip("'\"")
+        # if the grid filename supplied is already fully-qualified, leave it as is
+        head, tail = ntpath.split(grid_filename)                # split into pathname and base filename
+        
+        if head == '' or head == '.':                           # no path (or CWD) - add path as required
+            grid_filename = tail or ntpath.basename(head)
+            if compas_input_path_override == None:
+                grid_filename = os.getcwd() + '/' + grid_filename.strip("'\"")
+            else:
+                grid_filename = compas_input_path_override + '/' + grid_filename.strip("'\"")
 
     logfile_definitions = None                                  # logfile record definitions file name (e.g. 'logdefs.txt')
 
     if logfile_definitions != None:
-        if compas_input_path_override == None:
-            logfile_definitions = os.getcwd() + '/' + logfile_definitions.strip("'\"")
-        else:
-            logfile_definitions = compas_input_path_override + '/' + logfile_definitions.strip("'\"")
+        # if the grid filename supplied is already fully-qualified, leave it as is
+        head, tail = ntpath.split(grid_filename)                # split into pathname and base filename
+        
+        if head == '' or head == '.':                           # no path (or CWD) - add path as required
+            logfile_definitions = tail or ntpath.basename(head)
+            if compas_input_path_override == None:
+                logfile_definitions = os.getcwd() + '/' + logfile_definitions.strip("'\"")
+            else:
+                logfile_definitions = compas_input_path_override + '/' + logfile_definitions.strip("'\"")
 
     initial_mass    = None                                      # initial mass for SSE
     initial_mass_1  = None                                      # primary initial mass for BSE
@@ -117,6 +128,8 @@ class pythonProgramOptions:
 
     chemically_homogeneous_evolution = 'PESSIMISTIC'            # chemically homogeneous evolution.  Options are 'NONE', 'OPTIMISTIC' and 'PESSIMISTIC'
 
+    store_input_files = True                                    # store input files in output container?
+
     switch_log = False
 
     common_envelope_alpha = 1.0
@@ -319,6 +332,7 @@ def booleanChoices(self):
             self.errors_to_file,
             self.allow_rlof_at_birth,
             self.allow_touching_at_birth,
+            self.store_input_files,
             self.switch_log,
             self.check_photon_tiring_limit
         ]
@@ -345,6 +359,7 @@ def booleanCommands(self):
             '--errors-to-file',
             '--allow-rlof-at-birth',
             '--allow-touching-at-birth',
+            '--store-input-files',
             '--switch-log',
             '--check-photon-tiring-limit'
         ]
@@ -695,6 +710,8 @@ def generateCommandLineOptionsDict(self):
         for i in range(nBoolean):
             if booleanChoices[i] == True:
                 command.update({booleanCommands[i] : ''})
+            elif booleanChoices[i] == False:
+                command.update({booleanCommands[i] : 'False'})
 
         for i in range(nNumerical):
             if not numericalChoices[i] == None:

src/Log.cpp

@@ -389,6 +389,36 @@ void Log::Start(const string              p_LogBasePath,
                     m_Enabled = false;                                                                                      // fail
                 }
             }
+
+            // store input files if required
+            // use Boost to do the copy - copy_file() is available in standard c++17
+            if (OPTIONS->StoreInputFiles()) {                                                                               // user wants input files stored in output container?
+                                                                                                                            // yes
+                string dstPath = m_LogBasePath + "/" + m_LogContainerName + "/";                                            // destination path (output container)
+                if (!OPTIONS->GridFilename().empty()) {                                                                     // user specified a grid file?
+                    try {                                                                                                   // yes - copy it
+                        boost::filesystem::path srcPath(OPTIONS->GridFilename());                                           // grid file fully-qualified name
+                        string dstFn = dstPath + srcPath.filename().string();                                               // fully-qualified grid filename (inside container)
+                        boost::filesystem::copy_file(OPTIONS->GridFilename(), dstFn, boost::filesystem::copy_option::overwrite_if_exists); // copy grid file - overwrite any existing file (shouldn't be one, but just in case we want this one)
+                    } catch(const boost::filesystem::filesystem_error& e) {
+                        Squawk("ERROR: Unable to copy grid file " + OPTIONS->GridFilename() + " to output container " + dstPath); // announce error
+                        m_Enabled = false;                                                                                  // fail
+                    }
+                }
+
+                // if the user specified a logfile-definitions file, copy it to the output container
+
+                if (m_Enabled && !OPTIONS->LogfileDefinitionsFilename().empty()) {                                          // user specified a logfile-definitions file?
+                    try {                                                                                                   // yes - copy it
+                        boost::filesystem::path srcPath(OPTIONS->LogfileDefinitionsFilename());                             // logfile-definitions file fully-qualified name
+                        string dstFn = dstPath + srcPath.filename().string();                                               // fully-qualified logfile-definitions filename (inside container)
+                        boost::filesystem::copy_file(OPTIONS->LogfileDefinitionsFilename(), dstFn, boost::filesystem::copy_option::overwrite_if_exists); // copy logfile-definitions file - overwrite any existing file (shouldn't be one, but just in case we want this one)
+                    } catch(const boost::filesystem::filesystem_error& e) {
+                        Squawk("ERROR: Unable to copy logfile-definitions file " + OPTIONS->LogfileDefinitionsFilename() + " to output container " + dstPath); // announce error
+                        m_Enabled = false;                                                                                  // fail
+                    }
+                }
+            }
         }
     }
 }

src/Options.cpp

@@ -62,7 +62,7 @@
 /*    Read the explanations for each of the vectors in Options.h to get a better idea of  */
 /*    what they are for and where the new option should go.                               */
 /*                                                                                        */
-/* 10. Add the new option to the following structres in constants.h:                      */
+/* 10. Add the new option to the following structures in constants.h:                     */
 /*                                                                                        */
 /*        - enum class PROGRAM_OPTION                                                     */
 /*        - const COMPASUnorderedMap<PROGRAM_OPTION, std::string> PROGRAM_OPTION_LABEL    */
@@ -148,6 +148,8 @@ void Options::OptionValues::Initialise() {
 
     m_ShortHelp                                                     = true;
 
+    m_StoreInputFiles                                               = true;
+
     m_SwitchLog                                                     = false;
 
 
@@ -704,6 +706,11 @@ bool Options::AddOptions(OptionValues *p_Options, po::options_description *p_Opt
             po::value<bool>(&p_Options->m_RlofPrinting)->default_value(p_Options->m_RlofPrinting)->implicit_value(true),                                                                          
             ("Enable output parameters before/after RLOF (default = " + std::string(p_Options->m_RlofPrinting ? "TRUE" : "FALSE") + ")").c_str()
         )
+        (
+            "store-input-files",                                                
+            po::value<bool>(&p_Options->m_StoreInputFiles)->default_value(p_Options->m_StoreInputFiles)->implicit_value(true),                                                                          
+            ("Store input files in output container (default = " + std::string(p_Options->m_StoreInputFiles ? "TRUE" : "FALSE") + ")").c_str()
+        )
         (
             "switch-log",                                                
             po::value<bool>(&p_Options->m_SwitchLog)->default_value(p_Options->m_SwitchLog)->implicit_value(true),                                                                          

src/Options.h

@@ -195,6 +195,7 @@ class Options {
 
         "rlof-printing",
 
+        "store-input-files",
         "switch-log",
 
         "timestep-multiplier",
@@ -464,6 +465,7 @@ class Options {
 
         "semi-major-axis-distribution",
         "stellar-zeta-prescription",
+        "store-input-files",
         "switch-log",
 
         "use-mass-loss",
@@ -522,6 +524,7 @@ class Options {
         "random-seed",
         "rlof-printing",
 
+        "store-input-files",
         "switch-log",
 
         "version", "v"
@@ -573,6 +576,8 @@ class Options {
 
             bool                                                m_ShortHelp;                                                    // Flag to indicate whether user wants short help ('-h', just option names) or long help ('--help', plus descriptions)
 
+            bool                                                m_StoreInputFiles;                                              // Store input files in output container (default = true)
+
             bool                                                m_SwitchLog;                                                    // Print switch log details to file (default = false)
 
 
@@ -1291,7 +1296,8 @@ class Options {
     bool                                        RequestedHelp() const                                                   { return m_CmdLine.optionValues.m_VM["help"].as<bool>(); }
     bool                                        RequestedVersion() const                                                { return m_CmdLine.optionValues.m_VM["version"].as<bool>(); }
 
-    bool                                        SwitchLog() const                                                       { return OPT_VALUE("switch-log", m_SwitchLog, true); }
+    bool                                        StoreInputFiles() const                                                 { return m_CmdLine.optionValues.m_StoreInputFiles; }
+    bool                                        SwitchLog() const                                                       { return m_CmdLine.optionValues.m_SwitchLog; }
 
     ZETA_PRESCRIPTION                           StellarZetaPrescription() const                                         { return OPT_VALUE("stellar-zeta-prescription", m_StellarZetaPrescription.type, true); }
 

src/changelog.h

@@ -757,7 +757,12 @@
 // 02.20.02     JR - July 26, 2021  - Defect repair:
 //                                      - Add HDF5 support to logging code for SSE/BSE switch log files.  Support for HDF5 switch files was inadvertently not added when HDF5 file support as added in v02.18.00 for all standard log files.  Switch log files are 'special' (they have extra columns, not part of the 'standard' log file functionality), and that was missed.
 //                                      - Also removed '-lsz' from Makefile and Makefile.docker - library not required
+// 02.21.00     JR - July 28, 2021  - Enhancement and Defect Repairs:
+//                                      - Added code to copy any grid file and/or logfile-definitions file specified to output container.
+//                                      - Copying a large grid file could take time, and take up much space, so added new program option '--store-input-files' which is TRUE by default.  If FALSE, neither the grid file (if specified) nor the logfile-definitions file (if specified) will be copied to the output container (if TRUE, both will be copied (if specified)).
+//                                      - Fixed issue #600: changed pythonSubmit.py to treat fully-qualified grid filenames and fully-qualified logfile-definitions filenames correctly (i.e. don't add CWD if the filename is already fully-qualified).
+//                                      - Fixed issue #601: changed pythonSubmit.py to put all boolean parameters on the commandline, with "True" or "False" value.
 
-const std::string VERSION_STRING = "02.20.02";
+const std::string VERSION_STRING = "02.21.00";
 
 # endif // __changelog_h__