SRE-2932 Fix notifyBrokenBranch

* SRE-2932 Fix notifyBrokenBranch

vars/notifyBrokenBranch.grooy:
  This routine was doing a lookup of the enviroment Map which is not
  allowed in the pipeline-lib sandboxed operation.

vars/DocOnlyChange.groovy:
  Missing a String declaration that was causing log noise.


el9.2 is not available - use 9.4 instead
el8.6 is not available - prepare to upgrade to 8.8
el8.6 can be removed when daos 2.6 and master start using 8.8
opensuse-leap-15.4 not available - upgrade to 15.5

---------

Signed-off-by: John E. Malmberg <john.malmberg@hpe.com>
Signed-off-by: Tomasz Gromadzki <tomasz.gromadzki@hpe.com>
Co-authored-by: Tomasz Gromadzki <tomasz.gromadzki@hpe.com>

Author: John E. Malmberg

vars/distroVersion.groovy

@@ -33,8 +33,8 @@ String call(String distro, String branch) {
     return ['el8':      ['master': '8.8',
                          '2.4':    '8.8',
                          '2.6':    '8.8'],
-            'el9':      ['master': '9.2',
-                         '2.6':    '9.2'],
+            'el9':      ['master': '9.4',
+                         '2.6':    '9.4'],
             'leap15':   ['master': '15.6',
                          '2.4':    '15.6',
                          '2.6':    '15.6'],
@@ -48,8 +48,8 @@ assert(call('leap15', 'master') == '15.6')
 assert(call('el8', '2.4') == '8.8')
 assert(call('el8', '2.6') == '8.8')
 assert(call('el8', 'master') == '8.8')
-assert(call('el9', 'master') == '9.2')
-assert(call('el9', '2.6') == '9.2')
+assert(call('el9', 'master') == '9.4')
+assert(call('el9', '2.6') == '9.4')
 
 /* Uncomment to do further testing
 env = [:]

vars/docOnlyChange.groovy

@@ -17,6 +17,7 @@ boolean call(String target_branch) {
     if (cachedCommitPragma('Doc-only').toLowerCase() == 'false') {
         return false
     }
+    String script
     if (fileExists('ci/doc_only_change.sh')) {
         script = 'CHANGE_ID=' + env.CHANGE_ID +
                  ' TARGET_BRANCH=' + target_branch +

vars/getFunctionalTestStage.groovy

@@ -45,7 +45,7 @@ Map call(Map kwargs = [:]) {
 
     return {
         stage("${name}") {
-            // Get the tags for thge stage. Use either the build parameter, commit pragma, or
+            // Get the tags for the stage. Use either the build parameter, commit pragma, or
             // default tags. All tags are combined with the stage tags to ensure only tests that
             // 'fit' the cluster will be run.
             String tags = getFunctionalTags(

vars/notifyBrokenBranch.groovy

@@ -1,4 +1,5 @@
 // vars/notifyBrokenBranch.groovy
+/* groovylint-disable VariableName */
 
 /**
  * notifyBrokenBranch.groovy
@@ -12,37 +13,47 @@
 
 def call(Map config = [:]) {
 
+    String branches
     if (config['branches']) {
         branches = config['branches'].split()
     } else {
         branches = ["master"]
     }
 
-    if (!branches.contains(env.GIT_BRANCH)) {
+    // Needed this as a work around that env['GIT_BRANCH'] is blacklisted
+    // inside of pipeline-lib
+    String git_branch = env.GIT_BRANCH
+
+    if (!branches.contains(git_branch)) {
         return
     }
 
-    emailextDaos body: env.GIT_BRANCH + ' is broken and you are one of the people\n' +
+    emailextDaos body: git_branch + ' is broken and you are one of the people\n' +
                        'who have committed to it since it was last successful.  Please\n' +
-                       'investigate if your recent patch(es) to ' + env.GIT_BRANCH + '\n' +
+                       'investigate if your recent patch(es) to ' + git_branch + '\n' +
                        'are responsible for breaking it.\n\n' +
                        'See ' + env.BUILD_URL + ' for more details.',
                  recipientProviders: [
                      [$class: 'DevelopersRecipientProvider'],
                      [$class: 'RequesterRecipientProvider']
                  ],
-                 subject: 'Build broken on ' + env.GIT_BRANCH,
+                 subject: 'Build broken on ' + git_branch,
                  onPR: config['onPR']
 
-    def branch = env['GIT_BRANCH'].toUpperCase().replaceAll("-", "_")
-    def watchers = env["DAOS_STACK_${branch}_WATCHER"]
-
-    if (watchers != "null") {
-        emailextDaos body: env.GIT_BRANCH + ' is broken.\n\n' +
-                           'See ' + env.BUILD_URL + ' for more details.',
-                     to: watchers,
-                     subject: 'Build broken on ' + env.GIT_BRANCH,
-                     onPR: config['onPR']
+    String branch = git_branch.toUpperCase().replaceAll("-", "_")
+    // This will need to be implemented in trusted-pipe-line lib eventually
+    // as checking if environment variables exist is blacklisted in the
+    // groovy sandbox.
+    // for now we only have DAOS_STACK_MASTER_WATCHER
+    // def watchers = env["DAOS_STACK_${branch}_WATCHER"]
+    if (branch == 'MASTER') {
+        String watchers = env.DAOS_STACK_MASTER_WATCHER
+        if (watchers != "null") {
+            emailextDaos body: git_branch + ' is broken.\n\n' +
+                               'See ' + env.BUILD_URL + ' for more details.',
+                         to: watchers,
+                         subject: 'Build broken on ' + git_branch,
+                         onPR: config['onPR']
+        }
     }
-    
 }

vars/packageBuildingPipeline.groovy

@@ -535,7 +535,7 @@ void call(Map pipeline_args) {
                             }
                         }
                     } //stage('Build RPM on EL 9')
-                    stage('Build RPM on Leap 15.4') {
+                    stage('Build RPM on Leap 15.5') {
                         when {
                             beforeAgent true
                             expression { !skipStage() && distros.contains('leap15') }
@@ -555,19 +555,19 @@ void call(Map pipeline_args) {
                             sh label: 'Build package',
                                script: '''rm -rf artifacts/leap15/
                                           mkdir -p artifacts/leap15/
-                                          make CHROOT_NAME="opensuse-leap-15.4-x86_64" ''' +
+                                          make CHROOT_NAME="opensuse-leap-15.5-x86_64" ''' +
                                               'DISTRO_VERSION=' + parseStageInfo()['distro_version'] + ' ' +
                                        pipeline_args.get('make args', '') + ' chrootbuild ' +
                                        pipeline_args.get('add_make_targets', '')
                         }
                         post {
                             success {
-                                rpmlintMockResults('opensuse-leap-15.4-x86_64',
+                                rpmlintMockResults('opensuse-leap-15.5-x86_64',
                                                    pipeline_args.get('rpmlint_rpms_allow_errors', false),
                                                    pipeline_args.get('rpmlint_rpms_skip', false),
                                                    pipeline_args.get('make args', ''))
                                 sh label: 'Collect artifacts',
-                                   script: '''(cd /var/lib/mock/opensuse-leap-15.4-x86_64/result/ &&
+                                   script: '''(cd /var/lib/mock/opensuse-leap-15.5-x86_64/result/ &&
                                               cp -r . $OLDPWD/artifacts/leap15/)\n''' +
                                               pipeline_args.get('add_archiving_cmds', '').replace('<distro>',
                                                                                                   'leap15') +
@@ -584,7 +584,7 @@ void call(Map pipeline_args) {
                             }
                             unsuccessful {
                                 sh label: 'Build Log',
-                                   script: '''mockroot=/var/lib/mock/opensuse-leap-15.4-x86_64
+                                   script: '''mockroot=/var/lib/mock/opensuse-leap-15.5-x86_64
                                               ls -l $mockroot/result/
                                               cat $mockroot/result/{root,build}.log
                                               artdir=$PWD/artifacts/leap15
@@ -594,9 +594,9 @@ void call(Map pipeline_args) {
                             }
                             always {
                                 scrubSecret(pipeline_args['secret'],
-                                            '/var/lib/mock/opensuse-leap-15.4-x86_64/result/build.log')
+                                            '/var/lib/mock/opensuse-leap-15.5-x86_64/result/build.log')
                                 sh label: 'Collect config.log(s)',
-                                   script: '(if cd /var/lib/mock/opensuse-leap-15.4-x86_64/root/builddir/build/' +
+                                   script: '(if cd /var/lib/mock/opensuse-leap-15.5-x86_64/root/builddir/build/' +
                                           '''BUILD/*/; then
                                                    find . -name configure -printf %h\\\\n | \
                                                    while read dir; do

vars/parseStageInfo.groovy

@@ -79,6 +79,10 @@ Map call(Map config = [:]) {
             result['target'] = 'el8.6'
             result['distro_version'] = cachedCommitPragma('EL8.6-version', '8.6')
             new_ci_target = cachedCommitPragma('EL8.6-target', result['target'])
+        } else if (stage_name.contains('EL 8.8')) {
+            result['target'] = 'el8.8'
+            result['distro_version'] = cachedCommitPragma('EL8.8-version', '8.8')
+            new_ci_target = cachedCommitPragma('EL8.8-target', result['target'])
         } else if (stage_name.contains('EL 8')) {
             result['target'] = 'el8'
             result['distro_version'] = cachedCommitPragma('EL8-version',

vars/provisionNodes.groovy

@@ -39,7 +39,7 @@
                      May be used in the future as SUSE free development
                      licenses do currently allow them to be used for
                      CI automation, and SUSE also has offered site licenses.
-   Prefix "leap":    Specifically OpenSUSE leap oprating system builds of
+   Prefix "leap":    Specifically OpenSUSE leap operating system builds of
                      Suse Linux Enterprise Server.
 */
 /* groovylint-disable-next-line MethodSize */
@@ -122,6 +122,7 @@ Map call(Map config = [:]) {
 
     if (!fileExists('ci/provisioning/log_cleanup.sh') ||
         !fileExists('ci/provisioning/post_provision_config.sh')) {
+
         return provisionNodesV1(config)
     }
 

vars/sconsBuild.groovy

@@ -65,7 +65,7 @@ Map call(Map config = [:]) {
    * config['stash_files']  Filename containing list of test files to stash.
    * config['stash_opt'] Boolean, stash tar of /opt in preference to install/**.  Default false.
    *   If present, those files will be placed in a stash name
-   *   of based on parsing the evironment variables of the
+   *   of based on parsing the environment variables of the
    *   <target-compiler[-build_type]-test>.  Additional stashes
    *   will be created for "install" and "build_vars" with similar
    *   prefixes.

vars/skipStage.groovy

@@ -504,6 +504,20 @@ boolean call(Map config = [:]) {
                     !run_default_skipped_stage('test-el-8.6-rpms')) ||
                    (rpmTestVersion() != '') ||
                    stageAlreadyPassed()
+        case 'Test EL 8.8 RPMs':
+        case 'Test RPMs on EL 8.8':
+            return !paramsValue('CI_RPMS_el8.8_TEST', true) ||
+                   target_branch =~ branchTypeRE('weekly') ||
+                   skip_stage_pragma('build-el8-rpm') ||
+                   skip_stage_pragma('test') ||
+                   skip_stage_pragma('test-rpms') ||
+                   skip_stage_pragma('test-el-8.8-rpms') ||
+                   docOnlyChange(target_branch) ||
+                   (quickFunctional() &&
+                    !paramsValue('CI_RPMS_el8_8_TEST', true) &&
+                    !run_default_skipped_stage('test-el-8.8-rpms')) ||
+                   (rpmTestVersion() != '') ||
+                   stageAlreadyPassed()
         case 'Test Leap 15 RPMs':
         case 'Test Leap 15.2 RPMs':
             // Skip by default as it doesn't pass with Leap15.3 due to

vars/unitTest.groovy

@@ -26,7 +26,7 @@
    * config['description']       Description to report for SCM status.
    *                             Default env.STAGE_NAME.
    *
-   * config['failure_artifacts'] Failure aritfifacts to return.
+   * config['failure_artifacts'] Failure artifacts to return.
    *                             Default env.STAGE_NAME.
    *
    * config['ignore_failure']    Ignore test failures.  Default false.
@@ -43,10 +43,10 @@
    *
    * config['node_count']        Count of nodes that will actually be used
    *                             the test.  Default will be based on the
-   *                             enviroment variables for the stage.
+   *                             environment variables for the stage.
    *
    * config['stashes']           List of stashes to use.  Default will be
-   *                             baed on the environment variables for the
+   *                             based on the environment variables for the
    *                             stage.
    *
    * config['target']            Target distribution, such as 'centos7',
@@ -61,16 +61,16 @@
    *                             SSH_KEY_ARGS and NODELIST environment
    *                             variables set.
    *
-   * config['timeout_time']      Timelimit for test run, not including
+   * config['timeout_time']      Time limit for test run, not including
    *                             provisioning time.
    *                             Default is 120 Minutes.
    *
-   * config['timeout_units']     Timelimit units.  Default is minutes.
+   * config['timeout_units']     Time limit units.  Default is minutes.
    *
-   * config['unstash_opt']       Unstash -opt-tar instead of -opt,
+   * config['unstash_opt']       Un-stash -opt-tar instead of -opt,
    *                             default is false.
    *
-   * config['unstash_tests']     Unstash -tests, default is true.
+   * config['unstash_tests']     Un-stash -tests, default is true.
    */
 
 Map afterTest(Map config, Map testRunInfo) {
@@ -144,6 +144,7 @@ Map call(Map config = [:]) {
                           /([a-z]+)(.*)/)[0][1] + stage_info['distro_version'],
                  inst_repos: config.get('inst_repos', ''),
                  inst_rpms: inst_rpms)
+
     String target_stash = "${stage_info['target']}-${stage_info['compiler']}"
     if (stage_info['build_type']) {
         target_stash += '-' + stage_info['build_type']