Parallel tests over parallel node labels

#!groovy

/* Sample structure for parallelized steps over parallelized nodes in a DDSL pipeline (not 100% pure DDSL though)
 * Use-case : build and test of a PR to a FOSS project across a farm of different target OSes.
 * Note that practically there may be even more layers of loops (e.g. different compilers, bitnesses, autotools vs cmake, etc.)
 * Problem: BlueOcean has problem rendering this. 1.0.0-b12 shows nothing, and 1.0.0-b13 shows a single tall stack
 *          of green/red balls, often overlapped by largish step-tag texts (so the ball can not be clicked to get logs).
 * Copyright (C) 2016 by Jim Klimov on the terms of MIT license
 */

pipeline {
    agent label:""
    environment {
        PATH="/usr/lib/ccache:\$PATH"
        CC="/usr/lib/ccache/gcc"
        CXX="/usr/lib/ccache/g++"
        CCACHE_DIR="/home/jim/.ccache"
    }
    stages {
        stage ("SELECT-BUILDER") {
            steps {
                script {
                    def subbuilds = [:]
                    for (var_buildnode in ["hipster", "debian8"] ) {
                        def buildnode = "${var_buildnode}"
                        subbuilds["${buildnode}"] = {
                            node("${buildnode}") {
                                stage ("PREP-SCM@${buildnode}") {
                                        echo "Checkout to ${env.NODE_NAME} : ${env.WORKSPACE}"
                                        checkout scm
                                }

                                stage ("PREP-CLEAN@${buildnode}") {
                                        sh 'if [ -s Makefile ] ; then make -k clean || true; make -k distclean || true; fi; true'
                                        sh 'rm -f config.cache config.log config.status || true'
                                        sh 'git checkout -- scripts/DMF/dmfnutscan/*.dmf scripts/DMF/dmfsnmp/*.dmf || true'
                                }

                                stage ("PREP-AUTOGEN@${buildnode}") {
                                        echo 'Autogen'
                                        sh './autogen.sh'
                                }

                                stage ("Configure-DMF-quick@${buildnode}") {
                                        sh 'CCACHE_BASEDIR="`pwd`" ./configure --with-snmp --with-neon --with-dev --with-doc=skip --with-snmp_dmf=yes --with-dmfnutscan-regenerate=no --with-dmfsnmp-regenerate=no'
                                }

                                stage ("Build@${buildnode}") {
                                        sh 'CCACHE_BASEDIR="`pwd`" gmake -j 4 -k all || { echo ""; echo "================"; echo "=== REMAKE"; gmake -j1 all; }'
                                }

                                stage ("FanoutTests@${buildnode}") {
                                        script {
                                            def tag_stashed = "${env.BUILD_TAG}-${env.GIT_COMMIT}-${env.NODE_LABELS}".replace(' ','_').replace('%','_')
                                            stash("${tag_stashed}")
                                            deleteDir()
                                            def subtests = [:]
                                            for (makerecipe in [
 "distcheck-light"
,"distcheck-light-man"
,"distcheck-dmf-all-yes"
,"distcheck-dmf-no"
,"distcheck-dmf-warnings"
,"distcheck-dmf-features-REGEN_NO"
,"distcheck-dmf-features-LTDL_YES"
,"distcheck-dmf-features-LTDL_NO"
,"distcheck-dmf-features-LUA_YES"
,"distcheck-dmf-features-LUA_YESNO"
,"distcheck-dmf-features-LUA_NO"
,"distcheck-dmf-features-REGEN_YES"
                                            ] ) {
                                                def tag_recipe = "${makerecipe}"
                                                def tag_labels = "${env.NODE_LABELS}".replace(' ','_').replace('%','_')
                                                def node_labels = "${env.NODE_LABELS}".replace(' ',' && ')
                                                subtests["TEST-${tag_recipe}@${tag_labels}"] = {
                                                    node("${node_labels}") {
                                                        stage ("TEST@${buildnode}: ${tag_recipe}") {
                                                            echo "WIPE before UNSTASH for DISTCHECK"
                                                            deleteDir()
                                                            echo "UNSTASH for DISTCHECK"
                                                            unstash("${tag_stashed}")
                                                            echo "Reconfigure (fix up srcdir)"
                                                            sh 'CCACHE_BASEDIR="`pwd`" ./configure -C --with-snmp --with-neon --with-dev --with-doc=skip --with-snmp_dmf=yes --with-dmfnutscan-regenerate=no --with-dmfsnmp-regenerate=no'
                                                            echo "MAKE DISTCHECK : ${tag_recipe}"
                                                            sh "CCACHE_BASEDIR=\"`pwd`\" gmake ${tag_recipe}"
                                                            echo "WIPE after UNSTASH for DISTCHECK"
                                                            deleteDir()
                                                        }
                                                    }
                                                }
                                            }
                                            parallel subtests
                                        }
                                }
                            }
                        }
                    }
                    parallel subbuilds
                }
            }
        }
    }
}