From eb823a76216b8da28e0f9eac9143ce9a7f5af4fa Mon Sep 17 00:00:00 2001 From: Eiden Yoshida <47196116+eidenyoshida@users.noreply.github.com> Date: Tue, 18 Feb 2020 11:25:29 -0700 Subject: [PATCH] Refactor Jenkinsfiles to allow use of new docker containers (#170) [ROCm/rccl commit: edb863de6243dfe4af39f520471d7058001952fe] --- projects/rccl/.jenkins/common.groovy | 32 +++++++++ projects/rccl/.jenkins/precheckin.groovy | 82 ++++++++++++++++++++++ projects/rccl/Jenkinsfile | 87 ------------------------ 3 files changed, 114 insertions(+), 87 deletions(-) create mode 100644 projects/rccl/.jenkins/common.groovy create mode 100644 projects/rccl/.jenkins/precheckin.groovy delete mode 100644 projects/rccl/Jenkinsfile diff --git a/projects/rccl/.jenkins/common.groovy b/projects/rccl/.jenkins/common.groovy new file mode 100644 index 0000000000..7b01794c26 --- /dev/null +++ b/projects/rccl/.jenkins/common.groovy @@ -0,0 +1,32 @@ +// This file is for internal AMD use. +// If you are interested in running your own Jenkins, please raise a github issue for assistance. + +def runCompileCommand(platform, project, jobName) +{ + project.paths.construct_build_prefix() + + String hipclangArgs = jobName.contains('hipclang') ? '--hip-clang' : '' + def command = """#!/usr/bin/env bash + set -x + cd ${project.paths.project_build_prefix} + LD_LIBRARY_PATH=/opt/rocm/hcc/lib CXX= ${project.paths.build_command} -t {hipclangArgs} + """ + + platform.runCommand(this,command) +} + + +def runPackageCommand(platform, project, jobName) +{ + def command = """ + set -x + cd ${project.paths.project_build_prefix}/build/release + make package + rm -rf package && mkdir -p package + mv *.deb package/ + """ + platform.runCommand(this,command) + platform.archiveArtifacts(this, """${project.paths.project_build_prefix}/build/release/package/*.deb""") +} + +return this diff --git a/projects/rccl/.jenkins/precheckin.groovy b/projects/rccl/.jenkins/precheckin.groovy new file mode 100644 index 0000000000..11a7409b64 --- /dev/null +++ b/projects/rccl/.jenkins/precheckin.groovy @@ -0,0 +1,82 @@ +#!/usr/bin/env groovy +// This shared library is available at https://github.com/ROCmSoftwarePlatform/rocJENKINS/ +@Library('rocJenkins@pong') _ + +// This is file for internal AMD use. +// If you are interested in running your own Jenkins, please raise a github issue for assistance. + +import com.amd.project.* +import com.amd.docker.* +import java.nio.file.Path + +def runCI = +{ + nodeDetails, jobName-> + + def prj = new rocProject('rccl', 'PreCheckin') + prj.paths.build_command = './install.sh' + + // Define test architectures, optional rocm version argument is available + def nodes = new dockerNodes(nodeDetails, jobName, prj) + + boolean formatCheck = false + + def commonGroovy + + def compileCommand = + { + platform, project-> + + commonGroovy = load "${project.paths.project_src_prefix}/.jenkins/common.groovy" + commonGroovy.runCompileCommand(platform, project, jobName) + } + + + def packageCommand = + { + platform, project-> + + commonGroovy.runPackageCommand(platform, project, jobName) + } + + buildProject(prj, formatCheck, nodes.dockerArray, compileCommand, null, packageCommand) +} + +ci: { + String urlJobName = auxiliary.getTopJobName(env.BUILD_URL) + + def propertyList = ["compute-rocm-dkms-no-npi":[pipelineTriggers([cron('0 1 * * 0')])], + "compute-rocm-dkms-no-npi-hipclang":[pipelineTriggers([cron('0 1 * * 0')])], + "rocm-docker":[]] + propertyList = auxiliary.appendPropertyList(propertyList) + + def jobNameList = ["compute-rocm-dkms-no-npi":([ubuntu16:['any']]), + "compute-rocm-dkms-no-npi-hipclang":([ubuntu16:['any']]), + "rocm-docker":([ubuntu16:['any'],centos7:['any']])] + jobNameList = auxiliary.appendJobNameList(jobNameList) + + propertyList.each + { + jobName, property-> + if (urlJobName == jobName) + properties(auxiliary.addCommonProperties(property)) + } + + jobNameList.each + { + jobName, nodeDetails-> + if (urlJobName == jobName) + stage(jobName) { + runCI(nodeDetails, jobName) + } + } + + // For url job names that are not listed by the jobNameList i.e. compute-rocm-dkms-no-npi-1901 + if(!jobNameList.keySet().contains(urlJobName)) + { + properties(auxiliary.addCommonProperties([pipelineTriggers([cron('0 1 * * *')])])) + stage(urlJobName) { + runCI([ubuntu16:['gfx906']], urlJobName) + } + } +} \ No newline at end of file diff --git a/projects/rccl/Jenkinsfile b/projects/rccl/Jenkinsfile deleted file mode 100644 index fc340fcf0b..0000000000 --- a/projects/rccl/Jenkinsfile +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env groovy -// Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. -// This shared library is available at https://github.com/ROCmSoftwarePlatform/rccl -@Library('rocJenkins@rccl') _ - -// This is file for internal AMD use. -// If you are interested in running your own Jenkins, please raise a github issue for assistance. - -import com.amd.project.* -import com.amd.docker.* - -//////////////////////////////////////////////////////////////////////// -// Mostly generated from snippet generator 'properties; set job properties' -// Time-based triggers added to execute nightly tests, eg '30 2 * * *' means 2:30 AM -properties([ - pipelineTriggers([cron('0 1 * * *'), [$class: 'PeriodicFolderTrigger', interval: '5m']]), - buildDiscarder(logRotator( - artifactDaysToKeepStr: '', - artifactNumToKeepStr: '', - daysToKeepStr: '', - numToKeepStr: '10')), - disableConcurrentBuilds(), - [$class: 'CopyArtifactPermissionProperty', projectNames: '*'] - ]) - - -//////////////////////////////////////////////////////////////////////// -import java.nio.file.Path; - -rcclCI: -{ - - def rccl = new rocProject('rccl') - // customize for project - rccl.paths.build_command = './install.sh' - - // Define test architectures, optional rocm version argument is available - def nodes = new dockerNodes(['rccl && ubuntu'], rccl) - - boolean formatCheck = false - - def compileCommand = - { - platform, project-> - - project.paths.construct_build_prefix() - def command = """#!/usr/bin/env bash - set -x - cd ${project.paths.project_build_prefix} - LD_LIBRARY_PATH=/opt/rocm/hcc/lib CXX= ${project.paths.build_command} -t - """ - - platform.runCommand(this,command) - } - - def testCommand = - { - platform, project-> - - def command = """#!/usr/bin/env bash - set -x - cd ${project.paths.project_build_prefix}/build/release/test/ - HSA_FORCE_FINE_GRAIN_PCIE=1 ./UnitTests --gtest_output=xml --gtest_color=yes - """ - - platform.runCommand(this,command) - //junit "${project.paths.project_build_prefix}/build/release/*.xml" - } - - def packageCommand = - { - platform, project-> - - def command = """ - set -x - cd ${project.paths.project_build_prefix}/build/release - make package - rm -rf package && mkdir -p package - mv *.deb package/ - """ - platform.runCommand(this,command) - platform.archiveArtifacts(this, """${project.paths.project_build_prefix}/build/release/package/*.deb""") - } - - buildProject(rccl, formatCheck, nodes.dockerArray, compileCommand, testCommand, packageCommand) - -}