AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transfer files from RAD repository

Browse Source
This commit is contained in:
Brandon Potter
2024-07-01 09:57:08 -05:00
parent a30da178c1
commit ea8f264a11
382 changed files with 67034 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/continuous_integration/compile/Jenkinsfile
Vendored
+118
View File
@@ -0,0 +1,118 @@
pipeline {
agent { label 'sv-pdp-5' }
environment {
HSA_FORCE_FINE_GRAIN_PCIE = 1
MPI_HOME="/home/resperf/mpich-4.0.1/install/global"
PATH = "$MPI_HOME/bin:$PATH"
LD_LIBRARY_PATH = "$MPI_HOME/lib:$LD_LIBRARY_PATH"
build_dir = "builds/change-${GERRIT_CHANGE_NUMBER}-${GERRIT_PATCHSET_NUMBER}"
CMAKE_PREFIX_PATH = "/opt/rocm/lib/cmake"
}
stages {
stage('Synchronize Source Code') {
steps {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false]], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: '${GERRIT_REFSPEC}', url: 'ssh://gerritgit/rsch/ec/shmem']]]
}
}
stage('Make Build Directory') {
steps {
dir("library") {
sh "mkdir -p ${build_dir}"
}
}
}
stage('Build Source Code') {
parallel {
stage('RC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI_WF_COAL") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi_wf_coal install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
}
}
stage('RC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_MULTI") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
}
}
stage('DC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
}
}
stage('DC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
}
}
}
}
}
}
internal/continuous_integration/compile/compile_config.xml
+93
View File
@@ -0,0 +1,93 @@
<?xml version='1.1' encoding='UTF-8'?>
<flow-definition plugin="workflow-job@2.40">
<actions>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobAction plugin="pipeline-model-definition@1.8.4"/>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction plugin="pipeline-model-definition@1.8.4">
<jobProperties/>
<triggers/>
<parameters/>
<options/>
</org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction>
</actions>
<description></description>
<keepDependencies>false</keepDependencies>
<properties>
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger plugin="gerrit-trigger@2.33.0">
<spec></spec>
<gerritProjects>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
<compareType>PLAIN</compareType>
<pattern>rsch/ec/shmem</pattern>
<branches>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
<compareType>PLAIN</compareType>
<pattern>amd-master</pattern>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
</branches>
<disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
</gerritProjects>
<dynamicGerritProjects class="empty-list"/>
<skipVote>
<onSuccessful>true</onSuccessful>
<onFailed>true</onFailed>
<onUnstable>true</onUnstable>
<onNotBuilt>true</onNotBuilt>
<onAborted>true</onAborted>
</skipVote>
<silentMode>false</silentMode>
<enableTopicAssociation>false</enableTopicAssociation>
<notificationLevel></notificationLevel>
<silentStartMode>false</silentStartMode>
<escapeQuotes>true</escapeQuotes>
<nameAndEmailParameterMode>PLAIN</nameAndEmailParameterMode>
<dependencyJobsNames></dependencyJobsNames>
<commitMessageParameterMode>BASE64</commitMessageParameterMode>
<changeSubjectParameterMode>PLAIN</changeSubjectParameterMode>
<commentTextParameterMode>BASE64</commentTextParameterMode>
<buildStartMessage></buildStartMessage>
<buildFailureMessage></buildFailureMessage>
<buildSuccessfulMessage></buildSuccessfulMessage>
<buildUnstableMessage></buildUnstableMessage>
<buildNotBuiltMessage></buildNotBuiltMessage>
<buildAbortedMessage></buildAbortedMessage>
<buildUnsuccessfulFilepath></buildUnsuccessfulFilepath>
<customUrl></customUrl>
<serverName>amd-gerrit</serverName>
<triggerOnEvents>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
<commentAddedCommentContains>!COMPILE</commentAddedCommentContains>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
</triggerOnEvents>
<dynamicTriggerConfiguration>false</dynamicTriggerConfiguration>
<triggerConfigURL></triggerConfigURL>
<triggerInformationAction/>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
</triggers>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>ssh://gerritgit/rsch/ec/shmem</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>FETCH_HEAD</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions/>
</scm>
<scriptPath>internal/continuous_integration/compile/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
internal/continuous_integration/gdb/Jenkinsfile
Vendored
+221
View File
@@ -0,0 +1,221 @@
pipeline {
agent { label 'sv-pdp-5' }
environment {
HSA_FORCE_FINE_GRAIN_PCIE = 1
MPI_HOME="/home/resperf/mpich-4.0.1/install/global"
PATH = "$MPI_HOME/bin:$PATH"
LD_LIBRARY_PATH = "$MPI_HOME/lib:$LD_LIBRARY_PATH"
build_dir = "builds/change-${GERRIT_CHANGE_NUMBER}-${GERRIT_PATCHSET_NUMBER}"
CMAKE_PREFIX_PATH = "/opt/rocm/lib/cmake"
}
stages {
stage('Synchronize Source Code') {
steps {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false]], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: '${GERRIT_REFSPEC}', url: 'ssh://gerritgit/rsch/ec/shmem']]]
}
}
stage('Env Variables') {
steps {
sh 'printenv'
}
}
stage('Make Build Directory') {
steps {
dir("library") {
sh "mkdir -p ${build_dir}"
}
}
}
stage('Build Source Code') {
failFast true
parallel {
stage('RC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RC_SINGLE") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
//}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI_WF_COAL") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi_wf_coal install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RC_MULTI_WF_COAL") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
//}
}
}
stage('RC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RC_MULTI") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
//}
}
}
stage('DC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/DC_SINGLE") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
//}
}
}
stage('DC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/DC_MULTI") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
//}
}
}
}
}
stage('Run Tests') {
stages {
stage('RC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE true'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/RC_SINGLE/spts single_thread ${build_dir}/RC_SINGLE'
//}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI_WF_COAL true'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/spts multi_thread ${build_dir}/RC_MULTI_WF_COAL'
//}
}
}
stage('RC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI true'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RC_MULTI'
//}
}
}
stage('DC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/DC_SINGLE true'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/DC_SINGLE/spts single_thread ${build_dir}/DC_SINGLE'
//}
}
}
stage('DC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI true'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/DC_MULTI/spts multi_thread ${build_dir}/DC_MULTI'
//}
}
}
stage('RO_NET_BASIC') {
// RO_NET controlled at runtime, no need for a new build. Use RC_MULTI
steps {
dir("clients/functional_tests") {
sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver ro ${build_dir}/RO_NET_BASIC true'
}
//dir("internal/clients/spts") {
// sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
// sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RO_NET_BASIC'
//}
}
}
}
}
stage('Generate Checker Metadata') {
steps {
dir("library/${build_dir}") {
sh 'git fetch --tags'
sh 'git log --pretty=oneline remotes/origin/amd-master.. > changeset_delta.txt'
sh 'git log --pretty=oneline remotes/origin/amd-master~1..remotes/origin/amd-master >> changeset_delta.txt'
}
}
}
stage('Archive Artifacts') {
steps {
dir("library/${build_dir}") {
archiveArtifacts artifacts: 'changeset_delta.txt'
}
dir("clients/functional_tests/${build_dir}") {
archiveArtifacts artifacts: 'RC_SINGLE/**/*.log'
archiveArtifacts artifacts: 'RC_MULTI/**/*.log'
archiveArtifacts artifacts: 'DC_SINGLE/**/*.log'
archiveArtifacts artifacts: 'DC_MULTI/**/*.log'
archiveArtifacts artifacts: 'RO_NET_BASIC/**/*.log'
}
}
}
}
}
internal/continuous_integration/long/Jenkinsfile
Vendored
+413
View File
@@ -0,0 +1,413 @@
pipeline {
agent { label 'sv-pdp-5' }
environment {
build_dir = "builds/change-${GERRIT_CHANGE_NUMBER}-${GERRIT_PATCHSET_NUMBER}"
MPI_HOME="/home/resperf/mpich-4.0.1/install/global"
UCX_HOME="/home/resperf/ucx/install"
PATH="$MPI_HOME/bin:$UCX_HOME/bin:$PATH"
LD_LIBRARY_PATH="$MPI_HOME/lib:$UCX_HOME/lib:$LD_LIBRARY_PATH"
PKG_CONFIG_PATH="$MPI_HOME/lib/pkgconfig:$UCX_HOME/lib/pkgconfig"
CMAKE_PREFIX_PATH="/opt/rocm/lib/cmake"
UCX_WARN_UNUSED_ENV_VARS="n"
HSA_FORCE_FINE_GRAIN_PCIE=1
}
stages {
stage('Synchronize Source Code') {
steps {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false]], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: '${GERRIT_REFSPEC}', url: 'ssh://gerritgit/rsch/ec/shmem']]]
}
}
stage('Env Variables') {
steps {
sh 'printenv'
}
}
stage('Make Build Directory') {
steps {
dir("library") {
sh "mkdir -p ${build_dir}"
}
}
}
stage('Build Source Code') {
failFast true
parallel {
stage('RC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
dir("clients/sos_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI_WF_COAL") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi_wf_coal install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
dir("clients/sos_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
}
}
stage('RC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
dir("clients/sos_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_MULTI") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
}
}
stage('RC_SINGLE_DEBUG') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE_DEBUG") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single_debug install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE_DEBUG") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE_DEBUG/install'
}
dir("clients/sos_tests/${build_dir}/RC_SINGLE_DEBUG") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE_DEBUG/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE_DEBUG") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE_DEBUG/install'
}
}
}
stage('RC_SINGLE_PROFILE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE_PROFILE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single_profile install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE_PROFILE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE_PROFILE/install'
}
dir("clients/sos_tests/${build_dir}/RC_SINGLE_PROFILE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE_PROFILE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE_PROFILE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE_PROFILE/install'
}
}
}
stage('DC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
dir("clients/sos_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
}
}
stage('DC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
dir("clients/sos_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
}
}
stage('DC_MULTI_IPC') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI_IPC") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi_ipc install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI_IPC") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_IPC/install'
}
dir("clients/sos_tests/${build_dir}/DC_MULTI_IPC") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_IPC/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI_IPC") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI_IPC/install'
}
}
}
stage('DC_MULTI_DEBUG') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI_DEBUG") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi_debug install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI_DEBUG") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_DEBUG/install'
}
dir("clients/sos_tests/${build_dir}/DC_MULTI_DEBUG") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_DEBUG/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI_DEBUG") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI_DEBUG/install'
}
}
}
stage('DC_MULTI_PROFILE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI_PROFILE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi_profile install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI_PROFILE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_PROFILE/install'
}
dir("clients/sos_tests/${build_dir}/DC_MULTI_PROFILE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_PROFILE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI_PROFILE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI_PROFILE/install'
}
}
}
}
}
stage('Run Tests') {
stages {
stage('RC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE all ${build_dir}/RC_SINGLE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_SINGLE/spts single_thread ${build_dir}/RC_SINGLE'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI_WF_COAL'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL all ${build_dir}/RC_MULTI_WF_COAL'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/spts multi_thread ${build_dir}/RC_MULTI_WF_COAL'
}
}
}
stage('RC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_MULTI all ${build_dir}/RC_MULTI'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RC_MULTI'
}
}
}
stage('RC_SINGLE_DEBUG') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE_DEBUG/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE_DEBUG'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE_DEBUG all ${build_dir}/RC_SINGLE_DEBUG'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_SINGLE_DEBUG/spts single_thread ${build_dir}/RC_SINGLE_DEBUG'
}
}
}
stage('RC_SINGLE_PROFILE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE_PROFILE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE_PROFILE'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE_PROFILE all ${build_dir}/RC_SINGLE_PROFILE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_SINGLE_PROFILE/spts single_thread ${build_dir}/RC_SINGLE_PROFILE'
}
}
}
stage('DC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/DC_SINGLE'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE all ${build_dir}/DC_SINGLE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_SINGLE/spts single_thread ${build_dir}/DC_SINGLE'
}
}
}
stage('DC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_MULTI all ${build_dir}/DC_MULTI'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI/spts multi_thread ${build_dir}/DC_MULTI'
}
}
}
stage('DC_MULTI_IPC') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_IPC/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI_IPC'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_IPC all ${build_dir}/DC_MULTI_IPC'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI_IPC/spts multi_thread ${build_dir}/DC_MULTI_IPC'
}
}
}
stage('DC_MULTI_DEBUG') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_DEBUG/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI_DEBUG'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_DEBUG all ${build_dir}/DC_MULTI_DEBUG'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI_DEBUG/spts multi_thread ${build_dir}/DC_MULTI_DEBUG'
}
}
}
stage('DC_MULTI_PROFILE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_PROFILE/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI_PROFILE'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_PROFILE all ${build_dir}/DC_MULTI_PROFILE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI_PROFILE/spts multi_thread ${build_dir}/DC_MULTI_PROFILE'
}
}
}
stage('RO_NET_BASIC') {
// RO_NET controlled at runtime, no need for a new build. Use RC_MULTI
steps {
dir("clients/functional_tests") {
sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver ro ${build_dir}/RO_NET_BASIC'
}
dir("clients/sos_tests") {
sh 'ROC_SHMEM_RO=1 ./driver.sh ${build_dir}/RC_MULTI all ${build_dir}/RC_MULTI'
}
dir("internal/clients/spts") {
sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RO_NET_BASIC'
}
}
}
}
}
}
}
internal/continuous_integration/long/long_config.xml
+45
View File
@@ -0,0 +1,45 @@
<?xml version='1.1' encoding='UTF-8'?>
<flow-definition plugin="workflow-job@2.40">
<actions>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobAction plugin="pipeline-model-definition@1.8.4"/>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction plugin="pipeline-model-definition@1.8.4">
<jobProperties/>
<triggers/>
<parameters/>
<options/>
</org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction>
</actions>
<description></description>
<keepDependencies>false</keepDependencies>
<properties>
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<hudson.triggers.TimerTrigger>
<spec>H 22 * * *</spec>
</hudson.triggers.TimerTrigger>
</triggers>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>ssh://gerritgit/rsch/ec/shmem</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>*/amd-master</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions/>
</scm>
<scriptPath>internal/continuous_integration/nightly/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
internal/continuous_integration/nightly/Jenkinsfile
Vendored
+335
View File
@@ -0,0 +1,335 @@
pipeline {
agent { label 'sv-pdp-5' }
environment {
HSA_FORCE_FINE_GRAIN_PCIE = 1
MPI_HOME="/home/resperf/mpich-4.0.1/install/global"
PATH = "$MPI_HOME/bin:$PATH"
LD_LIBRARY_PATH = "$MPI_HOME/lib:$LD_LIBRARY_PATH"
build_dir = "builds/${BUILD_ID}"
CMAKE_PREFIX_PATH = "/opt/rocm/lib/cmake"
}
stages {
stage('Synchronize Source Code') {
steps {
git branch: 'amd-master', changelog: false, poll: false, url: 'ssh://gerritgit/rsch/ec/shmem'
}
}
stage('Make Build Directory') {
steps {
dir("library") {
sh "mkdir -p ${build_dir}"
}
}
}
stage('Build Source Code') {
parallel {
stage('RC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI_WF_COAL") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi_wf_coal install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
}
}
stage('RC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_MULTI") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
}
}
stage('RC_SINGLE_DEBUG') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE_DEBUG") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single_debug install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE_DEBUG") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE_DEBUG/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE_DEBUG") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE_DEBUG/install'
}
}
}
stage('RC_SINGLE_PROFILE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE_PROFILE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single_profile install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE_PROFILE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE_PROFILE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/RC_SINGLE_PROFILE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE_PROFILE/install'
}
}
}
stage('DC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
}
}
stage('DC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
}
}
stage('DC_MULTI_IPC') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI_IPC") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi_ipc install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI_IPC") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_IPC/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI_IPC") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI_IPC/install'
}
}
}
stage('DC_MULTI_DEBUG') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI_DEBUG") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi_debug install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI_DEBUG") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_DEBUG/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI_DEBUG") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI_DEBUG/install'
}
}
}
stage('DC_MULTI_PROFILE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI_PROFILE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi_profile install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI_PROFILE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI_PROFILE/install'
}
//===================== SPTS ==========================
dir("internal/clients/spts/${build_dir}/DC_MULTI_PROFILE") {
sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI_PROFILE/install'
}
}
}
}
}
stage('Run Tests') {
stages {
stage('RC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_SINGLE/spts single_thread ${build_dir}/RC_SINGLE'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI_WF_COAL'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/spts multi_thread ${build_dir}/RC_MULTI_WF_COAL'
}
}
}
stage('RC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RC_MULTI'
}
}
}
stage('RC_SINGLE_DEBUG') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE_DEBUG/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE_DEBUG'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_SINGLE_DEBUG/spts single_thread ${build_dir}/RC_SINGLE_DEBUG'
}
}
}
stage('RC_SINGLE_PROFILE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE_PROFILE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE_PROFILE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/RC_SINGLE_PROFILE/spts single_thread ${build_dir}/RC_SINGLE_PROFILE'
}
}
}
stage('DC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/DC_SINGLE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_SINGLE/spts single_thread ${build_dir}/DC_SINGLE'
}
}
}
stage('DC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI/spts multi_thread ${build_dir}/DC_MULTI'
}
}
}
stage('DC_MULTI_IPC') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_IPC/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI_IPC'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI_IPC/spts multi_thread ${build_dir}/DC_MULTI_IPC'
}
}
}
stage('DC_MULTI_DEBUG') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_DEBUG/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI_DEBUG'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI_DEBUG/spts multi_thread ${build_dir}/DC_MULTI_DEBUG'
}
}
}
stage('DC_MULTI_PROFILE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI_PROFILE/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI_PROFILE'
}
dir("internal/clients/spts") {
sh './driver.sh ${build_dir}/DC_MULTI_PROFILE/spts multi_thread ${build_dir}/DC_MULTI_PROFILE'
}
}
}
stage('RO_NET_BASIC') {
// RO_NET controlled at runtime, no need for a new build. Use RC_MULTI
steps {
dir("clients/functional_tests") {
sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver ro ${build_dir}/RO_NET_BASIC'
}
dir("internal/clients/spts") {
sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RO_NET_BASIC'
}
}
}
}
}
}
}
internal/continuous_integration/nightly/nightly_config.xml
+45
View File
@@ -0,0 +1,45 @@
<?xml version='1.1' encoding='UTF-8'?>
<flow-definition plugin="workflow-job@2.40">
<actions>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobAction plugin="pipeline-model-definition@1.8.4"/>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction plugin="pipeline-model-definition@1.8.4">
<jobProperties/>
<triggers/>
<parameters/>
<options/>
</org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction>
</actions>
<description></description>
<keepDependencies>false</keepDependencies>
<properties>
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<hudson.triggers.TimerTrigger>
<spec>H 22 * * *</spec>
</hudson.triggers.TimerTrigger>
</triggers>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>ssh://gerritgit/rsch/ec/shmem</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>*/amd-master</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions/>
</scm>
<scriptPath>internal/continuous_integration/nightly/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
internal/continuous_integration/short/Jenkinsfile
Vendored
+288
View File
@@ -0,0 +1,288 @@
pipeline {
agent { label 'sv-pdp-7' }
environment {
build_dir = "builds/change-${GERRIT_CHANGE_NUMBER}-${GERRIT_PATCHSET_NUMBER}"
MPI_HOME="/home/resperf/mpich/install"
UCX_HOME="/home/resperf/ucx/install"
PATH="$MPI_HOME/bin:$UCX_HOME/bin:$PATH"
LD_LIBRARY_PATH="$MPI_HOME/lib:$UCX_HOME/lib:$LD_LIBRARY_PATH"
PKG_CONFIG_PATH="$MPI_HOME/lib/pkgconfig:$UCX_HOME/lib/pkgconfig"
CMAKE_PREFIX_PATH="/opt/rocm/lib/cmake"
UCX_WARN_UNUSED_ENV_VARS="n"
HSA_FORCE_FINE_GRAIN_PCIE=1
UCX_TLS="rc"
ROC_SHMEM_USE_SQ_GPU_MEM=0
ROC_SHMEM_USE_CQ_GPU_MEM=0
ROC_SHMEM_NUM_BLOCKS=128
}
stages {
stage('Synchronize Source Code') {
steps {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false]], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: '${GERRIT_REFSPEC}', url: 'ssh://gerritgit/rsch/ec/shmem']]]
}
}
stage('Env Variables') {
steps {
sh 'printenv'
}
}
stage('Make Build Directory') {
steps {
dir("library") {
sh "mkdir -p ${build_dir}"
}
}
}
stage('Build Source Code') {
failFast true
parallel {
stage('RC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
dir("clients/sos_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RC_SINGLE") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
//}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI_WF_COAL") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi_wf_coal install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
dir("clients/sos_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RC_MULTI_WF_COAL") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
//}
}
}
stage('RC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
dir("clients/sos_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RC_MULTI") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
//}
}
}
stage('DC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
dir("clients/sos_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/DC_SINGLE") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
//}
}
}
stage('DC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
dir("clients/sos_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/DC_MULTI") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
//}
}
}
stage('RO_NET') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RO_NET") {
sh 'mkdir -p install'
sh '../../../build_configs/ro_net install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RO_NET") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RO_NET/install'
}
dir("clients/sos_tests/${build_dir}/RO_NET") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RO_NET/install'
}
//===================== SPTS ==========================
//dir("internal/clients/spts/${build_dir}/RO_NET") {
// sh '../../../build_configs/analyze_single_rocshmem ${WORKSPACE}/library/${build_dir}/RO_NET/install'
//}
}
}
}
}
stage('Run Tests') {
stages {
stage('RC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE short ${build_dir}/RC_SINGLE'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/RC_SINGLE/spts single_thread ${build_dir}/RC_SINGLE'
//}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI_WF_COAL'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL short ${build_dir}/RC_MULTI_WF_COAL'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/spts multi_thread ${build_dir}/RC_MULTI_WF_COAL'
//}
}
}
stage('RC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/RC_MULTI short ${build_dir}/RC_MULTI'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/RC_MULTI/spts multi_thread ${build_dir}/RC_MULTI'
//}
}
}
stage('DC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/DC_SINGLE'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE short ${build_dir}/DC_SINGLE'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/DC_SINGLE/spts single_thread ${build_dir}/DC_SINGLE'
//}
}
}
stage('DC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI'
}
dir("clients/sos_tests") {
sh './driver.sh ${build_dir}/DC_MULTI short ${build_dir}/DC_MULTI'
}
//dir("internal/clients/spts") {
// sh './driver.sh ${build_dir}/DC_MULTI/spts multi_thread ${build_dir}/DC_MULTI'
//}
}
}
stage('RO_NET') {
steps {
dir("clients/functional_tests") {
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RO_NET/rocshmem_example_driver ro ${build_dir}/RO_NET'
}
dir("clients/sos_tests") {
sh 'ROC_SHMEM_RO=1 ./driver.sh ${build_dir}/RO_NET short ${build_dir}/RO_NET'
}
//dir("internal/clients/spts") {
// sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RO_NET/spts multi_thread ${build_dir}/RO_NET'
//}
}
}
}
}
stage('Generate Checker Metadata') {
steps {
dir("library/${build_dir}") {
sh 'git fetch --tags'
sh 'git log --pretty=oneline remotes/origin/amd-master.. > changeset_delta.txt'
sh 'git log --pretty=oneline remotes/origin/amd-master~1..remotes/origin/amd-master >> changeset_delta.txt'
}
}
}
stage('Archive Artifacts') {
steps {
dir("library/${build_dir}") {
archiveArtifacts artifacts: 'changeset_delta.txt'
}
dir("clients/functional_tests/${build_dir}") {
archiveArtifacts artifacts: 'RC_SINGLE/**/*.log'
archiveArtifacts artifacts: 'RC_MULTI/**/*.log'
archiveArtifacts artifacts: 'DC_SINGLE/**/*.log'
archiveArtifacts artifacts: 'DC_MULTI/**/*.log'
archiveArtifacts artifacts: 'RO_NET/**/*.log'
}
}
}
}
post {
success {
build job: 'shmem_perf_check', wait: true
}
}
}
internal/continuous_integration/short/__init__.py
View File
internal/continuous_integration/short/absolute_path.py
+21
View File
@@ -0,0 +1,21 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import glob
import pprint
class PathGlobber():
def __init__(self, name, *partial_paths_to_concatenate):
self._search_path = ''
for partial_path in partial_paths_to_concatenate:
self._search_path += partial_path
self.dirs = []
self._name = name
def generate(self):
self.dirs = glob.glob(self._search_path, recursive=True)
def dump(self):
str_out = self._name
str_out += pprint.pformat(self.dirs, width=120)
str_out += '\n'
return str_out
internal/continuous_integration/short/archive_path.py
+15
View File
@@ -0,0 +1,15 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import absolute_path
import glob
class Archive(absolute_path.PathGlobber):
def __init__(self, args, name=''):
archive_path = args.archive_path
super().__init__(name, args.jenkins_path, archive_path,
args.benchmark_path)
def path_of_build(self, build_id):
path = self._search_path.replace('*/archive', build_id + '/archive')
path = glob.glob(path)
return path[0]
internal/continuous_integration/short/check_perf_delta.py
Executable
+54
View File
@@ -0,0 +1,54 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import parser
import dictionary
import archive_path
import checker
def main():
# This script accepts command line values, but has reasonable defaults
# needed to run as part of the CI infrastructure.
p = parser.Parser()
args = p.parse_command_line()
# Jenkins is configured to archive build artifacts in a directory.
# The 'archives' variable holds the set of directories for
# successful Jenkins builds (those which run to completion).
# Partitioning of successful builds is useful since we can ignore
# failed build directories while searching for performance data.
archives = archive_path.Archive(args)
archives.generate()
print(archives.dump())
# Jenkins records changeset information in a changeset_delta.txt file.
# We parse the changelog for the commit hash and save it into
# 'builds_to_changesets'.
build_to_changeset = dictionary.BuildToChangesetDict()
build_to_changeset.generate(archives.dirs)
print(build_to_changeset.dump())
# 'changeset_to_build' holds the changeset mappings with a
# list of build numbers that match the changeset value.
# Builds may be executed many times with the same changeset.
# The most recent build (identified by the largest build number) will
# be used to retrieve performance data.
changeset_to_build = dictionary.ChangesetToBuildDict()
changeset_to_build.generate(build_to_changeset)
print(changeset_to_build.dump())
# Jenkins is configured to dump Gerrit-esque relation chain changesets
# to an archived output file 'changeset-delta.txt'.
# The relation chain will be used to determine changeset performance
# data for each changeset in the relation chain (when possible).
build_to_relation_chain = dictionary.BuildToRelationChainDict()
build_to_relation_chain.generate(archives.dirs)
print(build_to_relation_chain.dump())
perf_checker = checker.Performance(args,
archives,
changeset_to_build,
build_to_relation_chain)
perf_checker.run()
if __name__ == '__main__':
main()
internal/continuous_integration/short/checker.py
+97
View File
@@ -0,0 +1,97 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import archive_path
import log
import dictionary
import report
import violation
class Performance():
def __init__(self, args, archives, changeset_to_build,
build_to_relation_chain):
self._args = args
self._archives = archives
self._changeset_to_build = changeset_to_build
self._build_to_relation_chain = build_to_relation_chain
self._build_id = build_to_relation_chain.most_recent_build()
self._archive_path = archives.path_of_build(self._build_id)
self._output = report.Report(self._build_id,
self._archive_path,
'performance_diff.txt')
def _other_build_id(self, other_changeset):
packed_id = [build_id for chng,
build_id in self._changeset_to_build.data.items()
if chng.startswith(other_changeset)]
# The 'packed_id' variable is a list containing lists.
# We need the content inside the packed_id data structure.
try:
build_id = packed_id[0][0]
return True, build_id
except IndexError:
# An index error can occur if builds in the relation chain
# have not been tested before attempting to test this
# changeset.
return False, 0
def _log_difference(self, log_filename, other_changeset,
other_archive_path, violations):
print('determining difference of log file ' + log_filename)
self._output.record(log_filename)
current_file_path = self._archive_path + '/' + log_filename
other_file_path = other_archive_path + '/' + log_filename
log_pair = log.Pair(current_file_path, other_file_path)
log_pair.calculate_differences()
latency_perc = [float(i.strip('%')) \
for i in log_pair.latency_percentage_differences]
max_latency = max(latency_perc)
violations.check(max_latency, other_changeset, log_filename)
self._output.record(log_pair.dump())
def _changeset_difference(self, current_changeset, other_changeset):
violations = violation.Threshold(self._args.latency_max, 'latency')
change_pair = '(' + current_changeset + ',' + other_changeset + ')'
print('comparing changesets ' + change_pair)
self._output.record(change_pair)
status, other_build_id = self._other_build_id(other_changeset)
if status == False:
message = 'skipping changeset ' + other_changeset
print(message)
self._output.record(message)
return violations
other_archive_path = self._archives.path_of_build(other_build_id)
print(self._archive_path)
print(other_archive_path)
for filename in self._args.logs:
self._log_difference(filename, other_changeset,
other_archive_path, violations)
print('\n')
return violations
def _calculate_performance_differences(self):
current_changeset = \
self._build_to_relation_chain.data[self._build_id][0]
other_changesets = \
self._build_to_relation_chain.data[self._build_id][1:]
for other_changeset in other_changesets:
violations = self._changeset_difference(current_changeset,
other_changeset)
# Only report on the last pairwise changeset combination.
# This combination represents the changeset being tested and
# the amd-master:HEAD.
violations.provide_violations_to_report(self._output)
def run(self):
self._output.open()
self._calculate_performance_differences()
internal/continuous_integration/short/dictionary.py
+163
View File
@@ -0,0 +1,163 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import abc
import os
import pprint
import subprocess
import sys
class BaseDict(metaclass=abc.ABCMeta):
def __init__(self):
self.data = {}
self._delimiter_path = 'archive'
self._changeset_delta_filename = 'changeset_delta.txt'
def _build_id(self, build_directory):
sub_directory_strings = build_directory.split('/')
word_count = 0
for word in sub_directory_strings:
if word == '':
continue
if word == self._delimiter_path:
break
word_count += 1
bld_id = sub_directory_strings[word_count]
return bld_id
def _open_changeset_delta_file(self, archive_directory):
build_directory, config_directory = os.path.split(archive_directory)
changeset_file_path = build_directory + '/' + \
self._changeset_delta_filename
try:
file_handle = open(changeset_file_path, 'r')
except:
sys.exit('failed to open: ' + changeset_file_path)
return file_handle
@abc.abstractmethod
def _changeset_delta_operations(self, file_handle, bld_id):
pass
def generate(self, archives):
for d in archives:
bld_id = self._build_id(d)
f = self._open_changeset_delta_file(d)
self._changeset_delta_operations(f, bld_id)
def most_recent_build(self):
build_id_strings = self.data.keys()
build_id_ints = list(map(int, build_id_strings))
most_recent_build_id_int = max(build_id_ints)
return str(most_recent_build_id_int)
def dump(self):
str_out = self._print_text
str_out += pprint.pformat(self.data, width=120)
str_out += '\n'
return str_out
class BuildToChangesetDict(BaseDict):
def __init__(self, name=''):
super().__init__()
self._print_text = name
def _changeset_delta_operations(self, file_handle, bld_id):
commit_line = file_handle.readline()
try:
commit_hash = commit_line.split()[0]
except IndexError:
commit_hash = None
if commit_hash != None:
self.data[bld_id] = commit_hash
class BuildToRelationChainDict(BaseDict):
def __init__(self, name=''):
super().__init__()
self._print_text = name
def _changeset_delta_operations(self, file_handle, bld_id):
changes = []
for line in file_handle:
changes.append(line.split()[0])
self.data[bld_id] = changes
class ChangesetToBuildDict():
def __init__(self, name=''):
self.data = {}
self._print_text = name
def _invert_dict(self, dictionary):
dict_with_duplicates = {}
for key, value in dictionary.data.items():
list_with_duplicates = dict_with_duplicates.get(value, [])
list_with_duplicates.append(key)
dict_with_duplicates[value] = list_with_duplicates
return dict_with_duplicates
def generate(self, dictionary):
self.data = self._invert_dict(dictionary)
def dump(self):
str_out = self._print_text
str_out += pprint.pformat(self.data, width=120)
str_out += '\n'
return str_out
class ChangelogToMostRecentBuild():
def __init__(self, name=''):
self._print_text = name
self._all_changesets = []
self._changesets_with_builds = []
self._changesets_without_builds = []
self.data = {}
def _build_id(self, changeset_to_build, changeset):
try:
build_id_strings = changeset_to_build.data[changeset]
build_id_ints = list(map(int, build_id_strings))
most_recent_build_id_int = max(build_id_ints)
build_id_str = str(most_recent_build_id_int)
except:
build_id_str = ''
return build_id_str
def _changelog(self):
# print git hash along with file modification stats
shellcmd = 'git log --pretty=tformat:"%H" --shortstat | '
# condense the output down to single line
shellcmd += "awk 'ORS=NR%3?\" \":\"\\n\"' | "
# parse out the git hash by itself
shellcmd += "awk '{print $1}'"
x = subprocess.getoutput(shellcmd)
self._all_changesets = x.split()
def _with_builds(self, changeset_to_build):
changesets = list(changeset_to_build.data.keys())
self._changesets_with_builds = changesets
def _without_builds(self):
self._changesets_without_builds = \
list(set(self._all_changesets) - \
set(self._changesets_with_builds))
def generate(self, changeset_to_build):
self._changelog()
self._with_builds(changeset_to_build)
self._without_builds()
for changeset in self._all_changesets:
if changeset in self._changesets_with_builds:
build = self._build_id(changeset_to_build, changeset)
self.data[changeset] = build
def dump(self):
str_out = self._print_text
str_out += 'git-log_changesets_in_order:\n'
str_out += pprint.pformat(self._all_changesets, width=120)
str_out += '\nfilesystem_with_builds:\n'
str_out += pprint.pformat(self._changesets_with_builds, width=120)
str_out += '\nfilesystem_without_builds:\n'
str_out += pprint.pformat(self._changesets_without_builds, width=120)
str_out += '\ngit-log_changesets_to_build-id_mappings:\n'
str_out += pprint.pformat(self.data, width=120)
str_out += '\n'
return str_out
internal/continuous_integration/short/log.py
+107
View File
@@ -0,0 +1,107 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import pprint
import re
import sys
class Log():
def __init__(self, logfile_abspath):
self._file_path = logfile_abspath
self.latency = []
self.bandwidth = []
# regex matches the latency and bandwidth lines in the log files
self._regex = '.*[0-9]+\.[0-9]+.*[0-9]\.[0-9].*'
def open(self):
try:
self._file_handle = open(self._file_path, 'r')
except:
sys.exit('failed to open: ' + self._file_path)
def parse(self):
for line in self._file_handle:
if re.match(self._regex, line):
entries = line.split()
self.latency.append(round(float(entries[0]), 4))
self.bandwidth.append(round(float(entries[1]), 4))
class Pair():
def __init__(self, first_logfile_abspath, second_logfile_abspath):
self.first = Log(first_logfile_abspath)
self.first.open()
self.first.parse()
self.second = Log(second_logfile_abspath)
self.second.open()
self.second.parse()
def _ratio(self, a, b):
diff = [round((x - y), 4) for x, y in zip(a, b)]
ratio = []
for numerator, denominator in zip(diff, a):
try:
ratio.append(round(numerator / denominator, 4))
except:
ratio.append(float(0.0000))
return ratio
def _percent(self, ratio):
perc = ['{0:.2%}'.format(x) for x in ratio]
return perc
def _percentage_difference(self, a, b):
ratio = self._ratio(a, b)
percent = self._percent(ratio)
return percent
def calculate_differences(self):
self.latency_percentage_differences = \
self._percentage_difference(self.first.latency,
self.second.latency)
self.bandwidth_percentage_differences = \
self._percentage_difference(self.first.bandwidth,
self.second.bandwidth)
def dump(self):
delim = ', '
output = '\tlatency:'
output += '\n\t\t'
output += delim.join(map(str, self.first.latency))
output += '\n\t\t'
output += delim.join(map(str, self.second.latency))
output += '\n\t\t'
output += delim.join(map(str, self.latency_percentage_differences))
output += '\n\tbandwidth:'
output += '\n\t\t'
output += delim.join(map(str, self.first.bandwidth))
output += '\n\t\t'
output += delim.join(map(str, self.second.bandwidth))
output += '\n\t\t'
output += delim.join(map(str, self.bandwidth_percentage_differences))
return output
class Tracker():
def __init__(self, args, archives):
self._args = args
self._archives = archives
self._data = {}
def add(self, changeset, most_recent_build_id):
archive_path = self._archives.path_of_build(most_recent_build_id)
for filename in self._args.logs:
abs_file_path = archive_path + '/' + filename
log = Log(abs_file_path)
log.open()
log.parse()
key = (changeset, filename)
self._data[key] = log
def dump(self):
out_str = ''
for key in self._data.keys():
log = self._data[key]
line_str = pprint.pformat(key, width=120)
line_str += ' = '
line_str += pprint.pformat(log.latency, width=120)
line_str += '\n'
out_str += line_str
return out_str
internal/continuous_integration/short/parser.py
+85
View File
@@ -0,0 +1,85 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import argparse
class Parser():
def __init__(self):
# A parent directory containing log file output from one of the
# configuration runs. The output directories are intended to
# be symmetric in naming with the various configurations supplied
# by the library's build_configs.
self._default_config = 'RC_SINGLE'
# The list of log files which need to be checked for performance
# differences.
self._default_logs = ['get.log',
'get_nbi.log',
'get_swarm.log',
'put.log',
'put_nbi.log']
# The maximum pairwise difference for the log file latencies.
self._default_latency_max = 5.0
# The minimum bandwidth difference for the log file bandwidths.
self._default_bandwidth_min = -50.0
# The Jenkins tester archives slave output on the master's
# filesystem which currently uses this top-level path (as the
# resperf account).
self._default_jenkins_path = \
'/proj/radl_extra/users/resperf/jenkins-2.192/'
# The performance tester runs as part of the 'short' job to
# verify that no performance degradation has occurred between
# commits. This archive path is the generic archive path
# for all of the builds. The Kleene star is used as a place
# holder for the Jenkins build number.
self._default_archive_path = \
'jobs/shmem_short/builds/*/archive/'
# The default benchmark path can be used to alter archive
# output placement. Currently, this is initialized to an empty
# string, but subsequently initialized to inject the config
# path.
self._default_benchmark_path = ''
def setup_options(self, argparser):
argparser.add_argument('-j',
dest='jenkins_path',
default=self._default_jenkins_path)
argparser.add_argument('-a',
dest='archive_path',
default=self._default_archive_path)
argparser.add_argument('-b',
dest='benchmark_path',
default=self._default_benchmark_path)
argparser.add_argument('-c',
dest='config',
default=self._default_config)
argparser.add_argument('-l',
dest='logs',
nargs='*',
default=self._default_logs)
argparser.add_argument('-x',
dest='latency_max',
type=float,
default=self._default_latency_max)
argparser.add_argument('-y',
dest='bandwidth_min',
type=float,
default=self._default_bandwidth_min)
argparser.add_argument('-o',
dest='one_changeset')
argparser.add_argument('-r',
dest='changeset_range',
nargs=2,
metavar=("most_recent_changeset", "least_recent_changeset"))
return argparser
def parse_command_line(self):
p = argparse.ArgumentParser()
p = self.setup_options(p)
args = p.parse_args()
args.benchmark_path = args.config + args.benchmark_path
return args
internal/continuous_integration/short/perf_config.xml
+30
View File
@@ -0,0 +1,30 @@
<?xml version='1.1' encoding='UTF-8'?>
<project>
<actions/>
<description>shmem performance delta checker</description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<assignedNode>master</assignedNode>
<canRoam>false</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders>
<hudson.tasks.Shell>
<command>/proj/radl_extra/users/resperf/jenkins-2.192/workspace/shmem_short@script/internal/continuous_integration/short/check_perf_delta.py -c &quot;RC_SINGLE&quot; -x &quot;15.0&quot; -l put.log put_nbi.log get.log get_nbi.log amo_add.log amo_fadd.log amo_fcswap.log amo_fetch.log amo_finc.log amo_inc.log ping_pong.log
/proj/radl_extra/users/resperf/jenkins-2.192/workspace/shmem_short@script/internal/continuous_integration/short/check_perf_delta.py -c &quot;RC_MULTI&quot; -x &quot;15.0&quot; -l put.log put_nbi.log get.log get_nbi.log amo_add.log amo_fadd.log amo_fcswap.log amo_fetch.log amo_finc.log amo_inc.log ping_pong.log get_swarm.log
/proj/radl_extra/users/resperf/jenkins-2.192/workspace/shmem_short@script/internal/continuous_integration/short/check_perf_delta.py -c &quot;DC_SINGLE&quot; -x &quot;15.0&quot; -l put.log put_nbi.log get.log get_nbi.log ping_pong.log
/proj/radl_extra/users/resperf/jenkins-2.192/workspace/shmem_short@script/internal/continuous_integration/short/check_perf_delta.py -c &quot;DC_MULTI&quot; -x &quot;15.0&quot; -l put.log put_nbi.log get.log get_nbi.log ping_pong.log get_swarm.log
/proj/radl_extra/users/resperf/jenkins-2.192/workspace/shmem_short@script/internal/continuous_integration/short/check_perf_delta.py -c &quot;RO_NET_BASIC&quot; -x &quot;75.0&quot; -l put.log put_nbi.log get.log get_nbi.log ping_pong.log
</command>
<configuredLocalRules/>
</hudson.tasks.Shell>
</builders>
<publishers/>
<buildWrappers>
<hudson.plugins.timestamper.TimestamperBuildWrapper plugin="timestamper@1.12"/>
</buildWrappers>
</project>
internal/continuous_integration/short/plot.py
Executable
+35
View File
@@ -0,0 +1,35 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import parser
import dictionary
import archive_path
import plotter
def main():
p = parser.Parser()
args = p.parse_command_line()
archives = archive_path.Archive(args)
archives.generate()
print(archives.dump())
build_to_changeset = dictionary.BuildToChangesetDict()
build_to_changeset.generate(archives.dirs)
print(build_to_changeset.dump())
changeset_to_build = dictionary.ChangesetToBuildDict()
changeset_to_build.generate(build_to_changeset)
print(changeset_to_build.dump())
plot = plotter.Plot(args,
archives,
changeset_to_build)
# either plot with all the changesets or the slice provided
plot.changeset_slice()
if (args.one_changeset):
plot.one_changeset_plot()
if __name__ == '__main__':
main()
internal/continuous_integration/short/plotter.R
Executable
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env Rscript
# load the required libraries:
library(tidyverse)
library(RColorBrewer)
library(optparse)
# declare some helper functions
ggpreview <- function (..., device = "png") {
fname <- tempfile(fileext = paste0(".", device))
ggplot2::ggsave(filename = fname, device = device, ...)
system2("open", fname)
invisible(NULL)
}
set_right_order <- function(df) {
# reverse the order of the rows so that oldest commit is first
df <- df %>% map_df(rev)
# ensure that ggplot plots the x-axis in the right order
df$Commit <- factor(df$Commit, levels = unique(df$Commit))
return(df)
}
plot_and_save <- function(df, xval, yval, title, subtitle, xlabel, filename) {
p <- ggplot(df, aes_string(x=xval, y=yval, group=1)) +
geom_line(size = 0.5, color=mycolors[1]) +
geom_point(size = 1.5, alpha = 1, color=mycolors[2]) +
theme_minimal() +
expand_limits(y=0) +
xlab(xlabel) +
ggtitle(title, subtitle = subtitle) +
theme(
axis.text.x = element_text(angle=90,hjust=1),
axis.title.y = element_blank()
) +
scale_fill_manual(values = mycolors)
#ggpreview(width=7.5, height=5, units="in", dpi=500)
ggsave(filename, p, device=pdf, dpi=500)
}
## Set up options ##
option_list = list(
make_option(c("-o", "--output"), type="character", default=NULL, action="store",
help="path (without trailing /) to a folder that will
contain the plots", metavar="folder-path"),
make_option(c("-a", "--changeset_a"), type="character", default=NULL, action="store",
help="beginning (inclusive) changeset of slice", metavar="changeset"),
make_option(c("-b", "--changeset_b"), type="character", default=NULL, action="store",
help="ending (inclusive) changeset of slice", metavar="changeset"),
make_option(c("-c", "--one_changeset"), type="character", default=NULL, action="store",
help="if set, will prepare plots for one changeset; if not, plots for a changeset slice")
)
## SCRIPT START ##
# parse the options
opt_parser <- OptionParser(option_list=option_list)
opts <- parse_args(opt_parser)
if (is.null(opts$output)) {
print_help(opt_parser)
stop("Please set the --output flag.", call.=FALSE)
}
slice_opt = 0
single_opt = 0
if (!is.null(opts$changeset_a) && !is.null(opts$changeset_b)) {
slice_opt = 1
}
if (!is.null(opts$one_changeset)) {
single_opt = 1
}
if ( (slice_opt && single_opt) || (!slice_opt && !single_opt) ) {
stop("Please supply a slice or a single changeset, not both.", call.= FALSE)
}
# choose color palette
mycolors <- brewer.pal(5, "Set2")
if (length(opts$one_changeset) > 0) {
## Plotting data for a single changeset ##
# read the files
non_amo <- read.csv("non_amo_one_changeset.csv", header=TRUE)
amo <- read.csv("amo_one_changeset.csv", header=TRUE)
ping_pong <- read.csv("ping_pong_one_changeset.csv", header=TRUE)
# ensure that ggplot plots the x-axis in the right order
non_amo$size <- factor(non_amo$size, levels = unique(non_amo$size))
amo$op <- factor(amo$op, levels = unique(amo$op))
# plot
non_amo_ops <- list("put","put_nbi","get","get_nbi")
for (op in non_amo_ops) {
plot_and_save(df=non_amo,
xval="size",
yval=op,
title=op,
subtitle="Latency (us)",
xlabel="Message size (bytes)",
filename=paste(opts$output,"/",op,"_changeset_",opts$one_changeset,".pdf", sep="")
)
}
# prepare data for plots with fixed message size and ops as x axis
non_amo$bsize <- paste("b",non_amo$size,sep="") # (so that the columns in non_amo_t start with a character)
non_amo_t <- setNames(data.frame(t(non_amo[,2:5])), non_amo[,6]) # transpose + set column names
non_amo_t$op <- colnames(non_amo[,2:5]) # make a column with operation names
sizes <- colnames(non_amo_t[,-(length(colnames(non_amo_t)))])
for (size in sizes) {
plot_and_save(df=non_amo_t,
xval="op",
yval=size,
title=paste(sub('.', '', size),"byte"),
subtitle="Latency (us)",
xlabel="Operation",
filename=paste(opts$output,"/",size,"_changeset_",opts$one_changeset,".pdf", sep="")
)
}
plot_and_save(df=amo,
xval="op",
yval="latency",
title="Atomics",
subtitle="Latency (us)",
xlabel="Operation",
filename=paste(opts$output,"/atomic_changeset_",opts$one_changeset,".pdf", sep="")
)
ping_pong$type <- c("ping_pong")
p<-ggplot(ping_pong, aes(x=type, y=latency, fill=type)) +
geom_bar(stat="identity", width=0.5) +
theme_minimal() +
ggtitle("Ping pong", subtitle = "Latency (us)") +
theme(
axis.title.y = element_blank(),
axis.text.y = element_blank(),
axis.title.x = element_blank(),
legend.position = "none"
) +
coord_flip() +
scale_fill_manual(values = mycolors)
#ggpreview(width=7.5, height=5, units="in", dpi=500)
ggsave(paste(opts$output,"/ping_pong_changeset_",opts$one_changeset,".pdf", sep=""), p, device=pdf, dpi=500)
} else {
## Plotting across a changeset slice ##
# read the files
put <- read.csv("put.csv", header=TRUE)
put_nbi <- read.csv("put_nbi.csv", header=TRUE)
get <- read.csv("get.csv", header=TRUE)
get_nbi <- read.csv("get_nbi.csv", header=TRUE)
amo <- read.csv("amo.csv", header=TRUE)
ping_pong <- read.csv("ping_pong.csv", header=TRUE)
# slice out the commits
start <- match(c(opts$changeset_a), put$Commit)
end <- match(c(opts$changeset_b), put$Commit)
# (start and end should be the same for all the frames) #
put <- put[start:end,]
put_nbi <- put_nbi[start:end,]
get <- get[start:end,]
get_nbi <- get_nbi[start:end,]
amo <- amo[start:end,]
ping_pong <- ping_pong[start:end,]
put <- set_right_order(put)
put_nbi <- set_right_order(put_nbi)
get <- set_right_order(get)
get_nbi <- set_right_order(get_nbi)
amo <- set_right_order(amo)
ping_pong <- set_right_order(ping_pong)
# plot
non_amo_ops <- list("put","put_nbi","get","get_nbi")
sizes_to_subtitle_map <- list("b1"="1 byte",
"b2"="2 bytes",
"b4"="4 bytes",
"b8"="8 bytes",
"b16"="16 bytes",
"b32"="32 bytes",
"b64"="64 bytes",
"b128"="128 bytes",
"b256"="256 bytes",
"b512"="512 bytes",
"b1024"="1024 bytes",
"b2048"="2048 bytes",
"b4096"="4096 bytes",
"b8192"="8192 bytes",
"b16384"="16384 bytes",
"b32768"="32768 bytes")
for (op in non_amo_ops) {
for (size in names(sizes_to_subtitle_map)) {
plot_and_save(df=eval(parse(text=op)),
xval="Commit",
yval=size,
title=op,
subtitle=paste("Latency (us) for ",sizes_to_subtitle_map[[size]],sep=""),
xlabel="Commit (older to newer)",
filename=paste(opts$output,"/",op,"_",size,".pdf", sep="")
)
}
}
amo_ops <- list("add","cswap","fadd","fcswap","fetch","finc","inc")
for (op in amo_ops) {
plot_and_save(df=amo,
xval="Commit",
yval=op,
title=op,
subtitle="Latency (us)",
xlabel="Commit (older to newer)",
filename=paste(opts$output,"/",op,".pdf", sep="")
)
}
plot_and_save(df=ping_pong,
xval="Commit",
yval="latency",
title="ping_pong",
subtitle="Latency (us)",
xlabel="Commit (older to newer)",
filename=paste(opts$output,"/","ping_pong.pdf", sep="")
)
}
## SCRIPT END ##
internal/continuous_integration/short/plotter.py
+295
View File
@@ -0,0 +1,295 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import dictionary
import log
#import matplotlib.pyplot
import numpy
import csv
import os
import subprocess
import sys
class Plot():
def __init__(self, args, archives, changeset_to_build):
self._args = args
self._archives = archives
self._changelog = dictionary.ChangelogToMostRecentBuild()
self._changelog.generate(changeset_to_build)
print(self._changelog.dump())
def abbreviate_changesets(self, changesets):
return [changeset[0:8] for changeset in changesets]
@staticmethod
def write_dict_to_file(tracker, field_names, file_name):
with open(file_name, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=field_names)
writer.writeheader()
writer.writerows(tracker)
@staticmethod
def check_and_add_to_dict(dictionary, key, array):
if len(array) > 0:
dictionary[key] = array[0]
else:
dictionary[key] = 0
def changeset_slice(self):
self._log_tracker = log.Tracker(self._args, self._archives)
for changeset in self._changelog._all_changesets:
if changeset in self._changelog.data.keys():
build_id = self._changelog.data[changeset]
self._log_tracker.add(changeset, build_id)
print(self._log_tracker.dump())
"""
separate out dictionaries based on operation
and prepare them in a format that works with
the csv module
"""
put_tracker = []
put_nbi_tracker = []
get_tracker = []
get_nbi_tracker = []
amo_tracker = []
ping_pong_tracker = []
prev_commit = list(self._log_tracker._data.keys())[0][0]
amo_dict = {}
for key, value in self._log_tracker._data.items():
if (key[1] == "put.log"):
put_tracker.append({'Commit':key[0][0:7],
'b1':value.latency[0],
'b2':value.latency[1],
'b4':value.latency[2],
'b8':value.latency[3],
'b16':value.latency[4],
'b32':value.latency[5],
'b64':value.latency[6],
'b128':value.latency[7],
'b256':value.latency[8],
'b512':value.latency[9],
'b1024':value.latency[10],
'b2048':value.latency[11],
'b4096':value.latency[12],
'b8192':value.latency[13],
'b16384':value.latency[14],
'b32768':value.latency[15]
})
if (key[1] == "put_nbi.log"):
put_nbi_tracker.append({'Commit':key[0][0:7],
'b1':value.latency[0],
'b2':value.latency[1],
'b4':value.latency[2],
'b8':value.latency[3],
'b16':value.latency[4],
'b32':value.latency[5],
'b64':value.latency[6],
'b128':value.latency[7],
'b256':value.latency[8],
'b512':value.latency[9],
'b1024':value.latency[10],
'b2048':value.latency[11],
'b4096':value.latency[12],
'b8192':value.latency[13],
'b16384':value.latency[14],
'b32768':value.latency[15]
})
if (key[1] == "get.log"):
get_tracker.append({'Commit':key[0][0:7],
'b1':value.latency[0],
'b2':value.latency[1],
'b4':value.latency[2],
'b8':value.latency[3],
'b16':value.latency[4],
'b32':value.latency[5],
'b64':value.latency[6],
'b128':value.latency[7],
'b256':value.latency[8],
'b512':value.latency[9],
'b1024':value.latency[10],
'b2048':value.latency[11],
'b4096':value.latency[12],
'b8192':value.latency[13],
'b16384':value.latency[14],
'b32768':value.latency[15]
})
if (key[1] == "get_nbi.log"):
get_nbi_tracker.append({'Commit':key[0][0:7],
'b1':value.latency[0],
'b2':value.latency[1],
'b4':value.latency[2],
'b8':value.latency[3],
'b16':value.latency[4],
'b32':value.latency[5],
'b64':value.latency[6],
'b128':value.latency[7],
'b256':value.latency[8],
'b512':value.latency[9],
'b1024':value.latency[10],
'b2048':value.latency[11],
'b4096':value.latency[12],
'b8192':value.latency[13],
'b16384':value.latency[14],
'b32768':value.latency[15]
})
if (key[1] == "ping_pong.log"):
ping_pong_tracker.append({'Commit':key[0][0:7],
'latency':value.latency[0]
})
# check to see if we have moved to a new commit
# if we have, store the dict in the amo_tracker
if (key[0] != prev_commit):
amo_dict['Commit'] = prev_commit[0:7]
amo_tracker.append(amo_dict.copy())
amo_dict.clear()
prev_commit = key[0]
if (key[1] == "amo_add.log"):
self.check_and_add_to_dict(amo_dict, 'add', value.latency)
if (key[1] == "amo_cswap.log"):
self.check_and_add_to_dict(amo_dict, 'cswap', value.latency)
if (key[1] == "amo_fadd.log"):
self.check_and_add_to_dict(amo_dict, 'fadd', value.latency)
if (key[1] == "amo_fcswap.log"):
self.check_and_add_to_dict(amo_dict, 'fcswap', value.latency)
if (key[1] == "amo_fetch.log"):
self.check_and_add_to_dict(amo_dict, 'fetch', value.latency)
if (key[1] == "amo_finc.log"):
self.check_and_add_to_dict(amo_dict, 'finc', value.latency)
if (key[1] == "amo_inc.log"):
self.check_and_add_to_dict(amo_dict, 'inc', value.latency)
# store the last commit's amo data
amo_dict['Commit'] = prev_commit[0:7]
amo_tracker.append(amo_dict.copy())
# write put results into a file:
size_field_names= ['Commit','b1','b2','b4','b8','b16','b32','b64','b128','b256','b512','b1024','b2048','b4096','b8192','b16384','b32768']
amo_field_names= ['Commit','add','cswap','fadd','fcswap','fetch','finc','inc']
ping_pong_field_names= ['Commit','latency']
self.write_dict_to_file(put_tracker, size_field_names, "put.csv")
self.write_dict_to_file(put_nbi_tracker, size_field_names, "put_nbi.csv")
self.write_dict_to_file(get_tracker, size_field_names, "get.csv")
self.write_dict_to_file(get_nbi_tracker, size_field_names, "get_nbi.csv")
self.write_dict_to_file(amo_tracker, amo_field_names, "amo.csv")
self.write_dict_to_file(ping_pong_tracker, ping_pong_field_names, "ping_pong.csv")
# make a directory and execute the R script to generate plots in that directory
current_dir = os.getcwd()
plot_dir = os.path.join(current_dir, 'plots')
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
changeset_a = list(self._log_tracker._data.keys())[0][0]
changeset_b = list(self._log_tracker._data.keys())[-1][0]
# check if the provided changesets are correct
if (self._args.changeset_range):
found_changeset_a = False
found_changeset_b = False
for key, value in self._log_tracker._data.items():
if (found_changeset_a and found_changeset_b):
break
if (not found_changeset_a):
if (self._args.changeset_range[0] == key[0]):
found_changeset_a = True
if (not found_changeset_b):
if (self._args.changeset_range[1] == key[0]):
found_changeset_b = True
if ((not found_changeset_a) and (not found_changeset_b)):
sys.exit("One of the specified changesets was not found. Please specify correct/complete commit IDs.")
else:
changeset_a = self._args.changeset_range[0]
changeset_b = self._args.changeset_range[1]
r_command = "Rscript ./plotter.R -o ./plots -a " + changeset_a[0:7] + " -b " + changeset_b[0:7]
print(r_command)
subprocess.check_call(r_command, shell=True)
def one_changeset_plot(self):
found_changeset = 0
non_amo_tracker = []
amo_tracker = []
ping_pong_tracker = []
for key, value in self._log_tracker._data.items():
if (key[0] == self._args.one_changeset):
found_changeset = 1
if (key[1] == "put.log"):
put_vals = value.latency
if (key[1] == "put_nbi.log"):
put_nbi_vals = value.latency
if (key[1] == "get.log"):
get_vals = value.latency
if (key[1] == "get_nbi.log"):
get_nbi_vals = value.latency
if (key[1] == "amo_add.log"):
amo_tracker.append({'op':'add',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_add.log"):
amo_tracker.append({'op':'add',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_cswap.log"):
amo_tracker.append({'op':'cswap',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_fadd.log"):
amo_tracker.append({'op':'fadd',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_fcswap.log"):
amo_tracker.append({'op':'fcswap',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_fetch.log"):
amo_tracker.append({'op':'fetch',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_finc.log"):
amo_tracker.append({'op':'finc',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "amo_inc.log"):
amo_tracker.append({'op':'inc',
'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (key[1] == "ping_pong.log"):
ping_pong_tracker.append({'latency': value.latency[0] if len(value.latency) > 0 else 0
})
if (not found_changeset):
sys.exit("The requested changeset was not found. Please specify correct/complete commit IDs.")
index = 0
for size in [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]:
non_amo_tracker.append({'size':size,
'put':put_vals[index],
'put_nbi':put_nbi_vals[index],
'get':get_vals[index],
'get_nbi':get_nbi_vals[index]
})
index = index + 1
# write results into a file:
non_amo_field_names= ['size','put','put_nbi','get','get_nbi']
amo_field_names= ['op','latency']
ping_pong_field_names= ['latency']
self.write_dict_to_file(non_amo_tracker, non_amo_field_names, "non_amo_one_changeset.csv")
self.write_dict_to_file(amo_tracker, amo_field_names, "amo_one_changeset.csv")
self.write_dict_to_file(ping_pong_tracker, ping_pong_field_names, "ping_pong_one_changeset.csv")
# call the R script with an option that tells it to plot figures for
r_command = "Rscript ./plotter.R -o ./plots -c " + self._args.one_changeset
print(r_command)
subprocess.check_call(r_command, shell=True)
internal/continuous_integration/short/report.py
+21
View File
@@ -0,0 +1,21 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import sys
class Report():
def __init__(self, identifier, path, filename):
self._identifier = identifier
self._path = path
self._filename = filename
def open(self):
print('opening report for ' + self._identifier)
try:
report_path = self._path + '/' + self._filename
print('report_path: ' + report_path)
self._file_handle = open(report_path, 'w')
except:
sys.exit('failed to open report: ' + report_path)
def record(self, message):
self._file_handle.write(message + '\n')
internal/continuous_integration/short/short_config.xml
+96
View File
@@ -0,0 +1,96 @@
<?xml version='1.1' encoding='UTF-8'?>
<flow-definition plugin="workflow-job@2.40">
<actions>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobAction plugin="pipeline-model-definition@1.8.4"/>
<org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction plugin="pipeline-model-definition@1.8.4">
<jobProperties/>
<triggers/>
<parameters/>
<options/>
</org.jenkinsci.plugins.pipeline.modeldefinition.actions.DeclarativeJobPropertyTrackerAction>
</actions>
<description></description>
<keepDependencies>false</keepDependencies>
<properties>
<org.jenkinsci.plugins.workflow.job.properties.DisableConcurrentBuildsJobProperty/>
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger plugin="gerrit-trigger@2.33.0">
<spec></spec>
<gerritProjects>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
<compareType>PLAIN</compareType>
<pattern>rsch/ec/shmem</pattern>
<branches>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
<compareType>PLAIN</compareType>
<pattern>amd-master</pattern>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
</branches>
<disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
</gerritProjects>
<dynamicGerritProjects class="empty-list"/>
<skipVote>
<onSuccessful>false</onSuccessful>
<onFailed>false</onFailed>
<onUnstable>false</onUnstable>
<onNotBuilt>false</onNotBuilt>
<onAborted>false</onAborted>
</skipVote>
<silentMode>false</silentMode>
<enableTopicAssociation>false</enableTopicAssociation>
<notificationLevel></notificationLevel>
<silentStartMode>false</silentStartMode>
<escapeQuotes>true</escapeQuotes>
<nameAndEmailParameterMode>PLAIN</nameAndEmailParameterMode>
<dependencyJobsNames>shmem_perf_check, </dependencyJobsNames>
<commitMessageParameterMode>BASE64</commitMessageParameterMode>
<changeSubjectParameterMode>PLAIN</changeSubjectParameterMode>
<commentTextParameterMode>BASE64</commentTextParameterMode>
<buildStartMessage></buildStartMessage>
<buildFailureMessage></buildFailureMessage>
<buildSuccessfulMessage></buildSuccessfulMessage>
<buildUnstableMessage></buildUnstableMessage>
<buildNotBuiltMessage></buildNotBuiltMessage>
<buildAbortedMessage></buildAbortedMessage>
<buildUnsuccessfulFilepath></buildUnsuccessfulFilepath>
<customUrl></customUrl>
<serverName>amd-gerrit</serverName>
<triggerOnEvents>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
<commentAddedCommentContains>!SHORT</commentAddedCommentContains>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
</triggerOnEvents>
<dynamicTriggerConfiguration>false</dynamicTriggerConfiguration>
<triggerConfigURL></triggerConfigURL>
<triggerInformationAction/>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
</triggers>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<name>origin</name>
<refspec>${GERRIT_REFSPEC}</refspec>
<url>ssh://gerritgit/rsch/ec/shmem</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>FETCH_HEAD</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions/>
</scm>
<scriptPath>internal/continuous_integration/short/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>
internal/continuous_integration/short/violation.py
+34
View File
@@ -0,0 +1,34 @@
#!/tool/pandora64/.package/python-3.8.0/bin/python3
import pprint
import report
import sys
class Threshold():
def __init__(self, maximum_threshold, violation_type):
self._violations = {}
self._maximum_threshold = maximum_threshold
self._violation_type = violation_type
def check(self, value, changeset, filename):
if value >= self._maximum_threshold:
key = changeset + '|' + filename + '|' + self._violation_type
self._violations[key] = value
print(key + ': ' + str(value) + '%')
def provide_violations_to_report(self, report):
if self.has_violations():
report.record('FAILURE')
report.record(self.dump())
sys.exit(1)
else:
report.record('SUCCESS')
sys.exit(0)
def has_violations(self):
return bool(self._violations)
def dump(self):
str_out = pprint.pformat(self._violations, width=120)
str_out += '\n'
return str_out
internal/continuous_integration/smoke/Jenkinsfile
Vendored
+151
View File
@@ -0,0 +1,151 @@
pipeline {
agent { label 'sv-pdp-5' }
environment {
HSA_FORCE_FINE_GRAIN_PCIE = 1
MPI_HOME="/home/resperf/mpich-4.0.1/install/global"
PATH = "$MPI_HOME/bin:$PATH"
LD_LIBRARY_PATH = "$MPI_HOME/lib:$LD_LIBRARY_PATH"
build_dir = "builds/change-${GERRIT_CHANGE_NUMBER}-${GERRIT_PATCHSET_NUMBER}"
CMAKE_PREFIX_PATH = "/opt/rocm/lib/cmake"
}
stages {
stage('Synchronize Source Code') {
steps {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false]], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: '${GERRIT_REFSPEC}', url: 'ssh://gerritgit/rsch/ec/shmem']]]
}
}
stage('Make Build Directory') {
steps {
dir("library") {
sh "mkdir -p ${build_dir}"
}
}
}
stage('Build Source Code') {
parallel {
stage('RC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_SINGLE/install'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI_WF_COAL") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi_wf_coal install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI_WF_COAL") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI_WF_COAL/install'
}
}
}
stage('RC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/RC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/rc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/RC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/RC_MULTI/install'
}
}
}
stage('DC_SINGLE') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_SINGLE") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_single install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_SINGLE") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_SINGLE/install'
}
}
}
stage('DC_MULTI') {
steps {
//===================== LIBRARY =======================
dir("library/${build_dir}/DC_MULTI") {
sh 'mkdir -p install'
sh '../../../build_configs/dc_multi install'
}
//===================== CLIENT ========================
dir("clients/functional_tests/${build_dir}/DC_MULTI") {
sh '../../../build_configs/release ${WORKSPACE}/library/${build_dir}/DC_MULTI/install'
}
}
}
}
}
stage('Run Tests') {
parallel {
stage('RC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/RC_SINGLE'
}
}
}
stage('RC_MULTI_WF_COAL') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI_WF_COAL/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI_WF_COAL'
}
}
}
stage('RC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/RC_MULTI'
}
}
}
stage('DC_SINGLE') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_SINGLE/rocshmem_example_driver single_thread ${build_dir}/DC_SINGLE'
}
}
}
stage('DC_MULTI') {
steps {
dir("clients/functional_tests") {
sh './driver.sh ${build_dir}/DC_MULTI/rocshmem_example_driver multi_thread ${build_dir}/DC_MULTI'
}
}
}
stage('RO_NET_BASIC') {
// RO_NET controlled at runtime, no need for a new build. Use RC_MULTI
steps {
dir("clients/functional_tests") {
sh 'mkdir -p ${build_dir}/RO_NET_BASIC'
sh 'ROC_SHMEM_RO=1 RO_NET_CPU_QUEUE=1 UCX_TLS=rc ./driver.sh ${build_dir}/RC_MULTI/rocshmem_example_driver ro ${build_dir}/RO_NET_BASIC'
}
}
}
}
}
}
}
internal/continuous_integration/smoke/smoke_config.xml
+85
View File
@@ -0,0 +1,85 @@
<?xml version='1.1' encoding='UTF-8'?>
<flow-definition plugin="workflow-job@2.40">
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties>
<org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
<triggers>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger plugin="gerrit-trigger@2.33.0">
<spec></spec>
<gerritProjects>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
<compareType>PLAIN</compareType>
<pattern>rsch/ec/shmem</pattern>
<branches>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
<compareType>PLAIN</compareType>
<pattern>amd-master</pattern>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
</branches>
<disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
</gerritProjects>
<dynamicGerritProjects class="empty-list"/>
<skipVote>
<onSuccessful>true</onSuccessful>
<onFailed>true</onFailed>
<onUnstable>true</onUnstable>
<onNotBuilt>true</onNotBuilt>
<onAborted>true</onAborted>
</skipVote>
<silentMode>false</silentMode>
<enableTopicAssociation>false</enableTopicAssociation>
<notificationLevel></notificationLevel>
<silentStartMode>false</silentStartMode>
<escapeQuotes>true</escapeQuotes>
<nameAndEmailParameterMode>PLAIN</nameAndEmailParameterMode>
<dependencyJobsNames></dependencyJobsNames>
<commitMessageParameterMode>BASE64</commitMessageParameterMode>
<changeSubjectParameterMode>PLAIN</changeSubjectParameterMode>
<commentTextParameterMode>BASE64</commentTextParameterMode>
<buildStartMessage></buildStartMessage>
<buildFailureMessage></buildFailureMessage>
<buildSuccessfulMessage></buildSuccessfulMessage>
<buildUnstableMessage></buildUnstableMessage>
<buildNotBuiltMessage></buildNotBuiltMessage>
<buildAbortedMessage></buildAbortedMessage>
<buildUnsuccessfulFilepath></buildUnsuccessfulFilepath>
<customUrl></customUrl>
<serverName>amd-gerrit</serverName>
<triggerOnEvents>
<com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
<commentAddedCommentContains>!SMOKE</commentAddedCommentContains>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginCommentAddedContainsEvent>
</triggerOnEvents>
<dynamicTriggerConfiguration>false</dynamicTriggerConfiguration>
<triggerConfigURL></triggerConfigURL>
<triggerInformationAction/>
</com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
</triggers>
</org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
</properties>
<definition class="org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition" plugin="workflow-cps@2.90">
<scm class="hudson.plugins.git.GitSCM" plugin="git@4.7.1">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>ssh://gerritgit/rsch/ec/shmem</url>
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>FETCH_HEAD</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<submoduleCfg class="empty-list"/>
<extensions/>
</scm>
<scriptPath>internal/continuous_integration/smoke/Jenkinsfile</scriptPath>
<lightweight>false</lightweight>
</definition>
<triggers/>
<disabled>false</disabled>
</flow-definition>