Merge proto files from multiprocess run into one file. (#63)

- Added script to merge multiprocess output automatically to one file when there are multiprocess proto files written into output folder
- Execute the merge multiprocess script from the rank 0 process
- Added the scripts folder path to env path, via setup-env.sh
- Installed merge_multiprocess_output.sh to /share/rocprofiler-systems/bin dir

Co-authored-by: David Galiffi <David.Galiffi@amd.com>

CMakeLists.txt

@@ -370,6 +370,11 @@ configure_file(
     ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}/${ROCPROFSYS_VERSION}
     @ONLY)
 
+configure_file(
+    ${PROJECT_SOURCE_DIR}/scripts/merge-multiprocess-output.sh
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/bin/merge-multiprocess-output.sh
+    COPYONLY)
+
 install(
     FILES ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/setup-env.sh
           ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/perfetto.cfg
@@ -387,6 +392,12 @@ install(
     DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}
     COMPONENT setup)
 
+install(
+    PROGRAMS
+        ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/bin/merge-multiprocess-output.sh
+    DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/bin/
+    COMPONENT setup)
+
 # ------------------------------------------------------------------------------#
 #
 # install

cmake/Templates/modulefile.in

@@ -11,6 +11,7 @@ set ROOT [file normalize [file dirname [file normalize ${ModulesCurrentModulefil
 setenv @PROJECT_NAME_UNDERSCORED@_ROOT "${ROOT}"
 prepend-path CMAKE_PREFIX_PATH "${ROOT}"
 prepend-path PATH "${ROOT}/bin"
+prepend-path PATH "${ROOT}/@CMAKE_INSTALL_DATAROOTDIR@/@PROJECT_NAME@/bin"
 prepend-path LD_LIBRARY_PATH "${ROOT}/@CMAKE_INSTALL_LIBDIR@"
 prepend-path PYTHONPATH "${ROOT}/@CMAKE_INSTALL_PYTHONDIR@"
 setenv @PROJECT_NAME_UNDERSCORED@_DIR "${ROOT}/@CMAKE_INSTALL_DATAROOTDIR@/cmake/@PROJECT_NAME@"

cmake/Templates/setup-env.sh.in

@@ -15,6 +15,7 @@ fi
 
 @PROJECT_NAME_UNDERSCORED@_ROOT=${BASEDIR}
 PATH=${BASEDIR}/bin:${PATH}
+PATH=${BASEDIR}/@CMAKE_INSTALL_DATAROOTDIR@/@PROJECT_NAME@/bin:${PATH}
 LD_LIBRARY_PATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@:${LD_LIBRARY_PATH}
 PYTHONPATH=${BASEDIR}/@CMAKE_INSTALL_PYTHONDIR@:${PYTHONPATH}
 CMAKE_PREFIX_PATH=${BASEDIR}:${CMAKE_PREFIX_PATH}

scripts/merge-multiprocess-output.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Check if the folder path is provided
+if [ -z "$1" ]; then
+  echo "Usage: $0 <folder_path>"
+  exit 1
+fi
+
+# Assign the folder path to a variable
+FOLDER_PATH=$1
+
+# Check if the folder exists
+if [ ! -d "$FOLDER_PATH" ]; then
+  echo "Error: Folder '$FOLDER_PATH' does not exist."
+  exit 1
+fi
+
+# Check if there are more than one .proto files
+PROTO_FILES=("$FOLDER_PATH"/*.proto)
+if [ ${#PROTO_FILES[@]} -le 1 ]; then
+  exit 0
+fi
+
+echo "Merging multiprocess files ..."
+# Check if all .proto files have been fully written or wait
+TIMEOUT=60  # Timeout in seconds
+for file in "${PROTO_FILES[@]}"; do
+  SECONDS=0
+  while lsof "$file" > /dev/null 2>&1; do
+    if [ $SECONDS -ge $TIMEOUT ]; then
+      echo "Timeout reached while waiting for $file to be released."
+      break
+    fi
+    echo "Waiting for $file to be released..."
+    sleep 1
+  done
+done
+
+# Output file name
+OUTPUT_FILE="merged.proto"
+
+# Merge all .proto files into one file
+cat "$FOLDER_PATH"/*.proto > "$FOLDER_PATH"/"$OUTPUT_FILE"
+
+echo "All multiprocess .proto files in '$FOLDER_PATH' have been merged into '$OUTPUT_FILE'."

source/lib/core/perfetto.cpp

@@ -263,6 +263,25 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
                 _timemory_manager->add_file_output("protobuf", "perfetto", _filename);
         }
         ofs.close();
+
+        if(dmp::rank() == 0)
+        {
+            const char* file_path   = _filename.c_str();
+            auto        folder_path = [](std::string_view _v) {
+                return tim::filepath::dirname(std::string(_v));
+            };
+            // Execute the merge script
+            std::string command =
+                "merge-multiprocess-output.sh '" + folder_path(file_path) + "'";
+            int result = system(command.c_str());
+            if(result != 0)
+            {
+                ROCPROFSYS_VERBOSE(0,
+                                   "Failed to execute merge-multiprocess-output.sh with "
+                                   "folder path: %s\n",
+                                   folder_path(file_path).c_str());
+            }
+        }
     }
     else if(dmp::rank() == 0)
     {