User regions in Python (#57)

* User regions in Python

* User-region testing + common submodule

- Updated examples/python/source.py to use user-regions
- Python-level user submodule
- Python-level common submodule
- clean-up of profiler python code
- extended source.py testing to include the user-regions

[ROCm/rocprofiler-systems commit: 4dd144a32c]

projects/rocprofiler-systems/source/python/CMakeLists.txt

@@ -88,6 +88,7 @@ target_link_libraries(
         omnitrace::omnitrace-compile-options
         omnitrace::omnitrace-lto
         omnitrace::omnitrace-dl-library
+        omnitrace::omnitrace-user-library
         omnitrace::omnitrace-python
         omnitrace::omnitrace-python-compile-options
         $<BUILD_INTERFACE:$<IF:$<BOOL:${OMNITRACE_BUILD_STATIC_LIBGCC}>,omnitrace::omnitrace-static-libgcc,>>

projects/rocprofiler-systems/source/python/libpyomnitrace.cpp

@@ -24,6 +24,7 @@
 #include "dl.hpp"
 #include "library/coverage.hpp"
 #include "library/impl/coverage.hpp"
+#include "omnitrace/user.h"
 
 #include <timemory/backends/process.hpp>
 #include <timemory/backends/threading.hpp>
@@ -62,6 +63,11 @@ namespace pycoverage
 py::module
 generate(py::module& _pymod);
 }
+namespace pyuser
+{
+py::module
+generate(py::module& _pymod);
+}
 }  // namespace pyomnitrace
 
 template <typename... Tp>
@@ -158,8 +164,8 @@ PYBIND11_MODULE(libpyomnitrace, omni)
 
     py::doc("omnitrace profiler for python");
     pyprofile::generate(omni);
-
     pycoverage::generate(omni);
+    pyuser::generate(omni);
 }
 
 //======================================================================================//
@@ -824,6 +830,33 @@ generate(py::module& _pymod)
     return _pycov;
 }
 }  // namespace pycoverage
+
+namespace pyuser
+{
+py::module
+generate(py::module& _pymod)
+{
+    py::module _pyuser = _pymod.def_submodule("user", "User instrumentation");
+
+    _pyuser.def("start_trace", &omnitrace_user_start_trace,
+                "Enable tracing on this thread and all subsequently created threads");
+    _pyuser.def("stop_trace", &omnitrace_user_stop_trace,
+                "Disable tracing on this thread and all subsequently created threads");
+    _pyuser.def(
+        "start_thread_trace", &omnitrace_user_start_thread_trace,
+        "Enable tracing on this thread. Does not apply to subsequently created threads");
+    _pyuser.def(
+        "stop_thread_trace", &omnitrace_user_stop_thread_trace,
+        "Enable tracing on this thread. Does not apply to subsequently created threads");
+    _pyuser.def("push_region", &omnitrace_user_push_region,
+                "Start a user-defined region");
+    _pyuser.def("pop_region", &omnitrace_user_pop_region, "Start a user-defined region");
+    _pyuser.def("error_string", &omnitrace_user_error_string,
+                "Return a descriptor for the provided error code");
+
+    return _pyuser;
+}
+}  // namespace pyuser
 }  // namespace pyomnitrace
 //
 //======================================================================================//

projects/rocprofiler-systems/source/python/omnitrace/__init__.py

@@ -36,6 +36,7 @@ This submodule imports the timemory Python function profiler
 
 try:
     from .libpyomnitrace import coverage
+    from . import user
     from .profiler import Profiler, FakeProfiler
     from .libpyomnitrace.profiler import (
         profiler_function,
@@ -67,6 +68,7 @@ try:
         "profile",
         "noprofile",
         "coverage",
+        "user",
     ]
 
     import atexit

projects/rocprofiler-systems/source/python/omnitrace/common.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python@_VERSION@
+# MIT License
+#
+# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+__author__ = "AMD Research"
+__copyright__ = "Copyright 2022, Advanced Micro Devices, Inc."
+__license__ = "MIT"
+__version__ = "@PROJECT_VERSION@"
+__maintainer__ = "AMD Research"
+__status__ = "Development"
+
+import os
+import sys
+
+from . import libpyomnitrace
+from .libpyomnitrace.profiler import profiler_init as _profiler_init
+from .libpyomnitrace.profiler import profiler_finalize as _profiler_fini
+
+
+__all__ = ["exec_", "_file", "_get_argv", "_initialize", "_finalize"]
+
+
+PY3 = sys.version_info[0] == 3
+
+# exec (from https://bitbucket.org/gutworth/six/):
+if PY3:
+    import builtins
+
+    exec_ = getattr(builtins, "exec")
+    del builtins
+else:
+
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+
+def _file(back=2, only_basename=True, use_dirname=False, noquotes=True):
+    """
+    Returns the file name
+    """
+
+    from os.path import basename, dirname
+
+    def get_fcode(back):
+        fname = "<module>"
+        try:
+            fname = sys._getframe(back).f_code.co_filename
+        except Exception as e:
+            print(e)
+            fname = "<module>"
+        return fname
+
+    result = None
+    if only_basename is True:
+        if use_dirname is True:
+            result = "{}".format(
+                join(
+                    basename(dirname(get_fcode(back))),
+                    basename(get_fcode(back)),
+                )
+            )
+        else:
+            result = "{}".format(basename(get_fcode(back)))
+    else:
+        result = "{}".format(get_fcode(back))
+
+    if noquotes is False:
+        result = "'{}'".format(result)
+
+    return result
+
+
+def _get_argv(init_file, argv=None):
+    if argv is None:
+        argv = sys.argv[:]
+
+    if "--" in argv:
+        _idx = argv.index("--")
+        argv = sys.argv[(_idx + 1) :]
+
+    if len(argv) > 1:
+        if argv[0] == "-m":
+            argv = argv[1:]
+        elif argv[0] == "-c":
+            argv[0] = os.path.basename(sys.executable)
+        else:
+            while len(argv) > 1 and argv[0].startswith("-"):
+                argv = argv[1:]
+                if os.path.exists(argv[0]):
+                    break
+    if len(argv) == 0:
+        argv = [init_file]
+    elif not os.path.exists(argv[0]):
+        argv[0] = init_file
+
+    return argv
+
+
+def _initialize(_file):
+    if not libpyomnitrace.is_initialized():
+        libpyomnitrace.initialize(_get_argv(_file))
+
+
+def _finalize():
+    if libpyomnitrace.is_initialized() and not libpyomnitrace.is_finalized():
+        _profiler_fini()

projects/rocprofiler-systems/source/python/omnitrace/profiler.py

@@ -35,6 +35,10 @@ import sys
 import threading
 from functools import wraps
 
+from .common import exec_
+from .common import _initialize
+from .common import _file
+
 from . import libpyomnitrace
 from .libpyomnitrace.profiler import (
     profiler_function as _profiler_function,
@@ -43,107 +47,17 @@ from .libpyomnitrace.profiler import config as _profiler_config
 from .libpyomnitrace.profiler import profiler_init as _profiler_init
 from .libpyomnitrace.profiler import profiler_finalize as _profiler_fini
 
-
 __all__ = ["profile", "config", "Profiler", "FakeProfiler", "Config"]
 
 
-#
-def _default_functor():
-    return True
-
-
-#
-PY3 = sys.version_info[0] == 3
-PY35 = PY3 and sys.version_info[1] >= 5
-
-# exec (from https://bitbucket.org/gutworth/six/):
-if PY3:
-    import builtins
-
-    exec_ = getattr(builtins, "exec")
-    del builtins
-else:
-
-    def exec_(_code_, _globs_=None, _locs_=None):
-        """Execute code in a namespace."""
-        if _globs_ is None:
-            frame = sys._getframe(1)
-            _globs_ = frame.f_globals
-            if _locs_ is None:
-                _locs_ = frame.f_locals
-            del frame
-        elif _locs_ is None:
-            _locs_ = _globs_
-        exec("""exec _code_ in _globs_, _locs_""")
-
-
 config = _profiler_config
 Config = _profiler_config
 
 
-def _file(back=2, only_basename=True, use_dirname=False, noquotes=True):
-    """
-    Returns the file name
-    """
-
-    from os.path import basename, dirname
-
-    def get_fcode(back):
-        fname = "<module>"
-        try:
-            fname = sys._getframe(back).f_code.co_filename
-        except Exception as e:
-            print(e)
-            fname = "<module>"
-        return fname
-
-    result = None
-    if only_basename is True:
-        if use_dirname is True:
-            result = "{}".format(
-                join(
-                    basename(dirname(get_fcode(back))),
-                    basename(get_fcode(back)),
-                )
-            )
-        else:
-            result = "{}".format(basename(get_fcode(back)))
-    else:
-        result = "{}".format(get_fcode(back))
-
-    if noquotes is False:
-        result = "'{}'".format(result)
-
-    return result
+def _default_functor():
+    return True
 
 
-def _get_argv(init_file, argv=None):
-    if argv is None:
-        argv = sys.argv[:]
-
-    if "--" in argv:
-        _idx = argv.index("--")
-        argv = sys.argv[(_idx + 1) :]
-
-    if len(argv) > 1:
-        if argv[0] == "-m":
-            argv = argv[1:]
-        elif argv[0] == "-c":
-            argv[0] = os.path.basename(sys.executable)
-        else:
-            while len(argv) > 1 and argv[0].startswith("-"):
-                argv = argv[1:]
-                if os.path.exists(argv[0]):
-                    break
-    if len(argv) == 0:
-        argv = [init_file]
-    elif not os.path.exists(argv[0]):
-        argv[0] = init_file
-
-    return argv
-
-
-#
 class Profiler:
     """Provides decorators and context-manager for the omnitrace profilers"""
 
@@ -152,15 +66,11 @@ class Profiler:
     # static variable
     _conditional_functor = _default_functor
 
-    # ---------------------------------------------------------------------------------- #
-    #
     @staticmethod
     def condition(functor):
         """Assign a function evaluating whether to enable the profiler"""
         Profiler._conditional_functor = functor
 
-    # ---------------------------------------------------------------------------------- #
-    #
     @staticmethod
     def is_enabled():
         """Checks whether the profiler is enabled"""
@@ -171,8 +81,6 @@ class Profiler:
             pass
         return False
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __init__(self, **kwargs):
         """ """
 
@@ -188,21 +96,16 @@ class Profiler:
         self._file = _file()
         self.debug = kwargs["debug"] if "debug" in kwargs else False
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __del__(self):
         """Make sure the profiler stops"""
 
         self.stop()
         sys.setprofile(self._original_function)
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def configure(self):
         """Initialize, configure the bundle, store original profiler function"""
 
-        if not libpyomnitrace.is_initialized():
-            libpyomnitrace.initialize(_get_argv(self._file))
+        _initialize(self._file)
 
         _profiler_init()
 
@@ -215,8 +118,6 @@ class Profiler:
         if self.debug:
             sys.stderr.write("Tracer configured...\n")
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def update(self):
         """Updates whether the profiler is already running based on whether the tracer
         is not already running, is enabled, and the function is not already set
@@ -229,8 +130,6 @@ class Profiler:
             and not libpyomnitrace.is_finalized()
         )
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def start(self):
         """Start the profiler explicitly"""
 
@@ -247,8 +146,6 @@ class Profiler:
         self._unset = self._unset + 1
         return self._unset
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def stop(self):
         """Stop the profiler explicitly"""
 
@@ -263,8 +160,6 @@ class Profiler:
 
         return self._unset
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __call__(self, func):
         """Decorator"""
 
@@ -281,15 +176,11 @@ class Profiler:
 
         return function_wrapper
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __enter__(self, *args, **kwargs):
         """Context manager start function"""
 
         self.start()
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __exit__(self, exec_type, exec_value, exec_tb):
         """Context manager stop function"""
 
@@ -300,8 +191,6 @@ class Profiler:
 
             traceback.print_exception(exec_type, exec_value, exec_tb, limit=5)
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def run(self, cmd):
         """Execute and profile a command"""
 
@@ -313,8 +202,6 @@ class Profiler:
         else:
             return self.runctx(" ".join(cmd), dict, dict)
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def runctx(self, cmd, globals, locals):
         """Profile a context"""
 
@@ -326,8 +213,6 @@ class Profiler:
 
         return self
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def runcall(self, func, *args, **kw):
         """Profile a single function call"""
 
@@ -344,26 +229,18 @@ profile = Profiler
 class FakeProfiler:
     """Provides dummy decorators and context-manager for the omnitrace profiler"""
 
-    # ---------------------------------------------------------------------------------- #
-    #
     @staticmethod
     def condition(functor):
         pass
 
-    # ---------------------------------------------------------------------------------- #
-    #
     @staticmethod
     def is_enabled():
         return False
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __init__(self, *args, **kwargs):
         """ """
         pass
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __call__(self, func):
         """Decorator"""
 
@@ -373,14 +250,10 @@ class FakeProfiler:
 
         return function_wrapper
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __enter__(self, *args, **kwargs):
         """Context manager begin"""
         pass
 
-    # ---------------------------------------------------------------------------------- #
-    #
     def __exit__(self, exec_type, exec_value, exec_tb):
         """Context manager end"""
 

projects/rocprofiler-systems/source/python/omnitrace/user.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python@_VERSION@
+# MIT License
+#
+# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+__author__ = "AMD Research"
+__copyright__ = "Copyright 2022, Advanced Micro Devices, Inc."
+__license__ = "MIT"
+__version__ = "@PROJECT_VERSION@"
+__maintainer__ = "AMD Research"
+__status__ = "Development"
+
+from functools import wraps
+
+from . import libpyomnitrace
+from .libpyomnitrace import user as _libuser
+from .libpyomnitrace.user import start_trace
+from .libpyomnitrace.user import start_thread_trace
+from .libpyomnitrace.user import stop_trace
+from .libpyomnitrace.user import stop_thread_trace
+from .libpyomnitrace.user import push_region
+from .libpyomnitrace.user import pop_region
+
+from .common import _initialize
+from .common import _file
+
+
+__all__ = [
+    "region",
+    "Region",
+    "start_trace",
+    "start_thread_trace",
+    "stop_trace",
+    "stop_thread_trace",
+    "push_region",
+    "pop_region",
+]
+
+
+class Region:
+    """Provides decorators and context-manager for the omnitrace user-defined regions"""
+
+    # static variable
+    _counter = 0
+
+    def __init__(self, _label):
+        """Stores the label"""
+        self._active = False
+        self._label = _label
+        self._count = 0
+        self._file = _file() if Region._counter == 0 else None
+
+    def __del__(self):
+        """Stops"""
+        self.stop()
+
+    def start(self):
+        """Start the region"""
+
+        if not self._active:
+            self._active = True
+            self._count = Region._counter
+            if self._file is not None:
+                _initialize(self._file)
+            Region._counter += 1
+            _libuser.push_region(self._label)
+
+    def stop(self):
+        """Stop the region"""
+
+        if self._active:
+            Region._counter -= 1
+            _count = Region._counter
+            self._active = False
+            if _count != self._count:
+                raise LogicError(
+                    f"{self._label} was not popped in the order it was pushed. Current stack number: {_count}, expected stack number: {self._count}"
+                )
+            _libuser.pop_region(self._label)
+
+    def __call__(self, func):
+        """Decorator"""
+
+        @wraps(func)
+        def function_wrapper(*args, **kwargs):
+            # start the region
+            self.start()
+            # execute the wrapped function
+            result = func(*args, **kwargs)
+            # stop the region
+            self.stop()
+            return result
+
+        return function_wrapper
+
+    def __enter__(self, *args, **kwargs):
+        """Context manager start function"""
+
+        self.start()
+
+    def __exit__(self, exec_type, exec_value, exec_tb):
+        """Context manager stop function"""
+
+        self.stop()
+
+        if exec_type is not None and exec_value is not None and exec_tb is not None:
+            import traceback
+
+            traceback.print_exception(exec_type, exec_value, exec_tb, limit=5)
+
+    def runcall(self, func, *args, **kwargs):
+        """Profile a single function call"""
+
+        try:
+            self.start()
+            return func(*args, **kwargs)
+        finally:
+            self.stop()
+
+
+region = Region