[rocprofiler-compute] Remove grafana and mongodb integration (#978)
* Remove grafana and mongodb integration * Remove grafana documentation assets * clarify changelog --------- Co-authored-by: Vignesh Edithal <Vignesh.Edithal@amd.com>
This commit is contained in:
xuchen-amd
GitHub
@@ -20,6 +20,11 @@ Full documentation for ROCm Compute Profiler is available at [https://rocm.docs.
|
||||
|
||||
|
||||
### Removed
|
||||
* Removed `database` mode from `rocprofiler-compute`. This is to move our focus from grafana
|
||||
and mongodb integration to other visualization methods such as:
|
||||
* Analysis DB based Visualizer (upcoming)
|
||||
* Plotly server based standalone GUI
|
||||
* Commandline based Textual User Interface
|
||||
|
||||
### Optimized
|
||||
|
||||
|
||||
@@ -445,7 +445,6 @@ if(${ENABLE_COVERAGE})
|
||||
test_L1_cache_counters
|
||||
test_num_xcds_spec_class
|
||||
test_num_xcds_cli_output
|
||||
test_db_connector
|
||||
test_utils
|
||||
test_autogen_config
|
||||
)
|
||||
@@ -542,12 +541,6 @@ install(
|
||||
COMPONENT main
|
||||
PATTERN "__pycache__" EXCLUDE
|
||||
)
|
||||
# grafana assets
|
||||
install(
|
||||
DIRECTORY grafana
|
||||
DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}
|
||||
COMPONENT main
|
||||
)
|
||||
# samples
|
||||
install(
|
||||
DIRECTORY sample
|
||||
|
||||
@@ -2214,114 +2214,6 @@
|
||||
<line number="210" hits="1"/>
|
||||
</lines>
|
||||
</class>
|
||||
<class name="db_connector.py" filename="src/utils/db_connector.py" complexity="0" line-rate="0.9515" branch-rate="0">
|
||||
<methods/>
|
||||
<lines>
|
||||
<line number="26" hits="1"/>
|
||||
<line number="27" hits="1"/>
|
||||
<line number="28" hits="1"/>
|
||||
<line number="29" hits="1"/>
|
||||
<line number="31" hits="1"/>
|
||||
<line number="32" hits="1"/>
|
||||
<line number="33" hits="1"/>
|
||||
<line number="35" hits="1"/>
|
||||
<line number="36" hits="1"/>
|
||||
<line number="43" hits="1"/>
|
||||
<line number="45" hits="1"/>
|
||||
<line number="48" hits="1"/>
|
||||
<line number="49" hits="1"/>
|
||||
<line number="50" hits="1"/>
|
||||
<line number="51" hits="1"/>
|
||||
<line number="52" hits="1"/>
|
||||
<line number="61" hits="1"/>
|
||||
<line number="64" hits="1"/>
|
||||
<line number="66" hits="1"/>
|
||||
<line number="67" hits="1"/>
|
||||
<line number="69" hits="1"/>
|
||||
<line number="70" hits="1"/>
|
||||
<line number="71" hits="1"/>
|
||||
<line number="72" hits="1"/>
|
||||
<line number="73" hits="1"/>
|
||||
<line number="74" hits="1"/>
|
||||
<line number="75" hits="1"/>
|
||||
<line number="76" hits="1"/>
|
||||
<line number="81" hits="1"/>
|
||||
<line number="85" hits="1"/>
|
||||
<line number="94" hits="1"/>
|
||||
<line number="95" hits="1"/>
|
||||
<line number="96" hits="1"/>
|
||||
<line number="97" hits="1"/>
|
||||
<line number="98" hits="1"/>
|
||||
<line number="99" hits="1"/>
|
||||
<line number="100" hits="1"/>
|
||||
<line number="101" hits="1"/>
|
||||
<line number="105" hits="1"/>
|
||||
<line number="106" hits="1"/>
|
||||
<line number="107" hits="1"/>
|
||||
<line number="110" hits="1"/>
|
||||
<line number="111" hits="1"/>
|
||||
<line number="112" hits="1"/>
|
||||
<line number="114" hits="1"/>
|
||||
<line number="123" hits="1"/>
|
||||
<line number="124" hits="1"/>
|
||||
<line number="125" hits="1"/>
|
||||
<line number="126" hits="1"/>
|
||||
<line number="127" hits="1"/>
|
||||
<line number="128" hits="1"/>
|
||||
<line number="130" hits="1"/>
|
||||
<line number="131" hits="1"/>
|
||||
<line number="132" hits="1"/>
|
||||
<line number="133" hits="1"/>
|
||||
<line number="134" hits="1"/>
|
||||
<line number="135" hits="1"/>
|
||||
<line number="136" hits="1"/>
|
||||
<line number="138" hits="1"/>
|
||||
<line number="139" hits="1"/>
|
||||
<line number="140" hits="1"/>
|
||||
<line number="143" hits="1"/>
|
||||
<line number="144" hits="1"/>
|
||||
<line number="145" hits="1"/>
|
||||
<line number="146" hits="1"/>
|
||||
<line number="148" hits="1"/>
|
||||
<line number="152" hits="1"/>
|
||||
<line number="153" hits="1"/>
|
||||
<line number="155" hits="1"/>
|
||||
<line number="156" hits="1"/>
|
||||
<line number="157" hits="1"/>
|
||||
<line number="160" hits="1"/>
|
||||
<line number="163" hits="1"/>
|
||||
<line number="164" hits="1"/>
|
||||
<line number="165" hits="1"/>
|
||||
<line number="166" hits="1"/>
|
||||
<line number="167" hits="1"/>
|
||||
<line number="172" hits="1"/>
|
||||
<line number="176" hits="1"/>
|
||||
<line number="180" hits="0"/>
|
||||
<line number="184" hits="0"/>
|
||||
<line number="188" hits="1"/>
|
||||
<line number="189" hits="1"/>
|
||||
<line number="195" hits="1"/>
|
||||
<line number="201" hits="1"/>
|
||||
<line number="202" hits="1"/>
|
||||
<line number="204" hits="1"/>
|
||||
<line number="208" hits="1"/>
|
||||
<line number="209" hits="0"/>
|
||||
<line number="212" hits="1"/>
|
||||
<line number="217" hits="1"/>
|
||||
<line number="218" hits="1"/>
|
||||
<line number="219" hits="1"/>
|
||||
<line number="220" hits="0"/>
|
||||
<line number="221" hits="0"/>
|
||||
<line number="223" hits="1"/>
|
||||
<line number="225" hits="1"/>
|
||||
<line number="228" hits="1"/>
|
||||
<line number="239" hits="1"/>
|
||||
<line number="242" hits="1"/>
|
||||
<line number="243" hits="1"/>
|
||||
<line number="244" hits="1"/>
|
||||
<line number="245" hits="1"/>
|
||||
</lines>
|
||||
</class>
|
||||
<class name="file_io.py" filename="src/utils/file_io.py" complexity="0" line-rate="0.6554" branch-rate="0">
|
||||
<methods/>
|
||||
<lines>
|
||||
|
||||
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 85 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 99 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 29 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 49 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 42 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 29 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 22 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 21 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 10 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 23 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 20 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 27 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 15 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 20 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 15 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 55 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 20 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 48 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 66 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 21 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 17 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 33 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 108 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 110 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 41 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 15 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 15 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 72 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 49 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 34 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 39 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 51 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 24 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 75 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 20 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 53 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 18 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 16 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 18 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 34 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 36 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 83 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 184 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 272 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 63 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 57 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 352 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 172 KiB |
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 58 KiB |
Plik diff jest za duży
Load Diff
@@ -1,7 +1,7 @@
|
||||
.. meta::
|
||||
:description: How to use ROCm Compute Profiler's analyze mode
|
||||
:keywords: ROCm Compute Profiler, ROCm, profiler, tool, Instinct, accelerator, AMD,
|
||||
Grafana, analysis, analyze mode
|
||||
analysis, analyze mode
|
||||
|
||||
************
|
||||
Analyze mode
|
||||
@@ -12,15 +12,10 @@ profiling. Your level of familiarity with the profiled application, computing
|
||||
environment, and experience with ROCm Compute Profiler should inform the analysis method you
|
||||
choose.
|
||||
|
||||
While analyzing with the CLI offers quick and straightforward access to ROCm Compute Profiler
|
||||
metrics from the terminal, Grafana's dashboard GUI adds an extra layer of
|
||||
readability and interactivity you might prefer.
|
||||
|
||||
See the following sections to explore ROCm Compute Profiler's analysis and visualization
|
||||
options.
|
||||
|
||||
* :doc:`cli`
|
||||
* :doc:`grafana-gui`
|
||||
* :doc:`standalone-gui`
|
||||
* :doc:`tui`
|
||||
|
||||
|
||||
@@ -8,9 +8,7 @@ Standalone GUI analysis
|
||||
|
||||
ROCm Compute Profiler's standalone analysis GUI is a lightweight web page that you can
|
||||
generate straight from the command line. The standalone analysis GUI is an
|
||||
alternative to the CLI if you want to explore profiling results visually, but
|
||||
without the additional setup requirements or server-side overhead of ROCm Compute Profiler's
|
||||
detailed :doc:`Grafana interface <grafana-gui>` option. This analysis
|
||||
alternative to the CLI if you want to explore profiling results visually. This analysis
|
||||
option is implemented as a simple `Flask <https://flask.palletsprojects.com>`_
|
||||
application that lets you view results from your preferred web browser.
|
||||
|
||||
@@ -90,6 +88,4 @@ particular kernels or dispatches. You should see the web page update with
|
||||
metrics specific to your selected filters.
|
||||
|
||||
Once a filter is applied, you'll see several additional sections become
|
||||
available with detailed metrics specific to that area of AMD hardware. These
|
||||
detailed sections mirror the data displayed in ROCm Compute Profiler's
|
||||
:doc:`Grafana interface <grafana-gui>`.
|
||||
available with detailed metrics specific to that area of AMD hardware.
|
||||
|
||||
@@ -99,25 +99,6 @@ workload path.
|
||||
|
||||
See :doc:`analyze/cli` for more detailed information.
|
||||
|
||||
.. _basic-analyze-grafana:
|
||||
|
||||
Analyze in the Grafana GUI
|
||||
--------------------------
|
||||
|
||||
To conduct a more in-depth analysis of profiling results, it's suggested to use
|
||||
a Grafana GUI with ROCm Compute Profiler. To interact with profiling results, import your
|
||||
data to the MongoDB instance included in the ROCm Compute Profiler Dockerfile. See
|
||||
:doc:`/install/grafana-setup`.
|
||||
|
||||
To interact with Grafana data, stored in the ROCm Compute Profiler database, enter
|
||||
``database`` :ref:`mode <modes-database>`; for example:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ rocprof-compute database --import [CONNECTION OPTIONS]
|
||||
|
||||
See :doc:`/how-to/analyze/grafana-gui` for more detailed information.
|
||||
|
||||
.. _modes:
|
||||
|
||||
Modes
|
||||
@@ -160,10 +141,6 @@ Analyze mode
|
||||
To generate a lightweight GUI interface, you can add the ``--gui`` flag to your
|
||||
analysis command.
|
||||
|
||||
This mode is a middle ground to the highly detailed ROCm Compute Profiler Grafana GUI and
|
||||
is great if you want immediate access to a hardware component you’re already
|
||||
familiar with.
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ rocprof-compute analyze --help
|
||||
@@ -179,26 +156,6 @@ Analyze mode
|
||||
See :doc:`analyze/mode` to learn about these modes in depth and to get started
|
||||
with analysis using ROCm Compute Profiler.
|
||||
|
||||
.. _modes-database:
|
||||
|
||||
Database mode
|
||||
-------------
|
||||
|
||||
``database``
|
||||
The Grafana analyzer GUI is built on a MongoDB database. ``--import``
|
||||
profiling results to the DB to interact with the workload in Grafana or
|
||||
``--remove`` the workload from the DB.
|
||||
|
||||
Connection options need to be specified. See :doc:`/how-to/analyze/grafana-gui` for
|
||||
more details.
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ rocprof-compute database --help
|
||||
|
||||
See :doc:`/install/grafana-setup` to learn about setting up a Grafana server and
|
||||
database instance to make your profiling data more digestible and shareable.
|
||||
|
||||
.. _global-options:
|
||||
|
||||
Global options
|
||||
@@ -249,14 +206,6 @@ The following table lists ROCm Compute Profiler's basic operations, their
|
||||
- ``profile``
|
||||
- ``--name``, ``--roof-only``, ``--roofline-data-type <data_type>``, ``-- <profile_cmd>``
|
||||
|
||||
* - :ref:`Import a workload to database <grafana-gui-import>`
|
||||
- ``database``
|
||||
- ``--import``, ``--host``, ``--username``, ``--workload``, ``--team``
|
||||
|
||||
* - :ref:`Remove a workload from database <grafana-gui-remove>`
|
||||
- ``database``
|
||||
- ``--remove``, ``--host``, ``--username``, ``--workload``, ``--team``
|
||||
|
||||
* - :doc:`Launch standalone GUI from CLI </how-to/analyze/standalone-gui>`
|
||||
- ``analyze``
|
||||
- ``--path``, ``--gui``
|
||||
|
||||
@@ -27,7 +27,6 @@ ROCm Compute Profiler is open source and hosted at `<https://github.com/ROCm/roc
|
||||
.. grid-item-card:: Install
|
||||
|
||||
* :doc:`Installation and deployment <install/core-install>`
|
||||
* :doc:`Grafana server for ROCm Compute Profiler <install/grafana-setup>`
|
||||
|
||||
.. grid-item::
|
||||
|
||||
@@ -48,8 +47,6 @@ in practice.
|
||||
|
||||
* :doc:`how-to/analyze/cli`
|
||||
|
||||
* :doc:`how-to/analyze/grafana-gui`
|
||||
|
||||
* :doc:`how-to/analyze/standalone-gui`
|
||||
|
||||
* :doc:`how-to/analyze/tui`
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
.. meta::
|
||||
:description: ROCm Compute Profiler installation and deployment
|
||||
:keywords: Omniperf, ROCm Compute Profiler, ROCm, tool, Instinct, accelerator, AMD,
|
||||
install, deploy, Grafana, client, configuration, modulefiles
|
||||
install, deploy, client, configuration, modulefiles
|
||||
|
||||
**********************************************
|
||||
Installing and deploying ROCm Compute Profiler
|
||||
**********************************************
|
||||
|
||||
ROCm Compute Profiler consists of two installation components.
|
||||
|
||||
* :ref:`ROCm Compute Profiler core installation <core-install>` (client-side)
|
||||
* :ref:`ROCm Compute Profiler core installation <core-install>`
|
||||
|
||||
* Provides the core application profiling capability.
|
||||
* Allows the collection of performance counters, filtering by hardware
|
||||
@@ -17,20 +15,6 @@ ROCm Compute Profiler consists of two installation components.
|
||||
* Provides a CLI-based analysis mode.
|
||||
* Provides a standalone web interface for importing analysis metrics.
|
||||
|
||||
* :doc:`Grafana server for ROCm Compute Profiler <grafana-setup>` (server-side) (*optional*)
|
||||
|
||||
* Hosts the MongoDB backend and Grafana instance.
|
||||
* Is packaged in a Docker container for easy setup.
|
||||
|
||||
Determine what you need to install based on how you would like to interact with
|
||||
ROCm Compute Profiler. See the following decision tree to help determine what installation is
|
||||
right for you.
|
||||
|
||||
.. image:: ../data/install/install-decision-tree.png
|
||||
:align: center
|
||||
:alt: Decision tree for installing and deploying ROCm Compute Profiler
|
||||
:width: 800
|
||||
|
||||
.. _core-install:
|
||||
|
||||
Core installation
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
.. meta::
|
||||
:description: ROCm Compute Profiler Grafana server installation and deployment
|
||||
:keywords: ROCm Compute Profiler, ROCm, profiler, tool, Instinct, accelerator, AMD,
|
||||
install, deploy, Grafana, server, configuration, GUI
|
||||
|
||||
***************************************************
|
||||
Setting up Grafana server for ROCm Compute Profiler
|
||||
***************************************************
|
||||
|
||||
.. warning::
|
||||
|
||||
Grafana and MongoDB functionality is deprecated and will be removed in a future release.
|
||||
|
||||
A Grafana server is *not required* to profile or analyze performance data
|
||||
from the CLI. It's a supplementary mechanism to help you import performance
|
||||
data and examine it in a detailed
|
||||
`Grafana <https://github.com/grafana/grafana>`_ dashboard GUI.
|
||||
|
||||
Learn about installing and configuring the main ROCm Compute Profiler tool in
|
||||
:ref:`core-install`.
|
||||
|
||||
Setting up a Grafana instance for ROCm Compute Profiler requires the following basic software
|
||||
dependencies.
|
||||
|
||||
* `Docker Engine <https://docs.docker.com/engine/install/>`_
|
||||
|
||||
The recommended process for enabling the server-side of ROCm Compute Profiler is to use the
|
||||
provided ``Dockerfile`` to build the Grafana and MongoDB instance.
|
||||
|
||||
.. _grafana-mongodb-setup:
|
||||
|
||||
Set up Grafana and MongoDB
|
||||
==========================
|
||||
|
||||
Once you've decided where to host the Grafana and MongoDB instance, complete the
|
||||
the following setup instructions.
|
||||
|
||||
Install MongoDB utilities
|
||||
-------------------------
|
||||
|
||||
ROCm Compute Profiler uses the
|
||||
`mongoimport <https://www.mongodb.com/docs/database-tools/mongoimport/>`_
|
||||
utility to upload data to your Grafana instance's backend database.
|
||||
|
||||
Use the following commands to install MongoDB utilities for Ubuntu 20.04.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb
|
||||
$ sudo apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb
|
||||
|
||||
.. note::
|
||||
|
||||
Find installation instructions for other distributions in
|
||||
`MongoDB Database Tools Downloads <https://www.mongodb.com/download-center/database-tools/releases/archive>`_.
|
||||
|
||||
.. _grafana-persistent-storage-setup:
|
||||
|
||||
Set up persistent storage
|
||||
-------------------------
|
||||
|
||||
Bind MongoDB to a directory on the host OS to create a local backup in case of a
|
||||
crash or reset. This is called *creating a persistent volume*.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo mkdir -p /usr/local/persist && cd /usr/local/persist/
|
||||
$ sudo mkdir -p grafana-storage mongodb
|
||||
$ sudo docker volume create --driver local --opt type=none --opt device=/usr/local/persist/grafana-storage --opt o=bind grafana-storage
|
||||
$ sudo docker volume create --driver local --opt type=none --opt device=/usr/local/persist/mongodb --opt o=bind grafana-mongo-db
|
||||
|
||||
.. _grafana-docker-container:
|
||||
|
||||
Build and launch the Docker container
|
||||
-------------------------------------
|
||||
|
||||
You're now ready to build your ``Dockerfile``. Navigate to your ROCm Compute Profiler install
|
||||
directory to begin.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ cd grafana
|
||||
$ sudo docker-compose build
|
||||
$ sudo docker-compose up -d
|
||||
|
||||
.. note::
|
||||
|
||||
To troubleshoot Docker container build failures related to certificate verification, try
|
||||
disabling any network proxy services on the host system. These proxy services can interfere
|
||||
with OpenSSL's ability to retrieve a correct certificate chain when the container accesses
|
||||
external websites.
|
||||
|
||||
The TCP ports for Grafana (``4000``) and MongoDB (``27017``) in the Docker
|
||||
container are mapped to ``14000`` and ``27018``, respectively, on the host side.
|
||||
|
||||
.. tip::
|
||||
|
||||
In the event that either your Grafana or MongoDB instance crashes fatally,
|
||||
just restart the server. Navigate to your install directory and run:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ sudo docker-compose down
|
||||
$ sudo docker-compose up -d
|
||||
|
||||
.. _grafana-dashboard-setup:
|
||||
|
||||
Set up the Grafana dashboard
|
||||
----------------------------
|
||||
|
||||
Once you've launched your Docker container you should be able to reach Grafana
|
||||
at ``http://<host-ip>:14000``. The default login credentials for your first-time
|
||||
Grafana setup are:
|
||||
|
||||
* **Username**: ``admin``
|
||||
* **Password**: ``admin``
|
||||
|
||||
.. figure:: ../data/install/grafana_welcome.png
|
||||
:align: center
|
||||
:alt: Grafana dashboard welcome screen
|
||||
:width: 800
|
||||
|
||||
Grafana's welcome screen.
|
||||
|
||||
.. _grafana-datasource-setup:
|
||||
|
||||
Configure the MongoDB data source
|
||||
---------------------------------
|
||||
|
||||
You must configure your MongoDB data source in Grafana before first-time use.
|
||||
Navigate to Grafana's **Configuration** page to add the "Omniperf Data"
|
||||
connection.
|
||||
|
||||
.. figure:: ../data/install/datasource_config.jpg
|
||||
:align: center
|
||||
:alt: Grafana data source configuration
|
||||
:width: 800
|
||||
|
||||
Grafana's Configuration page.
|
||||
|
||||
Configure the following fields in the data source settings.
|
||||
|
||||
.. list-table::
|
||||
:stub-columns: 1
|
||||
|
||||
* - HTTP URL
|
||||
- ``http://localhost:3333``
|
||||
|
||||
* - MongoDB URL
|
||||
- ``mongodb://temp:temp123@\<host-ip>:27018/admin?authSource=admin``
|
||||
|
||||
* - Database Name
|
||||
- ``admin``
|
||||
|
||||
After configuring these fields, click **Save & test** to make sure your
|
||||
connection is successful.
|
||||
|
||||
.. figure:: ../data/install/datasource_settings.jpg
|
||||
:align: center
|
||||
:alt: Grafana data source settings
|
||||
:width: 800
|
||||
|
||||
Grafana data source settings.
|
||||
|
||||
.. note::
|
||||
|
||||
To avoid potential DNS issues, you might need to use the actual IP address
|
||||
for the host node in the MongoDB URL.
|
||||
|
||||
.. _grafana-import-dashboard-file:
|
||||
|
||||
Import the ROCm Compute Profiler dashboard file
|
||||
-----------------------------------------------
|
||||
|
||||
From the **Create** → **Import** page, upload the dashboard file,
|
||||
``/dashboards/Omniperf_v{__VERSION__}_pub.json`` from the
|
||||
:doc:`ROCm Compute Profiler tarball <core-install>`.
|
||||
|
||||
Edit both the dashboard **Name** and the **Unique identifier (UID)** fields to
|
||||
uniquely identify the dashboard. Click **Import** to complete the process.
|
||||
|
||||
.. figure:: ../data/install/import_dashboard.png
|
||||
:align: center
|
||||
:alt: Grafana's import dashboard
|
||||
:width: 800
|
||||
|
||||
Grafana's Import dashboard.
|
||||
|
||||
.. _grafana-select-workload:
|
||||
|
||||
Select and load the ROCm Compute Profiler workload
|
||||
--------------------------------------------------
|
||||
|
||||
Once you have imported a dashboard you're ready to begin. Start by browsing
|
||||
available dashboards and selecting the dashboard you have just imported.
|
||||
|
||||
.. figure:: ../data/install/opening_dashboard.png
|
||||
:align: center
|
||||
:alt: Opening your ROCm Compute Profiler dashboard in Grafana
|
||||
:width: 800
|
||||
|
||||
Opening your ROCm Compute Profiler profiling dashboard in Grafana.
|
||||
|
||||
Remember that you need to upload workload data to the MongoDB backend before
|
||||
analyzing in your Grafana interface. See a detailed example of this in
|
||||
:ref:`grafana-gui-import`.
|
||||
|
||||
After a workload has been successfully uploaded, you should be able to select it
|
||||
from the workload dropdown located at the top of your Grafana dashboard.
|
||||
|
||||
.. figure:: ../data/install/grafana_workload_selection.png
|
||||
:align: center
|
||||
:alt: ROCm Compute Profiler workload selection in Grafana
|
||||
:width: 800
|
||||
|
||||
Selecting your ROCm Compute Profiler workload in Grafana.
|
||||
|
||||
For more information on how to use the Grafana interface for analysis see
|
||||
:doc:`/how-to/analyze/grafana-gui`.
|
||||
@@ -9,18 +9,6 @@ FAQ
|
||||
|
||||
Frequently asked questions and troubleshooting tips.
|
||||
|
||||
How do I export profiling data I have already generated using ROCm Compute Profiler?
|
||||
====================================================================================
|
||||
|
||||
To interact with the Grafana GUI, you must sync data with the MongoDB
|
||||
backend. You can do this using :ref:`database <modes-database>` mode.
|
||||
|
||||
Pass in the directory of your desired workload as follows.
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
$ rocprof-compute database --import -w <path-to-results> -H <hostname> -u <username> -t <team-name>
|
||||
|
||||
python ast error: 'Constant' object has no attribute 'kind'
|
||||
===========================================================
|
||||
|
||||
|
||||
@@ -12,8 +12,6 @@ subtrees:
|
||||
entries:
|
||||
- file: install/core-install.rst
|
||||
title: Installation and deployment
|
||||
- file: install/grafana-setup.rst
|
||||
title: Grafana server setup
|
||||
|
||||
- caption: How to
|
||||
entries:
|
||||
@@ -24,7 +22,6 @@ subtrees:
|
||||
- file: how-to/analyze/mode.rst
|
||||
entries:
|
||||
- file: how-to/analyze/cli.rst
|
||||
- file: how-to/analyze/grafana-gui.rst
|
||||
- file: how-to/analyze/standalone-gui.rst
|
||||
- file: how-to/analyze/tui.rst
|
||||
|
||||
|
||||
@@ -35,30 +35,11 @@ Counters are stored in a comma-separated-values format for further
|
||||
micro-benchmarks to acquire hierarchical roofline data. The roofline model is
|
||||
not available on accelerators pre-MI200.
|
||||
|
||||
Grafana server for ROCm Compute Profiler
|
||||
----------------------------------------
|
||||
|
||||
* **Grafana database import**: All raw performance counters are imported into
|
||||
a :ref:`backend MongoDB database <grafana-mongodb-setup>` to support
|
||||
analysis and visualization in the Grafana GUI. Compatibility with
|
||||
previously generated data using older ROCm Compute Profiler versions is not guaranteed.
|
||||
|
||||
* **Grafana analysis dashboard GUI**: The
|
||||
:doc:`Grafana dashboard <how-to/analyze/grafana-gui>` retrieves the raw
|
||||
counters information from the backend database. It displays the relevant
|
||||
performance metrics and visualization.
|
||||
|
||||
ROCm Compute Profiler standalone GUI analyzer
|
||||
---------------------------------------------
|
||||
|
||||
ROCm Compute Profiler provides a :doc:`standalone GUI <how-to/analyze/standalone-gui>` to
|
||||
enable basic performance analysis without the need to import data into a
|
||||
database instance. Find setup instructions in :doc:`install/grafana-setup`
|
||||
|
||||
.. image:: data/install/omniperf_server_vs_client_install.png
|
||||
:align: center
|
||||
:alt: Architectural design of ROCm Compute Profiler
|
||||
:width: 800
|
||||
enable basic performance analysis.
|
||||
|
||||
Features
|
||||
========
|
||||
@@ -70,51 +51,13 @@ Additionally, ROCm Compute Profiler provides in-depth memory chart analysis, roo
|
||||
analysis, baseline comparisons, and more, ensuring a thorough understanding of
|
||||
system performance.
|
||||
|
||||
ROCm Compute Profiler supports analysis through both the :doc:`command line </how-to/analyze/cli>` or a
|
||||
:doc:`GUI </how-to/analyze/grafana-gui>`. The following list describes ROCm Compute Profiler's features at a
|
||||
high level.
|
||||
ROCm Compute Profiler supports analysis through both the :doc:`command line </how-to/analyze/cli>`.
|
||||
The following list describes ROCm Compute Profiler's features at a high level.
|
||||
|
||||
* :doc:`Support for AMD Instinct MI300, MI200, and MI100 accelerators <reference/compatible-accelerators>`
|
||||
|
||||
* :doc:`Standalone GUI analyzer </how-to/analyze/standalone-gui>`
|
||||
|
||||
* :doc:`GUI analyzer via Grafana and MongoDB </how-to/analyze/grafana-gui>`
|
||||
|
||||
* :ref:`System Info panel <grafana-panel-sys-info>`
|
||||
|
||||
* :ref:`Kernel Statistic panel <grafana-panel-kernel-stats>`
|
||||
|
||||
* :ref:`System Speed-of-Light panel <grafana-panel-system-sol>`
|
||||
|
||||
* :ref:`Memory Chart Analysis panel <grafana-panel-memory-chart-analysis>`
|
||||
|
||||
* :ref:`Roofline Analysis panel <grafana-panel-roofline-analysis>` (Supported on MI200 and above architectures only)
|
||||
|
||||
* :ref:`Command Processor (CP) panel <grafana-panel-cp>`
|
||||
|
||||
* :ref:`Workgroup Manager (SPI) panel <grafana-panel-spi>`
|
||||
|
||||
* :ref:`Wavefront Launch panel <grafana-panel-wavefront>`
|
||||
|
||||
* :ref:`Compute Unit - Instruction Mix panel <grafana-panel-cu-instruction-mix>`
|
||||
|
||||
* :ref:`Compute Unit - Pipeline panel <grafana-panel-cu-compute-pipeline>`
|
||||
|
||||
* :ref:`Local Data Share (LDS) panel <grafana-panel-lds>`
|
||||
|
||||
* :ref:`Instruction Cache panel <grafana-panel-instruction-cache>`
|
||||
|
||||
* :ref:`Scalar L1D Cache panel <grafana-panel-sl1d-cache>`
|
||||
|
||||
* :ref:`L1 Address Processing Unit or Texture Addresser (TA) <grafana-panel-ta>`;
|
||||
and :ref:`L1 Backend Data Processing Unit or Texture Data (TD) <grafana-panel-td>` panels
|
||||
|
||||
* :ref:`Vector L1D Cache panel <grafana-panel-vl1d>`
|
||||
|
||||
* :ref:`L2 Cache panel <grafana-panel-l2-cache>`
|
||||
|
||||
* :ref:`L2 Cache (per-channel) panel <grafana-panel-l2-cache-per-channel>`
|
||||
|
||||
* :ref:`Filtering <filtering>` to reduce profiling time
|
||||
|
||||
* Filtering by dispatch
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
/dashboards
|
||||
@@ -1,73 +0,0 @@
|
||||
# -----------------------------------------------------------------------
|
||||
# NOTE:
|
||||
# Dependencies are not included as part of ROCm Compute Profiler.
|
||||
# It's the user's responsibility to accept any licensing implications
|
||||
# before building the project
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
FROM ubuntu:22.04
|
||||
WORKDIR /app
|
||||
|
||||
USER root
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV TZ "US/Chicago"
|
||||
ENV NVM_DIR /usr/local/nvm
|
||||
ENV NODE_VERSION 20.12.2
|
||||
|
||||
ADD plugins/rocprofiler-compute_plugin /var/lib/grafana/plugins/rocprofiler-compute_plugin
|
||||
|
||||
# Install Grafana and MongoDB Community Edition
|
||||
# Note: Grafana install is stubbed to 10.4.3
|
||||
RUN apt-get update && \
|
||||
apt-get install -y adduser libfontconfig1 musl wget && \
|
||||
wget -q https://dl.grafana.com/enterprise/release/grafana-enterprise_10.4.3_amd64.deb && \
|
||||
dpkg -i grafana-enterprise_10.4.3_amd64.deb && \
|
||||
apt-get install -y gnupg curl && \
|
||||
curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor && \
|
||||
echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list && \
|
||||
apt-get update && \
|
||||
apt-get install -y mongodb-org
|
||||
|
||||
RUN mkdir /usr/local/nvm && \
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \
|
||||
. $NVM_DIR/nvm.sh && \
|
||||
nvm install $NODE_VERSION && \
|
||||
nvm alias default $NODE_VERSION && \
|
||||
nvm use default
|
||||
|
||||
ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
|
||||
ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
|
||||
|
||||
RUN npm --version && \
|
||||
node --version
|
||||
|
||||
# Install Grafana plugins
|
||||
RUN apt-get install -y tzdata systemd apt-utils npm vim net-tools && \
|
||||
/usr/sbin/grafana-cli plugins install michaeldmoore-multistat-panel && \
|
||||
/usr/sbin/grafana-cli plugins install ae3e-plotly-panel && \
|
||||
/usr/sbin/grafana-cli plugins install natel-plotly-panel && \
|
||||
/usr/sbin/grafana-cli plugins install grafana-image-renderer && \
|
||||
/usr/sbin/grafana-cli plugins install aceiot-svg-panel && \
|
||||
chown root:grafana /etc/grafana && \
|
||||
cd /var/lib/grafana/plugins/rocprofiler-compute_plugin && \
|
||||
npm install && \
|
||||
npm run build && \
|
||||
apt-get autoremove -y && \
|
||||
apt-get autoclean -y && \
|
||||
sed -i "s/ bindIp.*/ bindIp: 0.0.0.0/" /etc/mongod.conf && \
|
||||
mkdir -p /var/lib/grafana && \
|
||||
touch /var/lib/grafana/grafana.lib && \
|
||||
chown grafana:grafana /var/lib/grafana/grafana.lib
|
||||
|
||||
# Overwrite grafana ini file
|
||||
COPY grafana.ini /etc/grafana
|
||||
|
||||
# Switch Grafana port to 4000
|
||||
RUN sed -i "s/^;http_port = 3000/http_port = 4000/" /etc/grafana/grafana.ini && \
|
||||
sed -i "s/^http_port = 3000/http_port = 4000/" /usr/share/grafana/conf/defaults.ini
|
||||
|
||||
# Starts mongo and grafana-server at startup
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT [ "/docker-entrypoint.sh" ]
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,43 +0,0 @@
|
||||
# -----------------------------------------------------------------------
|
||||
# NOTE:
|
||||
# Dependencies are not included as part of ROCm Compute Profiler.
|
||||
# It's the user's responsibility to accept any licensing implications
|
||||
# before building the project
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
version: "3.3"
|
||||
|
||||
services:
|
||||
web:
|
||||
image: rocprofiler-compute-grafana-v1.0
|
||||
container_name: rocprofiler-compute-grafana-v1.0
|
||||
restart: always
|
||||
build: .
|
||||
environment:
|
||||
- GF_PATHS_CONFIG="grafana/etc/grafana.ini"
|
||||
- GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=amd-rocprofiler-compute-data-plugin
|
||||
- GF_DEFAULT_APP_MODE=development
|
||||
ports:
|
||||
- "14000:4000"
|
||||
volumes:
|
||||
- grafana-storage:/var/lib/grafana
|
||||
stdin_open: true
|
||||
tty: true
|
||||
db_mongo:
|
||||
container_name: mongo
|
||||
image: mongo
|
||||
restart: always
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: temp
|
||||
MONGO_INITDB_ROOT_PASSWORD: temp123
|
||||
volumes:
|
||||
- grafana-mongo-db:/data/db
|
||||
ports:
|
||||
- "27018:27017"
|
||||
command: mongod --bind_ip 0.0.0.0
|
||||
|
||||
volumes:
|
||||
grafana-mongo-db:
|
||||
external: true
|
||||
grafana-storage:
|
||||
external: true
|
||||
@@ -1,37 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
##############################################################################bl
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2021 - 2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
##############################################################################el
|
||||
|
||||
pushd /var/lib/grafana/plugins/rocprofiler-compute_plugin
|
||||
npm run server &
|
||||
popd
|
||||
|
||||
service grafana-server start
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
exec bash
|
||||
else
|
||||
eval $@
|
||||
fi
|
||||
Plik diff jest za duży
Load Diff
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"esnext": true,
|
||||
"disallowImplicitTypeConversion": ["string"],
|
||||
"disallowKeywords": ["with"],
|
||||
"disallowMultipleLineBreaks": true,
|
||||
"disallowMixedSpacesAndTabs": true,
|
||||
"disallowTrailingWhitespace": true,
|
||||
"requireSpacesInFunctionExpression": {
|
||||
"beforeOpeningCurlyBrace": true
|
||||
},
|
||||
"disallowSpacesInsideArrayBrackets": true,
|
||||
"disallowSpacesInsideParentheses": true,
|
||||
"validateIndentation": 2
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "es5",
|
||||
"useTabs": true
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
## 1.0.0 (Unreleased)
|
||||
|
||||
Initial release.
|
||||
-88
@@ -1,88 +0,0 @@
|
||||
module.exports = function(grunt) {
|
||||
|
||||
require('load-grunt-tasks')(grunt);
|
||||
|
||||
grunt.loadNpmTasks('grunt-execute');
|
||||
grunt.loadNpmTasks('grunt-contrib-clean');
|
||||
|
||||
grunt.initConfig({
|
||||
|
||||
clean: ["dist"],
|
||||
|
||||
copy: {
|
||||
src_to_dist: {
|
||||
cwd: 'src',
|
||||
expand: true,
|
||||
src: ['**/*', '!**/*.js', '!**/*.scss'],
|
||||
dest: 'dist'
|
||||
},
|
||||
server_to_dist: {
|
||||
cwd: 'server',
|
||||
expand: true,
|
||||
src: ['**/*'],
|
||||
dest: 'dist/server'
|
||||
},
|
||||
pluginDef: {
|
||||
expand: true,
|
||||
src: ['README.md'],
|
||||
dest: 'dist'
|
||||
}
|
||||
},
|
||||
|
||||
watch: {
|
||||
rebuild_all: {
|
||||
files: ['src/**/*'],
|
||||
tasks: ['default'],
|
||||
options: {spawn: false}
|
||||
}
|
||||
},
|
||||
|
||||
babel: {
|
||||
options: {
|
||||
sourceMap: true,
|
||||
presets: ['@babel/preset-env']
|
||||
},
|
||||
dist: {
|
||||
// options: {
|
||||
// plugins: ['transform-es2015-modules-systemjs', 'transform-es2015-for-of']
|
||||
// },
|
||||
files: [{
|
||||
cwd: 'src',
|
||||
expand: true,
|
||||
src: ['**/*.js'],
|
||||
dest: 'dist',
|
||||
ext:'.js'
|
||||
}]
|
||||
},
|
||||
distTestNoSystemJs: {
|
||||
files: [{
|
||||
cwd: 'src',
|
||||
expand: true,
|
||||
src: ['**/*.js'],
|
||||
dest: 'dist/test',
|
||||
ext:'.js'
|
||||
}]
|
||||
},
|
||||
distTestsSpecsNoSystemJs: {
|
||||
files: [{
|
||||
expand: true,
|
||||
cwd: 'spec',
|
||||
src: ['**/*.js'],
|
||||
dest: 'dist/test/spec',
|
||||
ext:'.js'
|
||||
}]
|
||||
}
|
||||
},
|
||||
|
||||
mochaTest: {
|
||||
test: {
|
||||
options: {
|
||||
reporter: 'spec'
|
||||
},
|
||||
src: ['dist/test/spec/test-main.js', 'dist/test/spec/*_spec.js']
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
grunt.registerTask('default', ['clean', 'copy:src_to_dist', 'copy:server_to_dist', 'copy:pluginDef', 'babel', 'mochaTest']);
|
||||
};
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 JamesOsgood
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,19 +0,0 @@
|
||||
# ROCm Compute Profiler Data Source Plugin
|
||||
|
||||
This plugin allows users of ROCm Compute Profiler to connect their MongoDB database to for visualization in Grafana
|
||||
|
||||
## Info
|
||||
|
||||
This backend exposes the endpoints Grafana requires to create a custom data source.
|
||||
|
||||
An express server, located in the `server` folder exposes these endpoints and makes the proxy to the MongoDB.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Node.js
|
||||
|
||||
`sudo apt-get install -y npm`
|
||||
|
||||
## Useage
|
||||
|
||||
<img src="src/img/sample.PNG" alt="Sample Data Source" style="width: 500px;"/>
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"name": "amd-rocprofiler-compute-data-plugin",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "server/mongo-proxy.js",
|
||||
"scripts": {
|
||||
"build": "./node_modules/grunt-cli/bin/grunt",
|
||||
"test": "./node_modules/grunt-cli/bin/grunt mochaTest'",
|
||||
"server": "cd server && node mongo-proxy.js"
|
||||
},
|
||||
"author": "Audacious Software Group",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@babel/preset-env": "^7.24.5",
|
||||
"babel": "^6.23.0",
|
||||
"chai": "^4.3.6",
|
||||
"grunt": "^1.6.1",
|
||||
"grunt-babel": "^8.0.0",
|
||||
"grunt-cli": "^1.4.3",
|
||||
"grunt-contrib-clean": "^1.1.0",
|
||||
"grunt-contrib-copy": "^1.0.0",
|
||||
"grunt-contrib-uglify": "^5.2.1",
|
||||
"grunt-contrib-watch": "^1.1.0",
|
||||
"grunt-mocha-test": "^0.13.3",
|
||||
"jsdom": "^20.0.0",
|
||||
"liftoff": "^5.0.0",
|
||||
"load-grunt-tasks": "^3.5.2",
|
||||
"prunk": "^1.3.0",
|
||||
"q": "^1.5.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"body-parser": "^1.18.3",
|
||||
"config": "^1.30.0",
|
||||
"express": "^4.17.3",
|
||||
"fs": "0.0.1-security",
|
||||
"lodash": "^4.17.21",
|
||||
"mocha": "^10.4.0",
|
||||
"moment": "^2.29.4",
|
||||
"mongodb": "^4.12.1",
|
||||
"statman-stopwatch": "^2.7.0"
|
||||
},
|
||||
"_comments": "Dependencies are not included as part of ROCm Compute Profiler. It's the user's responsibility to accept any licensing implications before building the project."
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"server":
|
||||
{
|
||||
"port": 3333,
|
||||
"logRequests": false,
|
||||
"logQueries": false,
|
||||
"logTimings": false
|
||||
}
|
||||
}
|
||||
-647
@@ -1,647 +0,0 @@
|
||||
// ################################################################################
|
||||
// # Copyright (c) 2018 JamesOsgood
|
||||
// #
|
||||
// # Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// # of this software and associated documentation files (the "Software"), to deal
|
||||
// # in the Software without restriction, including without limitation the rights
|
||||
// # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// # copies of the Software, and to permit persons to whom the Software is
|
||||
// # furnished to do so, subject to the following conditions:
|
||||
// #
|
||||
// # The above copyright notice and this permission notice shall be included in all
|
||||
// # copies or substantial portions of the Software.
|
||||
// #
|
||||
// # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// # SOFTWARE.
|
||||
// ################################################################################
|
||||
var express = require('express');
|
||||
var bodyParser = require('body-parser');
|
||||
var _ = require('lodash');
|
||||
var app = express();
|
||||
const MongoClient = require('mongodb').MongoClient;
|
||||
const assert = require('assert');
|
||||
var config = require('config');
|
||||
var Stopwatch = require("statman-stopwatch");
|
||||
var moment = require('moment')
|
||||
const serverConfig = require('./config/default.json').server;
|
||||
|
||||
app.use(bodyParser.json({ limit: '5mb' }));
|
||||
|
||||
// Called via required 'testDataSource'
|
||||
app.all('/', function(req, res, next)
|
||||
{
|
||||
console.log("Entered /")
|
||||
logRequest(req.body, "/")
|
||||
setCORSHeaders(res);
|
||||
|
||||
MongoClient.connect(req.body.db.url, function(err, client)
|
||||
{
|
||||
if ( err != null )
|
||||
{
|
||||
res.send({ status : "error",
|
||||
display_status : "Error",
|
||||
message : 'MongoDB Connection Error: ' + err.message });
|
||||
}
|
||||
else
|
||||
{
|
||||
res.send( { status : "success",
|
||||
display_status : "Success",
|
||||
message : 'MongoDB Connection test OK' });
|
||||
}
|
||||
next()
|
||||
})
|
||||
});
|
||||
|
||||
// Called by template functions and to look up variables
|
||||
app.all('/search', function(req, res, next)
|
||||
{
|
||||
console.log("Entered /search")
|
||||
logRequest(req.body, "/search")
|
||||
setCORSHeaders(res);
|
||||
|
||||
// Generate an id to track requests
|
||||
const requestId = ++requestIdCounter
|
||||
// Add state for the queries in this request
|
||||
var queryStates = []
|
||||
requestsPending[requestId] = queryStates
|
||||
// Parse query string in target
|
||||
queryArgs = parseQuery(req.body.target, {})
|
||||
if (queryArgs.err != null)
|
||||
{
|
||||
queryError(requestId, queryArgs.err, next)
|
||||
}
|
||||
else
|
||||
{
|
||||
doTemplateQuery(requestId, queryArgs, req.body.db, res, next);
|
||||
}
|
||||
});
|
||||
|
||||
// State for queries in flight. As results come it, acts as a semaphore and sends the results back
|
||||
var requestIdCounter = 0
|
||||
// Map of request id -> array of results. Results is
|
||||
// { query, err, output }
|
||||
var requestsPending = {}
|
||||
|
||||
// Called when a query finishes with an error
|
||||
function queryError(requestId, err, next)
|
||||
{
|
||||
// We only 1 return error per query so it may have been removed from the list
|
||||
if ( requestId in requestsPending )
|
||||
{
|
||||
// Remove request
|
||||
delete requestsPending[requestId]
|
||||
// Send back error
|
||||
next(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Called when query finished
|
||||
function queryFinished(requestId, queryId, results, res, next)
|
||||
{
|
||||
// We only 1 return error per query so it may have been removed from the list
|
||||
if ( requestId in requestsPending )
|
||||
{
|
||||
var queryStatus = requestsPending[requestId]
|
||||
// Mark this as finished
|
||||
queryStatus[queryId].pending = false
|
||||
queryStatus[queryId].results = results
|
||||
|
||||
// See if we're all done
|
||||
var done = true
|
||||
for ( var i = 0; i < queryStatus.length; i++)
|
||||
{
|
||||
if (queryStatus[i].pending == true )
|
||||
{
|
||||
done = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If query done, send back results
|
||||
if (done)
|
||||
{
|
||||
// Concatenate results
|
||||
output = []
|
||||
for ( var i = 0; i < queryStatus.length; i++)
|
||||
{
|
||||
var queryResults = queryStatus[i].results
|
||||
var keys = Object.keys(queryResults)
|
||||
for (var k = 0; k < keys.length; k++)
|
||||
{
|
||||
var tg = keys[k]
|
||||
output.push(queryResults[tg])
|
||||
}
|
||||
}
|
||||
res.json(output);
|
||||
next()
|
||||
// Remove request
|
||||
delete requestsPending[requestId]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Called to get graph points
|
||||
app.all('/query', function(req, res, next)
|
||||
{
|
||||
console.log("Entered /query")
|
||||
logRequest(req.body, "/query")
|
||||
setCORSHeaders(res);
|
||||
|
||||
// Parse query string in target
|
||||
substitutions = { "$from" : new Date(req.body.range.from),
|
||||
"$to" : new Date(req.body.range.to),
|
||||
"$dateBucketCount" : getBucketCount(req.body.range.from, req.body.range.to, req.body.intervalMs)
|
||||
}
|
||||
|
||||
// Generate an id to track requests
|
||||
const requestId = ++requestIdCounter
|
||||
// Add state for the queries in this request
|
||||
var queryStates = []
|
||||
requestsPending[requestId] = queryStates
|
||||
var error = false
|
||||
|
||||
for ( var queryId = 0; queryId < req.body.targets.length && !error; queryId++)
|
||||
{
|
||||
tg = req.body.targets[queryId]
|
||||
queryArgs = parseQuery(tg.target, substitutions)
|
||||
queryArgs.type = tg.type
|
||||
// If we picked up any errors while parsing query
|
||||
if (queryArgs.err != null)
|
||||
{
|
||||
queryError(requestId, queryArgs.err, next)
|
||||
error = true
|
||||
}
|
||||
else
|
||||
{
|
||||
// Add to the state
|
||||
queryStates.push( { pending : true } )
|
||||
|
||||
// Run the query
|
||||
runAggregateQuery( requestId, queryId, req.body, queryArgs, res, next)
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
app.use(function(error, req, res, next)
|
||||
{
|
||||
// Any request to this server will get here, and will send an HTTP
|
||||
// response with the error message
|
||||
res.status(500).json({ message: error.message });
|
||||
});
|
||||
|
||||
app.listen(serverConfig.port);
|
||||
|
||||
console.log("Server is listening on port " + serverConfig.port);
|
||||
|
||||
function setCORSHeaders(res)
|
||||
{
|
||||
res.setHeader("Access-Control-Allow-Origin", "*");
|
||||
res.setHeader("Access-Control-Allow-Methods", "POST");
|
||||
res.setHeader("Access-Control-Allow-Headers", "accept, content-type");
|
||||
}
|
||||
|
||||
function forIn(obj, processFunc)
|
||||
{
|
||||
var key;
|
||||
for (key in obj)
|
||||
{
|
||||
var value = obj[key]
|
||||
processFunc(obj, key, value)
|
||||
if ( value != null && typeof(value) == "object")
|
||||
{
|
||||
forIn(value, processFunc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseQuery(query, substitutions)
|
||||
{
|
||||
doc = {}
|
||||
docs = []
|
||||
queryErrors = []
|
||||
|
||||
chosenDB = null
|
||||
|
||||
query = query.trim() // Gets rid of any leading or trailing space
|
||||
// TODO: Add a case to catch custom db name
|
||||
// if (query.substring(0,3) != "db.")
|
||||
// {
|
||||
// queryErrors.push("Query must start with db.")
|
||||
// return null
|
||||
// }
|
||||
var firstDotIndex = query.indexOf('.')
|
||||
if (query.substring(0,3) != "db.")
|
||||
{
|
||||
if(firstDotIndex == -1){
|
||||
queryErrors.push("Invalid database name format")
|
||||
return null
|
||||
}
|
||||
chosenDB = query.substring(0, firstDotIndex)
|
||||
}
|
||||
doc.db = chosenDB
|
||||
|
||||
|
||||
// Query is of the form db.<collection>.aggregate or db.<collection>.find
|
||||
// Split on the first ( after db.
|
||||
var openBracketIndex = query.indexOf('(', firstDotIndex)
|
||||
|
||||
if (openBracketIndex == -1)
|
||||
{
|
||||
queryErrors.push("Can't find opening bracket")
|
||||
}
|
||||
else
|
||||
{
|
||||
// Split the first bit - it's the collection name and operation ( must be aggregate )
|
||||
var parts = query.substring(firstDotIndex+1, openBracketIndex).split('.')
|
||||
|
||||
// Collection names can have .s so last part is operation, rest is the collection name
|
||||
if (parts.length >= 2)
|
||||
{
|
||||
doc.operation = parts.pop().trim()
|
||||
doc.collection = parts.join('.')
|
||||
}
|
||||
else
|
||||
{
|
||||
queryErrors.push("Invalid collection and operation syntax")
|
||||
}
|
||||
|
||||
// Args is the rest up to the last bracket
|
||||
const potentialPipelineEnding = ['])', '],'];
|
||||
|
||||
const pipelineEnd = potentialPipelineEnding.find(end => query.indexOf(end, openBracketIndex) !== -1);
|
||||
const pipelineEndIndex = query.indexOf(pipelineEnd, openBracketIndex)
|
||||
|
||||
if (!pipelineEnd)
|
||||
{
|
||||
queryErrors.push("Can't find last bracket")
|
||||
}
|
||||
else
|
||||
{
|
||||
var args = query.substring(openBracketIndex + 1, pipelineEndIndex)
|
||||
if ( doc.operation == 'aggregate')
|
||||
{
|
||||
// Wrap args in array syntax so we can check for optional options arg
|
||||
args = '[' + args + ']]'
|
||||
docs = ((raw) => {
|
||||
try {
|
||||
const data = JSON.parse(raw);
|
||||
return data;
|
||||
}
|
||||
catch (error) {
|
||||
return null;
|
||||
}
|
||||
})(args);
|
||||
|
||||
if(!docs) {
|
||||
queryErrors.push("Invalid query syntax");
|
||||
}
|
||||
|
||||
// First Arg is pipeline
|
||||
doc.pipeline = docs[0]
|
||||
// If we have 2 top level args, second is agg options
|
||||
if ( docs.length == 2 )
|
||||
{
|
||||
doc.agg_options = docs[1]
|
||||
}
|
||||
// Replace with substitutions
|
||||
for ( var i = 0; i < doc.pipeline.length; i++)
|
||||
{
|
||||
var stage = doc.pipeline[i]
|
||||
forIn(stage, function (obj, key, value)
|
||||
{
|
||||
if ( typeof(value) == "string" )
|
||||
{
|
||||
if ( value.startsWith("&") )
|
||||
{
|
||||
obj[key] = "$"+value.substring(1) // ie "$from" --> new Date(req.body.range.from)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
queryErrors.push("Unknown operation " + doc.operation + ", only aggregate supported")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (queryErrors.length > 0 )
|
||||
{
|
||||
doc.err = new Error('Failed to parse query - ' + queryErrors.join(':'))
|
||||
}
|
||||
|
||||
// Checker for debugging
|
||||
if (chosenDB == null){
|
||||
console.log("Chosen DB is DEFAULT");
|
||||
}
|
||||
else{
|
||||
console.log("chosenDB is " + doc.db);
|
||||
}
|
||||
console.log("Collection is "+ doc.collection);
|
||||
|
||||
return doc
|
||||
}
|
||||
|
||||
// Run an aggregate query. Must return documents of the form
|
||||
// { value : 0.34334, ts : <epoch time in seconds> }
|
||||
|
||||
function runAggregateQuery( requestId, queryId, body, queryArgs, res, next )
|
||||
{
|
||||
MongoClient.connect(body.db.url, function(err, client)
|
||||
{
|
||||
if ( err != null )
|
||||
{
|
||||
queryError(requestId, err, next)
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
// TODO: Use the other db name if it was provided
|
||||
// var assignedName = null
|
||||
// if(queryArgs.db == null){
|
||||
// assignedName = body.db.db
|
||||
// }
|
||||
// else{
|
||||
// assignedName = queryArgs.db
|
||||
// }
|
||||
// console.log("Assigned DB name is " + assignedName)
|
||||
// const db = client.db(assignedName);
|
||||
var secondDb = client.db(queryArgs.db)
|
||||
|
||||
// Get the documents collection
|
||||
const collection = secondDb.collection(queryArgs.collection);
|
||||
logQuery(queryArgs.pipeline, queryArgs.agg_options)
|
||||
// Track how long it takes to complete query
|
||||
var stopwatch = new Stopwatch(true)
|
||||
|
||||
collection.aggregate(queryArgs.pipeline, {allowDiskUse:true}).toArray(function(err, docs)
|
||||
{
|
||||
if ( err != null )
|
||||
{
|
||||
client.close();
|
||||
queryError(requestId, err, next)
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
var results = {}
|
||||
if ( queryArgs.type == 'timeserie' )
|
||||
{
|
||||
results = getTimeseriesResults(docs)
|
||||
}
|
||||
|
||||
// This is where rocprofiler-compute will go for most results
|
||||
else
|
||||
{
|
||||
results = getTableResults(docs)
|
||||
}
|
||||
|
||||
client.close();
|
||||
var elapsedTimeMs = stopwatch.stop()
|
||||
logTiming(body, elapsedTimeMs)
|
||||
// Mark query as finished - will send back results when all queries finished
|
||||
queryFinished(requestId, queryId, results, res, next)
|
||||
}
|
||||
catch(err)
|
||||
{
|
||||
queryError(requestId, err, next)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function getTableResults(docs)
|
||||
{
|
||||
var columns = {}
|
||||
|
||||
// Build superset of columns
|
||||
for ( var i = 0; i < docs.length; i++)
|
||||
{
|
||||
var doc = docs[i]
|
||||
// Go through all properties
|
||||
for (var propName in doc )
|
||||
{
|
||||
// See if we need to add a new column
|
||||
if ( !(propName in columns) )
|
||||
{
|
||||
columns[propName] =
|
||||
{
|
||||
text : propName,
|
||||
type : "text"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build return rows
|
||||
rows = []
|
||||
for ( var i = 0; i < docs.length; i++)
|
||||
{
|
||||
var doc = docs[i]
|
||||
row = []
|
||||
// All cols
|
||||
for ( var colName in columns )
|
||||
{
|
||||
var col = columns[colName]
|
||||
if ( col.text in doc )
|
||||
{
|
||||
row.push(doc[col.text])
|
||||
}
|
||||
else
|
||||
{
|
||||
row.push(null)
|
||||
}
|
||||
}
|
||||
rows.push(row)
|
||||
}
|
||||
|
||||
var results = {}
|
||||
results["table"] = {
|
||||
columns : Object.values(columns),
|
||||
rows : rows,
|
||||
type : "table"
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
function getTimeseriesResults(docs)
|
||||
{
|
||||
var results = {}
|
||||
for ( var i = 0; i < docs.length; i++)
|
||||
{
|
||||
var doc = docs[i]
|
||||
var tg = doc.name
|
||||
var dp = null
|
||||
if (tg in results)
|
||||
{
|
||||
dp = results[tg]
|
||||
}
|
||||
else
|
||||
{
|
||||
dp = { 'target' : tg, 'datapoints' : [] }
|
||||
results[tg] = dp
|
||||
}
|
||||
|
||||
results[tg].datapoints.push([doc['value'], doc['ts'].getTime()])
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// Runs a query to support templates. Must return documents of the form
|
||||
// { _id : <id> }
|
||||
function doTemplateQuery(requestId, queryArgs, db, res, next)
|
||||
{
|
||||
if ( queryArgs.err == null)
|
||||
{
|
||||
if(queryArgs.db != null){ //We're looking at a custom dbName.aggregate
|
||||
|
||||
// Use connect method to connect to the server
|
||||
MongoClient.connect(db.url, function(err, client)
|
||||
{
|
||||
if ( err != null )
|
||||
{
|
||||
queryError(requestId, err, next )
|
||||
}
|
||||
else
|
||||
{
|
||||
// Remove request from list
|
||||
if ( requestId in requestsPending )
|
||||
{
|
||||
delete requestsPending[requestId]
|
||||
}
|
||||
var secondDb = client.db(queryArgs.db);
|
||||
// Get the documents collection
|
||||
const collection = secondDb.collection(queryArgs.collection);
|
||||
|
||||
collection.aggregate(queryArgs.pipeline).toArray(function(err, result)
|
||||
{
|
||||
assert.equal(err, null)
|
||||
|
||||
output = []
|
||||
for ( var i = 0; i < result.length; i++)
|
||||
{
|
||||
var doc = result[i]
|
||||
output.push(doc["_id"])
|
||||
}
|
||||
res.json(output);
|
||||
client.close()
|
||||
next()
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
else{ //Then we're just looking at the default db.aggregate
|
||||
// Database Name
|
||||
const dbName = db.db
|
||||
|
||||
// Use connect method to connect to the server
|
||||
MongoClient.connect(db.url, function(err, client)
|
||||
{
|
||||
if ( err != null )
|
||||
{
|
||||
queryError(requestId, err, next )
|
||||
}
|
||||
else
|
||||
{
|
||||
// Remove request from list
|
||||
if ( requestId in requestsPending )
|
||||
{
|
||||
delete requestsPending[requestId]
|
||||
}
|
||||
const db = client.db(dbName);
|
||||
// Get the documents collection
|
||||
const collection = db.collection(queryArgs.collection);
|
||||
|
||||
collection.aggregate(queryArgs.pipeline).toArray(function(err, result)
|
||||
{
|
||||
assert.equal(err, null)
|
||||
|
||||
output = []
|
||||
for ( var i = 0; i < result.length; i++)
|
||||
{
|
||||
var doc = result[i]
|
||||
output.push(doc["_id"])
|
||||
}
|
||||
res.json(output);
|
||||
client.close()
|
||||
next()
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
next(queryArgs.err)
|
||||
}
|
||||
}
|
||||
|
||||
function logRequest(body, type)
|
||||
{
|
||||
if (serverConfig.logRequests)
|
||||
{
|
||||
console.log("REQUEST: " + type + ":\n" + JSON.stringify(body,null,2))
|
||||
}
|
||||
}
|
||||
|
||||
function logQuery(query, options)
|
||||
{
|
||||
if (serverConfig.logQueries)
|
||||
{
|
||||
console.log("Query:")
|
||||
console.log(JSON.stringify(query,null,2))
|
||||
if ( options != null )
|
||||
{
|
||||
console.log("Query Options:")
|
||||
console.log(JSON.stringify(options,null,2))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function logTiming(body, elapsedTimeMs)
|
||||
{
|
||||
if (serverConfig.logTimings)
|
||||
{
|
||||
var range = new Date(body.range.to) - new Date(body.range.from)
|
||||
var diff = moment.duration(range)
|
||||
|
||||
console.log("Request: " + intervalCount(diff, body.interval, body.intervalMs) + " - Returned in " + elapsedTimeMs.toFixed(2) + "ms")
|
||||
}
|
||||
}
|
||||
|
||||
// Take a range as a moment.duration and a grafana interval like 30s, 1m etc
|
||||
// And return the number of intervals that represents
|
||||
function intervalCount(range, intervalString, intervalMs)
|
||||
{
|
||||
// Convert everything to seconds
|
||||
var rangeSeconds = range.asSeconds()
|
||||
var intervalsInRange = rangeSeconds / (intervalMs / 1000)
|
||||
|
||||
var output = intervalsInRange.toFixed(0) + ' ' + intervalString + ' intervals'
|
||||
return output
|
||||
}
|
||||
|
||||
function getBucketCount(from, to, intervalMs)
|
||||
{
|
||||
var boundaries = []
|
||||
var current = new Date(from).getTime()
|
||||
var toMs = new Date(to).getTime()
|
||||
var count = 0
|
||||
while ( current < toMs )
|
||||
{
|
||||
current += intervalMs
|
||||
count++
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
-38
@@ -1,38 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>mongodb-grafana-proxy</string>
|
||||
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>PATH</key>
|
||||
<string>/usr/local/bin/:$PATH</string>
|
||||
</dict>
|
||||
|
||||
<key>WorkingDirectory</key>
|
||||
<string>/usr/local/var/lib/grafana/plugins/mongodb-grafana/dist/server</string>
|
||||
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/usr/local/lib/npm-packages/bin/forever</string>
|
||||
<string>-l</string>
|
||||
<string>/usr/local/var/lib/grafana/plugins/mongodb-grafana/dist/server/mongodb-proxy-forever.log</string>
|
||||
<string>-o</string>
|
||||
<string>/usr/local/var/lib/grafana/plugins/mongodb-grafana/dist/server/mongodb-proxy.log</string>
|
||||
<string>-e</string>
|
||||
<string>/usr/local/var/lib/grafana/plugins/mongodb-grafana/dist/server/mongodb-proxy-err.log</string>
|
||||
<string>--workingDir</string>
|
||||
<string>/usr/local/var/lib/grafana/plugins/mongodb-grafana/dist/server</string>
|
||||
<string>/usr/local/var/lib/grafana/plugins/mongodb-grafana/dist/server/mongodb-proxy.js</string>
|
||||
</array>
|
||||
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
|
||||
<key>KeepAlive</key>
|
||||
<false/>
|
||||
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -1,3 +0,0 @@
|
||||
.generic-datasource-query-row .query-keyword {
|
||||
width: 75px;
|
||||
}
|
||||
-147
@@ -1,147 +0,0 @@
|
||||
// ################################################################################
|
||||
// # Copyright (c) 2018 JamesOsgood
|
||||
// #
|
||||
// # Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// # of this software and associated documentation files (the "Software"), to deal
|
||||
// # in the Software without restriction, including without limitation the rights
|
||||
// # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// # copies of the Software, and to permit persons to whom the Software is
|
||||
// # furnished to do so, subject to the following conditions:
|
||||
// #
|
||||
// # The above copyright notice and this permission notice shall be included in all
|
||||
// # copies or substantial portions of the Software.
|
||||
// #
|
||||
// # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// # SOFTWARE.
|
||||
// ################################################################################
|
||||
|
||||
import _ from "lodash";
|
||||
|
||||
export class GenericDatasource {
|
||||
|
||||
constructor(instanceSettings, $q, backendSrv, templateSrv) {
|
||||
this.type = instanceSettings.type;
|
||||
this.url = instanceSettings.url;
|
||||
this.name = instanceSettings.name;
|
||||
this.db = { 'url' : instanceSettings.jsonData.mongodb_url, 'db' : instanceSettings.jsonData.mongodb_db }
|
||||
this.q = $q;
|
||||
this.backendSrv = backendSrv;
|
||||
this.templateSrv = templateSrv;
|
||||
this.withCredentials = instanceSettings.withCredentials;
|
||||
this.headers = {'Content-Type': 'application/json'};
|
||||
// Assert valid sign in parameter
|
||||
if (typeof instanceSettings.basicAuth === 'string' && instanceSettings.basicAuth.length > 0) {
|
||||
this.headers['Authorization'] = instanceSettings.basicAuth;
|
||||
}
|
||||
}
|
||||
|
||||
query(options) {
|
||||
var query = this.buildQueryParameters(options);
|
||||
query.targets = query.targets.filter(t => !t.hide);
|
||||
query.db = this.db
|
||||
|
||||
if (query.targets.length <= 0) {
|
||||
return this.q.when({data: []});
|
||||
}
|
||||
|
||||
return this.doRequest({
|
||||
url: this.url + '/query',
|
||||
data: query,
|
||||
method: 'POST'
|
||||
});
|
||||
}
|
||||
|
||||
// Testing whether connection is valid
|
||||
testDatasource() {
|
||||
return this.doRequest({
|
||||
url: this.url + '/',
|
||||
data : { db : this.db },
|
||||
method: 'POST',
|
||||
}).then(response => {
|
||||
if (response.status === 200) {
|
||||
return { status: response.data.status, message: response.data.message, title: response.data.display_status };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
annotationQuery(options) {
|
||||
var query = this.templateSrv.replace(options.annotation.query, {}, 'glob');
|
||||
var annotationQuery = {
|
||||
range: options.range,
|
||||
annotation: {
|
||||
name: options.annotation.name,
|
||||
datasource: options.annotation.datasource,
|
||||
enable: options.annotation.enable,
|
||||
iconColor: options.annotation.iconColor,
|
||||
query: query
|
||||
},
|
||||
rangeRaw: options.rangeRaw
|
||||
};
|
||||
|
||||
return this.doRequest({
|
||||
url: this.url + '/annotations',
|
||||
method: 'POST',
|
||||
data: annotationQuery
|
||||
}).then(result => {
|
||||
response.data.$$status = result.status;
|
||||
response.data.$$config = result.config;
|
||||
return result.data;
|
||||
});
|
||||
}
|
||||
|
||||
metricFindQuery(query) {
|
||||
var interpolated = {
|
||||
target: this.templateSrv.replace(query, null, '')
|
||||
};
|
||||
interpolated.db = this.db
|
||||
|
||||
return this.doRequest({
|
||||
url: this.url + '/search',
|
||||
data: interpolated,
|
||||
method: 'POST',
|
||||
}).then(this.mapToTextValue);
|
||||
}
|
||||
|
||||
mapToTextValue(result) {
|
||||
return _.map(result.data, (d, i) => {
|
||||
if (d && d.text && d.value) {
|
||||
return { text: d.text, value: d.value };
|
||||
} else if (_.isObject(d)) {
|
||||
return { text: d, value: i};
|
||||
}
|
||||
return { text: d, value: d };
|
||||
});
|
||||
}
|
||||
|
||||
doRequest(options) {
|
||||
options.withCredentials = this.withCredentials;
|
||||
options.headers = this.headers;
|
||||
|
||||
return this.backendSrv.datasourceRequest(options);
|
||||
}
|
||||
|
||||
buildQueryParameters(options) {
|
||||
//remove place holder targets
|
||||
options.targets = _.filter(options.targets, target => {
|
||||
return target.target !== 'select metric';
|
||||
});
|
||||
|
||||
var targets = _.map(options.targets, target => {
|
||||
return {
|
||||
target: this.templateSrv.replace(target.target, options.scopedVars, ''),
|
||||
refId: target.refId,
|
||||
hide: target.hide,
|
||||
type: target.type || 'timeserie'
|
||||
};
|
||||
});
|
||||
|
||||
options.targets = targets;
|
||||
|
||||
return options;
|
||||
}
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
<!-- ################################################################################
|
||||
# Copyright (c) 2018 JamesOsgood
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
################################################################################ -->
|
||||
|
||||
<h5 class="section-heading">Query</h5>
|
||||
<div class="gf-form-group">
|
||||
<div class="gf-form">
|
||||
<input type="text" class="gf-form-input" ng-model='ctrl.annotation.query' placeholder=""></input>
|
||||
</div>
|
||||
</div>
|
||||
-47
@@ -1,47 +0,0 @@
|
||||
<!-- ################################################################################
|
||||
# Copyright (c) 2018 JamesOsgood
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
################################################################################ -->
|
||||
|
||||
<datasource-http-settings current="ctrl.current">
|
||||
</datasource-http-settings>
|
||||
|
||||
<h3 class="page-heading">MongoDB details</h3>
|
||||
|
||||
<div class="gf-form-group">
|
||||
<div class="gf-form-inline">
|
||||
<div class="gf-form">
|
||||
<span class="gf-form-label width-10">MongoDB URL</span>
|
||||
<input class="gf-form-input width-30" type="text"
|
||||
ng-model='ctrl.current.jsonData.mongodb_url'
|
||||
placeholder="mongodb://localhost:27017" required>
|
||||
</input>
|
||||
</div>
|
||||
|
||||
<div class="gf-form">
|
||||
<!--TODO: Make this compatible with $db Grafana variable-->
|
||||
<span class="gf-form-label width-10">Database Name</span>
|
||||
<input class="gf-form-input width-30" type="text"
|
||||
ng-model='ctrl.current.jsonData.mongodb_db'
|
||||
placeholder="myDatabase" required>
|
||||
</input>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
<!-- ################################################################################
|
||||
# Copyright (c) 2018 JamesOsgood
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
################################################################################ -->
|
||||
|
||||
<query-editor-row query-ctrl="ctrl" class="generic-datasource-query-row" has-text-edit-mode="true">
|
||||
<div class="gf-form-inline">
|
||||
<div class="gf-form max-width-8">
|
||||
<!--Drop down menu for selecting query type-->
|
||||
<select class="gf-form-input" ng-model="ctrl.target.type" ng-options="f as f for f in ['table', 'timeseries']"></select>
|
||||
</div>
|
||||
|
||||
<div class="gf-form" ng-if="ctrl.target.rawQuery">
|
||||
<textarea class="gf-form-input" rows="5" cols="150" ng-model="ctrl.target.target" spellcheck="false" ng-blur="ctrl.onChangeInternal()" />
|
||||
</div>
|
||||
|
||||
<div ng-if="!ctrl.target.rawQuery">
|
||||
<div class="gf-form">
|
||||
<gf-form-dropdown model="ctrl.target.target"
|
||||
allow-custom="true"
|
||||
lookup-text="true"
|
||||
get-options="ctrl.getOptions($query)"
|
||||
on-change="ctrl.onChangeInternal()">
|
||||
</gf-form-dropdown>
|
||||
</div>
|
||||
</div>
|
||||
<div class="gf-form gf-form--auto">
|
||||
<div class="gf-form-label gf-form-label--auto"></div>
|
||||
</div>
|
||||
</div>
|
||||
</query-editor-row>
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
<!-- ################################################################################
|
||||
# Copyright (c) 2018 JamesOsgood
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
################################################################################ -->
|
||||
|
||||
<section class="grafana-metric-options" >
|
||||
<div class="gf-form">
|
||||
</div>
|
||||
</section>
|
||||
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 67 KiB |
BIN
Plik binarny nie jest wyświetlany.
|
Przed Szerokość: | Wysokość: | Rozmiar: 47 KiB |
-41
@@ -1,41 +0,0 @@
|
||||
// ################################################################################
|
||||
// # Copyright (c) 2018 JamesOsgood
|
||||
// #
|
||||
// # Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// # of this software and associated documentation files (the "Software"), to deal
|
||||
// # in the Software without restriction, including without limitation the rights
|
||||
// # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// # copies of the Software, and to permit persons to whom the Software is
|
||||
// # furnished to do so, subject to the following conditions:
|
||||
// #
|
||||
// # The above copyright notice and this permission notice shall be included in all
|
||||
// # copies or substantial portions of the Software.
|
||||
// #
|
||||
// # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// # SOFTWARE.
|
||||
// ################################################################################
|
||||
|
||||
import {GenericDatasource} from './datasource';
|
||||
import {GenericDatasourceQueryCtrl} from './query_ctrl';
|
||||
|
||||
class GenericConfigCtrl {}
|
||||
GenericConfigCtrl.templateUrl = 'html/config.html';
|
||||
|
||||
class GenericQueryOptionsCtrl {}
|
||||
GenericQueryOptionsCtrl.templateUrl = 'html/query.options.html';
|
||||
|
||||
class GenericAnnotationsQueryCtrl {}
|
||||
GenericAnnotationsQueryCtrl.templateUrl = 'html/annotations.editor.html'
|
||||
|
||||
export {
|
||||
GenericDatasource as Datasource,
|
||||
GenericDatasourceQueryCtrl as QueryCtrl,
|
||||
GenericConfigCtrl as ConfigCtrl,
|
||||
GenericQueryOptionsCtrl as QueryOptionsCtrl,
|
||||
GenericAnnotationsQueryCtrl as AnnotationsQueryCtrl
|
||||
};
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "ROCm Compute Profiler Data",
|
||||
"id": "amd-rocprofiler-compute-data-plugin",
|
||||
"type": "datasource",
|
||||
"backend": true,
|
||||
"partials": {
|
||||
"config": "public/app/plugins/datasource/simplejson/partials/config.html"
|
||||
},
|
||||
"metrics": true,
|
||||
"annotations": false,
|
||||
"info": {
|
||||
"description": "An ROCm Compute Profiler datasource build for MongoDB",
|
||||
"author": {
|
||||
"name": "Audacious Software Group",
|
||||
"url": ""
|
||||
},
|
||||
"logos": {
|
||||
"small": "img/rocprofiler-compute_circle.png",
|
||||
"large": "img/rocprofiler-compute_circle.png"
|
||||
},
|
||||
"version": "%VERSION%",
|
||||
"updated": "%TODAY%"
|
||||
},
|
||||
"dependencies": {
|
||||
"grafanaVersion": ">=7.0.0",
|
||||
"plugins": []
|
||||
}
|
||||
}
|
||||
-50
@@ -1,50 +0,0 @@
|
||||
// ################################################################################
|
||||
// # Copyright (c) 2018 JamesOsgood
|
||||
// #
|
||||
// # Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// # of this software and associated documentation files (the "Software"), to deal
|
||||
// # in the Software without restriction, including without limitation the rights
|
||||
// # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// # copies of the Software, and to permit persons to whom the Software is
|
||||
// # furnished to do so, subject to the following conditions:
|
||||
// #
|
||||
// # The above copyright notice and this permission notice shall be included in all
|
||||
// # copies or substantial portions of the Software.
|
||||
// #
|
||||
// # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// # SOFTWARE.
|
||||
// ################################################################################
|
||||
|
||||
import {QueryCtrl} from 'app/plugins/sdk';
|
||||
import './css/query-editor.css!'
|
||||
|
||||
export class GenericDatasourceQueryCtrl extends QueryCtrl {
|
||||
|
||||
constructor($scope, $injector) {
|
||||
super($scope, $injector);
|
||||
|
||||
this.scope = $scope;
|
||||
this.target.target = this.target.target || 'select metric';
|
||||
this.target.type = this.target.type || 'timeserie';
|
||||
this.target.rawQuery = true;
|
||||
}
|
||||
|
||||
getOptions(query) {
|
||||
return this.datasource.metricFindQuery(query || '');
|
||||
}
|
||||
|
||||
toggleEditorMode() {
|
||||
this.target.rawQuery = !this.target.rawQuery;
|
||||
}
|
||||
|
||||
onChangeInternal() {
|
||||
this.panelCtrl.refresh(); // Asks the panel to refresh data.
|
||||
}
|
||||
}
|
||||
|
||||
GenericDatasourceQueryCtrl.templateUrl = 'html/query.editor.html';
|
||||
@@ -21,11 +21,7 @@ extend-exclude = [
|
||||
".misc",
|
||||
"external",
|
||||
"build-rocprof_compute",
|
||||
# deprecated files
|
||||
"src/rocprof_compute_profile/profiler_rocprof_v1.py",
|
||||
"src/rocprof_compute_profile/profiler_rocprof_v2.py",
|
||||
"src/utils/db_connector.py",
|
||||
# WIP files
|
||||
"src/rocprof_compute_analyze/analysis_db.py",
|
||||
"src/rocprof_compute_tui/widgets/splitter.py"
|
||||
]
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ def add_general_group(
|
||||
default=rocprof_compute_home / "rocprof_compute_soc/analysis_configs/",
|
||||
)
|
||||
# Nowhere to load specs from in db mode
|
||||
if parser.usage and "database" not in parser.usage:
|
||||
if parser.usage:
|
||||
general_group.add_argument(
|
||||
"-s", "--specs", action="store_true", help="Print system specs and exit."
|
||||
)
|
||||
@@ -503,106 +503,6 @@ Examples:
|
||||
# help="\t\t\tNumber of iterations (DEFAULT: 10)"
|
||||
# )
|
||||
|
||||
## Database Command Line Options
|
||||
## ----------------------------
|
||||
db_parser = subparsers.add_parser(
|
||||
"database",
|
||||
help="Interact with rocprofiler-compute database",
|
||||
usage="""
|
||||
\nrocprof-compute database <interaction type> [connection options]
|
||||
|
||||
\n\n-------------------------------------------------------------------------------
|
||||
\nExamples:
|
||||
\n\trocprof-compute database --import -H pavii1 -u temp -t asw -w "workloads/vcopy/mi200/"
|
||||
\n\trocprof-compute database --remove -H pavii1 -u temp -w "rocprofiler-compute_asw_sample_mi200"
|
||||
\n-------------------------------------------------------------------------------\n
|
||||
""", # noqa: E501
|
||||
prog="tool",
|
||||
allow_abbrev=False,
|
||||
formatter_class=lambda prog: argparse.RawTextHelpFormatter(
|
||||
prog, max_help_position=40
|
||||
),
|
||||
)
|
||||
db_parser._optionals.title = "Help"
|
||||
|
||||
add_general_group(
|
||||
db_parser, rocprof_compute_home, supported_archs, rocprof_compute_version
|
||||
)
|
||||
interaction_group = db_parser.add_argument_group("Interaction Type")
|
||||
connection_group = db_parser.add_argument_group("Connection Options")
|
||||
|
||||
interaction_group.add_argument(
|
||||
"-i",
|
||||
"--import",
|
||||
required=False,
|
||||
dest="upload",
|
||||
action="store_true",
|
||||
help="\t\tImport workload to rocprofiler-compute DB",
|
||||
)
|
||||
interaction_group.add_argument(
|
||||
"-r",
|
||||
"--remove",
|
||||
required=False,
|
||||
dest="remove",
|
||||
action="store_true",
|
||||
help="\t\tRemove a workload from rocprofiler-compute DB",
|
||||
)
|
||||
|
||||
connection_group.add_argument(
|
||||
"-H",
|
||||
"--host",
|
||||
required=True,
|
||||
metavar="",
|
||||
help="\t\tName or IP address of the server host.",
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"-P",
|
||||
"--port",
|
||||
required=False,
|
||||
metavar="",
|
||||
help="\t\tTCP/IP Port. (DEFAULT: 27018)",
|
||||
default=27018,
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"-u",
|
||||
"--username",
|
||||
required=True,
|
||||
metavar="",
|
||||
help="\t\tUsername for authentication.",
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"-p",
|
||||
"--password",
|
||||
metavar="",
|
||||
help="\t\tThe user's password. (will be requested later if it's not set)",
|
||||
default="",
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"-t", "--team", required=False, metavar="", help="\t\tSpecify Team prefix."
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"-w",
|
||||
"--workload",
|
||||
required=True,
|
||||
metavar="",
|
||||
dest="workload",
|
||||
help=(
|
||||
"\t\tSpecify name of workload (to remove) or path to workload (to import)"
|
||||
),
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"--kernel-verbose",
|
||||
required=False,
|
||||
metavar="",
|
||||
help=(
|
||||
"\t\tSpecify Kernel Name verbose level 1-5. "
|
||||
"Lower the level, shorter the kernel name. "
|
||||
"(DEFAULT: 5) (DISABLE: 5)"
|
||||
),
|
||||
default=5,
|
||||
type=int,
|
||||
)
|
||||
|
||||
## Analyze Command Line Options
|
||||
## ----------------------------
|
||||
analyze_parser = subparsers.add_parser(
|
||||
|
||||
@@ -143,8 +143,6 @@ def main() -> None:
|
||||
# major rocprofiler-compute execution modes
|
||||
if mode == "profile":
|
||||
rocprof_compute.run_profiler()
|
||||
elif mode == "database":
|
||||
rocprof_compute.update_db()
|
||||
elif mode == "analyze":
|
||||
rocprof_compute.run_analysis()
|
||||
else:
|
||||
|
||||
@@ -444,31 +444,6 @@ class RocProfCompute:
|
||||
console_debug(f'time taken for "post_processing" was {post_duration} seconds')
|
||||
self.__soc[self.__mspec.gpu_arch].post_profiling()
|
||||
|
||||
@demarcate
|
||||
def update_db(self) -> None:
|
||||
self.print_graphic()
|
||||
|
||||
console_warning(
|
||||
"Database update mode is deprecated and will "
|
||||
"be removed in a future release "
|
||||
"and no fixes will be made for this mode."
|
||||
)
|
||||
|
||||
from utils.db_connector import DatabaseConnector
|
||||
|
||||
db_connection = DatabaseConnector(self.__args)
|
||||
|
||||
# -----------------------
|
||||
# run database workflow
|
||||
# -----------------------
|
||||
db_connection.pre_processing()
|
||||
if self.__args.upload:
|
||||
db_connection.db_import()
|
||||
else:
|
||||
db_connection.db_remove()
|
||||
|
||||
return
|
||||
|
||||
@demarcate
|
||||
def run_analysis(self) -> None:
|
||||
self.print_graphic()
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
##############################################################################
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2021 - 2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
##############################################################################
|
||||
|
||||
import getpass
|
||||
import os
|
||||
from abc import abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from pymongo import MongoClient
|
||||
from tqdm import tqdm
|
||||
|
||||
from utils.kernel_name_shortener import kernel_name_shortener
|
||||
from utils.logger import (
|
||||
console_debug,
|
||||
console_error,
|
||||
console_log,
|
||||
console_warning,
|
||||
demarcate,
|
||||
)
|
||||
from utils.utils import is_workload_empty
|
||||
|
||||
MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout
|
||||
|
||||
|
||||
class DatabaseConnector:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.cache = dict()
|
||||
self.connection_info = {
|
||||
"username": self.args.username,
|
||||
"password": self.args.password,
|
||||
"host": self.args.host,
|
||||
"port": str(self.args.port),
|
||||
"team": self.args.team,
|
||||
"workload": self.args.workload,
|
||||
"db": None,
|
||||
}
|
||||
self.interaction_type: str = (
|
||||
None # set to 'import' or 'remove' based on user arguments
|
||||
)
|
||||
self.client: MongoClient = None
|
||||
|
||||
@demarcate
|
||||
def prep_import(self):
|
||||
# Extract SoC and workload name from sysinfo.csv
|
||||
sys_info = str(Path(self.connection_info["workload"]).joinpath("sysinfo.csv"))
|
||||
if Path(sys_info).is_file():
|
||||
sys_info = pd.read_csv(sys_info)
|
||||
try:
|
||||
soc = sys_info["gpu_model"][0].strip()
|
||||
name = sys_info["workload_name"][0].strip()
|
||||
except KeyError as e:
|
||||
console_error(
|
||||
f"Outdated workload. "
|
||||
f"Cannot find {e} field. Please reprofile to update."
|
||||
)
|
||||
else:
|
||||
console_error(
|
||||
"database", "Unable to parse SoC and/or workload name from sysinfo.csv"
|
||||
)
|
||||
|
||||
self.connection_info["db"] = (
|
||||
"rocprofiler-compute_"
|
||||
+ str(self.args.team)
|
||||
+ "_"
|
||||
+ str(name)
|
||||
+ "_"
|
||||
+ str(soc)
|
||||
)
|
||||
|
||||
@demarcate
|
||||
def db_import(self):
|
||||
self.prep_import()
|
||||
i = 0
|
||||
file = "blank"
|
||||
for file in tqdm(os.listdir(self.connection_info["workload"])):
|
||||
if file.endswith(".csv"):
|
||||
console_log(
|
||||
"database",
|
||||
"Uploading: %s" % self.connection_info["workload"] + "/" + file,
|
||||
)
|
||||
try:
|
||||
fileName = file[0 : file.find(".")]
|
||||
data = pd.read_csv(self.connection_info["workload"] + "/" + file)
|
||||
|
||||
# Demangle original KernelNames
|
||||
kernel_name_shortener(data, self.args.kernel_verbose)
|
||||
data.reset_index(inplace=True)
|
||||
data_dict = data.to_dict("records")
|
||||
|
||||
client = MongoClient(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin".format(
|
||||
self.connection_info["username"],
|
||||
self.connection_info["password"],
|
||||
self.connection_info["host"],
|
||||
self.connection_info["port"],
|
||||
self.connection_info["db"],
|
||||
)
|
||||
)
|
||||
db = client[self.connection_info["db"]]
|
||||
collection = db[fileName]
|
||||
collection.insert_many(data_dict)
|
||||
i += 1
|
||||
except pd.errors.EmptyDataError:
|
||||
console_warning("database", "Skipping empty file: %s" % file)
|
||||
|
||||
console_log("database", "%s collections successfully added." % i)
|
||||
mydb = self.client["workload_names"]
|
||||
mycol = mydb["names"]
|
||||
value = {"name": self.connection_info["db"]}
|
||||
newValue = {"name": self.connection_info["db"]}
|
||||
mycol.replace_one(value, newValue, upsert=True)
|
||||
console_log("database", "Workload name uploaded.")
|
||||
|
||||
@demarcate
|
||||
def db_remove(self):
|
||||
db_to_remove = self.client[self.connection_info["workload"]]
|
||||
|
||||
# check the collection names on the database
|
||||
self.client.drop_database(db_to_remove)
|
||||
db = self.client["workload_names"]
|
||||
col = db["names"]
|
||||
col.delete_many({"name": self.connection_info["workload"]})
|
||||
|
||||
console_log(
|
||||
"database", "Successfully removed %s" % self.connection_info["workload"]
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def pre_processing(self):
|
||||
"""Perform any pre-processing steps prior to database conncetion."""
|
||||
console_debug("database", "pre-processing database connection")
|
||||
if not self.args.remove and not self.args.upload:
|
||||
console_error(
|
||||
"Either -i/--import or -r/--remove is required in database mode"
|
||||
)
|
||||
self.interaction_type = "import" if self.args.upload else "remove"
|
||||
|
||||
# Detect interaction type
|
||||
if self.interaction_type == "remove":
|
||||
console_debug("database", "validating arguments for --remove workflow")
|
||||
is_full_workload_name = self.args.workload.count("_") >= 3
|
||||
if not is_full_workload_name:
|
||||
console_error(
|
||||
"-w/--workload is not valid. Please use full workload name "
|
||||
"as seen in GUI when removing (i.e. "
|
||||
"rocprofiler-compute_asw_vcopy_mi200)"
|
||||
)
|
||||
if (
|
||||
self.connection_info["host"] == None
|
||||
or self.connection_info["username"] == None
|
||||
):
|
||||
console_error(
|
||||
"-H/--host and -u/--username are required when "
|
||||
"interaction type is set to %s" % self.interaction_type
|
||||
)
|
||||
if (
|
||||
self.connection_info["workload"] == "admin"
|
||||
or self.connection_info["workload"] == "local"
|
||||
):
|
||||
console_error(
|
||||
"Cannot remove %s. Try again." % self.connection_info["workload"]
|
||||
)
|
||||
else:
|
||||
console_debug("database", "validating arguments for --import workflow")
|
||||
if (
|
||||
self.connection_info["host"] == None
|
||||
or self.connection_info["team"] == None
|
||||
or self.connection_info["username"] == None
|
||||
or self.connection_info["workload"] == None
|
||||
):
|
||||
console_error(
|
||||
"-H/--host, -w/--workload, -u/--username, and -t/--team are all "
|
||||
"required when interaction type is set to %s"
|
||||
% self.interaction_type
|
||||
)
|
||||
|
||||
if Path(self.connection_info["workload"]).absolute().is_dir():
|
||||
is_workload_empty(self.connection_info["workload"])
|
||||
else:
|
||||
console_error(
|
||||
"--workload is invalid. Please pass path to a valid directory."
|
||||
)
|
||||
|
||||
if len(self.args.team) > 13:
|
||||
console_error("--team exceeds 13 character limit. Try again.")
|
||||
|
||||
# format path properly
|
||||
self.connection_info["workload"] = str(
|
||||
Path(self.connection_info["workload"]).absolute().resolve()
|
||||
)
|
||||
|
||||
# Detect password
|
||||
if self.connection_info["password"] == "":
|
||||
try:
|
||||
self.connection_info["password"] = getpass.getpass()
|
||||
except Exception as e:
|
||||
console_error("database", "PASSWORD ERROR %s" % e)
|
||||
else:
|
||||
console_log("database", "Password received")
|
||||
else:
|
||||
pass
|
||||
|
||||
# Establish client connection
|
||||
connection_str = (
|
||||
"mongodb://"
|
||||
+ self.connection_info["username"]
|
||||
+ ":"
|
||||
+ self.connection_info["password"]
|
||||
+ "@"
|
||||
+ self.connection_info["host"]
|
||||
+ ":"
|
||||
+ self.connection_info["port"]
|
||||
+ "/?authSource=admin"
|
||||
)
|
||||
self.client = MongoClient(
|
||||
connection_str, serverSelectionTimeoutMS=MAX_SERVER_SEL_DELAY
|
||||
)
|
||||
try:
|
||||
self.client.server_info()
|
||||
except Exception:
|
||||
console_error("database", "Unable to connect to the DB server.")
|
||||
@@ -1,414 +0,0 @@
|
||||
##############################################################################
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
##############################################################################
|
||||
|
||||
import logging
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from db_connector import DatabaseConnector
|
||||
|
||||
logging.TRACE = logging.DEBUG - 5
|
||||
logging.addLevelName(logging.TRACE, "TRACE")
|
||||
|
||||
|
||||
def trace_logger(message, *args, **kwargs):
|
||||
logging.log(logging.TRACE, message, *args, **kwargs)
|
||||
|
||||
|
||||
setattr(logging, "trace", trace_logger)
|
||||
|
||||
"""
|
||||
Tests for the DatabaseConnector class that tests almost methods with initialization,
|
||||
CSV import, database removal, and error handling.
|
||||
The tests use mocks instead of a real MongoDB server for speed and reliability.
|
||||
"""
|
||||
|
||||
|
||||
class TestDatabaseConnector:
|
||||
@pytest.fixture
|
||||
def mock_args_import(self):
|
||||
"""Mock arguments for import operation"""
|
||||
args = Mock()
|
||||
args.username = "test_user"
|
||||
args.password = "test_pass"
|
||||
args.host = "localhost"
|
||||
args.port = 27017
|
||||
args.team = "test_team"
|
||||
args.workload = "/app/tests/workloads/device_filter/MI100"
|
||||
args.upload = True
|
||||
args.remove = False
|
||||
args.kernel_verbose = False
|
||||
return args
|
||||
|
||||
@pytest.fixture
|
||||
def mock_args_remove(self):
|
||||
"""Mock arguments for remove operation"""
|
||||
args = Mock()
|
||||
args.username = "test_user"
|
||||
args.password = "test_pass"
|
||||
args.host = "localhost"
|
||||
args.port = 27017
|
||||
args.team = "test_team"
|
||||
args.workload = "rocprofiler-compute_test_team_workload_mi100"
|
||||
args.upload = False
|
||||
args.remove = True
|
||||
args.kernel_verbose = False
|
||||
return args
|
||||
|
||||
def test_init(self, mock_args_import):
|
||||
"""Test DatabaseConnector initialization"""
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
assert connector.args == mock_args_import
|
||||
assert isinstance(connector.cache, dict)
|
||||
assert len(connector.cache) == 0
|
||||
|
||||
expected_connection_info = {
|
||||
"username": "test_user",
|
||||
"password": "test_pass",
|
||||
"host": "localhost",
|
||||
"port": "27017",
|
||||
"team": "test_team",
|
||||
"workload": "/app/tests/workloads/device_filter/MI100",
|
||||
"db": None,
|
||||
}
|
||||
assert connector.connection_info == expected_connection_info
|
||||
assert connector.interaction_type is None
|
||||
assert connector.client is None
|
||||
|
||||
@patch("db_connector.pd.read_csv")
|
||||
@patch("db_connector.Path")
|
||||
def test_prep_import_success(self, mock_path, mock_read_csv, mock_args_import):
|
||||
"""Test successful prep_import"""
|
||||
# Setup mocks
|
||||
mock_path.return_value.joinpath.return_value = "/fake/path/sysinfo.csv"
|
||||
mock_path.return_value.is_file.return_value = True
|
||||
|
||||
mock_sysinfo = pd.DataFrame({
|
||||
"gpu_model": ["MI100 "],
|
||||
"workload_name": [" test_workload"],
|
||||
})
|
||||
mock_read_csv.return_value = mock_sysinfo
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
connector.prep_import()
|
||||
|
||||
expected_db = "rocprofiler-compute_test_team_test_workload_MI100"
|
||||
assert connector.connection_info["db"] == expected_db
|
||||
|
||||
@patch("db_connector.pd.read_csv")
|
||||
@patch("db_connector.Path")
|
||||
def test_prep_import_missing_file(self, mock_path, mock_read_csv, mock_args_import):
|
||||
"""Test prep_import when sysinfo.csv is missing"""
|
||||
mock_path.return_value.joinpath.return_value = "/fake/path/sysinfo.csv"
|
||||
mock_path.return_value.is_file.return_value = False
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch(
|
||||
"db_connector.console_error", side_effect=SystemExit(1)
|
||||
) as mock_console_error:
|
||||
with pytest.raises(SystemExit):
|
||||
connector.prep_import()
|
||||
|
||||
mock_console_error.assert_called_with(
|
||||
"database", "Unable to parse SoC and/or workload name from sysinfo.csv"
|
||||
)
|
||||
|
||||
@patch("db_connector.pd.read_csv")
|
||||
@patch("db_connector.Path")
|
||||
def test_prep_import_key_error(self, mock_path, mock_read_csv, mock_args_import):
|
||||
"""Test prep_import when required fields are missing"""
|
||||
mock_path.return_value.joinpath.return_value = "/fake/path/sysinfo.csv"
|
||||
mock_path.return_value.is_file.return_value = True
|
||||
|
||||
mock_sysinfo = pd.DataFrame({"other_column": ["value"]})
|
||||
mock_read_csv.return_value = mock_sysinfo
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch(
|
||||
"db_connector.console_error", side_effect=SystemExit(1)
|
||||
) as mock_console_error:
|
||||
with pytest.raises(SystemExit):
|
||||
connector.prep_import()
|
||||
|
||||
assert mock_console_error.called
|
||||
error_call = mock_console_error.call_args[0][0]
|
||||
assert "Outdated workload" in error_call
|
||||
|
||||
@patch("db_connector.tqdm")
|
||||
@patch("db_connector.os.listdir")
|
||||
@patch("db_connector.console_log")
|
||||
@patch("db_connector.console_warning")
|
||||
@patch("db_connector.kernel_name_shortener")
|
||||
@patch("db_connector.MongoClient")
|
||||
@patch("db_connector.pd.read_csv")
|
||||
def test_db_import_success(
|
||||
self,
|
||||
mock_read_csv,
|
||||
mock_mongo_client,
|
||||
mock_kernel_shortener,
|
||||
mock_console_warning,
|
||||
mock_console_log,
|
||||
mock_listdir,
|
||||
mock_tqdm,
|
||||
mock_args_import,
|
||||
):
|
||||
"""Test successful database import"""
|
||||
mock_listdir.return_value = ["test_data.csv", "empty_file.csv", "non_csv.txt"]
|
||||
mock_tqdm.return_value = mock_listdir.return_value
|
||||
|
||||
test_df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
|
||||
mock_read_csv.side_effect = [test_df, pd.errors.EmptyDataError()]
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_db = MagicMock()
|
||||
mock_collection = MagicMock()
|
||||
mock_workload_db = MagicMock()
|
||||
mock_workload_col = MagicMock()
|
||||
|
||||
mock_mongo_client.return_value = mock_client_instance
|
||||
mock_client_instance.__getitem__.side_effect = lambda x: {
|
||||
"rocprofiler-compute_test_team_test_workload_MI100": mock_db,
|
||||
"workload_names": mock_workload_db,
|
||||
}.get(x, mock_db)
|
||||
mock_db.__getitem__.return_value = mock_collection
|
||||
mock_workload_db.__getitem__.return_value = mock_workload_col
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
connector.connection_info["workload"] = "/fake/workload/path"
|
||||
connector.client = mock_client_instance
|
||||
|
||||
with patch.object(connector, "prep_import") as mock_prep:
|
||||
mock_prep.return_value = None
|
||||
connector.connection_info["db"] = (
|
||||
"rocprofiler-compute_test_team_test_workload_MI100"
|
||||
)
|
||||
|
||||
connector.db_import()
|
||||
|
||||
mock_collection.insert_many.assert_called_once()
|
||||
mock_workload_col.replace_one.assert_called_once()
|
||||
|
||||
@patch("db_connector.console_log")
|
||||
def test_db_remove_success(self, mock_console_log, mock_args_remove):
|
||||
"""Test successful database removal"""
|
||||
mock_client = MagicMock()
|
||||
mock_db_to_remove = MagicMock()
|
||||
mock_workload_names_db = MagicMock()
|
||||
mock_names_col = MagicMock()
|
||||
|
||||
mock_client.__getitem__.side_effect = lambda x: {
|
||||
"rocprofiler-compute_test_team_workload_mi100": mock_db_to_remove,
|
||||
"workload_names": mock_workload_names_db,
|
||||
}[x]
|
||||
mock_workload_names_db.__getitem__.return_value = mock_names_col
|
||||
mock_db_to_remove.list_collection_names.return_value = ["col1", "col2"]
|
||||
|
||||
connector = DatabaseConnector(mock_args_remove)
|
||||
connector.client = mock_client
|
||||
|
||||
connector.db_remove()
|
||||
|
||||
mock_client.drop_database.assert_called_once_with(mock_db_to_remove)
|
||||
mock_names_col.delete_many.assert_called_once_with({
|
||||
"name": "rocprofiler-compute_test_team_workload_mi100"
|
||||
})
|
||||
|
||||
def test_pre_processing_no_action_specified(self, mock_args_import):
|
||||
"""Test pre_processing when neither upload nor remove is specified"""
|
||||
mock_args_import.upload = False
|
||||
mock_args_import.remove = False
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
def test_pre_processing_remove_invalid_workload_name(self, mock_args_remove):
|
||||
"""Test pre_processing remove with invalid workload name"""
|
||||
mock_args_remove.workload = "invalid_name"
|
||||
|
||||
connector = DatabaseConnector(mock_args_remove)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
def test_pre_processing_remove_missing_host_username(self, mock_args_remove):
|
||||
"""Test pre_processing remove with missing host/username"""
|
||||
mock_args_remove.host = None
|
||||
mock_args_remove.username = None
|
||||
|
||||
connector = DatabaseConnector(mock_args_remove)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
def test_pre_processing_remove_protected_database(self, mock_args_remove):
|
||||
"""Test pre_processing remove with protected database names"""
|
||||
mock_args_remove.workload = "admin"
|
||||
|
||||
connector = DatabaseConnector(mock_args_remove)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
@patch("db_connector.Path")
|
||||
@patch("db_connector.is_workload_empty")
|
||||
@patch("db_connector.getpass.getpass")
|
||||
@patch("db_connector.console_log")
|
||||
@patch("db_connector.MongoClient")
|
||||
def test_pre_processing_import_password_prompt_success(
|
||||
self,
|
||||
mock_mongo_client,
|
||||
mock_console_log,
|
||||
mock_getpass,
|
||||
mock_is_workload_empty,
|
||||
mock_path,
|
||||
mock_args_import,
|
||||
):
|
||||
"""Test pre_processing import with password prompt success"""
|
||||
mock_args_import.password = ""
|
||||
mock_getpass.return_value = "prompted_password"
|
||||
|
||||
mock_path.return_value.absolute.return_value.is_dir.return_value = True
|
||||
mock_path.return_value.absolute.return_value.resolve.return_value = (
|
||||
"/resolved/path"
|
||||
)
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_mongo_client.return_value = mock_client_instance
|
||||
mock_client_instance.server_info.return_value = {}
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
connector.pre_processing()
|
||||
|
||||
mock_getpass.assert_called_once()
|
||||
mock_console_log.assert_called_with("database", "Password received")
|
||||
|
||||
@patch("db_connector.Path")
|
||||
@patch("db_connector.is_workload_empty")
|
||||
@patch("db_connector.MongoClient")
|
||||
def test_pre_processing_import_connection_failure(
|
||||
self, mock_mongo_client, mock_is_workload_empty, mock_path, mock_args_import
|
||||
):
|
||||
"""Test pre_processing import with MongoDB connection failure"""
|
||||
mock_path.return_value.absolute.return_value.is_dir.return_value = True
|
||||
mock_path.return_value.absolute.return_value.resolve.return_value = (
|
||||
"/resolved/path"
|
||||
)
|
||||
|
||||
mock_client_instance = MagicMock()
|
||||
mock_mongo_client.return_value = mock_client_instance
|
||||
mock_client_instance.server_info.side_effect = Exception("Connection failed")
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
@patch("db_connector.Path")
|
||||
@patch("db_connector.is_workload_empty")
|
||||
def test_pre_processing_import_missing_required_fields(
|
||||
self, mock_is_workload_empty, mock_path, mock_args_import
|
||||
):
|
||||
"""Test pre_processing import with missing required fields"""
|
||||
mock_args_import.host = None
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
@patch("db_connector.Path")
|
||||
def test_pre_processing_import_invalid_workload_path(
|
||||
self, mock_path, mock_args_import
|
||||
):
|
||||
"""Test pre_processing import with invalid workload path"""
|
||||
mock_path.return_value.absolute.return_value.is_dir.return_value = False
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
def test_pre_processing_import_team_name_too_long(self, mock_args_import):
|
||||
"""Test pre_processing import with team name exceeding limit"""
|
||||
mock_args_import.team = "this_team_name_is_way_too_long"
|
||||
|
||||
connector = DatabaseConnector(mock_args_import)
|
||||
|
||||
with patch("db_connector.console_error", side_effect=SystemExit(1)):
|
||||
with pytest.raises(SystemExit):
|
||||
connector.pre_processing()
|
||||
|
||||
|
||||
class TestDatabaseConnectorIntegration:
|
||||
"""Simple integration test"""
|
||||
|
||||
@patch("db_connector.Path")
|
||||
@patch("db_connector.pd.read_csv")
|
||||
def test_prep_import_with_real_workload_path(self, mock_read_csv, mock_path):
|
||||
"""Test prep_import with actual workload path structure"""
|
||||
args = Mock()
|
||||
args.username = "test_user"
|
||||
args.password = "test_pass"
|
||||
args.host = "localhost"
|
||||
args.port = 27017
|
||||
args.team = "test_team"
|
||||
args.workload = "/app/tests/workloads/device_filter/MI100"
|
||||
args.upload = True
|
||||
args.remove = False
|
||||
args.kernel_verbose = False
|
||||
|
||||
mock_path.return_value.joinpath.return_value = (
|
||||
"/app/tests/workloads/device_filter/MI100/sysinfo.csv"
|
||||
)
|
||||
mock_path.return_value.is_file.return_value = True
|
||||
|
||||
mock_sysinfo = pd.DataFrame({
|
||||
"gpu_model": ["MI100"],
|
||||
"workload_name": ["device_filter"],
|
||||
})
|
||||
mock_read_csv.return_value = mock_sysinfo
|
||||
|
||||
connector = DatabaseConnector(args)
|
||||
connector.prep_import()
|
||||
|
||||
expected_db = "rocprofiler-compute_test_team_device_filter_MI100"
|
||||
assert connector.connection_info["db"] == expected_db
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Plik diff jest za duży
Load Diff
Reference in New Issue
Block a user