colramos-amd
a1371462ba
Signed-off-by: colramos-amd <colramos@amd.com>
[ROCm/rocprofiler-compute commit: 5bf38a4fed]
224 строки
8.6 KiB
Python
224 строки
8.6 KiB
Python
##############################################################################bl
|
|
# MIT License
|
|
#
|
|
# Copyright (c) 2021 - 2024 Advanced Micro Devices, Inc. All Rights Reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
##############################################################################el
|
|
|
|
from abc import ABC, abstractmethod
|
|
from utils.utils import is_workload_empty, demarcate, console_error, console_log, console_warning, console_debug
|
|
from pymongo import MongoClient
|
|
from tqdm import tqdm
|
|
|
|
import os
|
|
import getpass
|
|
import pandas as pd
|
|
|
|
MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout
|
|
|
|
|
|
class DatabaseConnector:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.cache = dict()
|
|
self.connection_info = {
|
|
"username": self.args.username,
|
|
"password": self.args.password,
|
|
"host": self.args.host,
|
|
"port": str(self.args.port),
|
|
"team": self.args.team,
|
|
"workload": self.args.workload,
|
|
"db": None,
|
|
}
|
|
self.interaction_type: str = (
|
|
None # set to 'import' or 'remove' based on user arguments
|
|
)
|
|
self.client: MongoClient = None
|
|
|
|
@demarcate
|
|
def prep_import(self):
|
|
|
|
# Extract SoC and workload name from sysinfo.csv
|
|
sys_info = os.path.join(self.connection_info["workload"], "sysinfo.csv")
|
|
if os.path.isfile(sys_info):
|
|
sys_info = pd.read_csv(sys_info)
|
|
soc = sys_info["name"][0]
|
|
name = sys_info["workload_name"][0]
|
|
else:
|
|
console_error("[database] Unable to parse SoC and/or workload name from sysinfo.csv")
|
|
|
|
self.connection_info["db"] = (
|
|
"omniperf_" + str(self.args.team) + "_" + str(name) + "_" + str(soc)
|
|
)
|
|
|
|
@demarcate
|
|
def db_import(self):
|
|
self.prep_import()
|
|
i = 0
|
|
file = "blank"
|
|
for file in tqdm(os.listdir(self.connection_info["workload"])):
|
|
if file.endswith(".csv"):
|
|
console_log(
|
|
"database",
|
|
"Uploading: %s" % self.connection_info["workload"] + "/" + file
|
|
)
|
|
try:
|
|
fileName = file[0 : file.find(".")]
|
|
cmd = (
|
|
"mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline"
|
|
).format(
|
|
self.connection_info["username"],
|
|
self.connection_info["password"],
|
|
self.connection_info["host"],
|
|
self.connection_info["port"],
|
|
self.connection_info["db"],
|
|
self.connection_info["workload"] + "/" + file,
|
|
fileName,
|
|
)
|
|
os.system(cmd)
|
|
i += 1
|
|
except pd.errors.EmptyDataError:
|
|
console_warning("[database] Skipping empty file: %s" % file)
|
|
|
|
console_log(
|
|
"database",
|
|
"%s collections successfully added." % i
|
|
)
|
|
mydb = self.client["workload_names"]
|
|
mycol = mydb["names"]
|
|
value = {"name": self.connection_info["db"]}
|
|
newValue = {"name": self.connection_info["db"]}
|
|
mycol.replace_one(value, newValue, upsert=True)
|
|
console_log(
|
|
"database",
|
|
"Workload name uploaded."
|
|
)
|
|
|
|
@demarcate
|
|
def db_remove(self):
|
|
db_to_remove = self.client[self.connection_info["workload"]]
|
|
|
|
# check the collection names on the database
|
|
col_list = db_to_remove.list_collection_names()
|
|
self.client.drop_database(db_to_remove)
|
|
db = self.client["workload_names"]
|
|
col = db["names"]
|
|
col.delete_many({"name": self.connection_info['workload']})
|
|
|
|
console_log(
|
|
"database",
|
|
"Successfully removed %s" % self.connection_info['workload']
|
|
)
|
|
|
|
|
|
@abstractmethod
|
|
def pre_processing(self):
|
|
"""Perform any pre-processing steps prior to database conncetion.
|
|
"""
|
|
console_debug(
|
|
"database",
|
|
"pre-processing database connection"
|
|
)
|
|
if not self.args.remove and not self.args.upload:
|
|
console_error("Either -i/--import or -r/--remove is required in database mode")
|
|
self.interaction_type = 'import' if self.args.upload else 'remove'
|
|
|
|
# Detect interaction type
|
|
if self.interaction_type == 'remove':
|
|
console_debug(
|
|
"database",
|
|
"validating arguments for --remove workflow"
|
|
)
|
|
is_full_workload_name = self.args.workload.count("_") >= 3
|
|
if not is_full_workload_name:
|
|
console_error("-w/--workload is not valid. Please use full workload name as seen in GUI when removing (i.e. omniperf_asw_vcopy_mi200)")
|
|
if self.connection_info['host'] == None or self.connection_info['username'] == None:
|
|
console_error("-H/--host and -u/--username are required when interaction type is set to %s" % self.interaction_type)
|
|
if self.connection_info['workload'] == "admin" or self.connection_info['workload'] == "local":
|
|
console_error("Cannot remove %s. Try again." % self.connection_info['workload'])
|
|
else:
|
|
console_debug(
|
|
"database",
|
|
"validating arguments for --import workflow"
|
|
)
|
|
if (
|
|
self.connection_info["host"] == None
|
|
or self.connection_info["team"] == None
|
|
or self.connection_info["username"] == None
|
|
or self.connection_info["workload"] == None
|
|
):
|
|
console_error("-H/--host, -w/--workload, -u/--username, and -t/--team are all required when interaction type is set to %s" % self.interaction_type)
|
|
|
|
if os.path.isdir(os.path.abspath(self.connection_info["workload"])):
|
|
is_workload_empty(self.connection_info["workload"])
|
|
else:
|
|
console_error("--workload is invalid. Please pass path to a valid directory.")
|
|
|
|
if len(self.args.team) > 13:
|
|
console_error("--team exceeds 13 character limit. Try again.")
|
|
|
|
# format path properly
|
|
self.connection_info["workload"] = os.path.abspath(
|
|
self.connection_info["workload"]
|
|
)
|
|
|
|
# Detect password
|
|
if self.connection_info["password"] == "":
|
|
try:
|
|
self.connection_info["password"] = getpass.getpass()
|
|
except Exception as e:
|
|
console_error(
|
|
"database",
|
|
"PASSWORD ERROR %s" % e
|
|
)
|
|
else:
|
|
console_log(
|
|
"database",
|
|
"Password recieved"
|
|
)
|
|
else:
|
|
password = self.connection_info["password"]
|
|
|
|
# Establish client connection
|
|
connection_str = (
|
|
"mongodb://"
|
|
+ self.connection_info["username"]
|
|
+ ":"
|
|
+ self.connection_info["password"]
|
|
+ "@"
|
|
+ self.connection_info["host"]
|
|
+ ":"
|
|
+ self.connection_info["port"]
|
|
+ "/?authSource=admin"
|
|
)
|
|
self.client = MongoClient(
|
|
connection_str, serverSelectionTimeoutMS=MAX_SERVER_SEL_DELAY
|
|
)
|
|
try:
|
|
self.client.server_info()
|
|
except:
|
|
console_error(
|
|
"database",
|
|
"Unable to connect to the DB server."
|
|
)
|
|
|
|
|
|
|