diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index 1fd022bd2c..75afb7289c 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -1093,10 +1093,15 @@ typedef enum { */ typedef enum { AMDSMI_UTILIZATION_COUNTER_FIRST = 0, - //!< GFX Activity + //!< Corse grain activity counters AMDSMI_COARSE_GRAIN_GFX_ACTIVITY = AMDSMI_UTILIZATION_COUNTER_FIRST, AMDSMI_COARSE_GRAIN_MEM_ACTIVITY, //!< Memory Activity - AMDSMI_UTILIZATION_COUNTER_LAST = AMDSMI_COARSE_GRAIN_MEM_ACTIVITY + AMDSMI_COARSE_DECODER_ACTIVITY, //!< Decoder Activity + //!< Fine grain activity counters + AMDSMI_FINE_GRAIN_GFX_ACTIVITY = 100, + AMDSMI_FINE_GRAIN_MEM_ACTIVITY = 101, + AMDSMI_FINE_DECODER_ACTIVITY = 102, + AMDSMI_UTILIZATION_COUNTER_LAST = AMDSMI_FINE_DECODER_ACTIVITY } amdsmi_utilization_counter_type_t; /** @@ -1111,11 +1116,16 @@ typedef enum { /** * @brief The utilization counter data */ +//! The max number of values per counter type +#define AMDSMI_MAX_UTILIZATION_VALUES 4 typedef struct { - amdsmi_utilization_counter_type_t type; //!< Utilization counter type - uint64_t value; //!< Utilization counter value + amdsmi_utilization_counter_type_t type; //!< Utilization counter type + uint64_t value; //!< Coarse grain activity counter value (average) + uint64_t fine_value[AMDSMI_MAX_UTILIZATION_VALUES]; //!< Utilization counter value + uint16_t fine_value_count; } amdsmi_utilization_counter_t; + /** * @brief Reserved Memory Page Record */ @@ -3162,7 +3172,7 @@ amdsmi_status_t amdsmi_set_gpu_clk_range(amdsmi_processor_handle processor_handl * * @platform{gpu_bm_linux} @platform{guest_1vf} * - * @details Given a processor handle @p processor_handle, a clock type @p clk_type, + * @details Given a processor handle @p processor_handle, a clock type @p clk_type, * a value @p clk_value needs to be set, and the @p level indicates min or max * clock you want to set, this function the clock limit. * diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index db00f18226..a7360b66fd 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -386,6 +386,10 @@ class AmdSmiIoLinkType(IntEnum): class AmdSmiUtilizationCounterType(IntEnum): COARSE_GRAIN_GFX_ACTIVITY = amdsmi_wrapper.AMDSMI_COARSE_GRAIN_GFX_ACTIVITY COARSE_GRAIN_MEM_ACTIVITY = amdsmi_wrapper.AMDSMI_COARSE_GRAIN_MEM_ACTIVITY + COARSE_DECODER_ACTIVITY = amdsmi_wrapper.AMDSMI_COARSE_DECODER_ACTIVITY + FINE_GRAIN_GFX_ACTIVITY = amdsmi_wrapper.AMDSMI_FINE_GRAIN_GFX_ACTIVITY + FINE_GRAIN_MEM_ACTIVITY = amdsmi_wrapper.AMDSMI_FINE_GRAIN_MEM_ACTIVITY + FINE_DECODER_ACTIVITY = amdsmi_wrapper.AMDSMI_FINE_DECODER_ACTIVITY UTILIZATION_COUNTER_FIRST = amdsmi_wrapper.AMDSMI_UTILIZATION_COUNTER_FIRST UTILIZATION_COUNTER_LAST = amdsmi_wrapper.AMDSMI_UTILIZATION_COUNTER_LAST @@ -3395,7 +3399,7 @@ def amdsmi_get_utilization_count( if counter_type == "AMDSMI_UTILIZATION_COUNTER_FIRST": counter_type = "AMDSMI_COARSE_GRAIN_GPU_ACTIVITY" if counter_type == "AMDSMI_UTILIZATION_COUNTER_LAST": - counter_type = "AMDSMI_COARSE_GRAIN_MEM_ACTIVITY" + counter_type = "AMDSMI_FINE_DECODER_ACTIVITY" result.append( {"type": counter_type, "value": util_counter_list[index].value}) diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index c7650bb4aa..77138d266a 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -1452,12 +1452,20 @@ amdsmi_utilization_counter_type_t__enumvalues = { 0: 'AMDSMI_UTILIZATION_COUNTER_FIRST', 0: 'AMDSMI_COARSE_GRAIN_GFX_ACTIVITY', 1: 'AMDSMI_COARSE_GRAIN_MEM_ACTIVITY', - 1: 'AMDSMI_UTILIZATION_COUNTER_LAST', + 2: 'AMDSMI_COARSE_DECODER_ACTIVITY', + 100: 'AMDSMI_FINE_GRAIN_GFX_ACTIVITY', + 101: 'AMDSMI_FINE_GRAIN_MEM_ACTIVITY', + 102: 'AMDSMI_FINE_DECODER_ACTIVITY', + 102: 'AMDSMI_UTILIZATION_COUNTER_LAST', } AMDSMI_UTILIZATION_COUNTER_FIRST = 0 AMDSMI_COARSE_GRAIN_GFX_ACTIVITY = 0 AMDSMI_COARSE_GRAIN_MEM_ACTIVITY = 1 -AMDSMI_UTILIZATION_COUNTER_LAST = 1 +AMDSMI_COARSE_DECODER_ACTIVITY = 2 +AMDSMI_FINE_GRAIN_GFX_ACTIVITY = 100 +AMDSMI_FINE_GRAIN_MEM_ACTIVITY = 101 +AMDSMI_FINE_DECODER_ACTIVITY = 102 +AMDSMI_UTILIZATION_COUNTER_LAST = 102 amdsmi_utilization_counter_type_t = ctypes.c_uint32 # enum # values for enumeration 'amdsmi_power_type_t' @@ -1478,6 +1486,9 @@ struct_amdsmi_utilization_counter_t._fields_ = [ ('type', amdsmi_utilization_counter_type_t), ('PADDING_0', ctypes.c_ubyte * 4), ('value', ctypes.c_uint64), + ('fine_value', ctypes.c_uint64 * 4), + ('fine_value_count', ctypes.c_uint16), + ('PADDING_1', ctypes.c_ubyte * 6), ] amdsmi_utilization_counter_t = struct_amdsmi_utilization_counter_t @@ -2411,6 +2422,7 @@ __all__ = \ 'AMDSMI_CLK_TYPE_SYS', 'AMDSMI_CLK_TYPE_VCLK0', 'AMDSMI_CLK_TYPE_VCLK1', 'AMDSMI_CLK_TYPE__MAX', 'AMDSMI_CNTR_CMD_START', 'AMDSMI_CNTR_CMD_STOP', + 'AMDSMI_COARSE_DECODER_ACTIVITY', 'AMDSMI_COARSE_GRAIN_GFX_ACTIVITY', 'AMDSMI_COARSE_GRAIN_MEM_ACTIVITY', 'AMDSMI_COMPUTE_PARTITION_CPX', 'AMDSMI_COMPUTE_PARTITION_DPX', @@ -2443,25 +2455,27 @@ __all__ = \ 'AMDSMI_EVT_NOTIF_GPU_PRE_RESET', 'AMDSMI_EVT_NOTIF_LAST', 'AMDSMI_EVT_NOTIF_NONE', 'AMDSMI_EVT_NOTIF_RING_HANG', 'AMDSMI_EVT_NOTIF_THERMAL_THROTTLE', 'AMDSMI_EVT_NOTIF_VMFAULT', - 'AMDSMI_FREQ_IND_INVALID', 'AMDSMI_FREQ_IND_MAX', - 'AMDSMI_FREQ_IND_MIN', 'AMDSMI_FW_ID_ASD', 'AMDSMI_FW_ID_CP_CE', - 'AMDSMI_FW_ID_CP_ME', 'AMDSMI_FW_ID_CP_MEC1', - 'AMDSMI_FW_ID_CP_MEC2', 'AMDSMI_FW_ID_CP_MEC_JT1', - 'AMDSMI_FW_ID_CP_MEC_JT2', 'AMDSMI_FW_ID_CP_MES', - 'AMDSMI_FW_ID_CP_PFP', 'AMDSMI_FW_ID_CP_PM4', 'AMDSMI_FW_ID_DFC', - 'AMDSMI_FW_ID_DMCU', 'AMDSMI_FW_ID_DMCU_ERAM', - 'AMDSMI_FW_ID_DMCU_ISR', 'AMDSMI_FW_ID_DRV_CAP', - 'AMDSMI_FW_ID_FIRST', 'AMDSMI_FW_ID_IMU_DRAM', - 'AMDSMI_FW_ID_IMU_IRAM', 'AMDSMI_FW_ID_ISP', 'AMDSMI_FW_ID_MC', - 'AMDSMI_FW_ID_MES_KIQ', 'AMDSMI_FW_ID_MES_STACK', - 'AMDSMI_FW_ID_MES_THREAD1', 'AMDSMI_FW_ID_MES_THREAD1_STACK', - 'AMDSMI_FW_ID_MMSCH', 'AMDSMI_FW_ID_PM', 'AMDSMI_FW_ID_PPTABLE', - 'AMDSMI_FW_ID_PSP_BL', 'AMDSMI_FW_ID_PSP_DBG', - 'AMDSMI_FW_ID_PSP_INTF', 'AMDSMI_FW_ID_PSP_KEYDB', - 'AMDSMI_FW_ID_PSP_SOC', 'AMDSMI_FW_ID_PSP_SOSDRV', - 'AMDSMI_FW_ID_PSP_SPL', 'AMDSMI_FW_ID_PSP_SYSDRV', - 'AMDSMI_FW_ID_PSP_TOC', 'AMDSMI_FW_ID_REG_ACCESS_WHITELIST', - 'AMDSMI_FW_ID_RLC', 'AMDSMI_FW_ID_RLCV_LX7', 'AMDSMI_FW_ID_RLC_P', + 'AMDSMI_FINE_DECODER_ACTIVITY', 'AMDSMI_FINE_GRAIN_GFX_ACTIVITY', + 'AMDSMI_FINE_GRAIN_MEM_ACTIVITY', 'AMDSMI_FREQ_IND_INVALID', + 'AMDSMI_FREQ_IND_MAX', 'AMDSMI_FREQ_IND_MIN', 'AMDSMI_FW_ID_ASD', + 'AMDSMI_FW_ID_CP_CE', 'AMDSMI_FW_ID_CP_ME', + 'AMDSMI_FW_ID_CP_MEC1', 'AMDSMI_FW_ID_CP_MEC2', + 'AMDSMI_FW_ID_CP_MEC_JT1', 'AMDSMI_FW_ID_CP_MEC_JT2', + 'AMDSMI_FW_ID_CP_MES', 'AMDSMI_FW_ID_CP_PFP', + 'AMDSMI_FW_ID_CP_PM4', 'AMDSMI_FW_ID_DFC', 'AMDSMI_FW_ID_DMCU', + 'AMDSMI_FW_ID_DMCU_ERAM', 'AMDSMI_FW_ID_DMCU_ISR', + 'AMDSMI_FW_ID_DRV_CAP', 'AMDSMI_FW_ID_FIRST', + 'AMDSMI_FW_ID_IMU_DRAM', 'AMDSMI_FW_ID_IMU_IRAM', + 'AMDSMI_FW_ID_ISP', 'AMDSMI_FW_ID_MC', 'AMDSMI_FW_ID_MES_KIQ', + 'AMDSMI_FW_ID_MES_STACK', 'AMDSMI_FW_ID_MES_THREAD1', + 'AMDSMI_FW_ID_MES_THREAD1_STACK', 'AMDSMI_FW_ID_MMSCH', + 'AMDSMI_FW_ID_PM', 'AMDSMI_FW_ID_PPTABLE', 'AMDSMI_FW_ID_PSP_BL', + 'AMDSMI_FW_ID_PSP_DBG', 'AMDSMI_FW_ID_PSP_INTF', + 'AMDSMI_FW_ID_PSP_KEYDB', 'AMDSMI_FW_ID_PSP_SOC', + 'AMDSMI_FW_ID_PSP_SOSDRV', 'AMDSMI_FW_ID_PSP_SPL', + 'AMDSMI_FW_ID_PSP_SYSDRV', 'AMDSMI_FW_ID_PSP_TOC', + 'AMDSMI_FW_ID_REG_ACCESS_WHITELIST', 'AMDSMI_FW_ID_RLC', + 'AMDSMI_FW_ID_RLCV_LX7', 'AMDSMI_FW_ID_RLC_P', 'AMDSMI_FW_ID_RLC_RESTORE_LIST_CNTL', 'AMDSMI_FW_ID_RLC_RESTORE_LIST_GPM_MEM', 'AMDSMI_FW_ID_RLC_RESTORE_LIST_SRM_MEM', diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h index e53527b1ce..a9dcaa18ce 100755 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h @@ -765,10 +765,15 @@ typedef enum _RSMI_IO_LINK_TYPE { */ typedef enum { RSMI_UTILIZATION_COUNTER_FIRST = 0, - //!< GFX Activity + //!< Corse grain activity counters RSMI_COARSE_GRAIN_GFX_ACTIVITY = RSMI_UTILIZATION_COUNTER_FIRST, RSMI_COARSE_GRAIN_MEM_ACTIVITY, //!< Memory Activity - RSMI_UTILIZATION_COUNTER_LAST = RSMI_COARSE_GRAIN_MEM_ACTIVITY + RSMI_COARSE_DECODER_ACTIVITY, //!< Decoder Activity + //!< Fine grain activity counters + RSMI_FINE_GRAIN_GFX_ACTIVITY = 100, + RSMI_FINE_GRAIN_MEM_ACTIVITY = 101, + RSMI_FINE_DECODER_ACTIVITY = 102, + RSMI_UTILIZATION_COUNTER_LAST = RSMI_FINE_DECODER_ACTIVITY } RSMI_UTILIZATION_COUNTER_TYPE; /** @@ -783,11 +788,16 @@ typedef enum { /** * @brief The utilization counter data */ +//! The max number of values per counter type +#define RSMI_MAX_UTILIZATION_VALUES 4 typedef struct { - RSMI_UTILIZATION_COUNTER_TYPE type; //!< Utilization counter type - uint64_t value; //!< Utilization counter value + RSMI_UTILIZATION_COUNTER_TYPE type; //!< Utilization counter type + uint64_t value; //!< Coarse grain activity counter value (average) + uint64_t fine_value[RSMI_MAX_UTILIZATION_VALUES]; //!< Utilization counter value (individual values) + uint16_t fine_value_count; } rsmi_utilization_counter_t; + /** * @brief Reserved Memory Page Record */ diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi.cc b/projects/amdsmi/rocm_smi/src/rocm_smi.cc index c38d97480a..1526e5832c 100755 --- a/projects/amdsmi/rocm_smi/src/rocm_smi.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi.cc @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -2087,7 +2088,7 @@ rsmi_status_t rsmi_dev_process_isolation_get(uint32_t dv_ind, } /* - for 4 partition: enforce isolation is enabled on partition 2 and + for 4 partition: enforce isolation is enabled on partition 2 and disabled on partitions 0, 1, 3. $ cat /sys/class/drm/cardX/device/enforce_isolation 0 0 1 0 @@ -4209,6 +4210,7 @@ rsmi_utilization_count_get(uint32_t dv_ind, rsmi_status_t ret; rsmi_gpu_metrics_t gpu_metrics; uint32_t val_ui32; + uint16_t val_counter(0); ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics); if (ret != RSMI_STATUS_SUCCESS) { @@ -4217,21 +4219,61 @@ rsmi_utilization_count_get(uint32_t dv_ind, for (uint32_t index = 0 ; index < count; index++) { switch (utilization_counters[index].type) { + case RSMI_COARSE_GRAIN_GFX_ACTIVITY: - val_ui32 = gpu_metrics.gfx_activity_acc; + case RSMI_FINE_GRAIN_GFX_ACTIVITY: + val_counter = 1; + utilization_counters[index].value = gpu_metrics.gfx_activity_acc; + utilization_counters[index].fine_value[0] = + (gpu_metrics.gfx_activity_acc != std::numeric_limits::max()) + ? gpu_metrics.gfx_activity_acc : std::numeric_limits::max(); + utilization_counters[index].fine_value_count = + (gpu_metrics.gfx_activity_acc == std::numeric_limits::max()) + ? 0 : val_counter; break; + case RSMI_COARSE_GRAIN_MEM_ACTIVITY: - val_ui32 = gpu_metrics.mem_activity_acc; + case RSMI_FINE_GRAIN_MEM_ACTIVITY: + val_counter = 1; + utilization_counters[index].value = gpu_metrics.mem_activity_acc; + utilization_counters[index].fine_value[0] = + (gpu_metrics.mem_activity_acc != std::numeric_limits::max()) + ? gpu_metrics.mem_activity_acc : std::numeric_limits::max(); + utilization_counters[index].fine_value_count = + (gpu_metrics.mem_activity_acc == std::numeric_limits::max()) + ? 0 : val_counter; break; + + case RSMI_COARSE_DECODER_ACTIVITY: + case RSMI_FINE_DECODER_ACTIVITY: + { + auto value_count = uint16_t(0); + auto value_accum = uint64_t(0); + for (const auto& elem : gpu_metrics.vcn_activity) { + if (elem != std::numeric_limits::max()) { + ++value_count; + value_accum += elem; + } + + if (utilization_counters[index].type == RSMI_UTILIZATION_COUNTER_TYPE::RSMI_FINE_DECODER_ACTIVITY) { + utilization_counters[index].fine_value[value_count] = elem; + } + } + + utilization_counters[index].value = 0; + utilization_counters[index].fine_value_count = value_count; + if (utilization_counters[index].type == RSMI_UTILIZATION_COUNTER_TYPE::RSMI_COARSE_DECODER_ACTIVITY) { + if (value_count > 0) { + utilization_counters[index].value = (value_accum / value_count); + } + } + } + break; + default: return RSMI_STATUS_INVALID_ARGS; } - if (val_ui32 == UINT32_MAX) { - return RSMI_STATUS_NOT_SUPPORTED; - } - utilization_counters[index].value = val_ui32; } - *timestamp = gpu_metrics.system_clock_counter; return ret; diff --git a/projects/amdsmi/tests/amd_smi_test/functional/metrics_counter_read.cc b/projects/amdsmi/tests/amd_smi_test/functional/metrics_counter_read.cc index 28225637d5..868f046dfb 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/metrics_counter_read.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/metrics_counter_read.cc @@ -49,6 +49,7 @@ #include #include #include +#include #include #include "amd_smi/amdsmi.h" @@ -132,28 +133,95 @@ void TestMetricsCounterRead::Run(void) { ASSERT_EQ(err, AMDSMI_STATUS_INVAL); // Coarse Grain counters - amdsmi_utilization_counter_t utilization_counters[2]; + constexpr uint32_t kUTILIZATION_COUNTERS(3); + amdsmi_utilization_counter_t utilization_counters[kUTILIZATION_COUNTERS]; utilization_counters[0].type = AMDSMI_COARSE_GRAIN_GFX_ACTIVITY; utilization_counters[1].type = AMDSMI_COARSE_GRAIN_MEM_ACTIVITY; + utilization_counters[2].type = AMDSMI_COARSE_DECODER_ACTIVITY; + err = amdsmi_get_utilization_count(processor_handles_[i], utilization_counters, - 2, ×tamp); + kUTILIZATION_COUNTERS, ×tamp); if (err != AMDSMI_STATUS_SUCCESS) { if (err == AMDSMI_STATUS_NOT_SUPPORTED) { IF_VERB(STANDARD) { std::cout << "\t**" << - "Not supported on this machine" << std::endl; + "amdsmi_get_utilization_count(): Not supported on this machine" << std::endl; return; } } } else { CHK_ERR_ASRT(err); IF_VERB(STANDARD) { - std::cout << std::dec << "gfx_activity=" - << utilization_counters[0].value << '\n'; - std::cout << std::dec << "mem_activity=" - << utilization_counters[1].value << '\n'; - std::cout << std::dec << "timestamp=" - << timestamp << '\n'; + std::cout << "\n\namdsmi_get_utilization_count() : COARSE GRAIN ACTIVITIES" << "\n"; + for (auto idx = uint32_t(0); idx < kUTILIZATION_COUNTERS; ++idx) { + switch (utilization_counters[idx].type) { + case AMDSMI_COARSE_GRAIN_GFX_ACTIVITY: + std::cout << "-> gfx_activity: [" << utilization_counters[idx].fine_value_count << "]" << "\n"; + break; + + case AMDSMI_COARSE_GRAIN_MEM_ACTIVITY: + std::cout << "-> mem_activity: [" << utilization_counters[idx].fine_value_count << "]" << "\n";; + break; + + case AMDSMI_COARSE_DECODER_ACTIVITY: + std::cout << "-> decoder_activity: [" << utilization_counters[idx].fine_value_count << "]" << "\n"; + break; + + default: + break; + } + + for (auto val_idx = uint16_t(0); val_idx < utilization_counters[idx].fine_value_count; ++val_idx) { + std::cout << "\t" << std::dec << utilization_counters[idx].value << "\n"; + } + } + + std::cout << std::dec << "timestamp=" << timestamp << '\n'; + } + } + + // Fine Grain counters + utilization_counters[0].type = AMDSMI_FINE_GRAIN_GFX_ACTIVITY; + utilization_counters[1].type = AMDSMI_FINE_GRAIN_MEM_ACTIVITY; + utilization_counters[2].type = AMDSMI_FINE_DECODER_ACTIVITY; + err = amdsmi_get_utilization_count(processor_handles_[i], utilization_counters, + kUTILIZATION_COUNTERS, ×tamp); + if (err != AMDSMI_STATUS_SUCCESS) { + if (err == AMDSMI_STATUS_NOT_SUPPORTED) { + IF_VERB(STANDARD) { + std::cout << "\t**" << + "amdsmi_get_utilization_count(): Not supported on this machine" << std::endl; + return; + } + } + } else { + CHK_ERR_ASRT(err); + IF_VERB(STANDARD) { + std::cout << "\n\namdsmi_get_utilization_count() : FINE GRAIN ACTIVITIES" << "\n"; + for (auto idx = uint32_t(0); idx < kUTILIZATION_COUNTERS; ++idx) { + switch (utilization_counters[idx].type) { + case AMDSMI_FINE_GRAIN_GFX_ACTIVITY: + std::cout << "-> gfx_activity: [" << utilization_counters[idx].fine_value_count << "]" << "\n"; + break; + + case AMDSMI_FINE_GRAIN_MEM_ACTIVITY: + std::cout << "-> mem_activity: [" << utilization_counters[idx].fine_value_count << "]" << "\n";; + break; + + case AMDSMI_FINE_DECODER_ACTIVITY: + std::cout << "-> decoder_activity: [" << utilization_counters[idx].fine_value_count << "]" << "\n"; + break; + + default: + break; + } + + for (auto val_idx = uint16_t(0); val_idx < utilization_counters[idx].fine_value_count; ++val_idx) { + std::cout << "\t" << std::dec << utilization_counters[idx].fine_value[val_idx] << "\n"; + } + } + + std::cout << std::dec << "timestamp=" << timestamp << '\n'; } }