Skip to content

OpenVINO: PRD-compliant device enumeration and memory reporting for --list-devices #256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
haarika-madaka wants to merge 1 commit into
base: master
Choose a base branch
from
Open

OpenVINO: PRD-compliant device enumeration and memory reporting for --list-devices #256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 57 additions & 9 deletions ggml/src/ggml-openvino/ggml-openvino-extra.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "ggml-impl.h"
#include "ggml.h"

#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
Expand All @@ -15,6 +16,46 @@ ov::Core & ov_singleton_core() {
return core;
}

static bool has_prefix(const std::string & s, const std::string & prefix) {
return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin());
}

static bool is_virtual_routing_device(const std::string & device_name) {
return has_prefix(device_name, "AUTO") || has_prefix(device_name, "MULTI") || has_prefix(device_name, "HETERO");
}

static std::vector<std::string> ov_enumerate_devices() {
std::vector<std::string> result;

for (const auto & device : ov_singleton_core().get_available_devices()) {
if (!is_virtual_routing_device(device)) {
result.push_back(device);
}
}

if (result.empty()) {
result.push_back("CPU");
}

std::sort(result.begin(), result.end());
result.erase(std::unique(result.begin(), result.end()), result.end());
return result;
}

static std::string resolve_openvino_device_name(const std::vector<std::string> & available_devices,
const std::string & requested) {
if (available_devices.empty()) {
return "CPU";
}

auto it = std::find(available_devices.begin(), available_devices.end(), requested);
if (it != available_devices.end()) {
return *it;
}

return "CPU";
}

// =====================================================
// Device Configuration Implementations
// =====================================================
Expand Down Expand Up @@ -54,16 +95,17 @@ void ggml_openvino_device_config::init() {
}
}

device_name = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU");
auto available_devices = ov_singleton_core().get_available_devices();
if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) {
GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to CPU\n", device_name.c_str());
device_name = "CPU";
const std::string requested_device = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU");
available_devices = ov_enumerate_devices();
device_name = resolve_openvino_device_name(available_devices, requested_device);
if (device_name != requested_device) {
GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to %s\n", requested_device.c_str(),
device_name.c_str());
}
is_npu = (device_name == "NPU");
is_npu = has_prefix(device_name, "NPU");

const char * cache_dir = ggml_openvino_getenv_str("GGML_OPENVINO_CACHE_DIR");
if (device_name == "NPU") {
if (has_prefix(device_name, "NPU")) {
compile_config = {
{"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES" },
{"NPU_USE_NPUW", "YES" },
Expand All @@ -85,7 +127,7 @@ void ggml_openvino_device_config::init() {
}

// Initialize remote context with queue sharing for GPU
if (device_name == "GPU") {
if (has_prefix(device_name, "GPU")) {
// Create OpenCL context and queue
cl_int err;
cl_platform_id platform;
Expand Down Expand Up @@ -120,7 +162,7 @@ void ggml_openvino_device_config::init() {

// Release the context (queue keeps a reference)
clReleaseContext(cl_ctx);
} else if (device_name == "NPU") {
} else if (has_prefix(device_name, "NPU")) {
// remote tensor is not used for NPU yet
// remote_context = ov_singleton_core().get_default_context(device_name);
}
Expand Down Expand Up @@ -151,6 +193,12 @@ const std::string & ggml_openvino_get_device_name() {
return ggml_openvino_get_device_config().device_name;
}

std::vector<std::string> ggml_openvino_get_available_devices() {
auto & config = ggml_openvino_get_device_config();
config.init();
return config.available_devices;
}

// Get the value of a GGML_OPENVINO_* env var as a string. Returns
// default_value when the var is unset or set to an empty string.
const char * ggml_openvino_getenv_str(const char * var, const char * default_value) {
Expand Down
4 changes: 4 additions & 0 deletions ggml/src/ggml-openvino/ggml-openvino-extra.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL();

struct ggml_openvino_device_config {
std::string device_name = "CPU";
std::vector<std::string> available_devices;
bool is_npu = false;
bool initialized = false;
std::optional<ov::RemoteContext> remote_context;
Expand All @@ -80,6 +81,9 @@ void ggml_openvino_init_device_config();
// Get the device name
const std::string & ggml_openvino_get_device_name();

// Get all available physical OpenVINO devices
std::vector<std::string> ggml_openvino_get_available_devices();

// Environment variable accessors. All GGML_OPENVINO_* env vars are read once
// during backend init and cached on the device config; consumers must go
// through these helpers (never call ::getenv directly) so behavior stays
Expand Down
132 changes: 128 additions & 4 deletions ggml/src/ggml-openvino/ggml-openvino.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
#include <openvino/runtime/allocator.hpp>
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
#include <openvino/runtime/intel_npu/level_zero/level_zero.hpp>
#include <openvino/runtime/properties.hpp>
#include <openvino/runtime/tensor.hpp>
#include <algorithm>
#include <map>
#include <set>
#include <string>
#include <vector>
Expand Down Expand Up @@ -656,7 +659,7 @@ static const ggml_backend_i ggml_backend_openvino_interface = {
};

int ggml_backend_openvino_get_device_count() {
return 1;
return (int) ggml_openvino_get_available_devices().size();
}

static ggml_guid_t ggml_backend_openvino_guid(void) {
Expand Down Expand Up @@ -716,8 +719,90 @@ struct ggml_backend_openvino_device_context {
int device;
std::string name;
std::string description;
size_t total_memory;
};

static bool ov_device_has_prefix(const std::string & s, const std::string & prefix) {
return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin());
}

static std::string ov_device_description_from_name(const std::string & device_name) {
std::string description = device_name;
try {
description = ov_singleton_core().get_property(device_name, ov::device::full_name);
} catch (...) {
return device_name;
}

if (ov_device_has_prefix(device_name, "NPU")) {
try {
const std::string arch = ov_singleton_core().get_property(device_name, "DEVICE_ARCHITECTURE").as<std::string>();
if (!arch.empty()) {
description += " (NPU " + arch + ")";
}
} catch (...) {
}
}

return description;
}

static bool ov_try_get_size_t_property(const std::string & device, const std::string & property, size_t & out) {
try {
const ov::Any value = ov_singleton_core().get_property(device, property);
if (value.is<size_t>()) {
out = value.as<size_t>();
return true;
}
if (value.is<uint64_t>()) {
out = (size_t) value.as<uint64_t>();
return true;
}
if (value.is<unsigned long long>()) {
out = (size_t) value.as<unsigned long long>();
return true;
}
if (value.is<int64_t>()) {
const int64_t v = value.as<int64_t>();
if (v >= 0) {
out = (size_t) v;
return true;
}
}
} catch (...) {
}
return false;
}

static bool ov_try_get_gpu_used_memory(const std::string & device, size_t & out) {
out = 0;
try {
const ov::Any stats_any = ov_singleton_core().get_property(device, "GPU_MEMORY_STATISTICS");
if (stats_any.is<std::map<std::string, uint64_t>>()) {
const auto stats = stats_any.as<std::map<std::string, uint64_t>>();
for (const auto & kv : stats) {
out += (size_t) kv.second;
}
return true;
}
if (stats_any.is<ov::AnyMap>()) {
const auto stats = stats_any.as<ov::AnyMap>();
for (const auto & kv : stats) {
if (kv.second.is<size_t>()) {
out += kv.second.as<size_t>();
} else if (kv.second.is<uint64_t>()) {
out += (size_t) kv.second.as<uint64_t>();
} else if (kv.second.is<unsigned long long>()) {
out += (size_t) kv.second.as<unsigned long long>();
}
}
return true;
}
} catch (...) {
}
return false;
}

static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) {
ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;
return ctx->name.c_str();
Expand All @@ -729,6 +814,34 @@ static const char * ggml_backend_openvino_device_get_description(ggml_backend_de
}

static void ggml_backend_openvino_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;

if (ov_device_has_prefix(ctx->name, "GPU")) {
size_t used = 0;
if (ctx->total_memory == 0 || !ov_try_get_gpu_used_memory(ctx->name, used)) {
*total = 0;
*free = 0;
return;
}

*total = ctx->total_memory;
*free = (used >= *total) ? 0 : (*total - used);
return;
}

if (ov_device_has_prefix(ctx->name, "NPU")) {
size_t allocated = 0;
if (ctx->total_memory == 0 || !ov_try_get_size_t_property(ctx->name, "NPU_DEVICE_ALLOC_MEM_SIZE", allocated)) {
*total = 0;
*free = 0;
return;
}

*total = ctx->total_memory;
*free = (allocated >= *total) ? 0 : (*total - allocated);
return;
}

#ifdef _WIN32
MEMORYSTATUSEX status;
status.dwLength = sizeof(status);
Expand Down Expand Up @@ -1174,6 +1287,11 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
GGML_ASSERT(dev->reg != nullptr);

ggml_backend_openvino_device_context * dev_ctx = (ggml_backend_openvino_device_context *) dev->context;
if (dev_ctx->name != ggml_openvino_get_device_name()) {
return false;
}

static std::unordered_set<ggml_type> supported_types{
GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0,
GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K};
Expand Down Expand Up @@ -1351,15 +1469,21 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void) {
std::lock_guard<std::mutex> lock(mutex);
if (!initialized) {
ggml_openvino_init();
const std::vector<std::string> openvino_devices = ggml_openvino_get_available_devices();

ggml_backend_openvino_reg_context * ctx = new ggml_backend_openvino_reg_context;

for (int i = 0; i < ggml_backend_openvino_get_device_count(); i++) {
ggml_backend_openvino_device_context * dev_ctx = new ggml_backend_openvino_device_context;
dev_ctx->device = i;
dev_ctx->name = GGML_OPENVINO_NAME + std::to_string(i);

dev_ctx->description = ov::get_openvino_version().description;
dev_ctx->name = openvino_devices[i];
dev_ctx->description = ov_device_description_from_name(dev_ctx->name);
dev_ctx->total_memory = 0;
if (ov_device_has_prefix(dev_ctx->name, "GPU")) {
ov_try_get_size_t_property(dev_ctx->name, "GPU_DEVICE_TOTAL_MEM_SIZE", dev_ctx->total_memory);
} else if (ov_device_has_prefix(dev_ctx->name, "NPU")) {
ov_try_get_size_t_property(dev_ctx->name, "NPU_DEVICE_TOTAL_MEM_SIZE", dev_ctx->total_memory);
}

ggml_backend_dev_t dev =
new ggml_backend_device{/* .interface = */ ggml_backend_openvino_device_interface,
Expand Down
Loading