Description: Allow gpu nvml/rsmi/oneapi when plugin .so exists
 When Slurm is built without HAVE_NVML/HAVE_RSMI/HAVE_ONEAPI but
 AutoDetect=nvml|rsmi|oneapi is set, check for the plugin .so in
 PluginDir. If present, try dlopen of the vendor library; on success
 use that GPU type. Add gpu_plugin_so_exists_in_dir() and unit tests.
Author: Deepak Jain <deepujain@gmail.com>
Origin: upstream
Bug: https://support.schedmd.com/show_bug.cgi?id=15909
Forwarded: not-needed
Last-Update: 2026-10-18

--- a/src/interfaces/gpu.c
+++ b/src/interfaces/gpu.c
@@ -34,10 +34,14 @@
 \*****************************************************************************/
 
 #include <dlfcn.h>
+#include <stdbool.h>
+#include <sys/stat.h>
 
 #include "src/common/assoc_mgr.h"
 #include "src/common/list.h"
 #include "src/common/plugin.h"
+#include "src/common/read_config.h"
+#include "src/common/xstring.h"
 #include "src/common/xmalloc.h"
 
 #include "src/interfaces/gpu.h"
@@ -76,6 +80,66 @@
 static void *ext_lib_handle = NULL;
 
 /*
+ * Check if the GPU plugin .so exists in the given plugindir (e.g. gpu_nvml.so).
+ * plugindir may be colon-separated. Allows split-builds (Bug 15909).
+ * Exposed for unit tests.
+ */
+bool gpu_plugin_so_exists_in_dir(const char *plugindir, const char *type_name)
+{
+	char *dir_array = NULL, *head = NULL, *path = NULL;
+	char so_name[256];
+	size_t i;
+	struct stat st;
+	bool found = false;
+
+	if (!plugindir || !type_name)
+		return false;
+
+	snprintf(so_name, sizeof(so_name), "%s.so", type_name);
+	for (i = 0; so_name[i]; i++) {
+		if (so_name[i] == '/')
+			so_name[i] = '_';
+	}
+
+	dir_array = xstrdup(plugindir);
+	head = dir_array;
+
+	for (i = 0;; i++) {
+		bool got_colon = false;
+
+		if (dir_array[i] == ':') {
+			dir_array[i] = '\0';
+			got_colon = true;
+		} else if (dir_array[i] != '\0') {
+			continue;
+		}
+
+		path = xstrdup_printf("%s/%s", head, so_name);
+		if ((stat(path, &st) == 0) && S_ISREG(st.st_mode)) {
+			found = true;
+			xfree(path);
+			break;
+		}
+		xfree(path);
+
+		if (got_colon)
+			head = dir_array + i + 1;
+		else
+			break;
+	}
+
+	xfree(dir_array);
+	return found;
+}
+
+static bool _gpu_plugin_so_exists(const char *type_name)
+{
+	if (!slurm_conf.plugindir)
+		return false;
+	return gpu_plugin_so_exists_in_dir(slurm_conf.plugindir, type_name);
+}
+
+/*
  * Parses fake_gpus_file for fake GPU devices and adds them to gres_list_system
  *
  * The file format is: <type>|<sys_cpu_count>|<cpu_range>|<links>|<device_file>
@@ -239,7 +303,21 @@
 		else
 			return "gpu/nvml";
 #else
-		info("We were configured to autodetect nvml functionality, but we weren't able to find that lib when Slurm was configured.");
+		if (_gpu_plugin_so_exists("gpu/nvml")) {
+			(void) dlerror();
+			if (!(ext_lib_handle = dlopen("libnvidia-ml.so",
+						      RTLD_NOW | RTLD_GLOBAL)) &&
+			    !(ext_lib_handle = dlopen("libnvidia-ml.so.1",
+						      RTLD_NOW | RTLD_GLOBAL)))
+				info("gpu_nvml.so found but libnvidia-ml not "
+				     "found. %s", dlerror());
+			else
+				return "gpu/nvml";
+		} else {
+			info("We were configured to autodetect nvml functionality, "
+			     "but we weren't able to find that lib when Slurm "
+			     "was configured.");
+		}
 #endif
 	} else if (autodetect_flags & GRES_AUTODETECT_GPU_RSMI) {
 #ifdef HAVE_RSMI
@@ -251,7 +329,18 @@
 		else
 			return "gpu/rsmi";
 #else
-		info("Configured with rsmi, but rsmi isn't enabled during the build.");
+		if (_gpu_plugin_so_exists("gpu/rsmi")) {
+			(void) dlerror();
+			if (!(ext_lib_handle = dlopen("librocm_smi64.so",
+						      RTLD_NOW | RTLD_GLOBAL)))
+				info("gpu_rsmi.so found but librocm_smi64 not "
+				     "found. %s", dlerror());
+			else
+				return "gpu/rsmi";
+		} else {
+			info("Configured with rsmi, but rsmi isn't enabled "
+			     "during the build.");
+		}
 #endif
 	} else if (autodetect_flags & GRES_AUTODETECT_GPU_ONEAPI) {
 #ifdef HAVE_ONEAPI
@@ -263,7 +352,18 @@
 		else
 			return "gpu/oneapi";
 #else
-		info("Configured with oneAPI, but oneAPI isn't enabled during the build.");
+		if (_gpu_plugin_so_exists("gpu/oneapi")) {
+			(void) dlerror();
+			if (!(ext_lib_handle = dlopen("libze_loader.so",
+						      RTLD_NOW | RTLD_GLOBAL)))
+				info("gpu_oneapi.so found but libze_loader not "
+				     "found. %s", dlerror());
+			else
+				return "gpu/oneapi";
+		} else {
+			info("Configured with oneAPI, but oneAPI isn't enabled "
+			     "during the build.");
+		}
 #endif
 	} else if (autodetect_flags & GRES_AUTODETECT_GPU_NRT) {
 		return "gpu/nrt";
--- a/src/interfaces/gpu.h
+++ b/src/interfaces/gpu.h
@@ -36,6 +36,7 @@
 #ifndef _INTERFACES_GPU_H
 #define _INTERFACES_GPU_H
 
+#include <stdbool.h>
 #include "slurm/slurm.h"
 #include "src/interfaces/gres.h"
 #include "src/interfaces/acct_gather.h"
@@ -48,6 +49,10 @@
 	acct_gather_energy_t energy;
 } gpu_status_t;
 
+/* For unit tests: check if plugin .so exists in given dir(s). */
+extern bool gpu_plugin_so_exists_in_dir(const char *plugindir,
+                                       const char *type_name);
+
 /*
  * Load the GPU plugin selected by gres.conf's Autodetect= setting.
  *
--- a/testsuite/slurm_unit/common/Makefile.am
+++ b/testsuite/slurm_unit/common/Makefile.am
@@ -22,6 +22,7 @@
 TESTS += xhash-test \
 	 data-test \
 	 dns-test \
+	 gpu-test \
 	 http-test \
 	 serializer-test \
 	 sluid-test \
@@ -40,6 +41,8 @@
 data_test_LDADD   = $(LDADD) @CHECK_LIBS@
 dns_test_CFLAGS  = $(MYCFLAGS)
 dns_test_LDADD   = $(LDADD) @CHECK_LIBS@
+gpu_test_CFLAGS  = $(MYCFLAGS)
+gpu_test_LDADD   = $(LDADD) @CHECK_LIBS@
 http_test_CFLAGS  = $(MYCFLAGS)
 http_test_LDADD   = $(LDADD) @CHECK_LIBS@
 serializer_test_CFLAGS  = $(MYCFLAGS)
--- /dev/null
+++ b/testsuite/slurm_unit/common/gpu-test.c
@@ -0,0 +1,164 @@
+/*****************************************************************************\
+ *  Copyright (C) SchedMD LLC.
+ *
+ *  This file is part of Slurm, a resource management program.
+ *  For details, see <https://slurm.schedmd.com/>.
+ *  Please also read the included file: DISCLAIMER.
+ *
+ *  Slurm is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU General Public License as published by the Free
+ *  Software Foundation; either version 2 of the License, or (at your option)
+ *  any later version.
+ *
+ *  In addition, as a special exception, the copyright holders give permission
+ *  to link the code of portions of this program with the OpenSSL library under
+ *  certain conditions as described in each individual source file, and
+ *  distribute linked combinations including the two. You must obey the GNU
+ *  General Public License in all respects for all of the code used other than
+ *  OpenSSL. If you modify file(s) with this exception, you may extend this
+ *  exception to your version of the file(s), but you are not obligated to do
+ *  so. If you do not wish to do so, delete this exception statement from your
+ *  version.  If you delete this exception statement from all source files in
+ *  the program, then also delete it here.
+ *
+ *  Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
+ *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ *  details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with Slurm; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
+\*****************************************************************************/
+
+#include <check.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "src/common/log.h"
+#include "src/interfaces/gpu.h"
+
+/* Test gpu_plugin_so_exists_in_dir (Bug 15909). */
+
+START_TEST(test_plugin_so_exists_null)
+{
+	ck_assert(!gpu_plugin_so_exists_in_dir(NULL, "gpu/nvml"));
+	ck_assert(!gpu_plugin_so_exists_in_dir("/tmp", NULL));
+	ck_assert(!gpu_plugin_so_exists_in_dir(NULL, NULL));
+}
+END_TEST
+
+START_TEST(test_plugin_so_exists_single_dir)
+{
+	char dir[] = "/tmp/gpu-test.XXXXXX";
+	char path[256];
+	int fd;
+
+	ck_assert(mkdtemp(dir) != NULL);
+
+	/* type "gpu/nvml" -> gpu_nvml.so */
+	snprintf(path, sizeof(path), "%s/gpu_nvml.so", dir);
+	fd = open(path, O_CREAT | O_WRONLY, 0600);
+	ck_assert_msg(fd >= 0, "open %s: %s", path, strerror(errno));
+	close(fd);
+
+	ck_assert_msg(gpu_plugin_so_exists_in_dir(dir, "gpu/nvml"),
+		      "gpu_nvml.so in %s", dir);
+	ck_assert_msg(!gpu_plugin_so_exists_in_dir(dir, "gpu/rsmi"),
+		      "gpu_rsmi.so not in %s", dir);
+	ck_assert_msg(!gpu_plugin_so_exists_in_dir(dir, "gpu/oneapi"),
+		      "gpu_oneapi.so not in %s", dir);
+
+	unlink(path);
+	rmdir(dir);
+}
+END_TEST
+
+START_TEST(test_plugin_so_exists_colon_sep)
+{
+	char dir1[] = "/tmp/gpu-test1.XXXXXX";
+	char dir2[] = "/tmp/gpu-test2.XXXXXX";
+	char *plugindir = NULL;
+	char path[256];
+	int fd;
+
+	ck_assert(mkdtemp(dir1) != NULL);
+	ck_assert(mkdtemp(dir2) != NULL);
+
+	/* Create gpu_rsmi.so in second dir only. */
+	snprintf(path, sizeof(path), "%s/gpu_rsmi.so", dir2);
+	fd = open(path, O_CREAT | O_WRONLY, 0600);
+	ck_assert_msg(fd >= 0, "open %s: %s", path, strerror(errno));
+	close(fd);
+
+	plugindir = malloc(strlen(dir1) + 1 + strlen(dir2) + 1);
+	ck_assert(plugindir != NULL);
+	sprintf(plugindir, "%s:%s", dir1, dir2);
+
+	ck_assert_msg(gpu_plugin_so_exists_in_dir(plugindir, "gpu/rsmi"),
+		      "gpu_rsmi.so in second dir of %s", plugindir);
+	ck_assert_msg(!gpu_plugin_so_exists_in_dir(plugindir, "gpu/nvml"),
+		      "gpu_nvml.so not in %s", plugindir);
+
+	free(plugindir);
+	unlink(path);
+	rmdir(dir2);
+	rmdir(dir1);
+}
+END_TEST
+
+START_TEST(test_plugin_so_exists_type_to_so_name)
+{
+	char dir[] = "/tmp/gpu-test.XXXXXX";
+	char path[256];
+	int fd;
+
+	ck_assert(mkdtemp(dir) != NULL);
+
+	/* gpu/oneapi -> gpu_oneapi.so */
+	snprintf(path, sizeof(path), "%s/gpu_oneapi.so", dir);
+	fd = open(path, O_CREAT | O_WRONLY, 0600);
+	ck_assert_msg(fd >= 0, "open %s: %s", path, strerror(errno));
+	close(fd);
+
+	ck_assert_msg(gpu_plugin_so_exists_in_dir(dir, "gpu/oneapi"),
+		      "gpu/oneapi -> gpu_oneapi.so in %s", dir);
+
+	unlink(path);
+	rmdir(dir);
+}
+END_TEST
+
+Suite *gpu_suite(void)
+{
+	Suite *s = suite_create("gpu");
+	TCase *tc_core = tcase_create("gpu_plugin_so_exists_in_dir");
+
+	tcase_add_test(tc_core, test_plugin_so_exists_null);
+	tcase_add_test(tc_core, test_plugin_so_exists_single_dir);
+	tcase_add_test(tc_core, test_plugin_so_exists_colon_sep);
+	tcase_add_test(tc_core, test_plugin_so_exists_type_to_so_name);
+	suite_add_tcase(s, tc_core);
+	return s;
+}
+
+int main(int argc, char *argv[])
+{
+	int number_failed;
+	log_options_t log_opts = LOG_OPTS_INITIALIZER;
+
+	log_opts.stderr_level = LOG_LEVEL_DEBUG5;
+	log_init("gpu-test", log_opts, 0, NULL);
+
+	SRunner *sr = srunner_create(gpu_suite());
+	srunner_run_all(sr, CK_ENV);
+	number_failed = srunner_ntests_failed(sr);
+	srunner_free(sr);
+
+	log_fini();
+	return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
