From 4bea04d43659ac545de6163e20a81a316b32f6bf Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Wed, 11 Nov 2026 17:13:35 +0100 Subject: [PATCH] fix(slurm): honor slurm_oversubscribe in the batch partition The slurm.conf template reassigned slurm_oversubscribe from slurm_manage_gpus right before the PartitionName line, so a value set in the inventory was never used: OverSubscribe was always NO with slurm_manage_gpus true and EXCLUSIVE otherwise. Move that expression into the role default and drop the reassignment. The rendered default is unchanged for both slurm_manage_gpus values, and an explicit slurm_oversubscribe now reaches slurm.conf. Add tests/slurm-conf/test_partition.py, which renders the template in memory for the default and for explicit settings. Signed-off-by: Dezhen Lu --- roles/slurm/defaults/main.yml | 3 +- roles/slurm/templates/etc/slurm/slurm.conf | 1 - tests/slurm-conf/test_partition.py | 52 ++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 tests/slurm-conf/test_partition.py diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 35d2f9e94..0c8565cb7 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -70,7 +70,8 @@ slurm_conf_symlink: false # Controls the ability of the partition to execute more than one job at a time on each resource (node, socket or core depending upon the value of SelectTypeParameters). If resources are to be over-subscribed, avoiding memory over-subscription is very important. SelectTypeParameters should be configured to treat memory as a consumable resource and the --mem option should be used for job allocations. Sharing of resources is typically useful only when using gang scheduling (PreemptMode=suspend,gang). Possible values for OverSubscribe are "EXCLUSIVE", "FORCE", "YES", and "NO". Note that a value of "YES" or "FORCE" can negatively impact performance for systems with many thousands of running jobs. The default value is "NO". For more information see the following web pages: # Sets partition OverSubscribe # To avoid sharing nodes, set to "EXCLUSIVE" -slurm_oversubscribe: "NO" +# Default: "NO" when slurm_manage_gpus is true, otherwise "EXCLUSIVE" +slurm_oversubscribe: "{{ (slurm_manage_gpus == true) | ternary('NO', 'EXCLUSIVE') }}" # Maximum run time limit for jobs. Format is minutes, minutes:seconds, hours:minutes:seconds, days-hours, days-hours:minutes, days-hours:minutes:seconds or "UNLIMITED". Time resolution is one minute and second values are rounded up to the next minute. This limit does not apply to jobs executed by SlurmUser or user root. # Sets: MaxTime={{ slurm_max_job_timelimit }} diff --git a/roles/slurm/templates/etc/slurm/slurm.conf b/roles/slurm/templates/etc/slurm/slurm.conf index 91d946007..da5156353 100644 --- a/roles/slurm/templates/etc/slurm/slurm.conf +++ b/roles/slurm/templates/etc/slurm/slurm.conf @@ -153,5 +153,4 @@ GresTypes=gpu # hardcoding the partitions and default memory per node # TODO: automatically define the partitions by resource # TODO: set DefMemPerCPU = TotalMemory / LogicalCPUs -{% set slurm_oversubscribe = (slurm_manage_gpus == true)|ternary('NO', 'EXCLUSIVE') %} PartitionName=batch Nodes=ALL Default=YES DefMemPerCPU=0 State=UP OverSubscribe={{ slurm_oversubscribe }} {%- if slurm_max_job_timelimit is defined %} MaxTime={{ slurm_max_job_timelimit }} {%- else %} MaxTime=INFINITE {% endif -%} {%- if slurm_default_job_timelimit is defined %} DefaultTime={{ slurm_default_job_timelimit }} {% endif -%} diff --git a/tests/slurm-conf/test_partition.py b/tests/slurm-conf/test_partition.py new file mode 100644 index 000000000..ac2605788 --- /dev/null +++ b/tests/slurm-conf/test_partition.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Inert slurm.conf partition regressions; run with Ansible's Python. + +The role template is rendered in memory with fixture facts. No host is +contacted and nothing is written. +""" +import os +from pathlib import Path +import unittest + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar +import yaml + +ROOT = Path(os.environ.get("DEEPOPS_TEST_ROOT", Path(__file__).resolve().parents[2])) +ROLE = ROOT / "roles/slurm" +DEFAULTS = yaml.safe_load((ROLE / "defaults/main.yml").read_text()) +TEMPLATE = (ROLE / "templates" / DEFAULTS["slurm_conf_template"]).read_text() +FACTS = {"ansible_local": { + "memory": {"total_mb": 1000}, + "topology": {"cpu_topology": {"sockets": "1", "cores_per_socket": "4", + "threads_per_core": "2", "logical_cpus": "8"}, + "gpu_topology": ["0-7"]}}} + + +class Partition(unittest.TestCase): + def partition(self, **overrides): + variables = {**DEFAULTS, + "groups": {"slurm-master": ["login01"], "slurm-node": ["node01"]}, + "hostvars": {"node01": FACTS}, **overrides} + rendered = Templar(loader=DataLoader(), variables=variables).template(TEMPLATE) + lines = [line for line in rendered.splitlines() if line.startswith("PartitionName=")] + self.assertEqual(len(lines), 1, rendered) + return dict(field.split("=", 1) for field in lines[0].split()) + + def test_oversubscribe_default_follows_gpu_management(self): + for manage_gpus, expected in ((True, "NO"), (False, "EXCLUSIVE")): + with self.subTest(slurm_manage_gpus=manage_gpus): + partition = self.partition(slurm_manage_gpus=manage_gpus) + self.assertEqual(partition["OverSubscribe"], expected) + + def test_oversubscribe_setting_is_used(self): + for manage_gpus in (True, False): + for value in ("NO", "EXCLUSIVE", "YES", "FORCE"): + with self.subTest(slurm_manage_gpus=manage_gpus, slurm_oversubscribe=value): + partition = self.partition(slurm_manage_gpus=manage_gpus, + slurm_oversubscribe=value) + self.assertEqual(partition["OverSubscribe"], value) + + +if __name__ == "__main__": + unittest.main()