diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 35d2f9e94..0c8565cb7 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -70,7 +70,8 @@ slurm_conf_symlink: false # Controls the ability of the partition to execute more than one job at a time on each resource (node, socket or core depending upon the value of SelectTypeParameters). If resources are to be over-subscribed, avoiding memory over-subscription is very important. SelectTypeParameters should be configured to treat memory as a consumable resource and the --mem option should be used for job allocations. Sharing of resources is typically useful only when using gang scheduling (PreemptMode=suspend,gang). Possible values for OverSubscribe are "EXCLUSIVE", "FORCE", "YES", and "NO". Note that a value of "YES" or "FORCE" can negatively impact performance for systems with many thousands of running jobs. The default value is "NO". For more information see the following web pages: # Sets partition OverSubscribe # To avoid sharing nodes, set to "EXCLUSIVE" -slurm_oversubscribe: "NO" +# Default: "NO" when slurm_manage_gpus is true, otherwise "EXCLUSIVE" +slurm_oversubscribe: "{{ (slurm_manage_gpus == true) | ternary('NO', 'EXCLUSIVE') }}" # Maximum run time limit for jobs. Format is minutes, minutes:seconds, hours:minutes:seconds, days-hours, days-hours:minutes, days-hours:minutes:seconds or "UNLIMITED". Time resolution is one minute and second values are rounded up to the next minute. This limit does not apply to jobs executed by SlurmUser or user root. # Sets: MaxTime={{ slurm_max_job_timelimit }} diff --git a/roles/slurm/templates/etc/slurm/slurm.conf b/roles/slurm/templates/etc/slurm/slurm.conf index 91d946007..da5156353 100644 --- a/roles/slurm/templates/etc/slurm/slurm.conf +++ b/roles/slurm/templates/etc/slurm/slurm.conf @@ -153,5 +153,4 @@ GresTypes=gpu # hardcoding the partitions and default memory per node # TODO: automatically define the partitions by resource # TODO: set DefMemPerCPU = TotalMemory / LogicalCPUs -{% set slurm_oversubscribe = (slurm_manage_gpus == true)|ternary('NO', 'EXCLUSIVE') %} PartitionName=batch Nodes=ALL Default=YES DefMemPerCPU=0 State=UP OverSubscribe={{ slurm_oversubscribe }} {%- if slurm_max_job_timelimit is defined %} MaxTime={{ slurm_max_job_timelimit }} {%- else %} MaxTime=INFINITE {% endif -%} {%- if slurm_default_job_timelimit is defined %} DefaultTime={{ slurm_default_job_timelimit }} {% endif -%} diff --git a/tests/slurm-conf/test_partition.py b/tests/slurm-conf/test_partition.py new file mode 100644 index 000000000..ac2605788 --- /dev/null +++ b/tests/slurm-conf/test_partition.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Inert slurm.conf partition regressions; run with Ansible's Python. + +The role template is rendered in memory with fixture facts. No host is +contacted and nothing is written. +""" +import os +from pathlib import Path +import unittest + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar +import yaml + +ROOT = Path(os.environ.get("DEEPOPS_TEST_ROOT", Path(__file__).resolve().parents[2])) +ROLE = ROOT / "roles/slurm" +DEFAULTS = yaml.safe_load((ROLE / "defaults/main.yml").read_text()) +TEMPLATE = (ROLE / "templates" / DEFAULTS["slurm_conf_template"]).read_text() +FACTS = {"ansible_local": { + "memory": {"total_mb": 1000}, + "topology": {"cpu_topology": {"sockets": "1", "cores_per_socket": "4", + "threads_per_core": "2", "logical_cpus": "8"}, + "gpu_topology": ["0-7"]}}} + + +class Partition(unittest.TestCase): + def partition(self, **overrides): + variables = {**DEFAULTS, + "groups": {"slurm-master": ["login01"], "slurm-node": ["node01"]}, + "hostvars": {"node01": FACTS}, **overrides} + rendered = Templar(loader=DataLoader(), variables=variables).template(TEMPLATE) + lines = [line for line in rendered.splitlines() if line.startswith("PartitionName=")] + self.assertEqual(len(lines), 1, rendered) + return dict(field.split("=", 1) for field in lines[0].split()) + + def test_oversubscribe_default_follows_gpu_management(self): + for manage_gpus, expected in ((True, "NO"), (False, "EXCLUSIVE")): + with self.subTest(slurm_manage_gpus=manage_gpus): + partition = self.partition(slurm_manage_gpus=manage_gpus) + self.assertEqual(partition["OverSubscribe"], expected) + + def test_oversubscribe_setting_is_used(self): + for manage_gpus in (True, False): + for value in ("NO", "EXCLUSIVE", "YES", "FORCE"): + with self.subTest(slurm_manage_gpus=manage_gpus, slurm_oversubscribe=value): + partition = self.partition(slurm_manage_gpus=manage_gpus, + slurm_oversubscribe=value) + self.assertEqual(partition["OverSubscribe"], value) + + +if __name__ == "__main__": + unittest.main()