Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion roles/slurm/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ slurm_conf_symlink: false
# Controls the ability of the partition to execute more than one job at a time on each resource (node, socket or core depending upon the value of SelectTypeParameters). If resources are to be over-subscribed, avoiding memory over-subscription is very important. SelectTypeParameters should be configured to treat memory as a consumable resource and the --mem option should be used for job allocations. Sharing of resources is typically useful only when using gang scheduling (PreemptMode=suspend,gang). Possible values for OverSubscribe are "EXCLUSIVE", "FORCE", "YES", and "NO". Note that a value of "YES" or "FORCE" can negatively impact performance for systems with many thousands of running jobs. The default value is "NO". For more information see the following web pages:
# Sets partition OverSubscribe
# To avoid sharing nodes, set to "EXCLUSIVE"
slurm_oversubscribe: "NO"
# Default: "NO" when slurm_manage_gpus is true, otherwise "EXCLUSIVE"
slurm_oversubscribe: "{{ (slurm_manage_gpus == true) | ternary('NO', 'EXCLUSIVE') }}"

# Maximum run time limit for jobs. Format is minutes, minutes:seconds, hours:minutes:seconds, days-hours, days-hours:minutes, days-hours:minutes:seconds or "UNLIMITED". Time resolution is one minute and second values are rounded up to the next minute. This limit does not apply to jobs executed by SlurmUser or user root.
# Sets: MaxTime={{ slurm_max_job_timelimit }}
Expand Down
1 change: 0 additions & 1 deletion roles/slurm/templates/etc/slurm/slurm.conf
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,4 @@ GresTypes=gpu
# hardcoding the partitions and default memory per node
# TODO: automatically define the partitions by resource
# TODO: set DefMemPerCPU = TotalMemory / LogicalCPUs
{% set slurm_oversubscribe = (slurm_manage_gpus == true)|ternary('NO', 'EXCLUSIVE') %}
PartitionName=batch Nodes=ALL Default=YES DefMemPerCPU=0 State=UP OverSubscribe={{ slurm_oversubscribe }} {%- if slurm_max_job_timelimit is defined %} MaxTime={{ slurm_max_job_timelimit }} {%- else %} MaxTime=INFINITE {% endif -%} {%- if slurm_default_job_timelimit is defined %} DefaultTime={{ slurm_default_job_timelimit }} {% endif -%}
52 changes: 52 additions & 0 deletions tests/slurm-conf/test_partition.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Inert slurm.conf partition regressions; run with Ansible's Python.

The role template is rendered in memory with fixture facts. No host is
contacted and nothing is written.
"""
import os
from pathlib import Path
import unittest

from ansible.parsing.dataloader import DataLoader
from ansible.template import Templar
import yaml

ROOT = Path(os.environ.get("DEEPOPS_TEST_ROOT", Path(__file__).resolve().parents[2]))
ROLE = ROOT / "roles/slurm"
DEFAULTS = yaml.safe_load((ROLE / "defaults/main.yml").read_text())
TEMPLATE = (ROLE / "templates" / DEFAULTS["slurm_conf_template"]).read_text()
FACTS = {"ansible_local": {
"memory": {"total_mb": 1000},
"topology": {"cpu_topology": {"sockets": "1", "cores_per_socket": "4",
"threads_per_core": "2", "logical_cpus": "8"},
"gpu_topology": ["0-7"]}}}


class Partition(unittest.TestCase):
def partition(self, **overrides):
variables = {**DEFAULTS,
"groups": {"slurm-master": ["login01"], "slurm-node": ["node01"]},
"hostvars": {"node01": FACTS}, **overrides}
rendered = Templar(loader=DataLoader(), variables=variables).template(TEMPLATE)
lines = [line for line in rendered.splitlines() if line.startswith("PartitionName=")]
self.assertEqual(len(lines), 1, rendered)
return dict(field.split("=", 1) for field in lines[0].split())

def test_oversubscribe_default_follows_gpu_management(self):
for manage_gpus, expected in ((True, "NO"), (False, "EXCLUSIVE")):
with self.subTest(slurm_manage_gpus=manage_gpus):
partition = self.partition(slurm_manage_gpus=manage_gpus)
self.assertEqual(partition["OverSubscribe"], expected)

def test_oversubscribe_setting_is_used(self):
for manage_gpus in (True, False):
for value in ("NO", "EXCLUSIVE", "YES", "FORCE"):
with self.subTest(slurm_manage_gpus=manage_gpus, slurm_oversubscribe=value):
partition = self.partition(slurm_manage_gpus=manage_gpus,
slurm_oversubscribe=value)
self.assertEqual(partition["OverSubscribe"], value)


if __name__ == "__main__":
unittest.main()
Loading