Skip to content

Vendor info enhancement - #321

Open
jaspals3123 wants to merge 4 commits into
developmentfrom
jaspal_vendor_info_enhancements
Open

jaspals3123 wants to merge 4 commits into
developmentfrom
jaspal_vendor_info_enhancements

Conversation

@jaspals3123

@jaspals3123 jaspals3123 commented Oct 19, 2026 •

Copy link
Copy Markdown
Collaborator
{
  "name": "vendor info checks",
  "plugins": {
    "AmdSmiPlugin": {
      "collection_args": {},
      "analysis_args": {
        "min_hbm_gib_per_gpu": 288,
        "ecc_required": true
      }
    },
    "BiosPlugin": {
      "collection_args": {},
      "analysis_args": {
        "exp_server_vendor": "Supermicro",
        "exp_server_model": "AS -1234GS-ABC-LCC"
      }
    },
    "DeviceEnumerationPlugin": {
      "collection_args": {},
      "analysis_args": {
        "cpu_architecture": "x86_64",
        "min_cpu_cores": 128
      }
    },
    "MemoryPlugin": {
      "collection_args": {},
      "analysis_args": {
        "minimum_system_memory_gib": 3072,
        "memory_threshold": "4Ti",
        "ratio": 0.99
      }
    }
  }
}

Comment on lines +59 to +66
CMD_CPU_INFO_WINDOWS = (
"powershell -NoProfile -Command "
'"$p = @(Get-CimInstance -ClassName Win32_Processor); '
"$cores = ($p | Measure-Object -Property NumberOfCores -Sum).Sum; "
"$map = @{0='i686'; 5='arm'; 6='ia64'; 9='x86_64'; 12='aarch64'}; "
"$arch = $map[[int]$p[0].Architecture]; "
"Write-Output ('Architecture=' + $arch); "
"Write-Output ('Cores=' + $cores)\""

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
CMD_CPU_INFO_WINDOWS = (
"powershell -NoProfile -Command "
'"$p = @(Get-CimInstance -ClassName Win32_Processor); '
"$cores = ($p | Measure-Object -Property NumberOfCores -Sum).Sum; "
"$map = @{0='i686'; 5='arm'; 6='ia64'; 9='x86_64'; 12='aarch64'}; "
"$arch = $map[[int]$p[0].Architecture]; "
"Write-Output ('Architecture=' + $arch); "
"Write-Output ('Cores=' + $cores)\""
CMD_CPU_INFO_WINDOWS = (
"powershell -NoProfile -Command "
"'$p = @(Get-CimInstance -ClassName Win32_Processor); "
"$cores = ($p | Measure-Object -Property NumberOfCores -Sum).Sum; "
"$map = @{0='i686'; 5='arm'; 6='ia64'; 9='x86_64'; 12='aarch64'}; "
"$arch = $map[[int]$p[0].Architecture]; "
"Write-Output ('Architecture=' + $arch); "
"Write-Output ('Cores=' + $cores)'"
)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

single quotes so it works on openssh and powershell on windows

Comment on lines +675 to +693
def check_umc_ras_enabled(self, amdsmi_static_data: list[AmdSmiStatic]) -> None:
"""Require UMC RAS ENABLED. That is the in-band proxy for HBM ECC being on."""
actual_by_gpu: dict[int, str] = {}
for gpu_data in amdsmi_static_data:
umc_state = _umc_ecc_state(gpu_data.ras.ecc_block_state)
if umc_state != EccState.ENABLED:
shown = umc_state.value if isinstance(umc_state, EccState) else "missing"
actual_by_gpu[gpu_data.gpu] = shown
if not actual_by_gpu:
return
desc, details = _gpu_mismatch_description("UMC RAS (HBM ECC)", "ENABLED", actual_by_gpu)
self._log_event(
category=EventCategory.RAS,
description=desc,
priority=EventPriority.ERROR,
data={"umc_ras_by_gpu": actual_by_gpu, "details": details},
console_log=True,
)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
def check_umc_ras_enabled(self, amdsmi_static_data: list[AmdSmiStatic]) -> None:
"""Require UMC RAS ENABLED. That is the in-band proxy for HBM ECC being on."""
actual_by_gpu: dict[int, str] = {}
for gpu_data in amdsmi_static_data:
umc_state = _umc_ecc_state(gpu_data.ras.ecc_block_state)
if umc_state != EccState.ENABLED:
shown = umc_state.value if isinstance(umc_state, EccState) else "missing"
actual_by_gpu[gpu_data.gpu] = shown
if not actual_by_gpu:
return
desc, details = _gpu_mismatch_description("UMC RAS (HBM ECC)", "ENABLED", actual_by_gpu)
self._log_event(
category=EventCategory.RAS,
description=desc,
priority=EventPriority.ERROR,
data={"umc_ras_by_gpu": actual_by_gpu, "details": details},
console_log=True,
)
def check_umc_ras_enabled(self, amdsmi_static_data: list[AmdSmiStatic]) -> None:
"""Require UMC RAS to be enabled when per-block data is available.
Args:
amdsmi_static_data: Static amd-smi data for each GPU.
Returns:
None.
"""
unavailable: list[int] = []
not_enabled: dict[int, str] = {}
for gpu_data in amdsmi_static_data:
block_states = gpu_data.ras.ecc_block_state
if not isinstance(block_states, dict):
unavailable.append(gpu_data.gpu)
continue
umc_state = _umc_ecc_state(block_states)
if umc_state is None:
unavailable.append(gpu_data.gpu)
elif umc_state != EccState.ENABLED:
not_enabled[gpu_data.gpu] = umc_state.value
if unavailable:
self._log_event(
category=EventCategory.RAS,
description="UMC RAS state is unavailable for one or more GPUs",
priority=EventPriority.ERROR,
data={"gpus": unavailable},
console_log=True,
)
if not_enabled:
desc, details = _gpu_mismatch_description(
"UMC RAS (HBM ECC)",
"ENABLED",
not_enabled,
)
self._log_event(
category=EventCategory.RAS,
description=desc,
priority=EventPriority.ERROR,
data={"umc_ras_by_gpu": not_enabled, "details": details},
console_log=True,
)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this will distinguish:
String/absent block data → cannot validate UMC.
UMC explicitly DISABLED or another state → definite ECC configuration failure.
UMC explicitly ENABLED → passes.
otherwise on older output every GPU fails when UMC RAS is enabled

},
console_log=True,
)
return self.result

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This return skips minimum_free_memory_percent and the max-usage check whenever the installed-memory gate fails. Record this failure and continue through the other checks, then set the final status once. Do not let a later passing check overwrite this error with ExecutionStatus.OK.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants