From 566ebe8ef4dc65bb427b9b56e99a1db73cd24adf Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Wed, 11 Nov 2026 17:14:37 +0100 Subject: [PATCH] fix(k8s): show the logs of every GPU test pod in verify_gpu.sh The loop that prints the test pod logs counted with $i but indexed the pod array with $k, which is never set. The index was always 0, so the script printed the log of the first completed pod once per GPU and never showed the other pods. Iterate over the completed pods instead, and add a regression case to tests/verify-gpu/test_verify_gpu.py. Co-Authored-By: Claude Fable 5.2 Signed-off-by: Dezhen Lu --- scripts/k8s/verify_gpu.sh | 6 ++---- tests/verify-gpu/test_verify_gpu.py | 8 ++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/k8s/verify_gpu.sh b/scripts/k8s/verify_gpu.sh index 1cc39d57a..184c43e79 100755 --- a/scripts/k8s/verify_gpu.sh +++ b/scripts/k8s/verify_gpu.sh @@ -72,10 +72,8 @@ string_array=($pods_output) number_pods=${#string_array[@]} # loop through all pod from each node -i=1 -while [ $i -le $total_gpus ]; do - kubectl -n ${CLUSTER_VERIFY_NS} logs -f ${string_array[$k]} - let i=i+1 +for pod in "${string_array[@]}"; do + kubectl -n ${CLUSTER_VERIFY_NS} logs -f ${pod} done echo "Number of Nodes: ${number_gpu_nodes}" diff --git a/tests/verify-gpu/test_verify_gpu.py b/tests/verify-gpu/test_verify_gpu.py index 6a3315fa0..bfb086625 100644 --- a/tests/verify-gpu/test_verify_gpu.py +++ b/tests/verify-gpu/test_verify_gpu.py @@ -78,6 +78,14 @@ def test_positive_capacity_keeps_existing_job_flow(self): self.assertTrue(any("create -f" in call for call in calls)) self.assertEqual(calls[-1], "delete ns fixture-gpu-verify") + def test_logs_are_read_once_from_every_completed_pod(self): + for capacities, count in (([1], 1), ([None, 2, 1], 3)): + with self.subTest(capacities=capacities): + result, calls, _, _ = self.run_script(capacities) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + logged = [call.split()[-1] for call in calls if " logs -f " in call] + self.assertEqual(logged, [f"cluster-gpu-tests-{i}" for i in range(count)]) + if __name__ == "__main__": unittest.main()