diff --git a/scripts/k8s/verify_gpu.sh b/scripts/k8s/verify_gpu.sh index 1cc39d57a..184c43e79 100755 --- a/scripts/k8s/verify_gpu.sh +++ b/scripts/k8s/verify_gpu.sh @@ -72,10 +72,8 @@ string_array=($pods_output) number_pods=${#string_array[@]} # loop through all pod from each node -i=1 -while [ $i -le $total_gpus ]; do - kubectl -n ${CLUSTER_VERIFY_NS} logs -f ${string_array[$k]} - let i=i+1 +for pod in "${string_array[@]}"; do + kubectl -n ${CLUSTER_VERIFY_NS} logs -f ${pod} done echo "Number of Nodes: ${number_gpu_nodes}" diff --git a/tests/verify-gpu/test_verify_gpu.py b/tests/verify-gpu/test_verify_gpu.py index 6a3315fa0..bfb086625 100644 --- a/tests/verify-gpu/test_verify_gpu.py +++ b/tests/verify-gpu/test_verify_gpu.py @@ -78,6 +78,14 @@ def test_positive_capacity_keeps_existing_job_flow(self): self.assertTrue(any("create -f" in call for call in calls)) self.assertEqual(calls[-1], "delete ns fixture-gpu-verify") + def test_logs_are_read_once_from_every_completed_pod(self): + for capacities, count in (([1], 1), ([None, 2, 1], 3)): + with self.subTest(capacities=capacities): + result, calls, _, _ = self.run_script(capacities) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + logged = [call.split()[-1] for call in calls if " logs -f " in call] + self.assertEqual(logged, [f"cluster-gpu-tests-{i}" for i in range(count)]) + if __name__ == "__main__": unittest.main()