diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index d5b150f3dd4a..72982e926461 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2450,9 +2450,11 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG // `|| true` so a missing file (submission never wrote it) yields // an empty string and reaches the typed throw below, instead of a // generic "script returned exit code 1" the classifier can't act on. + // No `2>/dev/null`: under a csh login shell it would send stdout to + // /dev/null and always return an empty job ID. slurmJobId = Utils.exec( pipeline, - script: Utils.sshUserCmd(remote, "\"cat ${jobWorkspace}/slurm_job_id.txt 2>/dev/null || true\""), + script: Utils.sshUserCmd(remote, "\"cat ${jobWorkspace}/slurm_job_id.txt || true\""), returnStdout: true, numRetries: 3 ).trim() @@ -2616,8 +2618,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def sshStatCmd = Utils.sshUserCmd(remote, "\"srun --overlap ${srunAccountArg}--quiet --jobid='${slurmJobId}' --ntasks=1 stat -c %Y '${remoteWorkspaceTrk}/results.xml' || echo 0\"") def scpXmlCmd = scpFromRemoteCmd(remote, "${remoteWorkspaceTrk}/results*.xml", "${stageName}/") def scpUnfinishedCmd = scpFromRemoteCmd(remote, "${remoteWorkspaceTrk}/unfinished_test.txt", "${stageName}/") - def sshRefreshCacheCmd = Utils.sshUserCmd(remote, "\"ls '${remoteWorkspaceTrk}/' > /dev/null 2>&1; ls -la '${remoteWorkspaceTrk}/results.xml' > /dev/null 2>&1 || true\"") - def sshListPerfCmd = Utils.sshUserCmd(remote, "\"find '${remoteWorkspaceTrk}' -maxdepth 1 -type d \\( -name 'aggr*' -o -name 'disagg*' \\) -print 2>/dev/null || true\"") + // The sh blocks below paste these two commands between single quotes, so they + // must not contain single quotes (a `bash -c '...'` wrapper would break too). + // sshd runs the remote part under svc_tensorrt's login shell, which is csh on + // some clusters (e.g. oci-jhb, oci-aga); csh rejects unmatched globs and has no + // `2>`. So keep redirects and filters on the local side of the ssh call. + def sshRefreshCacheCmd = Utils.sshUserCmd(remote, "\"ls -la ${remoteWorkspaceTrk}/ ${remoteWorkspaceTrk}/results.xml\"") + " > /dev/null 2>&1 || true" + def sshListPerfCmd = Utils.sshUserCmd(remote, "\"find ${remoteWorkspaceTrk} -mindepth 1 -maxdepth 1 -type d\"") + " | grep -e /aggr -e /disagg" def scpPerfTemplate = scpFromRemoteCmd(remote, "PERF_FOLDER_PLACEHOLDER", "${stageName}/") sh "rm -f ${pytestDoneFile}" withCredentials([usernamePassword(