From c4a8bd34b41b2be26426ea01aafc69d41260cce5 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Thu, 7 Mar 2019 16:41:03 +0100 Subject: Make postprocessing scripts more resilient --- parasort.sh | 24 ++++++++++++++++-------- postprocess.sh | 27 ++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/parasort.sh b/parasort.sh index 6b54449..6565d61 100644 --- a/parasort.sh +++ b/parasort.sh @@ -7,14 +7,22 @@ function ctrl_c() { echo "Killed." } -cd output || exit 1 -mkdir -p ../sorted +if ! test -d work; then + printf "This directory contains no work/ directory.\n" + [ -d ../work ] && printf "(but .. does)\n" + exit 1 +fi -for a in *[05]; do echo "$a" >&2; ../merge_entries $a > ../sorted/$a; done & -for a in *[16]; do echo " $a" >&2; ../merge_entries $a > ../sorted/$a; done & -for a in *[27]; do echo " $a" >&2; ../merge_entries $a > ../sorted/$a; done & -for a in *[38]; do echo " $a" >&2; ../merge_entries $a > ../sorted/$a; done & -for a in *[49]; do echo " $a" >&2; ../merge_entries $a > ../sorted/$a; done & -for a in brken *_; do echo " $a" >&2; ../merge_entries $a > ../sorted/$a; done & +export PATH=${PATH}:`pwd -P`/bin/ + +mkdir -p work/sorted +cd work/output || exit 1 + +for a in *[05]; do echo "$a" >&2; merge_entries $a > ../sorted/$a; done & +for a in *[16]; do echo " $a" >&2; merge_entries $a > ../sorted/$a; done & +for a in *[27]; do echo " $a" >&2; merge_entries $a > ../sorted/$a; done & +for a in *[38]; do echo " $a" >&2; merge_entries $a > ../sorted/$a; done & +for a in *[49]; do echo " $a" >&2; merge_entries $a > ../sorted/$a; done & +for a in brken *_; do echo " $a" >&2; merge_entries $a > ../sorted/$a; done & wait diff --git a/postprocess.sh b/postprocess.sh index 7c4b4e6..9316357 100644 --- a/postprocess.sh +++ b/postprocess.sh @@ -2,11 +2,28 @@ export LANG=C export LC_ALL=C export LC_CTYPE=C -sudo sysctl kern.maxfilesperproc=65563 -sudo sysctl kern.maxfiles=65563 + +mfpp=`sysctl -n kern.maxfilesperproc` +mf=`sysctl -n kern.maxfiles` + + +if [ $mfpp -lt 65563 -o $mf -lt 65563 ]; then + printf "System resources not tuned, %d and %d please run:\n" $mfpp $mf + printf "sudo sysctl kern.maxfilesperproc=65563\n" + printf "sudo sysctl kern.maxfiles=65563\n" + exit +fi + ulimit -n 65563 -for a in 199[56789]_Q? 20*_Q?; do ./sort_plz $a; done +export PATH=${PATH}:`pwd -P`/bin/ + +if ! test -d work; then + printf "This directory contains no work/ directory.\n" + [ -d ../work ] && printf "(but .. does)\n" + exit 1 +fi -rm -rf sorted -mkdir sorted +cd work || exit 1 + +for a in 199[56789]_Q? 20*_Q?; do ./sort_plz $a; done -- cgit v1.2.3