aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar Jonas Gunz <himself@jonasgunz.de> 2023-09-02 00:16:13 +0200
committerGravatar Jonas Gunz <himself@jonasgunz.de> 2023-09-02 00:16:13 +0200
commit983b1225a5504c271be3e10d5da5675fdcaf2cf4 (patch)
treed36dfdefcbb601fc6d0bac38e5c1fee6a1428172
parent157cfb53784304d7a6acf583ba4ded451b79e309 (diff)
downloadmeteo_toolbox-983b1225a5504c271be3e10d5da5675fdcaf2cf4.tar.gz
icon_downoad multi threading
-rwxr-xr-xicon_download.sh25
1 files changed, 20 insertions, 5 deletions
diff --git a/icon_download.sh b/icon_download.sh
index 462685c..aaa4be9 100755
--- a/icon_download.sh
+++ b/icon_download.sh
@@ -4,18 +4,21 @@
# <BASE>/<RUN>/<PARAMETER>/icon-d2_regular-lat-lon_pressure-level_<INIT>_<OFFSET>_<LEVEL>_<PARAMETER>
+NPROC=$(nproc)
+
OUTDIR=dwd_icon-d2
MODEL=icon-d2
MODEL_LONG=icon-d2_germany
BASE="http://opendata.dwd.de/weather/nwp"
-RUN="00"
+RUN="18"
PARAMETERS=( "t" "relhum" "u" "v" "fi" )
# tot_prec and cape_ml/cin_ml is in 15min intervals and screws with xygrib
PARAMETERS_SINGLE_LEVEL=( "w_ctmax" )
PRESSURE_LEVELS=( "1000" "975" "950" "850" "700" "600" "500" "400" "300" "250" "200" )
-OFFSETS=( "000" "003" "006" "009" "012" "015" "018" "024" )
+OFFSETS=( "000" "003" "006" "009" "012" "015" "018" "024" "027" "030" "033" "036" "039" "042" "045" "048" )
DATE=$(date +%Y%m%d)
+DATE=20230901
mkdir -p $OUTDIR
@@ -24,28 +27,40 @@ echo -n > "$OUTDIR/index.txt"
for OFFSET in "${OFFSETS[@]}"; do
for PARAMETER in "${PARAMETERS[@]}"; do
for LEVEL in "${PRESSURE_LEVELS[@]}"; do
+ while [ $(pgrep -c -P$$) -gt $NPROC ]; do
+ sleep 1
+ done
+
URL="$BASE/$MODEL/grib/$RUN/$PARAMETER/${MODEL_LONG}_regular-lat-lon_pressure-level_${DATE}${RUN}_${OFFSET}_${LEVEL}_${PARAMETER}.grib2.bz2"
BNAME=$(basename "$URL")
echo Getting "$URL"
echo "${BNAME%.bz2}" >> $OUTDIR/index.txt
- wget -q --directory-prefix=$OUTDIR "$URL" || echo FAILED!
+ ( wget -q --directory-prefix=$OUTDIR "$URL" || echo FAILED! ) &
done
done
for PARAMETER in "${PARAMETERS_SINGLE_LEVEL[@]}"; do
+ while [ $(pgrep -c -P$$) -gt $NPROC ]; do
+ sleep 1
+ done
+
URL="$BASE/$MODEL/grib/$RUN/$PARAMETER/${MODEL_LONG}_regular-lat-lon_single-level_${DATE}${RUN}_${OFFSET}_2d_${PARAMETER}.grib2.bz2"
BNAME=$(basename "$URL")
echo Getting "$URL"
echo "${BNAME%.bz2}" >> $OUTDIR/index.txt
- wget -q --directory-prefix=$OUTDIR "$URL" || echo FAILED!
+ ( wget -q --directory-prefix=$OUTDIR "$URL" || echo FAILED! ) &
done
done
+while [ $(pgrep -c -P$$) -gt 0 ]; do
+ sleep 1
+done
+
echo Done downloading. Decompressing...
for F in $OUTDIR/*.grib2.bz2; do
- while [ $(pgrep -c -P$$) -gt 5 ]; do
+ while [ $(pgrep -c -P$$) -gt $NPROC ]; do
sleep 1
done