From 983b1225a5504c271be3e10d5da5675fdcaf2cf4 Mon Sep 17 00:00:00 2001
From: Jonas Gunz <himself@jonasgunz.de>
Date: Sat, 2 Sep 2023 00:16:13 +0200
Subject: icon_downoad multi threading

---
 icon_download.sh | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

(limited to 'icon_download.sh')

diff --git a/icon_download.sh b/icon_download.sh
index 462685c..aaa4be9 100755
--- a/icon_download.sh
+++ b/icon_download.sh
@@ -4,18 +4,21 @@
 
 # <BASE>/<RUN>/<PARAMETER>/icon-d2_regular-lat-lon_pressure-level_<INIT>_<OFFSET>_<LEVEL>_<PARAMETER>
 
+NPROC=$(nproc)
+
 OUTDIR=dwd_icon-d2
 MODEL=icon-d2
 MODEL_LONG=icon-d2_germany
 BASE="http://opendata.dwd.de/weather/nwp"
 
-RUN="00"
+RUN="18"
 PARAMETERS=( "t" "relhum" "u" "v" "fi" )
 # tot_prec and cape_ml/cin_ml is in 15min intervals and screws with xygrib
 PARAMETERS_SINGLE_LEVEL=( "w_ctmax" )
 PRESSURE_LEVELS=( "1000" "975" "950" "850" "700" "600" "500" "400" "300" "250" "200" )
-OFFSETS=( "000" "003" "006" "009" "012" "015" "018" "024" )
+OFFSETS=( "000" "003" "006" "009" "012" "015" "018" "024" "027" "030" "033" "036"  "039" "042" "045" "048" )
 DATE=$(date +%Y%m%d)
+DATE=20230901
 
 mkdir -p $OUTDIR
 
@@ -24,28 +27,40 @@ echo -n > "$OUTDIR/index.txt"
 for OFFSET in "${OFFSETS[@]}"; do
 	for PARAMETER in "${PARAMETERS[@]}"; do
 		for LEVEL in "${PRESSURE_LEVELS[@]}"; do
+			while [ $(pgrep -c -P$$) -gt $NPROC ]; do
+				sleep 1
+			done
+
 			URL="$BASE/$MODEL/grib/$RUN/$PARAMETER/${MODEL_LONG}_regular-lat-lon_pressure-level_${DATE}${RUN}_${OFFSET}_${LEVEL}_${PARAMETER}.grib2.bz2"
 			BNAME=$(basename "$URL")
 			echo Getting "$URL"
 			echo "${BNAME%.bz2}" >> $OUTDIR/index.txt
-			wget -q --directory-prefix=$OUTDIR  "$URL" || echo FAILED!
+			( wget -q --directory-prefix=$OUTDIR  "$URL" || echo FAILED! ) &
 
 		done
 	done
 
 	for PARAMETER in "${PARAMETERS_SINGLE_LEVEL[@]}"; do
+			while [ $(pgrep -c -P$$) -gt $NPROC ]; do
+				sleep 1
+			done
+
 			URL="$BASE/$MODEL/grib/$RUN/$PARAMETER/${MODEL_LONG}_regular-lat-lon_single-level_${DATE}${RUN}_${OFFSET}_2d_${PARAMETER}.grib2.bz2"
 			BNAME=$(basename "$URL")
 			echo Getting "$URL"
 			echo "${BNAME%.bz2}" >> $OUTDIR/index.txt
-			wget -q --directory-prefix=$OUTDIR  "$URL" || echo FAILED!
+			( wget -q --directory-prefix=$OUTDIR  "$URL" || echo FAILED! ) &
 	done
 done
 
+while [ $(pgrep -c -P$$) -gt 0 ]; do
+	sleep 1
+done
+
 echo Done downloading. Decompressing...
 
 for F in $OUTDIR/*.grib2.bz2; do
-	while [ $(pgrep -c -P$$) -gt 5 ]; do
+	while [ $(pgrep -c -P$$) -gt $NPROC ]; do
 		sleep 1
 	done
 
-- 
cgit v1.2.3