implemented aggreator

5 files changed, 46 insertions, 11 deletions

diff --git a/download.py b/aggregator/dwd_icon.py
index fdaa3f4..60c8433 100755
--- a/download.py
+++ b/aggregator/dwd_icon.py
@@ -11,6 +11,8 @@ from multiprocessing.pool import ThreadPool
 
 import subprocess
 
+import xarray as xr
+
 import misc
 
 BASE='https://opendata.dwd.de/weather/nwp'
@@ -86,12 +88,16 @@ def download_dwd_gribs(config, date, run, target):
     if res.returncode != 0:
         print('rm failed with: ', res.stderr)
 
-def load_data(config):
+def load_data(name, config):
     run, date = get_current_run()
-    target = os.path.join(config['output'], f'combined_{date}_{run}.grib2')
+    target = os.path.join(config['output'], f'{name}_{date}_{run}.grib2')
 
     if not os.path.exists(target):
         download_dwd_gribs(config, date, run, target)
+    else:
+        print(f'{target} alreasy exists. Using the cached version.')
+
+    return xr.load_dataset(target, engine='cfgrib')
 
 config = {
     'output':'dwd_icon-eu',
@@ -116,5 +122,5 @@ config = {
 }
 
 if __name__ == '__main__':
-    load_data(config)
+    load_data('test_icon_eu', config)
 
diff --git a/config.yaml b/config.yaml
index 8a882ca..d849625 100644
--- a/config.yaml
+++ b/config.yaml
@@ -1,8 +1,28 @@
 ---
 index: web/data/index.json
+aggregator:
+  icon_eu:
+    module: aggregator.dwd_icon
+    output: dwd_icon_cache
+    model: icon-eu
+    model_long: icon-eu_europe
+    parameter_caps_in_filename: true
+    pressure_level_parameters:
+      - t
+      - relhum
+      - u
+      - v
+      - fi
+      - clc
+    single_level_parameters:
+      - pmsl
+      - t_2m
+      - relhum_2m
+    pressure_levels: [1000, 950, 925, 900, 875, 850, 825, 800, 775, 700, 600, 500, 400, 300, 250, 200, 150, 100]
+    steps: [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48]
 plotter:
   - module: 'plotter.vertical_from_grib'
-    source: dwd_icon-eu/combined.grib2
+    aggregator: icon_eu
     output: web/data
     plots:
       - lat: 47.96
@@ -10,7 +30,7 @@ plotter:
         name: Antersberg
         analysis: lcl
   - module: 'plotter.horizontal'
-    source: dwd_icon-eu/combined.grib2
+    aggregator: icon_eu
     output: web/data
     plots:
       - name: t_fi_850
diff --git a/plotter/horizontal.py b/plotter/horizontal.py
index e83b683..fa5f9df 100755
--- a/plotter/horizontal.py
+++ b/plotter/horizontal.py
@@ -33,10 +33,8 @@ config = {
     ]
 }
 
-def run(source, plots, output='.'):
+def run(data, plots, output='.'):
     misc.create_output_dir(output)
-    data = xr.load_dataset(source, engine='cfgrib')
-
     index = []
 
     for plot in plots:
diff --git a/plotter/vertical_from_grib.py b/plotter/vertical_from_grib.py
index 929782e..9f927a7 100755
--- a/plotter/vertical_from_grib.py
+++ b/plotter/vertical_from_grib.py
@@ -25,10 +25,8 @@ config = {
     ]
 }
 
-def run(source, plots, output='.'):
+def run(data, plots, output='.'):
     misc.create_output_dir(output)
-    data = xr.load_dataset(source, engine='cfgrib')
-
     index = []
 
     for plot in plots:
diff --git a/run.py b/run.py
index de6a078..fa0f598 100755
--- a/run.py
+++ b/run.py
@@ -26,6 +26,19 @@ for plotter in conf['plotter']:
     modname = plotter['module']
     del plotter['module']
 
+    if 'aggregator' in plotter:
+        aggname = plotter['aggregator']
+        del plotter['aggregator']
+        aggconf = conf['aggregator'][aggname]
+        classname = aggconf['module']
+        # We should prbly not delete it like in the plotter, since it is not a deepcopy
+        # and may be used again.
+
+        agg = __import__(classname, fromlist=[None])
+
+        # TODO: figure out a way to use **aggconf instead.
+        plotter['data'] = agg.load_data(aggname, aggconf)
+
     mod = __import__(modname, fromlist=[None])
     index.extend(mod.run(**plotter))