diff options
Diffstat (limited to 'aggregator/dwd_icon.py')
| -rwxr-xr-x | aggregator/dwd_icon.py | 149 |
1 files changed, 0 insertions, 149 deletions
diff --git a/aggregator/dwd_icon.py b/aggregator/dwd_icon.py deleted file mode 100755 index 6df905c..0000000 --- a/aggregator/dwd_icon.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 - -import requests -import datetime -import pytz -import requests -import os - -from multiprocessing import cpu_count -from multiprocessing.pool import ThreadPool - -import subprocess - -import xarray as xr - -import misc - -BASE='https://opendata.dwd.de/weather/nwp' - -def get_current_run(): - # we allow up to 3h of slack for DWD to upload the latest run - tz = pytz.timezone('UTC') - now = datetime.datetime.now(datetime.timezone.utc) - corrected = now - datetime.timedelta(hours=3) - - run = int(corrected.hour / 6) * 6 - - return (f'{run:02d}', corrected.strftime('%Y%m%d')) - -def download_url(args): - url, dest = args - r = requests.get(url) - try: - with open(dest, 'wb') as f: - f.write(r.content) - print(f'Downloaded {dest}') - except Exception as e: - print(f'Failed to download {dest}:\n', e) - -def unpack_bz2(dest): - res = subprocess.run(['bzip2', '-df', dest]) - if res.returncode != 0: - print(f'There was an error unpacking {dest}:', res.stderr) - -def download_dwd_gribs( - date, run, target, output, model, steps, model_long, - pressure_level_parameters, parameter_caps_in_filename, - single_level_parameters, pressure_levels -): - misc.create_output_dir(output) - - to_download = [] - - for step in steps: - step_str = f'{step:03d}' - - for parameter in pressure_level_parameters: - parameter2 = parameter.upper() if parameter_caps_in_filename else parameter - - for level in pressure_levels: - filename = f'{model_long}_regular-lat-lon_pressure-level_{date}{run}_{step_str}_{level}_{parameter2}.grib2.bz2' - URL = f'{BASE}/{model}/grib/{run}/{parameter}/{filename}' - - to_download.append((URL, os.path.join(output, filename))) - - for parameter in single_level_parameters: - parameter2 = parameter.upper() if parameter_caps_in_filename else parameter - filename = f'{model_long}_regular-lat-lon_single-level_{date}{run}_{step_str}_{parameter2}.grib2.bz2' - URL = f'{BASE}/{model}/grib/{run}/{parameter}/{filename}' - - to_download.append((URL, os.path.join(output, filename))) - - - for _ in ThreadPool(cpu_count()).imap_unordered(download_url, to_download): - pass - - print('Done Downloading. Uncompressing...') - - for _ in ThreadPool(cpu_count()).imap_unordered(unpack_bz2, [dest for _, dest in to_download]): - pass - - downloaded_gribs = [dest.removesuffix('.bz2') for _, dest in to_download] - - res = subprocess.run(['grib_copy'] + downloaded_gribs + [target]) - if res.returncode != 0: - print('grib_copy failed with: ', res.stderr) - - res = subprocess.run(['rm', '-f'] + downloaded_gribs) - if res.returncode != 0: - print('rm failed with: ', res.stderr) - -def clean_output_dir(directory, target): - to_delete = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] - if target in to_delete: - del to_delete[to_delete.index(target)] - - for f in to_delete: - os.unlink(os.path.join(directory, f)) - -def load_data(name, output, description = None, clean = False, force_filename = None, **kwargs): - target = force_filename - - if target is None: - run, date = get_current_run() - filename = f'{name}_{date}_{run}.grib2' - target = os.path.join(output, filename) - - if not os.path.exists(target): - download_dwd_gribs(date, run, target, output, **kwargs) - else: - print(f'{target} already exists. Using the cached version.') - - if clean: - clean_output_dir(output, filename) - - # we drop heightAboveGround to allow 2m and 10m values to be merged down to one dataset - ds = xr.load_dataset(target, engine='cfgrib', drop_variables='heightAboveGround') - - if description is not None: - ds.attrs['_description'] = description - return ds - - -debug_config = { - 'output':'dwd_icon-eu', - 'model':'icon-eu', - 'model_long':'icon-eu_europe', - 'clean': True, - 'parameter_caps_in_filename':True, - 'pressure_level_parameters': [ - 't', - 'relhum', - 'u', - 'v', - 'fi', - 'clc' - ], - 'single_level_parameters': [ - 'pmsl', - 't_2m', - 'relhum_2m' - ], - 'pressure_levels':[ 1000, 950, 925, 900, 875, 850, 825, 800, 775, 700, 600, 500, 400, 300, 250, 200, 150, 100 ], - 'steps':[0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48] -} - -if __name__ == '__main__': - load_data('test_icon_eu', **debug_config) - |