From 7f07e7bb772c310b3f34ce93e4a20d7e4ed48821 Mon Sep 17 00:00:00 2001 From: Eugen Betke Date: Fri, 14 Jun 2024 11:20:51 +0000 Subject: [PATCH 1/9] ECC-1755: Add sub hourly support --- tools/grib_to_netcdf.cc | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/tools/grib_to_netcdf.cc b/tools/grib_to_netcdf.cc index 0dad044ab..8ee4097d9 100644 --- a/tools/grib_to_netcdf.cc +++ b/tools/grib_to_netcdf.cc @@ -137,6 +137,21 @@ static bool eq_time(const char* l, const char* r) return false; } +const char* get_step_units_longname(const char* step_units) +{ + if (step_units == NULL) + return NULL; + if (strcmp(step_units, "s") == 0) + return "seconds"; + if (strcmp(step_units, "m") == 0) + return "minutes"; + if (strcmp(step_units, "h") == 0) + return "hours"; + + Assert(0); + return NULL; +} + static value* new_value(const char* name) { value* v = (value*)calloc(sizeof(value), 1); @@ -1974,6 +1989,7 @@ static void validation_time(request* r) double v; long julian = 0; const char* step_units = NULL; + double step_units_factor = 1.0; long nstep = count_values(r, "step"); long ndate = count_values(r, "date"); @@ -2051,13 +2067,15 @@ static void validation_time(request* r) "Cannot convert stepUnits of '%s'. Only hours, minutes and seconds supported.", step_units); } if (STR_EQUAL("m", step_units)) { - step /= 60; + step_units_factor = 60.0; } else if (STR_EQUAL("s", step_units)) { - step /= 3600; + step_units_factor = 3600.0; } } - v = julian * 24.0 + fcmonthdays * 24.0 + time / 100.0 + step * 1.0; + + v = (julian * 24.0 + fcmonthdays * 24.0 + time / 100.0) * step_units_factor + step * 1.0; + grib_context_log(ctx, GRIB_LOG_DEBUG, "grib_to_netcdf: date=%ld, julian=%ld, fcmonthdays=%ld, time=%ld, step=%g, validation=%.3f", date, julian, fcmonthdays, time, step, v); set_value(r, "_validation", "%lf", v); set_value(r, "_juliandate", "%ld", julian); @@ -2065,7 +2083,7 @@ static void validation_time(request* r) if (!julianrefdate) julianrefdate = grib_date_to_julian(setup.refdate); - set_value(r, "_validationtime", "%lf", v - julianrefdate * 24.0); + set_value(r, "_validationtime", "%lf", v - julianrefdate * 24.0 * step_units_factor); /* Remove minutes from TIME */ if (ntime) @@ -3007,12 +3025,15 @@ static int define_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid, datase } } + const char* step_units = get_value(data_r, "stepUnits", 0); + const char* step_units_longname= get_step_units_longname(step_units); + if (strcmp(axis, "time") == 0) { bool onedtime = (count_values(cube, "date") == 0 && count_values(cube, "step") == 0); - snprintf(u, sizeof(u), "hours since 0000-00-00 00:00:00.0"); + snprintf(u, sizeof(u), "%s since 0000-00-00 00:00:00.0", step_units_longname); longname = "reference_time"; if (setup.usevalidtime || onedtime) { - snprintf(u, sizeof(u), "hours since %ld-%02ld-%02ld 00:00:00.0", setup.refdate / 10000, (setup.refdate % 10000) / 100, (setup.refdate % 100)); + snprintf(u, sizeof(u), "%s since %ld-%02ld-%02ld 00:00:00.0", step_units_longname, setup.refdate / 10000, (setup.refdate % 10000) / 100, (setup.refdate % 100)); longname = "time"; } if (setup.climatology) { @@ -3023,7 +3044,7 @@ static int define_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid, datase } if (strcmp(axis, "step") == 0) { - units = "hours"; + units = step_units; longname = "time_step"; if (count_values(cube, "date") == 0 && count_values(cube, "time") == 0) { const char* d = get_value(data_r, "date", 0); From 8f4b8e6a46cb0f574f501dc9633f94a170d7cab9 Mon Sep 17 00:00:00 2001 From: Eugen Betke Date: Mon, 17 Jun 2024 16:09:06 +0000 Subject: [PATCH 2/9] ECC-1755: Add tests --- tests/grib_to_netcdf.sh | 79 ++++++++++++++++++++++++++--------------- tools/grib_to_netcdf.cc | 15 ++++++-- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index ba3589329..f5201dad2 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -24,9 +24,11 @@ fi label="grib_to_netcdf_test" tempGrib=temp.${label}.grib +tempGrib2=temp2.${label}.grib tempNetcdf=temp.${label}.nc tempText=temp.${label}.txt tempDir=temp.${label}.dir +tempFilter=temp.${label}.filter have_netcdf4=0 @@ -36,6 +38,53 @@ if command -v "ncdump" >/dev/null 2>&1; then NC_DUMPER="ncdump" fi +echo "Test ECC-1755: Test sub-hourly support ..." +# ------------------------------------------------------------ +# ECC-1755: Sub-hourly support +# Minutes: +cat > $tempFilter < $tempGrib +${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib +${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2 +${NC_DUMPER} -t -v time $tempNetcdf > $tempText +cat $tempText +grep -q 'time:units = "minutes since 1900-01-01 00:00:00.0" ;' $tempText +grep -q 'time = "2008-02-06 12", "2008-02-06 12:15", "2008-02-06 12:30" ;' $tempText + +# Seconds: +cat > $tempFilter < $tempGrib +${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib + +# Please set the reference date to avoid an out-of-range error with the integer value. +${tools_dir}/grib_to_netcdf -R 20080206 -o $tempNetcdf $tempGrib2 +${NC_DUMPER} -t -v time $tempNetcdf > $tempText +cat $tempText +grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText +grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30" ;' $tempText + +# The operation should fail because the time value exceeded the maximum limit of 2,147,483,647. +if ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2; then + echo "Time values are out of range. Should fail." + exit 1 +fi + echo "Test ECC-1041: One parameter with different expvers ..." # ------------------------------------------------------------ @@ -110,14 +159,6 @@ ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib ${tools_dir}/grib_set -s productDefinitionTemplateNumber=31 $sample2 $tempGrib ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib -ECCODES_DEBUG=-1 ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib - - -# The -u option -input=${data_dir}/sample.grib2 -${tools_dir}/grib_to_netcdf -u time -o $tempNetcdf $input - - echo "Test different resolutions ..." # ------------------------------------ # This should fail as messages have different resolutions @@ -178,26 +219,6 @@ set -e grep -q "Wrong message length" $tempText -# Non-GRIB input -input=$data_dir/bufr/aaen_55.bufr -set +e -${tools_dir}/grib_to_netcdf -o $tempNetcdf $input > $tempText 2>&1 -status=$? -set -e -[ $status -ne 0 ] -grep -q "Input does not contain any field" $tempText - - -# Bad reference date -input=$data_dir/sample.grib2 -set +e -${tools_dir}/grib_to_netcdf -Rxxx -o $tempNetcdf $input > $tempText 2>&1 -status=$? -set -e -[ $status -ne 0 ] -grep -q "Invalid reference date" $tempText - - # Validity time check export GRIB_TO_NETCDF_CHECKVALIDTIME=0 ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib @@ -206,4 +227,4 @@ unset GRIB_TO_NETCDF_CHECKVALIDTIME # Clean up -rm -f $tempNetcdf $tempGrib $tempText +rm -f $tempNetcdf $tempGrib $tempGrib2 $tempText $tempFilter diff --git a/tools/grib_to_netcdf.cc b/tools/grib_to_netcdf.cc index 8ee4097d9..223429f92 100644 --- a/tools/grib_to_netcdf.cc +++ b/tools/grib_to_netcdf.cc @@ -3320,8 +3320,13 @@ static int fill_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid) for (j = 0; j < n; ++j) values[j] = monthnumber(get_value(cube, axis, j)); else - for (j = 0; j < n; ++j) - values[j] = grib_date_to_julian(atol(get_value(cube, axis, j))) - grib_date_to_julian(setup.refdate); + for (j = 0; j < n; ++j) { + long date = grib_date_to_julian(atol(get_value(cube, axis, j))); + long refdate = grib_date_to_julian(setup.refdate); + fprintf(stderr, "date=%ld, refdate=%ld\n", date, refdate); + values[j] = date - refdate; + //values[j] = grib_date_to_julian(atol(get_value(cube, axis, j))) - grib_date_to_julian(setup.refdate); + } } else { for (j = 0; j < n; ++j) { @@ -3329,6 +3334,12 @@ static int fill_netcdf_dimensions(hypercube* h, fieldset* fs, int ncid) const char* sv = get_value(cube, axis, j); if (is_number(sv)) { lv = atol(sv); /* Detect error? */ + constexpr long maxint = 2147483647; /* 2^31 - 1, max size of NC_INT*/ + if (lv > maxint) { + grib_context_log(ctx, GRIB_LOG_ERROR, "Value %ld for %s is too large. ", lv, axis); + grib_context_log(ctx, GRIB_LOG_ERROR, "Consider using the option: \"-R date\". Reference date in the format YYYYMMDD. Default value 19000101.", setup.refdate); + exit(1); + } } else { /* ECC-725: Convert string-valued dimension to integer From 57cb9706e286b59d4b9ccda031a33813afa18cce Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 16:42:54 +0000 Subject: [PATCH 3/9] ECC-1755: Fix tests --- tests/grib_to_netcdf.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index f5201dad2..0afabddc7 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -54,10 +54,11 @@ input=${data_dir}/sample.grib2 cat $input $input $input > $tempGrib ${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2 -${NC_DUMPER} -t -v time $tempNetcdf > $tempText -cat $tempText -grep -q 'time:units = "minutes since 1900-01-01 00:00:00.0" ;' $tempText -grep -q 'time = "2008-02-06 12", "2008-02-06 12:15", "2008-02-06 12:30" ;' $tempText +if test "x$NC_DUMPER" != "x"; then + ${NC_DUMPER} -t -v time $tempNetcdf > $tempText + grep -q 'time:units = "minutes since 1900-01-01 00:00:00.0" ;' $tempText + grep -q 'time = "2008-02-06 12", "2008-02-06 12:15", "2008-02-06 12:30" ;' $tempText +fi # Seconds: cat > $tempFilter < $tempText -cat $tempText -grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText -grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30" ;' $tempText +if test "x$NC_DUMPER" != "x"; then + ${NC_DUMPER} -t -v time $tempNetcdf > $tempText + grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText + grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30" ;' $tempText +fi # The operation should fail because the time value exceeded the maximum limit of 2,147,483,647. if ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2; then From 13c2b40aa65028b3439d6d14484b0571ea0bded9 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 16:52:40 +0000 Subject: [PATCH 4/9] ECC-1755: Cleanup --- tools/grib_to_netcdf.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/grib_to_netcdf.cc b/tools/grib_to_netcdf.cc index 223429f92..5b356683e 100644 --- a/tools/grib_to_netcdf.cc +++ b/tools/grib_to_netcdf.cc @@ -137,7 +137,7 @@ static bool eq_time(const char* l, const char* r) return false; } -const char* get_step_units_longname(const char* step_units) +static const char* get_step_units_longname(const char* step_units) { if (step_units == NULL) return NULL; @@ -148,7 +148,7 @@ const char* get_step_units_longname(const char* step_units) if (strcmp(step_units, "h") == 0) return "hours"; - Assert(0); + Assert(!"Unknown step units"); return NULL; } From ad0072c46fc3c992dd2d107f57e26c5415417e52 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 16:53:01 +0000 Subject: [PATCH 5/9] ECC-1755: Test issues (try 01) --- tests/grib_to_netcdf.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index 0afabddc7..7119be282 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -56,8 +56,9 @@ ${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib2 if test "x$NC_DUMPER" != "x"; then ${NC_DUMPER} -t -v time $tempNetcdf > $tempText + cat $tempText grep -q 'time:units = "minutes since 1900-01-01 00:00:00.0" ;' $tempText - grep -q 'time = "2008-02-06 12", "2008-02-06 12:15", "2008-02-06 12:30" ;' $tempText + grep -q 'time = "2008-02-06 12", "2008-02-06 12:15", "2008-02-06 12:30"' $tempText fi # Seconds: From 36b2e1a19c3aca1a7fee2ae81c99c459152286dd Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 17:02:34 +0000 Subject: [PATCH 6/9] ECC-1755: Test issues (try 02) --- tests/grib_to_netcdf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index 7119be282..2d5e7fd38 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -79,7 +79,7 @@ ${tools_dir}/grib_to_netcdf -R 20080206 -o $tempNetcdf $tempGrib2 if test "x$NC_DUMPER" != "x"; then ${NC_DUMPER} -t -v time $tempNetcdf > $tempText grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText - grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30" ;' $tempText + grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30"' $tempText fi # The operation should fail because the time value exceeded the maximum limit of 2,147,483,647. From d7bff50657bd964017c718fa7b0a1c7b0ffcef8d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 17:57:09 +0000 Subject: [PATCH 7/9] ECC-1755: Test issues (try 03) --- tests/grib_to_netcdf.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index 2d5e7fd38..b824aad23 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -78,6 +78,7 @@ ${tools_dir}/grib_filter -o $tempGrib2 $tempFilter $tempGrib ${tools_dir}/grib_to_netcdf -R 20080206 -o $tempNetcdf $tempGrib2 if test "x$NC_DUMPER" != "x"; then ${NC_DUMPER} -t -v time $tempNetcdf > $tempText + cat $tempText grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30"' $tempText fi From c41dfd29d852789d48fe2008e5d8a175454cd540 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 18:09:06 +0000 Subject: [PATCH 8/9] ECC-1755: Test issues (try 04) --- tests/grib_to_netcdf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index b824aad23..87d695b87 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -80,7 +80,7 @@ if test "x$NC_DUMPER" != "x"; then ${NC_DUMPER} -t -v time $tempNetcdf > $tempText cat $tempText grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText - grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15", "2008-02-06 12:00:30"' $tempText + grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15.*", "2008-02-06 12:00:30' $tempText fi # The operation should fail because the time value exceeded the maximum limit of 2,147,483,647. From c1fcbf6fb6b839c80593c1b0b286cd77da507b6d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 17 Jun 2024 18:20:10 +0000 Subject: [PATCH 9/9] ECC-1755: Test issues (try 05) --- tests/grib_to_netcdf.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index 87d695b87..b4ec08deb 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -80,7 +80,9 @@ if test "x$NC_DUMPER" != "x"; then ${NC_DUMPER} -t -v time $tempNetcdf > $tempText cat $tempText grep -q 'time:units = "seconds since 2008-02-06 00:00:00.0" ;' $tempText - grep -q 'time = "2008-02-06 12", "2008-02-06 12:00:15.*", "2008-02-06 12:00:30' $tempText + grep -q 'time = "2008-02-06 12' $tempText + grep -q '"2008-02-06 12:00:15' $tempText + grep -q '"2008-02-06 12:00:30' $tempText fi # The operation should fail because the time value exceeded the maximum limit of 2,147,483,647.