[WIP] Don't merge!!! Testbed-14

.travis.yml

@@ -1,10 +1,6 @@
 language: python
 env:
-- DJANGO=">=1.4,<1.5" GDAL=1.10 DB=spatialite
-- DJANGO=">=1.5,<1.6" GDAL=1.10 DB=spatialite
-- DJANGO=">=1.6,<1.7" GDAL=1.10 DB=spatialite
-- DJANGO=">=1.7,<1.8" GDAL=1.10 DB=spatialite
-- DJANGO=">=1.8,<1.9" GDAL=1.10 DB=spatialite
+- DJANGO=">=1.11,<1.12" GDAL=1.10 DB=spatialite
 python:
 - '2.7'
 services:

autotest/autotest/data/landsat8/LC81590302016105LGN00_MTL.txt

@@ -0,0 +1,209 @@
+GROUP = L1_METADATA_FILE
+  GROUP = METADATA_FILE_INFO
+    ORIGIN = "Image courtesy of the U.S. Geological Survey"
+    REQUEST_ID = "0501604140764_00028"
+    LANDSAT_SCENE_ID = "LC81590302016105LGN00"
+    FILE_DATE = 2016-04-14T16:50:23Z
+    STATION_ID = "LGN"
+    PROCESSING_SOFTWARE_VERSION = "LPGS_2.6.0"
+  END_GROUP = METADATA_FILE_INFO
+  GROUP = PRODUCT_METADATA
+    DATA_TYPE = "L1T"
+    ELEVATION_SOURCE = "GLS2000"
+    OUTPUT_FORMAT = "GEOTIFF"
+    SPACECRAFT_ID = "LANDSAT_8"
+    SENSOR_ID = "OLI_TIRS"
+    WRS_PATH = 159
+    WRS_ROW = 30
+    NADIR_OFFNADIR = "NADIR"
+    TARGET_WRS_PATH = 159
+    TARGET_WRS_ROW = 30
+    DATE_ACQUIRED = 2016-04-14
+    SCENE_CENTER_TIME = "06:34:48.4539210Z"
+    CORNER_UL_LAT_PRODUCT = 44.23347
+    CORNER_UL_LON_PRODUCT = 60.51815
+    CORNER_UR_LAT_PRODUCT = 44.25950
+    CORNER_UR_LON_PRODUCT = 63.46352
+    CORNER_LL_LAT_PRODUCT = 42.08237
+    CORNER_LL_LON_PRODUCT = 60.60377
+    CORNER_LR_LAT_PRODUCT = 42.10652
+    CORNER_LR_LON_PRODUCT = 63.44751
+    CORNER_UL_PROJECTION_X_PRODUCT = 301800.000
+    CORNER_UL_PROJECTION_Y_PRODUCT = 4900800.000
+    CORNER_UR_PROJECTION_X_PRODUCT = 537000.000
+    CORNER_UR_PROJECTION_Y_PRODUCT = 4900800.000
+    CORNER_LL_PROJECTION_X_PRODUCT = 301800.000
+    CORNER_LL_PROJECTION_Y_PRODUCT = 4661700.000
+    CORNER_LR_PROJECTION_X_PRODUCT = 537000.000
+    CORNER_LR_PROJECTION_Y_PRODUCT = 4661700.000
+    PANCHROMATIC_LINES = 15941
+    PANCHROMATIC_SAMPLES = 15681
+    REFLECTIVE_LINES = 7971
+    REFLECTIVE_SAMPLES = 7841
+    THERMAL_LINES = 7971
+    THERMAL_SAMPLES = 7841
+    FILE_NAME_BAND_1 = "LC81590302016105LGN00_B1.TIF"
+    FILE_NAME_BAND_2 = "LC81590302016105LGN00_B2.TIF"
+    FILE_NAME_BAND_3 = "LC81590302016105LGN00_B3.TIF"
+    FILE_NAME_BAND_4 = "LC81590302016105LGN00_B4.TIF"
+    FILE_NAME_BAND_5 = "LC81590302016105LGN00_B5.TIF"
+    FILE_NAME_BAND_6 = "LC81590302016105LGN00_B6.TIF"
+    FILE_NAME_BAND_7 = "LC81590302016105LGN00_B7.TIF"
+    FILE_NAME_BAND_8 = "LC81590302016105LGN00_B8.TIF"
+    FILE_NAME_BAND_9 = "LC81590302016105LGN00_B9.TIF"
+    FILE_NAME_BAND_10 = "LC81590302016105LGN00_B10.TIF"
+    FILE_NAME_BAND_11 = "LC81590302016105LGN00_B11.TIF"
+    FILE_NAME_BAND_QUALITY = "LC81590302016105LGN00_BQA.TIF"
+    METADATA_FILE_NAME = "LC81590302016105LGN00_MTL.txt"
+    BPF_NAME_OLI = "LO8BPF20160414062301_20160414070651.02"
+    BPF_NAME_TIRS = "LT8BPF20160407235542_20160408000419.01"
+    CPF_NAME = "L8CPF20160401_20160630.01"
+    RLUT_FILE_NAME = "L8RLUT20150303_20431231v11.h5"
+  END_GROUP = PRODUCT_METADATA
+  GROUP = IMAGE_ATTRIBUTES
+    CLOUD_COVER = 1.86
+    CLOUD_COVER_LAND = 1.86
+    IMAGE_QUALITY_OLI = 9
+    IMAGE_QUALITY_TIRS = 9
+    TIRS_SSM_POSITION_STATUS = "ESTIMATED"
+    ROLL_ANGLE = -0.001
+    SUN_AZIMUTH = 147.62833634
+    SUN_ELEVATION = 52.40004988
+    EARTH_SUN_DISTANCE = 1.0030918
+    GROUND_CONTROL_POINTS_VERSION = 3
+    GROUND_CONTROL_POINTS_MODEL = 345
+    GEOMETRIC_RMSE_MODEL = 6.748
+    GEOMETRIC_RMSE_MODEL_Y = 4.584
+    GEOMETRIC_RMSE_MODEL_X = 4.952
+    GROUND_CONTROL_POINTS_VERIFY = 63
+    GEOMETRIC_RMSE_VERIFY = 3.185
+  END_GROUP = IMAGE_ATTRIBUTES
+  GROUP = MIN_MAX_RADIANCE
+    RADIANCE_MAXIMUM_BAND_1 = 755.38452
+    RADIANCE_MINIMUM_BAND_1 = -62.37990
+    RADIANCE_MAXIMUM_BAND_2 = 773.52295
+    RADIANCE_MINIMUM_BAND_2 = -63.87778
+    RADIANCE_MAXIMUM_BAND_3 = 712.79480
+    RADIANCE_MINIMUM_BAND_3 = -58.86282
+    RADIANCE_MAXIMUM_BAND_4 = 601.06873
+    RADIANCE_MINIMUM_BAND_4 = -49.63645
+    RADIANCE_MAXIMUM_BAND_5 = 367.82407
+    RADIANCE_MINIMUM_BAND_5 = -30.37503
+    RADIANCE_MAXIMUM_BAND_6 = 91.47450
+    RADIANCE_MINIMUM_BAND_6 = -7.55399
+    RADIANCE_MAXIMUM_BAND_7 = 30.83180
+    RADIANCE_MINIMUM_BAND_7 = -2.54610
+    RADIANCE_MAXIMUM_BAND_8 = 680.24438
+    RADIANCE_MINIMUM_BAND_8 = -56.17480
+    RADIANCE_MAXIMUM_BAND_9 = 143.75398
+    RADIANCE_MINIMUM_BAND_9 = -11.87125
+    RADIANCE_MAXIMUM_BAND_10 = 0.10000
+    RADIANCE_MINIMUM_BAND_10 = 0.10000
+    RADIANCE_MAXIMUM_BAND_11 = 0.10000
+    RADIANCE_MINIMUM_BAND_11 = 0.10000
+  END_GROUP = MIN_MAX_RADIANCE
+  GROUP = MIN_MAX_REFLECTANCE
+    REFLECTANCE_MAXIMUM_BAND_1 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_1 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_2 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_2 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_3 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_3 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_4 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_4 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_5 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_5 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_6 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_6 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_7 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_7 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_8 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_8 = -0.099980
+    REFLECTANCE_MAXIMUM_BAND_9 = 1.210700
+    REFLECTANCE_MINIMUM_BAND_9 = -0.099980
+  END_GROUP = MIN_MAX_REFLECTANCE
+  GROUP = MIN_MAX_PIXEL_VALUE
+    QUANTIZE_CAL_MAX_BAND_1 = 65535
+    QUANTIZE_CAL_MIN_BAND_1 = 1
+    QUANTIZE_CAL_MAX_BAND_2 = 65535
+    QUANTIZE_CAL_MIN_BAND_2 = 1
+    QUANTIZE_CAL_MAX_BAND_3 = 65535
+    QUANTIZE_CAL_MIN_BAND_3 = 1
+    QUANTIZE_CAL_MAX_BAND_4 = 65535
+    QUANTIZE_CAL_MIN_BAND_4 = 1
+    QUANTIZE_CAL_MAX_BAND_5 = 65535
+    QUANTIZE_CAL_MIN_BAND_5 = 1
+    QUANTIZE_CAL_MAX_BAND_6 = 65535
+    QUANTIZE_CAL_MIN_BAND_6 = 1
+    QUANTIZE_CAL_MAX_BAND_7 = 65535
+    QUANTIZE_CAL_MIN_BAND_7 = 1
+    QUANTIZE_CAL_MAX_BAND_8 = 65535
+    QUANTIZE_CAL_MIN_BAND_8 = 1
+    QUANTIZE_CAL_MAX_BAND_9 = 65535
+    QUANTIZE_CAL_MIN_BAND_9 = 1
+    QUANTIZE_CAL_MAX_BAND_10 = 65535
+    QUANTIZE_CAL_MIN_BAND_10 = 1
+    QUANTIZE_CAL_MAX_BAND_11 = 65535
+    QUANTIZE_CAL_MIN_BAND_11 = 1
+  END_GROUP = MIN_MAX_PIXEL_VALUE
+  GROUP = RADIOMETRIC_RESCALING
+    RADIANCE_MULT_BAND_1 = 1.2478E-02
+    RADIANCE_MULT_BAND_2 = 1.2778E-02
+    RADIANCE_MULT_BAND_3 = 1.1775E-02
+    RADIANCE_MULT_BAND_4 = 9.9293E-03
+    RADIANCE_MULT_BAND_5 = 6.0762E-03
+    RADIANCE_MULT_BAND_6 = 1.5111E-03
+    RADIANCE_MULT_BAND_7 = 5.0932E-04
+    RADIANCE_MULT_BAND_8 = 1.1237E-02
+    RADIANCE_MULT_BAND_9 = 2.3747E-03
+    RADIANCE_MULT_BAND_10 = 0.0000E+00
+    RADIANCE_MULT_BAND_11 = 0.0000E+00
+    RADIANCE_ADD_BAND_1 = -62.39238
+    RADIANCE_ADD_BAND_2 = -63.89055
+    RADIANCE_ADD_BAND_3 = -58.87460
+    RADIANCE_ADD_BAND_4 = -49.64638
+    RADIANCE_ADD_BAND_5 = -30.38111
+    RADIANCE_ADD_BAND_6 = -7.55551
+    RADIANCE_ADD_BAND_7 = -2.54661
+    RADIANCE_ADD_BAND_8 = -56.18604
+    RADIANCE_ADD_BAND_9 = -11.87363
+    RADIANCE_ADD_BAND_10 = 0.10000
+    RADIANCE_ADD_BAND_11 = 0.10000
+    REFLECTANCE_MULT_BAND_1 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_2 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_3 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_4 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_5 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_6 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_7 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_8 = 2.0000E-05
+    REFLECTANCE_MULT_BAND_9 = 2.0000E-05
+    REFLECTANCE_ADD_BAND_1 = -0.100000
+    REFLECTANCE_ADD_BAND_2 = -0.100000
+    REFLECTANCE_ADD_BAND_3 = -0.100000
+    REFLECTANCE_ADD_BAND_4 = -0.100000
+    REFLECTANCE_ADD_BAND_5 = -0.100000
+    REFLECTANCE_ADD_BAND_6 = -0.100000
+    REFLECTANCE_ADD_BAND_7 = -0.100000
+    REFLECTANCE_ADD_BAND_8 = -0.100000
+    REFLECTANCE_ADD_BAND_9 = -0.100000
+  END_GROUP = RADIOMETRIC_RESCALING
+  GROUP = TIRS_THERMAL_CONSTANTS
+    K1_CONSTANT_BAND_10 = 774.8853
+    K1_CONSTANT_BAND_11 = 480.8883
+    K2_CONSTANT_BAND_10 = 1321.0789
+    K2_CONSTANT_BAND_11 = 1201.1442
+  END_GROUP = TIRS_THERMAL_CONSTANTS
+  GROUP = PROJECTION_PARAMETERS
+    MAP_PROJECTION = "UTM"
+    DATUM = "WGS84"
+    ELLIPSOID = "WGS84"
+    UTM_ZONE = 41
+    GRID_CELL_SIZE_PANCHROMATIC = 15.00
+    GRID_CELL_SIZE_REFLECTIVE = 30.00
+    GRID_CELL_SIZE_THERMAL = 30.00
+    ORIENTATION = "NORTH_UP"
+    RESAMPLING_OPTION = "CUBIC_CONVOLUTION"
+  END_GROUP = PROJECTION_PARAMETERS
+END_GROUP = L1_METADATA_FILE
+END

autotest/autotest/data/rgb_definition.json

@@ -0,0 +1,48 @@
+[{
+    "bands": [
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Red Channel",
+            "gdal_interpretation": "RedBand",
+            "identifier": "red",
+            "name": "red",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1"
+        },
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Green Channel",
+            "gdal_interpretation": "GreenBand",
+            "identifier": "green",
+            "name": "green",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1"
+        },
+        {
+            "definition": "http://www.opengis.net/def/property/OGC/0/Radiance",
+            "description": "Blue Channel",
+            "gdal_interpretation": "BlueBand",
+            "identifier": "blue",
+            "name": "blue",
+            "nil_values": [
+                {
+                    "reason": "http://www.opengis.net/def/nil/OGC/0/unknown",
+                    "value": 0
+                }
+            ],
+            "uom": "W.m-2.Sr-1"
+        }
+    ],
+    "data_type": "Byte",
+    "name": "RGB"
+}]

autotest/autotest/data/sentinel2/create_sentinel_types.sh

@@ -0,0 +1,18 @@
+python manage.py coveragetype create S2MSI1C_B01 --field-type B01 B01 "Solar irradiance" "W/m2/um" 1913.57
+python manage.py coveragetype create S2MSI1C_B02 --field-type B02 B02 "Solar irradiance" "W/m2/um" 1941.63
+python manage.py coveragetype create S2MSI1C_B03 --field-type B03 B03 "Solar irradiance" "W/m2/um" 1822.61
+python manage.py coveragetype create S2MSI1C_B04 --field-type B04 B04 "Solar irradiance" "W/m2/um" 1512.79
+python manage.py coveragetype create S2MSI1C_B05 --field-type B05 B05 "Solar irradiance" "W/m2/um" 1425.56
+python manage.py coveragetype create S2MSI1C_B06 --field-type B06 B06 "Solar irradiance" "W/m2/um" 1288.32
+python manage.py coveragetype create S2MSI1C_B07 --field-type B07 B07 "Solar irradiance" "W/m2/um" 1163.19
+python manage.py coveragetype create S2MSI1C_B08 --field-type B08 B08 "Solar irradiance" "W/m2/um" 1036.39
+python manage.py coveragetype create S2MSI1C_B8A --field-type B8A B8A "Solar irradiance" "W/m2/um" 955.19
+python manage.py coveragetype create S2MSI1C_B09 --field-type B09 B09 "Solar irradiance" "W/m2/um" 813.04
+python manage.py coveragetype create S2MSI1C_B10 --field-type B10 B10 "Solar irradiance" "W/m2/um" 367.15
+python manage.py coveragetype create S2MSI1C_B11 --field-type B11 B11 "Solar irradiance" "W/m2/um" 245.59
+python manage.py coveragetype create S2MSI1C_B12 --field-type B12 B12 "Solar irradiance" "W/m2/um" 85.25
+
+python manage.py producttype create S2MSI1C \
+    -c S2MSI1C_B01 -c S2MSI1C_B02 -c S2MSI1C_B03 -c S2MSI1C_B04 -c S2MSI1C_B05 -c S2MSI1C_B06 -c S2MSI1C_B07 -c S2MSI1C_B08 -c S2MSI1C_B8A -c S2MSI1C_B09 -c S2MSI1C_B10 -c S2MSI1C_B11 -c S2MSI1C_B12 \
+    -m clouds -m no_data
+

autotest/autotest/data/sentinel2/register_sentinel.sh

@@ -0,0 +1,50 @@
+product_path=$1
+browses_path=$2
+
+
+img_data=$(unzip -Z -2 $product_path | grep jp2$)
+out_browse_image=${browses_path}/$(basename "$product_path").tif
+
+# create browse image if it does not yet exist
+[ -e $out_browse_image ] || {
+    tci_path=$(echo "$img_data" | grep TCI.jp2)
+
+    unzip -j $product_path $tci_path -d /tmp/ -u > /dev/null
+
+    gdal_translate /tmp/$(basename "$tci_path") $out_browse_image \
+        -co TILED=YES --config GDAL_CACHEMAX 1000 --config GDAL_NUM_THREADS 4 \
+        -co COMPRESS=LZW
+
+    gdaladdo --config COMPRESS_OVERVIEW LZW $out_browse_image 2 4 8 16 32 64 128 256
+
+    rm /tmp/$(basename "$tci_path")
+}
+
+
+
+# actually register the product
+product_id=$(
+    python manage.py product register \
+        --package $product_path --product-type S2MSI1C \
+        --no-browses \
+        --replace --print-identifier
+)
+
+# register the generated browse
+python manage.py browse register $product_id ${browses_path}/$(basename "$product_path").tif
+
+# insert the product in the collection
+python manage.py collection insert S2MSI1C $product_id
+
+# img_data=$(python manage.py product discover $product_id "*/GRANULE/*/IMG_DATA/*.jp2" 2> /dev/null)
+
+for band in B01 B02 B03 B04 B05 B06 B07 B08 B8A B09 B10 B11 B12 ; do
+    # echo "*/GRANULE/*/IMG_DATA/*$band.jp2" '*/GRANULE/*/IMG_DATA/*$band.jp2'
+    # python manage.py product discover $PRODUCT_ID "*/GRANULE/*/IMG_DATA/*$band.jp2" 2> /dev/null
+    coverage_path=$(echo "$img_data" | grep ${band}.jp2)
+    python manage.py coverage register \
+        -d $product_path $coverage_path --coverage-type S2MSI1C_${band} \
+        --identifier ${product_id}_${band} --product ${product_id} --replace
+done
+
+python manage.py collection summary S2MSI1C