Merge branch 'master' into remove_homography_binary

MISS3D · Mar 1, 2019 · 68ad552 · 68ad552
2 parents 03dcab7 + cfec24e
commit 68ad552
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 119 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -29,7 +29,8 @@ install:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install GDAL==$(gdal-config --version | awk -F'[.]' '{print $1"."$2}'); fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget http://www.kyngchaos.com/files/software/frameworks/GDAL_Complete-2.1.dmg; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then X=$(echo `hdiutil mount GDAL_Complete-2.1.dmg | tail -1 | awk '{$1=$2=""; print $0}'` | xargs -0 echo) && sudo installer -pkg "${X}/"GDAL\ Complete.pkg -target /; export PATH="/Library/Frameworks/GDAL.framework/Programs:$PATH"; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install numpy scipy rasterio utm bs4 requests; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install numpy scipy utm bs4 requests; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install rasterio[test]; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install GDAL==$(gdal-config --version | awk -F'[.]' '{print $1"."$2}') --global-option build_ext --global-option=`gdal-config --cflags` --global-option build_ext --global-option=-L`gdal-config  --prefix`/unix/lib/; fi
 
 script:

diff --git a/s2p.py b/s2p.py
@@ -685,35 +685,16 @@ def global_dsm(tiles):
                              "%s %s %s" % (projwin, out_conf_vrt, out_conf_tif)]))
 
 
-# ALL_STEPS is a ordonned dictionary : key = 'stepname' : value = is_distributed (True/False)
-# initialization : pass in a sequence of tuples
-ALL_STEPS = [('initialisation', False),
-             ('local-pointing', True),
-             ('global-pointing', False),
-             ('rectification', True),
-             ('matching', True),
-             ('triangulation', True),
-             ('disparity-to-height', True),
-             ('global-mean-heights', False),
-             ('heights-to-ply', True),
-             ('local-dsm-rasterization', True),
-             ('global-dsm-rasterization', False) ]
-ALL_STEPS = collections.OrderedDict(ALL_STEPS)
-
-
-def main(user_cfg, steps=ALL_STEPS):
+def main(user_cfg):
     """
     Launch the s2p pipeline with the parameters given in a json file.
 
     Args:
         user_cfg: user config dictionary
-        steps: either a string (single step) or a list of strings (several
-            steps)
     """
     common.print_elapsed_time.t0 = datetime.datetime.now()
     initialization.build_cfg(user_cfg)
-    if 'initialisation' in steps:
-        initialization.make_dirs()
+    initialization.make_dirs()
 
     # multiprocessing setup
     nb_workers = multiprocessing.cpu_count()  # nb of available cores
@@ -722,69 +703,67 @@ def main(user_cfg, steps=ALL_STEPS):
 
     tw, th = initialization.adjust_tile_size()
     tiles_txt = os.path.join(cfg['out_dir'],'tiles.txt')
-    create_masks = 'initialisation' in steps
-    tiles = initialization.tiles_full_info(tw, th, tiles_txt, create_masks)
+    tiles = initialization.tiles_full_info(tw, th, tiles_txt, create_masks=True)
 
-    if 'initialisation' in steps:
-        # Write the list of json files to outdir/tiles.txt
-        with open(tiles_txt,'w') as f:
-            for t in tiles:
-                f.write(t['json']+os.linesep)
+    # initialisation step:
+    # Write the list of json files to outdir/tiles.txt
+    with open(tiles_txt,'w') as f:
+        for t in tiles:
+            f.write(t['json']+os.linesep)
 
     n = len(cfg['images'])
     tiles_pairs = [(t, i) for i in range(1, n) for t in tiles]
 
-    if 'local-pointing' in steps:
-        print('correcting pointing locally...')
-        parallel.launch_calls(pointing_correction, tiles_pairs, nb_workers)
+    # local-pointing step:
+    print('correcting pointing locally...')
+    parallel.launch_calls(pointing_correction, tiles_pairs, nb_workers)
 
-    if 'global-pointing' in steps:
-        print('correcting pointing globally...')
-        global_pointing_correction(tiles)
-        common.print_elapsed_time()
+    # global-pointing step:
+    print('correcting pointing globally...')
+    global_pointing_correction(tiles)
+    common.print_elapsed_time()
 
-    if 'rectification' in steps:
-        print('rectifying tiles...')
-        parallel.launch_calls(rectification_pair, tiles_pairs, nb_workers)
+    # rectification step:
+    print('rectifying tiles...')
+    parallel.launch_calls(rectification_pair, tiles_pairs, nb_workers)
 
-    if 'matching' in steps:
-        print('running stereo matching...')
-        parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers)
+    # matching step:
+    print('running stereo matching...')
+    parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers)
 
     if n > 2 and cfg['triangulation_mode'] == 'pairwise':
-        if 'disparity-to-height' in steps:
-            print('computing height maps...')
-            parallel.launch_calls(disparity_to_height, tiles_pairs, nb_workers)
+        # disparity-to-height step:
+        print('computing height maps...')
+        parallel.launch_calls(disparity_to_height, tiles_pairs, nb_workers)
 
-            print('computing local pairwise height offsets...')
-            parallel.launch_calls(mean_heights, tiles, nb_workers)
+        print('computing local pairwise height offsets...')
+        parallel.launch_calls(mean_heights, tiles, nb_workers)
 
-        if 'global-mean-heights' in steps:
-            print('computing global pairwise height offsets...')
-            global_mean_heights(tiles)
-
-        if 'heights-to-ply' in steps:
-            print('merging height maps and computing point clouds...')
-            parallel.launch_calls(heights_to_ply, tiles, nb_workers)
+        # global-mean-heights step:
+        print('computing global pairwise height offsets...')
+        global_mean_heights(tiles)
 
+        # heights-to-ply step:
+        print('merging height maps and computing point clouds...')
+        parallel.launch_calls(heights_to_ply, tiles, nb_workers)
     else:
-        if 'triangulation' in steps:
-            print('triangulating tiles...')
-            if cfg['triangulation_mode'] == 'geometric':
-                parallel.launch_calls(multidisparities_to_ply, tiles, nb_workers)
-            elif cfg['triangulation_mode'] == 'pairwise':
-                parallel.launch_calls(disparity_to_ply, tiles, nb_workers)
-            else:
-                raise ValueError("possible values for 'triangulation_mode' : 'pairwise' or 'geometric'")
-
-    if 'local-dsm-rasterization' in steps:
-        print('computing DSM by tile...')
-        parallel.launch_calls(plys_to_dsm, tiles, nb_workers)
-
-    if 'global-dsm-rasterization' in steps:
-        print('computing global DSM...')
-        global_dsm(tiles)
-        common.print_elapsed_time()
+        # triangulation step:
+        print('triangulating tiles...')
+        if cfg['triangulation_mode'] == 'geometric':
+            parallel.launch_calls(multidisparities_to_ply, tiles, nb_workers)
+        elif cfg['triangulation_mode'] == 'pairwise':
+            parallel.launch_calls(disparity_to_ply, tiles, nb_workers)
+        else:
+            raise ValueError("possible values for 'triangulation_mode' : 'pairwise' or 'geometric'")
+
+    # local-dsm-rasterization step:
+    print('computing DSM by tile...')
+    parallel.launch_calls(plys_to_dsm, tiles, nb_workers)
+
+    # global-dsm-rasterization step:
+    print('computing global DSM...')
+    global_dsm(tiles)
+    common.print_elapsed_time()
 
     # cleanup
     common.garbage_cleanup()
@@ -844,13 +823,11 @@ def read_config_file(config_file):
                         help=('path to a json file containing the paths to '
                               'input and output files and the algorithm '
                               'parameters'))
-    parser.add_argument('--step', type=str, choices=ALL_STEPS,
-                        default=ALL_STEPS)
     args = parser.parse_args()
 
     user_cfg = read_config_file(args.config)
 
-    main(user_cfg, args.step)
+    main(user_cfg)
 
     # Backup input file for sanity check
     if not args.config.startswith(os.path.abspath(cfg['out_dir']+os.sep)):

diff --git a/s2p_test.py b/s2p_test.py
@@ -200,50 +200,6 @@ def end2end(config,ref_dsm,absmean_tol=0.025,percentile_tol=1.):
 
     end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
 
-def end2end_cluster(config):
-    print('Configuration file: ',config)
-
-    print('Running end2end in sequential mode to get reference DSM ...')
-
-    test_cfg = s2p.read_config_file(config)
-    test_cfg['skip_existing'] = True
-    s2p.main(test_cfg)
-
-    outdir = test_cfg['out_dir']
-    expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
-    print('Running end2end in cluster mode ...')
-    test_cfg_cluster = dict()
-    test_cfg_cluster.update(test_cfg)
-    test_cfg_cluster['out_dir'] = test_cfg_cluster['out_dir'] + "_cluster"
-    test_cfg_cluster['skip_existing'] = True
-
-    print("Running initialisation step ...")
-    s2p.main(test_cfg_cluster,["initialisation"])
-
-    # Retrieve tiles list
-    outdir = test_cfg_cluster['out_dir']
-    tiles_file = os.path.join(outdir,'tiles.txt')
-
-    tiles = s2p.read_tiles(tiles_file)
-
-    print('Found '+str(len(tiles))+' tiles to process')
-
-    for step in s2p.ALL_STEPS:
-        if s2p.ALL_STEPS[step] is True:
-            print('Running %s on each tile...' % step)
-            for tile in tiles:
-                print('tile : %s' % tile)
-                tile_cfg_cluster = s2p.read_config_file(tile)
-                s2p.main(tile_cfg_cluster, [step])
-        else:
-            print('Running %s...' % step)
-            print('test_cfg_cluster : %s' % test_cfg_cluster)
-            s2p.main(test_cfg_cluster, [step])
-
-    computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
-
-    end2end_compare_dsm(computed,expected,0,0)
-
 def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.):
 
     test_cfg = s2p.read_config_file(config)
@@ -271,7 +227,6 @@ def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.):
                     ('unit_matches_from_rpc', (unit_matches_from_rpc,[])),
                     ('end2end_pair', (end2end, ['testdata/input_pair/config.json','testdata/expected_output/pair/dsm.tif',0.025,1])),
                     ('end2end_triplet', (end2end, ['testdata/input_triplet/config.json','testdata/expected_output/triplet/dsm.tif',0.05,2])),
-                    ('end2end_cluster', (end2end_cluster, ['testdata/input_triplet/config.json'])),
                     ('end2end_mosaic', (end2end_mosaic, ['testdata/input_triplet/config.json','testdata/expected_output/triplet/height_map.tif',0.05,2])),
                     ('end2end_geometric', (end2end, ['testdata/input_triplet/config_geo.json', 'testdata/expected_output/triplet/dsm_geo.tif',0.05,2])),
                     ('unit_distributed_plyflatten', (unit_distributed_plyflatten, ['testdata/input_triplet/config.json']))]