370 - Bringing OpenSfM V1 and porting changes from 360

README.md

@@ -1,3 +1,11 @@
+# This is OpenDroneMap's OpenSfM fork
+
+This fork is pretty close to upstream OpenSfM:OpenSfM, but includes changes specific to ODM.
+
+Please note that all such changes are released under the AGPLv3 license, not BSD.
+
+---
+
 ![OpenSfM](doc/images/logo_small.png)
 
 
@@ -137,6 +145,8 @@ Add `--dense` to include the dense stages, or `--resume` to recover an interrupt
 * [Benchmarking](doc/benchmark.md)
 
 ## ⚖️ License
-OpenSfM is BSD-style licensed, as found in the LICENSE file.
+Upstream OpenSfM is BSD-style licensed.
+
+> Changes specific to this OpenDroneMap fork are released under the AGPLv3 license, as found in the LICENSE file.
 
 Example data in the README is under [Creative Commons CC-BY 4.0 License](https://creativecommons.org/licenses/by/4.0/) by Wingtra AG, 8045 Zürich, Switzerland.

bin/rolling_shutter_plot

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+import argparse
+import os.path
+
+import matplotlib.pyplot as plt
+from matplotlib.collections import PatchCollection
+from matplotlib.patches import Wedge
+
+from opensfm import dataset
+from opensfm import features
+from opensfm import io
+
+
+def plot_features(image, points, orig_points):
+    h, w, d = image.shape
+    pixels = features.denormalized_image_coordinates(points, w, h)
+    pixels_orig = features.denormalized_image_coordinates(orig_points, w, h)
+
+    points[:, 2] = 0.001
+    sizes = points[:, 2] * max(w, h)
+    angles = points[:, 3]
+
+    ax = plt.axes()
+    ax.imshow(image)
+
+    patches = []
+    for p, s, a in zip(pixels, sizes, angles):
+        patches.append(Wedge(p, s, a + 1, a - 1))
+
+    orig_points[:, 2] = 0.001
+    sizes_orig = orig_points[:, 2] * max(w, h)
+    angles_orig = orig_points[:, 3]
+    orig_patches = []
+
+    for p, s, a in zip(pixels_orig, sizes_orig, angles_orig):
+        orig_patches.append(Wedge(p, s, a + 1, a - 1))
+
+    collection = PatchCollection(patches, alpha=0.5, edgecolor='b', facecolor='w')
+    orig_collection = PatchCollection(orig_patches, alpha=0.5, edgecolor='r', facecolor='w')
+
+    ax.add_collection(orig_collection)
+    ax.add_collection(collection)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="""
+    Plot detected features
+    NOTE that we require to backup the original features e.g.:
+    1. Backup: cp -r path/to/dataset/features path/to/dataset/features.orig
+    2. Correct: bin/opensfm correct_rolling_shutter path/to/dataset
+    3. Plot: python bin/rolling_shutter_plot path/to/dataset --image image.jpg
+    """)
+    parser.add_argument('dataset',
+                        help='path to the dataset to be processed')
+    parser.add_argument('--image',
+                        help='name of the image to show')
+    parser.add_argument('--save_figs',
+                        help='save figures instead of showing them',
+                        action='store_true')
+    args = parser.parse_args()
+
+    data = dataset.DataSet(args.dataset)
+
+    images = [args.image] if args.image else data.images()
+    for image in images:
+        # Load the corrected features
+        features_data = data.load_features(image)
+
+        # Manually load the original features from features.orig
+        features_orig_path = os.path.join(data.data_path, "features.orig", image + ".features.npz")
+        features_data_orig = None
+        if os.path.exists(features_orig_path):
+            with data.io_handler.open_rb(features_orig_path) as f:
+                features_data_orig = features.FeaturesData.from_file(f, data.config)
+
+        if not features_data:
+            continue
+        if not features_data_orig:
+            print(f"Skipping {image} because features.orig not found. Did you backup features to features.orig before running correct_rolling_shutter?")
+            continue
+
+        points = features_data.points
+        orig_points = features_data_orig.points
+
+        print("plotting {0}/{1} points".format(len(points), len(orig_points)))
+        plt.figure()
+        plt.title('Image: ' + image)
+        fig = plot_features(data.load_image(image), points, orig_points)
+
+        if args.save_figs:
+            p = os.path.join(args.dataset, 'plot_features')
+            io.mkdir_p(p)
+            plt.savefig(os.path.join(p, image + '.jpg'), dpi=1000)
+            plt.close()
+        else:
+            plt.show()

opensfm/actions/export_bundler.py

@@ -68,11 +68,7 @@ def export_bundler(
             if shot_id in shots:
                 shot = shots[shot_id]
                 camera = shot.camera
-                if shot.camera.projection_type == "brown":
-                    # Will approximate Brown model, not optimal
-                    focal_normalized = camera.focal_x
-                else:
-                    focal_normalized = camera.focal
+                focal_normalized = camera.focal
                 scale = max(camera.width, camera.height)
                 focal = focal_normalized * scale
                 k1 = camera.k1

opensfm/actions/export_geocoords.py

@@ -9,6 +9,7 @@
 from opensfm import io, types
 from opensfm import geo
 from opensfm.dataset import DataSet, UndistortedDataSet
+from opensfm.reconstruction import bundle_shot_poses
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -21,6 +22,8 @@ def run_dataset(
     reconstruction: bool,
     dense: bool,
     output: str,
+    offset=(0, 0),
+    mode="affine"
 ) -> None:
     """Export reconstructions in geographic coordinates
 
@@ -31,6 +34,8 @@ def run_dataset(
         reconstruction : export reconstruction.json
         dense : export dense point cloud (depthmaps/merged.ply)
         output : path of the output file relative to the dataset
+        offset : offset to substract from the translation (optional)
+        mode : Method of georeferencing (optional) [affine|projected]
 
     """
 
@@ -42,6 +47,8 @@ def run_dataset(
 
     projection = geo.construct_proj_transformer(proj, inverse=True)
     t = geo.get_proj_transform_matrix(reference, projection)
+    t[0, 3] -= offset[0]
+    t[1, 3] -= offset[1]
 
     if transformation:
         output = output or "geocoords_transformation.txt"
@@ -56,8 +63,28 @@ def run_dataset(
 
     if reconstruction:
         reconstructions = data.load_reconstruction()
-        for r in reconstructions:
-            geo.transform_reconstruction_with_proj(r, projection)
+        if mode == "affine":
+            for r in reconstructions:
+                geo.transform_reconstruction_with_matrix(r, t)
+        elif mode == "projected":
+            camera_priors = data.load_camera_models()
+            rig_camera_priors = data.load_rig_cameras()
+            for r in reconstructions:
+                geo.transform_reconstruction_with_proj(r, projection)
+                if offset != (0, 0):
+                    for point in r.points.values():
+                        point.coordinates[0] -= offset[0]
+                        point.coordinates[1] -= offset[1]
+                    for rig_instance in r.rig_instances.values():
+                        o = rig_instance.pose.get_origin()
+                        rig_instance.pose.set_origin(
+                            [o[0] - offset[0], o[1] - offset[1], o[2]])
+
+                logger.info("Bundle shot poses")
+                bundle_shot_poses(r, set(r.shots.keys()), camera_priors, rig_camera_priors, data.config)
+        else:
+            raise Exception(f"Invalid mode: {mode}")
+
         output = output or "reconstruction.geocoords.json"
         data.save_reconstruction(reconstructions, output)
 

opensfm/actions/export_openmvs.py

@@ -45,6 +45,8 @@ def export(
             )
             exporter.add_camera(str(camera.id), K, w, h)
 
+    shots_map = {}
+
     for shot in reconstruction.shots.values():
         if export_only is not None and shot.id not in export_only:
             continue
@@ -55,6 +57,8 @@ def export(
             if not os.path.isfile(mask_path):
                 mask_path = ""
 
+            shots_map[str(shot.id)] = shot
+
             exporter.add_shot(
                 str(os.path.abspath(image_path)),
                 str(os.path.abspath(mask_path)),
@@ -64,6 +68,15 @@ def export(
                 shot.pose.get_origin(),
             )
 
+    def positive_point_depth(point, shot_id):
+        if not shot_id in shots_map:
+            return False
+
+        shot = shots_map[shot_id]
+
+        # Is point in front of the camera?
+        return shot.pose.transform(point.coordinates)[2] > 0
+
     for point in reconstruction.points.values():
         observations = tracks_manager.get_track_observations(point.id)
 
@@ -72,6 +85,9 @@ def export(
         else:
             shots = list(observations)
 
+        if shots:
+            shots = [s for s in shots if positive_point_depth(point, s)]
+
         if shots:
             coordinates = np.array(point.coordinates, dtype=np.float64)
             exporter.add_point(coordinates, shots)

opensfm/actions/export_visualsfm.py

@@ -54,11 +54,7 @@ def export(
         shot_size_cache[shot.id] = udata.undistorted_image_size(shot.id)
         shot_index[shot.id] = i
         i += 1
-        if shot.camera.projection_type == "brown":
-            # Will approximate Brown model, not optimal
-            focal_normalized = (shot.camera.focal_x + shot.camera.focal_y) / 2.0
-        else:
-            focal_normalized = shot.camera.focal
+        focal_normalized = shot.camera.focal
 
         words = [
             image_path(shot.id, udata),
@@ -82,11 +78,11 @@ def export(
         skipped_points = 0
         lines.append("")
         points = reconstruction.points
+        shots_keys = list(reconstruction.shots.keys())
         lines.append(str(len(points)))
         points_count_index = len(lines) - 1
 
         for point_id, point in points.items():
-            shots = reconstruction.shots
             coord = point.coordinates
             color = list(map(int, point.color))
 
@@ -97,7 +93,7 @@ def export(
                 if export_only is not None and shot_key not in export_only:
                     continue
 
-                if shot_key in shots.keys():
+                if shot_key in shots_keys:
                     v = obs.point
                     x = (0.5 + v[0]) * shot_size_cache[shot_key][1]
                     y = (0.5 + v[1]) * shot_size_cache[shot_key][0]

opensfm/actions/extract_metadata.py

@@ -93,6 +93,9 @@ def _extract_exif(image: str, data: DataSetBase) -> Dict[str, Any]:
     ):
         d["model"] = f"unknown_{image}"
 
+    if data.config.get("camera_projection_type") != 'AUTO':
+        d['projection_type'] = data.config['camera_projection_type'].lower()
+
     d["camera"] = exif.camera_id(d)
 
     return d

opensfm/actions/reconstruct.py

@@ -1,16 +1,26 @@
 # pyre-strict
 import gc
+from typing import Optional
 from opensfm import io, reconstruction
 from opensfm.dataset_base import DataSetBase
 
 
 def run_dataset(
-    data: DataSetBase, algorithm: reconstruction.ReconstructionAlgorithm
+    data: DataSetBase, algorithm: Optional[reconstruction.ReconstructionAlgorithm] = None
 ) -> None:
     """Compute the SfM reconstruction."""
 
     tracks_manager = data.load_tracks_manager()
 
+    if algorithm is None:
+        algo_str = data.config.get("reconstruction_algorithm", "incremental").lower()
+        try:
+            algorithm = reconstruction.ReconstructionAlgorithm(algo_str.lower())
+        except ValueError:
+            raise RuntimeError(
+                f"Unsupported algorithm for reconstruction '{algo_str}'. Allowed values: {list(v.value for v in reconstruction.ReconstructionAlgorithm)}"
+            )
+
     if algorithm == reconstruction.ReconstructionAlgorithm.INCREMENTAL:
         report, reconstructions = reconstruction.incremental_reconstruction(
             data, tracks_manager
@@ -19,6 +29,10 @@ def run_dataset(
         report, reconstructions = reconstruction.triangulation_reconstruction(
             data, tracks_manager
         )
+    elif algorithm == reconstruction.ReconstructionAlgorithm.PLANAR:
+        report, reconstructions = reconstruction.planar_reconstruction(
+            data, tracks_manager
+        )
     else:
         raise RuntimeError(
             f"Unsupported algorithm for reconstruction {algorithm}")

opensfm/actions/undistort.py

@@ -1,6 +1,7 @@
 # pyre-strict
 import os
-from typing import Optional
+from typing import Callable, Optional
+import numpy as np
 
 from opensfm import dataset, undistort
 from opensfm.dataset import DataSet
@@ -12,6 +13,7 @@ def run_dataset(
     reconstruction_index: int = 0,
     tracks: Optional[str] = None,
     output: str = "undistorted",
+    imageFilter: Optional[Callable[[str, np.ndarray], np.ndarray]] = None,
     skip_images: bool = False,
 ) -> None:
     """Export reconstruction to NVM_V3 format from VisualSfM
@@ -36,5 +38,5 @@ def run_dataset(
     if reconstructions:
         r = reconstructions[reconstruction_index]
         undistort.undistort_reconstruction_with_images(
-            tracks_manager, r, data, udata, skip_images
+            tracks_manager, r, data, udata, imageFilter, skip_images
         )

opensfm/commands/export_geocoords.py

@@ -20,6 +20,8 @@ def run_impl(self, dataset: DataSet, args: argparse.Namespace) -> None:
             args.reconstruction,
             args.dense,
             args.output,
+            (args.offset_x, args.offset_y),
+            args.mode,
         )
 
     def add_arguments_impl(self, parser: argparse.ArgumentParser) -> None:
@@ -51,3 +53,16 @@ def add_arguments_impl(self, parser: argparse.ArgumentParser) -> None:
         parser.add_argument(
             "--output", help="Path of the output file relative to the dataset"
         )
+        parser.add_argument(
+            "--offset-x", type=float, help="Value to add to the final translation X axis", default=0.0
+        )
+        parser.add_argument(
+            "--offset-y", type=float, help="Value to add to the final translation Y axis", default=0.0
+        )        
+        parser.add_argument(
+            "--mode",
+            help="Georeferencing method",
+            type=str,
+            choices=["affine", "projected"],
+            default="affine",
+        )

opensfm/commands/reconstruct.py

@@ -21,5 +21,5 @@ def add_arguments_impl(self, parser: argparse.ArgumentParser) -> None:
             help="SfM algorithm to use to run reconstrution",
             type=str,
             choices=[k.value for k in reconstruction.ReconstructionAlgorithm],
-            default=reconstruction.ReconstructionAlgorithm.INCREMENTAL.value,
+            default=None,
         )

opensfm/commands/undistort.py

@@ -18,7 +18,7 @@ def run_impl(self, dataset: DataSet, args: argparse.Namespace) -> None:
             args.reconstruction_index,
             args.tracks,
             args.output,
-            args.skip_images,
+            skip_images=args.skip_images,
         )
 
     def add_arguments_impl(self, parser: argparse.ArgumentParser) -> None: