diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 91a94b656..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,124 +0,0 @@ -# specify version format -version: "{build}" - -image: - - Visual Studio 2019 - - ubuntu1804 - - macos - -platform: - - x64 - -# specify custom environment variables -environment: - APPVEYOR_SAVE_CACHE_ON_ERROR: true - -# build configuration, i.e. Debug, Release, etc. -configuration: - - Debug - - Release - -# branches to build -branches: - # blacklist - except: - - gh-pages - -for: -- - #------------------ - # Windows 10 - #------------------ - matrix: - only: - - image: Visual Studio 2019 - # scripts that are called at very beginning, before repo cloning - init: - - ver - - cmake --version - - msbuild /version - # scripts that run after cloning repository - install: - # update vcpkg - - cd C:\tools\vcpkg - - git pull - - .\bootstrap-vcpkg.bat - - if "%platform%"=="Win32" set VCPKG_ARCH=x86-windows - - if "%platform%"=="x64" set VCPKG_ARCH=x64-windows - # remove outdated versions - - vcpkg remove --outdated --recurse - # install dependencies - - vcpkg install --recurse --triplet %VCPKG_ARCH% zlib boost-iostreams boost-program-options boost-system boost-serialization eigen3 cgal[core] opencv glew glfw3 - - cd "%APPVEYOR_BUILD_FOLDER%" - # preserve contents of selected directories and files across project builds - cache: - - 'C:\tools\vcpkg\installed' - build_script: - - if "%platform%"=="Win32" set CMAKE_ARCH=-A"x86" - - if "%platform%"=="x64" set CMAKE_ARCH=-A"x64" - - git clone https://github.com/cdcseacave/VCG.git - - mkdir bin && cd bin - - cmake -G"Visual Studio 16 2019" %CMAKE_ARCH% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_TOOLCHAIN_FILE="C:\tools\vcpkg\scripts\buildsystems\vcpkg.cmake" -DVCG_ROOT="$APPVEYOR_BUILD_FOLDER/VCG" .. - - cmake --build . --target ALL_BUILD --config %Configuration% -- /maxcpucount:4 -- - #------------------ - # Ubuntu 18.04 LTS - #------------------ - matrix: - only: - - image: ubuntu1804 - # scripts that are called at very beginning, before repo cloning - init: - - lsb_release -a - - cmake --version - - gcc -v - # scripts that run after cloning repository - install: - - sudo apt-get update -qq && sudo apt-get install -qq - - sudo apt-get -y install build-essential git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev libxmu-dev libxi-dev - - sudo apt-get -y install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev - - sudo apt-get -y install libopencv-dev libcgal-dev libcgal-qt5-dev libatlas-base-dev - - sudo apt-get -y install freeglut3-dev libglew-dev libglfw3-dev - # preserve contents of selected directories and files across project builds - cache: - - '/usr/lib/x86_64-linux-gnu/' - build_script: - - git clone https://github.com/cdcseacave/VCG.git - - git clone --single-branch --branch 3.4 https://gitlab.com/libeigen/eigen.git - - mkdir eigen_build && cd eigen_build - - cmake . ../eigen - - make && sudo make install - - cd .. - - mkdir bin && cd bin - - cmake -DCMAKE_BUILD_TYPE=$Configuration -DVCG_ROOT="$APPVEYOR_BUILD_FOLDER/VCG" .. - - make -- - #------------------ - # MacOS - #------------------ - matrix: - only: - - image: macos - # scripts that are called at very beginning, before repo cloning - init: - - system_profiler SPSoftwareDataType - - cmake --version - - gcc -v - # scripts that run after cloning repository - install: - - brew update - - printf "#%s/bin/bash\nbrew install libomp boost eigen opencv cgal glew glfw3\nexit 0\n" "!" > install.sh - - chmod +x install.sh - - ./install.sh - # preserve contents of selected directories and files across project builds - cache: - - '/usr/local/Cellar/' - build_script: - - git clone https://github.com/cdcseacave/VCG.git - - mkdir bin && cd bin - - cmake -DCMAKE_BUILD_TYPE=$CONFIGURATION -DVCG_ROOT="$APPVEYOR_BUILD_FOLDER/VCG" .. - - make - -on_success: - - cmd: 7z a OpenMVS_x64.7z "C:\projects\openmvs\bin\bin\vc16\x64\%Configuration%\*.exe" "C:\projects\openmvs\bin\bin\vc16\x64\%Configuration%\*.dll" - - cmd: appveyor PushArtifact OpenMVS_x64.7z diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml new file mode 100644 index 000000000..7c327e6e9 --- /dev/null +++ b/.github/workflows/continuous_integration.yml @@ -0,0 +1,81 @@ +name: Continuous Integration +run-name: ${{ github.actor }} is building OpenMVS + +on: + push: + branches: [master, develop] + pull_request: + branches: [master, develop] + # Allows to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + CTEST_OUTPUT_ON_FAILURE: 1 + +defaults: + run: + shell: bash + +jobs: + build-tests: + name: Build on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + include: + - os: windows-latest + triplet: x64-windows-release + build-type: Release + - os: ubuntu-latest + triplet: x64-linux-release + build-type: Release + - os: macos-latest + triplet: x64-osx + build-type: Release + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Restore artifacts, or setup vcpkg for building artifacts + uses: lukka/run-vcpkg@v10.6 + with: + vcpkgDirectory: '${{ github.workspace }}/vcpkg' + vcpkgGitCommitId: '4cb4a5c5ddcb9de0c83c85837ee6974c8333f032' + + - name: Install Ubuntu dependencies + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update -y + sudo apt-get install -y autoconf-archive libxmu-dev libdbus-1-dev libxtst-dev libxi-dev libxinerama-dev libxcursor-dev xorg-dev libgl-dev libglu1-mesa-dev pkg-config + + - name: Install macOS dependencies + if: matrix.os == 'macOS-latest' + run: | + brew install automake autoconf-archive + + - name: Configure CMake + run: | + cmake -S . -B make -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} + + - name: Build + working-directory: ./make + run: | + rm -rf ../vcpkg/buildtrees + rm -rf ../vcpkg/downloads + cmake --build . --config ${{ matrix.build-type }} --parallel $(nproc) + + - name: Unit Tests + working-directory: ./make + run: | + ctest -j$(nproc) --build-config ${{ matrix.build-type }} + + - name: Deploy Windows release + if: matrix.os == 'windows-latest' + uses: actions/upload-artifact@v3 + with: + name: OpenMVS_Windows_Release_x64 + path: | + ${{ github.workspace }}/make/bin/**/x64 + !${{ github.workspace }}/make/bin/**/*.exp diff --git a/.gitignore b/.gitignore index a12459285..c301e5a4e 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ CMakeSettings.json .vscode/ out/ bin*/ +make*/ diff --git a/BUILD.md b/BUILD.md index 880d4b780..28fda265c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -2,14 +2,15 @@ Dependencies ------------ -*OpenMVS* relies on a number of open source libraries, some of which are optional. For details on customizing the build process, see the compilation instructions. +*OpenMVS* relies on a number of open source libraries, some optional, which are managed automatically by [vcpkg](https://github.com/Microsoft/vcpkg). For details on customizing the build process, see the build instructions. * [Eigen](http://eigen.tuxfamily.org) version 3.4 or higher * [OpenCV](http://opencv.org) version 2.4 or higher -* [Ceres](http://ceres-solver.org) version 1.10 or higher +* [Ceres](http://ceres-solver.org) version 1.10 or higher (optional) * [CGAL](http://www.cgal.org) version 4.2 or higher * [Boost](http://www.boost.org) version 1.56 or higher * [VCG](http://vcg.isti.cnr.it/vcglib) -* [GLFW](http://www.glfw.org) +* [CUDA](https://developer.nvidia.com/cuda-downloads) (optional) +* [GLFW](http://www.glfw.org) (optional) ------------------ Build instructions @@ -18,127 +19,29 @@ Build instructions Required tools: * [CMake](http://www.cmake.org) * [git](https://git-scm.com) -* C/C++ compiler like Visual Studio or GCC +* C/C++ compiler like Visual Studio 2019, GCC or Clang -------------------- -Windows compilation -------------------- +The dependencies can be fetched and built automatically using `vcpkg` on all major platform, by setting the environment variable `VCPKG_ROOT` to point to its path or by using the `cmake` parameter `-DCMAKE_TOOLCHAIN_FILE=[path to vcpkg]/scripts/buildsystems/vcpkg.cmake`. -Visual Studio 2008 or newer are supported. Please note that the development is done mainly on Windows, so this platform build is well tested. The latest pre-built binaries for fast testing can be download from [here](https://github.com/cdcseacave/openMVS_sample/releases/latest). Visual Studio 2017 and dependencies automation tool [vcpkg](https://github.com/Microsoft/vcpkg) are used in this example. +The latest pre-built stable binaries can be download from [here](https://github.com/cdcseacave/openMVS_sample/releases/latest). ``` -#Make a toplevel directory for deps & build & src somewhere: -mkdir OpenMVS -cd OpenMVS - -#Get and install dependencies using vcpkg; -#choose the desired triplet, like "x64-windows", by setting the VCPKG_DEFAULT_TRIPLET environment variable or by specifying it after each package: -vcpkg install zlib boost-iostreams boost-program-options boost-system boost-serialization eigen3 cgal[core] opencv vcglib glew glfw3 - -#Get VCGLib (Required): -git clone https://github.com/cdcseacave/VCG.git - -#Get and unpack OpenMVS in OpenMVS/src: -git clone https://github.com/cdcseacave/openMVS.git src +#Clone OpenMVS +git clone --recurse-submodules https://github.com/cdcseacave/openMVS.git #Make build directory: -mkdir build -cd build - -#Run CMake, where VCPKG_ROOT environment variable points to the root of vcpkg installation: -cmake . ..\src -G "Visual Studio 15 2017 Win64" -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows - -#Open the solution in MSVC and build it -``` - ------------------ -Linux compilation ------------------ - -[Ubuntu](http://www.ubuntu.com) 16.04 is used next as the example linux distribution. - -``` -#Prepare and empty machine for building: -sudo apt-get update -qq && sudo apt-get install -qq -sudo apt-get -y install git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev -main_path=`pwd` - -#Eigen (Required) -git clone https://gitlab.com/libeigen/eigen.git --branch 3.4 -mkdir eigen_build && cd eigen_build -cmake . ../eigen -make && sudo make install -cd .. - -#Boost (Required) -sudo apt-get -y install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev - -#OpenCV (Required) -sudo apt-get -y install libopencv-dev - -#CGAL (Required) -sudo apt-get -y install libcgal-dev libcgal-qt5-dev - -#VCGLib (Required) -git clone https://github.com/cdcseacave/VCG.git vcglib - -#Ceres (Optional) -sudo apt-get -y install libatlas-base-dev libsuitesparse-dev -git clone https://ceres-solver.googlesource.com/ceres-solver ceres-solver -mkdir ceres_build && cd ceres_build -cmake . ../ceres-solver/ -DMINIGLOG=ON -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -make -j2 && sudo make install -cd .. - -#GLFW3 (Optional) -sudo apt-get -y install freeglut3-dev libglew-dev libglfw3-dev - -#OpenMVS -git clone https://github.com/cdcseacave/openMVS.git openMVS -mkdir openMVS_build && cd openMVS_build -cmake . ../openMVS -DCMAKE_BUILD_TYPE=Release -DVCG_ROOT="$main_path/vcglib" - -#If you want to use OpenMVS as shared library, add to the CMake command: --DBUILD_SHARED_LIBS=ON - -#Install OpenMVS library (optional): -make -j2 && sudo make install -``` - --------------------- -Mac OS X compilation --------------------- - -Install dependencies, run CMake and make. - -``` -#Install dependencies -brew update -brew install boost eigen opencv cgal -main_path=`pwd` - -#GLFW3 (Optional) -brew install glew glfw3 - -#VCGLib (Required) -git clone https://github.com/cdcseacave/VCG.git vcglib - -#Getting the OpenMVS sources: -git clone https://github.com/cdcseacave/openMVS.git - -#Build OpenMVS -mkdir openMVS_build && cd openMVS_build -cmake . ../openMVS -DCMAKE_BUILD_TYPE=Release -DVCG_ROOT="$main_path/vcglib" +cd openMVS +mkdir make +cd make -#Alternatively, build using XCode -cmake . ../openMVS -G "Xcode" -DCMAKE_BUILD_TYPE=Release -DVCG_ROOT="$main_path/vcglib" -xcodebuild -configuration Release +#Run CMake: +cmake .. -#If you want to use OpenMVS as shared library, add to the CMake command: --DBUILD_SHARED_LIBS=ON +#Build: +cmake --build . -j4 #Install OpenMVS library (optional): -make && sudo make install +cmake --install . ``` ------------------- @@ -150,7 +53,7 @@ In order to use *OpenMVS* as a third-party library in your project, first compil vcpkg install openmvs ``` -And inside your project CMake script, use: +Inside your project CMake script, use: ``` find_package(OpenMVS) if(OpenMVS_FOUND) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d921e826..39ec359ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,8 +30,8 @@ ENDIF() PROJECT(OpenMVS) SET(OpenMVS_MAJOR_VERSION 2) -SET(OpenMVS_MINOR_VERSION 0) -SET(OpenMVS_PATCH_VERSION 1) +SET(OpenMVS_MINOR_VERSION 1) +SET(OpenMVS_PATCH_VERSION 0) SET(OpenMVS_VERSION ${OpenMVS_MAJOR_VERSION}.${OpenMVS_MINOR_VERSION}.${OpenMVS_PATCH_VERSION}) # List configuration options @@ -47,6 +47,7 @@ SET(OpenMVS_USE_FAST_INVSQRT OFF CACHE BOOL "Use an optimized code to compute th SET(OpenMVS_USE_FAST_CBRT ON CACHE BOOL "Use an optimized code to compute the cubic root") SET(OpenMVS_USE_SSE ON CACHE BOOL "Enable SSE optimizations") SET(OpenMVS_MAX_CUDA_COMPATIBILITY OFF CACHE BOOL "Build for maximum CUDA device compatibility") +SET(OpenMVS_ENABLE_TESTS ON CACHE BOOL "Enable test code") # Define helper functions and macros. INCLUDE(build/Utils.cmake) @@ -108,6 +109,13 @@ if(OpenMVS_USE_CUDA) message("Working around windows build failure with visual studio. Visual studio 16.10 introduced a compiler bug with compilng CUDA code with C++14. Set the cuda standard to 17 as a workaround.") set(CMAKE_CUDA_STANDARD 17) endif() + EXECUTE_PROCESS(COMMAND "${CMAKE_CUDA_COMPILER}" --list-gpu-arch + OUTPUT_VARIABLE LIST_GPU_ARCH + ERROR_QUIET) + if(NOT LIST_GPU_ARCH AND OpenMVS_MAX_CUDA_COMPATIBILITY) + message(WARNING "Cannot compile for max CUDA compatibility, nvcc does not support --list-gpu-arch") + set(OpenMVS_MAX_CUDA_COMPATIBILITY NO) + endif() if(NOT OpenMVS_MAX_CUDA_COMPATIBILITY) if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) SET(CMAKE_CUDA_ARCHITECTURES 75) @@ -117,9 +125,6 @@ if(OpenMVS_USE_CUDA) # https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ # Extract list of arch and gencodes - EXECUTE_PROCESS(COMMAND "${CMAKE_CUDA_COMPILER}" --list-gpu-arch - OUTPUT_VARIABLE LIST_GPU_ARCH - ERROR_QUIET) STRING(REPLACE "\r" "" LIST_GPU_ARCH ${LIST_GPU_ARCH}) STRING(REPLACE "\n" ";" LIST_GPU_ARCH ${LIST_GPU_ARCH}) @@ -172,6 +177,10 @@ if(Boost_FOUND) LIST(APPEND OpenMVS_DEFINITIONS -D_USE_BOOST) ADD_DEFINITIONS(${Boost_DEFINITIONS}) LINK_DIRECTORIES(${Boost_LIBRARY_DIRS}) + if(NOT MSVC AND DEFINED CMAKE_TOOLCHAIN_FILE) + # work around this missing library link in vcpkg + LIST(APPEND Boost_LIBRARIES zstd) + endif() SET(_USE_BOOST TRUE) endif() @@ -237,9 +246,17 @@ if(OpenMVS_BUILD_TOOLS) endif() ADD_SUBDIRECTORY(docs) +if (OpenMVS_ENABLE_TESTS) + # enable testing functionality + ENABLE_TESTING() + # define tests + ADD_TEST(NAME UnitTests COMMAND $ "0") + ADD_TEST(NAME PipelineTest COMMAND $ "1") +endif() + # Export the package for use from the build-tree # (this registers the build-tree with a global CMake-registry) -export(PACKAGE OpenMVS) +EXPORT(PACKAGE OpenMVS) # Install the export set for use with the install-tree INSTALL(EXPORT OpenMVSTargets diff --git a/MvsScalablePipeline.py b/MvsScalablePipeline.py new file mode 100644 index 000000000..e5a426d54 --- /dev/null +++ b/MvsScalablePipeline.py @@ -0,0 +1,180 @@ +#!/usr/bin/python3 +# -*- encoding: utf-8 -*- +""" +This script helps with OpenMVS scalable pipeline. + +Starting from a SfM solution stored into a MVS project accompanied by the undistorted images, +the fist step is to compute all depth maps without fusion: + +DensifyPointCloud scene.mvs --fusion-mode 1 + +Next split the scene in sub-scenes using the area parameter, which is related to the inverse of GSD; +it is a bit non intuitive, but normally it should remain constant for a desired memory limit; +for example you can use the bellow value to limit memory usage to ~16GB: + +DensifyPointCloud scene.mvs --sub-scene-area 660000 + +disable depth-maps re-filtering by creating a file Densify.ini with just this line: + +Optimize = 0 + +and call fusion on each of the sub-scenes like: + +DensifyPointCloud scene_0000.mvs --dense-config-file Densify.ini +............ +DensifyPointCloud scene_000n.mvs --dense-config-file Densify.ini + +This script helps to automate the process of calling DensifyPointCloud/ReconstructMesh on all sub-scenes. + +usage: MvsScalablePipeline.py openMVS_module input_scene + +ex: MvsScalablePipeline.py DensifyPointCloud scene_XXXX.mvs --number-views-fuse 2 +""" + +import os +import subprocess +import sys +import argparse +import glob + +DEBUG = False + +if sys.platform.startswith('win'): + PATH_DELIM = ';' + FOLDER_DELIM = '\\' +else: + PATH_DELIM = ':' + FOLDER_DELIM = '/' + + +def whereis(afile): + """ + return directory in which afile is, None if not found. Look in PATH + """ + if sys.platform.startswith('win'): + cmd = "where" + else: + cmd = "which" + try: + ret = subprocess.run([cmd, afile], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + return os.path.split(ret.stdout.decode())[0] + except subprocess.CalledProcessError: + return None + + +def find(afile): + """ + As whereis look only for executable on linux, this find look for all file type + """ + for d in os.environ['PATH'].split(PATH_DELIM): + if os.path.isfile(os.path.join(d, afile)): + return d + return None + +# Try to find openMVS binaries in PATH +OPENMVS_BIN = whereis("ReconstructMesh") + +# Ask user for openMVS directory if not found +if not OPENMVS_BIN: + OPENMVS_BIN = input("openMVS binary folder?\n") + + +# HELPERS for terminal colors +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) +NO_EFFECT, BOLD, UNDERLINE, BLINK, INVERSE, HIDDEN = (0, 1, 4, 5, 7, 8) + + +# from Python cookbook, #475186 +def has_colours(stream): + ''' + Return stream colours capability + ''' + if not hasattr(stream, "isatty"): + return False + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + curses.setupterm() + return curses.tigetnum("colors") > 2 + except Exception: + # guess false in case of error + return False + +HAS_COLOURS = has_colours(sys.stdout) + + +def printout(text, colour=WHITE, background=BLACK, effect=NO_EFFECT): + """ + print() with colour + """ + if HAS_COLOURS: + seq = "\x1b[%d;%d;%dm" % (effect, 30+colour, 40+background) + text + "\x1b[0m" + sys.stdout.write(seq+'\r\n') + else: + sys.stdout.write(text+'\r\n') + + +# store config and data in +class ConfContainer: + """ + Container for all the config variables + """ + def __init__(self): + pass + +CONF = ConfContainer() + +# ARGS +PARSER = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="Scalable MVS reconstruction with these steps: \r\n" + + "MvsScalablePipeline.py openMVS_module input_scene \r\n" + ) +PARSER.add_argument('openMVS_module', + help="the OpenMVS module to use: DensifyPointCloud, ReconstructMesh, etc.") +PARSER.add_argument('input_scene', + help="the scene name reg to process: scene_XXXX.mvs") +PARSER.add_argument('passthrough', nargs=argparse.REMAINDER, help="Option to be passed to command lines") + +PARSER.parse_args(namespace=CONF) # store args in the ConfContainer + +suffix = os.path.basename(CONF.input_scene).replace('scene_XXXX','') +CONF.input_scene = CONF.input_scene.replace('_dense','').replace('_mesh','').replace('_refine','').replace('_texture','') + +# Absolute path for input directory +if len(CONF.input_scene) < 10 or CONF.input_scene[-9:] != '_XXXX.mvs': + sys.exit("%s: invalid scene name" % CONF.input_scene) + +match CONF.openMVS_module: + case 'ReconstructMesh': + moduleSuffix = '_mesh.mvs' + case 'RefineMesh': + moduleSuffix = '_refine.mvs' + case 'TextureMesh': + moduleSuffix = '_texture.mvs' + case _: + moduleSuffix = '_dense.mvs' + +printout("# Module {} start #".format(CONF.openMVS_module), colour=RED, effect=BOLD) +for scene_name in glob.glob(os.path.abspath(os.path.join(os.path.dirname(CONF.input_scene), 'scene_[0-9][0-9][0-9][0-9]'+suffix))): + if os.path.exists(os.path.splitext(scene_name)[0] + moduleSuffix) == False: + printout("# Process: %s" % os.path.basename(scene_name), colour=GREEN, effect=NO_EFFECT) + + # create a commandline for the current step + cmdline = [os.path.join(OPENMVS_BIN, CONF.openMVS_module), scene_name] + CONF.passthrough + print('Cmd: ' + ' '.join(cmdline)) + + if not DEBUG: + # Launch the current step + try: + pStep = subprocess.Popen(cmdline) + pStep.wait() + if pStep.returncode != 0: + printout("# Warning: step failed", colour=RED, effect=BOLD) + except KeyboardInterrupt: + sys.exit('\r\nProcess canceled by user, all files remains') + else: + print('\t'.join(cmdline)) + +printout("# Module {} end #".format(CONF.openMVS_module), colour=RED, effect=BOLD) diff --git a/README.md b/README.md index 01386cec7..9cad520bb 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ See the complete [documentation](https://github.com/cdcseacave/openMVS/wiki) on ## Build -See the [building](https://github.com/cdcseacave/openMVS/wiki/Building) wiki page. Windows and Ubuntu x64 continuous integration status [![Build Status](https://ci.appveyor.com/api/projects/status/github/cdcseacave/openmvs?branch=master&svg=true)](https://ci.appveyor.com/project/cdcseacave/openmvs) -Automatic Windows x64 binary builds can be found for each commit on its Appveyor Artifacts page. +See the [building](https://github.com/cdcseacave/openMVS/wiki/Building) wiki page. Windows, Ubuntu and MacOS x64 continuous integration status [![Continuous Integration](https://github.com/cdcseacave/openMVS/actions/workflows/continuous_integration.yml/badge.svg)](https://github.com/cdcseacave/openMVS/actions/workflows/continuous_integration.yml) +Automatic Windows x64 binary builds can be found for each commit on its Artifacts page. ## Example diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index b24044839..7f7ea6584 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -1,8 +1,13 @@ # Add applications ADD_SUBDIRECTORY(InterfaceCOLMAP) ADD_SUBDIRECTORY(InterfaceMetashape) +ADD_SUBDIRECTORY(InterfaceMVSNet) ADD_SUBDIRECTORY(DensifyPointCloud) ADD_SUBDIRECTORY(ReconstructMesh) ADD_SUBDIRECTORY(RefineMesh) ADD_SUBDIRECTORY(TextureMesh) +ADD_SUBDIRECTORY(TransformScene) ADD_SUBDIRECTORY(Viewer) +if(OpenMVS_ENABLE_TESTS) + ADD_SUBDIRECTORY(Tests) +endif() diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index 556deb452..949314db9 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -57,6 +57,9 @@ String strDenseConfigFileName; String strExportDepthMapsName; float fMaxSubsceneArea; float fSampleMesh; +float fBorderROI; +bool bCrop2ROI; +int nEstimateROI; int nFusionMode; int thFilterPointCloud; int nExportNumViews; @@ -100,17 +103,24 @@ bool Initialize(size_t argc, LPCTSTR* argv) // group of options allowed both on command line and in config file #ifdef _USE_CUDA const unsigned nNumViewsDefault(8); + const unsigned numIters(4); #else const unsigned nNumViewsDefault(5); + const unsigned numIters(3); #endif unsigned nResolutionLevel; unsigned nMaxResolution; unsigned nMinResolution; unsigned nNumViews; unsigned nMinViewsFuse; + unsigned nSubResolutionLevels; + unsigned nEstimationIters; + unsigned nEstimationGeometricIters; unsigned nEstimateColors; unsigned nEstimateNormals; + unsigned nOptimize; int nIgnoreMaskLabel; + bool bRemoveDmaps; boost::program_options::options_description config("Densify options"); config.add_options() ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input filename containing camera poses and image list") @@ -120,17 +130,25 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("resolution-level", boost::program_options::value(&nResolutionLevel)->default_value(1), "how many times to scale down the images before point cloud computation") ("max-resolution", boost::program_options::value(&nMaxResolution)->default_value(2560), "do not scale images higher than this resolution") ("min-resolution", boost::program_options::value(&nMinResolution)->default_value(640), "do not scale images lower than this resolution") + ("sub-resolution-levels", boost::program_options::value(&nSubResolutionLevels)->default_value(2), "number of patch-match sub-resolution iterations (0 - disabled)") ("number-views", boost::program_options::value(&nNumViews)->default_value(nNumViewsDefault), "number of views used for depth-map estimation (0 - all neighbor views available)") ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier (<2 - only merge depth-maps)") ("ignore-mask-label", boost::program_options::value(&nIgnoreMaskLabel)->default_value(-1), "integer value for the label to ignore in the segmentation mask (<0 - disabled)") + ("iters", boost::program_options::value(&nEstimationIters)->default_value(numIters), "number of patch-match iterations") + ("geometric-iters", boost::program_options::value(&nEstimationGeometricIters)->default_value(2), "number of geometric consistent patch-match iterations (0 - disabled)") ("estimate-colors", boost::program_options::value(&nEstimateColors)->default_value(2), "estimate the colors for the dense point-cloud (0 - disabled, 1 - final, 2 - estimate)") ("estimate-normals", boost::program_options::value(&nEstimateNormals)->default_value(2), "estimate the normals for the dense point-cloud (0 - disabled, 1 - final, 2 - estimate)") ("sub-scene-area", boost::program_options::value(&OPT::fMaxSubsceneArea)->default_value(0.f), "split the scene in sub-scenes such that each sub-scene surface does not exceed the given maximum sampling area (0 - disabled)") ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(0.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") - ("fusion-mode", boost::program_options::value(&OPT::nFusionMode)->default_value(0), "depth map fusion mode (-2 - fuse disparity-maps, -1 - export disparity-maps only, 0 - depth-maps & fusion, 1 - export depth-maps only)") + ("fusion-mode", boost::program_options::value(&OPT::nFusionMode)->default_value(0), "depth-maps fusion mode (-2 - fuse disparity-maps, -1 - export disparity-maps only, 0 - depth-maps & fusion, 1 - export depth-maps only)") + ("postprocess-dmaps", boost::program_options::value(&nOptimize)->default_value(7), "flags used to filter the depth-maps after estimation (0 - disabled, 1 - remove-speckles, 2 - fill-gaps, 4 - adjust-filter)") ("filter-point-cloud", boost::program_options::value(&OPT::thFilterPointCloud)->default_value(0), "filter dense point-cloud based on visibility (0 - disabled)") - ("export-number-views", boost::program_options::value(&OPT::nExportNumViews)->default_value(0), "export points with >= number of views (0 - disabled)") - ; + ("export-number-views", boost::program_options::value(&OPT::nExportNumViews)->default_value(0), "export points with >= number of views (0 - disabled, <0 - save MVS project too)") + ("roi-border", boost::program_options::value(&OPT::fBorderROI)->default_value(0), "add a border to the region-of-interest when cropping the scene (0 - disabled, >0 - percentage, <0 - absolute)") + ("estimate-roi", boost::program_options::value(&OPT::nEstimateROI)->default_value(2), "estimate and set region-of-interest (0 - disabled, 1 - enabled, 2 - adaptive)") + ("crop-to-roi", boost::program_options::value(&OPT::bCrop2ROI)->default_value(true), "crop scene using the region-of-interest") + ("remove-dmaps", boost::program_options::value(&bRemoveDmaps)->default_value(false), "remove depth-maps after fusion") + ; // hidden options, allowed both on command line and // in config file, but will not be shown to the user @@ -205,11 +223,16 @@ bool Initialize(size_t argc, LPCTSTR* argv) OPTDENSE::nResolutionLevel = nResolutionLevel; OPTDENSE::nMaxResolution = nMaxResolution; OPTDENSE::nMinResolution = nMinResolution; + OPTDENSE::nSubResolutionLevels = nSubResolutionLevels; OPTDENSE::nNumViews = nNumViews; OPTDENSE::nMinViewsFuse = nMinViewsFuse; + OPTDENSE::nEstimationIters = nEstimationIters; + OPTDENSE::nEstimationGeometricIters = nEstimationGeometricIters; OPTDENSE::nEstimateColors = nEstimateColors; OPTDENSE::nEstimateNormals = nEstimateNormals; + OPTDENSE::nOptimize = nOptimize; OPTDENSE::nIgnoreMaskLabel = nIgnoreMaskLabel; + OPTDENSE::bRemoveDmaps = bRemoveDmaps; if (!bValidConfig && !OPT::strDenseConfigFileName.empty()) OPTDENSE::oConfig.Save(OPT::strDenseConfigFileName); @@ -257,14 +280,14 @@ int main(int argc, LPCTSTR* argv) Scene scene(OPT::nMaxThreads); if (OPT::fSampleMesh != 0) { // sample input mesh and export the obtained point-cloud - if (!scene.mesh.Load(MAKE_PATH_SAFE(OPT::strInputFileName))) + if (!scene.Load(MAKE_PATH_SAFE(OPT::strInputFileName), true) || scene.mesh.IsEmpty()) return EXIT_FAILURE; TD_TIMER_START(); PointCloud pointcloud; if (OPT::fSampleMesh > 0) scene.mesh.SamplePoints(OPT::fSampleMesh, 0, pointcloud); else - scene.mesh.SamplePoints((unsigned)ROUND2INT(-OPT::fSampleMesh), pointcloud); + scene.mesh.SamplePoints(ROUND2INT(-OPT::fSampleMesh), pointcloud); VERBOSE("Sample mesh completed: %u points (%s)", pointcloud.GetSize(), TD_TIMER_GET_FMT().c_str()); pointcloud.Save(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))+_T(".ply")); Finalize(); @@ -273,29 +296,27 @@ int main(int argc, LPCTSTR* argv) // load and estimate a dense point-cloud if (!scene.Load(MAKE_PATH_SAFE(OPT::strInputFileName))) return EXIT_FAILURE; - if (!OPT::strExportROIFileName.empty() && scene.IsBounded()) { - std::ofstream fs(MAKE_PATH_SAFE(OPT::strExportROIFileName)); + if (!OPT::strImportROIFileName.empty()) { + std::ifstream fs(MAKE_PATH_SAFE(OPT::strImportROIFileName)); if (!fs) return EXIT_FAILURE; - fs << scene.obb; + fs >> scene.obb; + scene.Save(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))+_T(".mvs"), (ARCHIVE_TYPE)OPT::nArchiveType); Finalize(); return EXIT_SUCCESS; } - if (!OPT::strImportROIFileName.empty()) { - std::ifstream fs(MAKE_PATH_SAFE(OPT::strImportROIFileName)); + if (!scene.IsBounded()) + scene.EstimateROI(OPT::nEstimateROI, 1.1f); + if (!OPT::strExportROIFileName.empty() && scene.IsBounded()) { + std::ofstream fs(MAKE_PATH_SAFE(OPT::strExportROIFileName)); if (!fs) return EXIT_FAILURE; - fs >> scene.obb; - scene.Save(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))+_T(".mvs"), (ARCHIVE_TYPE)OPT::nArchiveType); + fs << scene.obb; Finalize(); return EXIT_SUCCESS; } if (!OPT::strMeshFileName.empty()) scene.mesh.Load(MAKE_PATH_SAFE(OPT::strMeshFileName)); - if (scene.pointcloud.IsEmpty() && OPT::strViewNeighborsFileName.empty() && scene.mesh.IsEmpty()) { - VERBOSE("error: empty initial point-cloud"); - return EXIT_FAILURE; - } if (!OPT::strViewNeighborsFileName.empty()) scene.LoadViewNeighbors(MAKE_PATH_SAFE(OPT::strViewNeighborsFileName)); if (!OPT::strOutputViewNeighborsFileName.empty()) { @@ -334,14 +355,27 @@ int main(int argc, LPCTSTR* argv) } if (OPT::nExportNumViews && scene.pointcloud.IsValid()) { // export point-cloud containing only points with N+ views - const String baseFileName(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))); - scene.pointcloud.SaveNViews(baseFileName+String::FormatString(_T("_%dviews.ply"), OPT::nExportNumViews), (IIndex)OPT::nExportNumViews); + const String baseFileName(MAKE_PATH_SAFE(Util::getFileFullName(OPT::strOutputFileName))+ + String::FormatString(_T("_%dviews"), ABS(OPT::nExportNumViews))); + if (OPT::nExportNumViews > 0) { + // export point-cloud containing only points with N+ views + scene.pointcloud.SaveNViews(baseFileName+_T(".ply"), (IIndex)OPT::nExportNumViews); + } else { + // save scene and export point-cloud containing only points with N+ views + scene.pointcloud.RemoveMinViews((IIndex)-OPT::nExportNumViews); + scene.Save(baseFileName+_T(".mvs"), (ARCHIVE_TYPE)OPT::nArchiveType); + scene.pointcloud.Save(baseFileName+_T(".ply")); + } Finalize(); return EXIT_SUCCESS; } if ((ARCHIVE_TYPE)OPT::nArchiveType != ARCHIVE_MVS) { + #if TD_VERBOSE != TD_VERBOSE_OFF + if (VERBOSITY_LEVEL > 1 && !scene.pointcloud.IsEmpty()) + scene.pointcloud.PrintStatistics(scene.images.data(), &scene.obb); + #endif TD_TIMER_START(); - if (!scene.DenseReconstruction(OPT::nFusionMode)) { + if (!scene.DenseReconstruction(OPT::nFusionMode, OPT::bCrop2ROI, OPT::fBorderROI)) { if (ABS(OPT::nFusionMode) != 1) return EXIT_FAILURE; VERBOSE("Depth-maps estimated (%s)", TD_TIMER_GET_FMT().c_str()); diff --git a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp index e4888ca9a..751e2e7d8 100644 --- a/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp +++ b/apps/InterfaceCOLMAP/InterfaceCOLMAP.cpp @@ -130,7 +130,6 @@ bool Initialize(size_t argc, LPCTSTR* argv) boost::program_options::store(boost::program_options::command_line_parser((int)argc, argv).options(cmdline_options).positional(p).run(), OPT::vm); boost::program_options::notify(OPT::vm); Util::ensureValidPath(OPT::strInputFileName); - WORKING_FOLDER = (File::isFolder(OPT::strInputFileName) ? OPT::strInputFileName : Util::getFilePath(OPT::strInputFileName)); INIT_WORKING_FOLDER; // parse configuration file std::ifstream ifs(MAKE_PATH_SAFE(OPT::strConfigFileName)); @@ -860,9 +859,10 @@ bool ImportScene(const String& strFolder, const String& strOutFolder, Interface& }, normalMap); } MVS::ConfidenceMap confMap; + MVS::ViewsMap viewsMap; const auto depthMM(std::minmax_element(colDepthMap.data_.cbegin(), colDepthMap.data_.cend())); const MVS::Depth dMin(*depthMM.first), dMax(*depthMM.second); - if (!ExportDepthDataRaw(strOutFolder+String::FormatString("depth%04u.dmap", image.ID), MAKE_PATH_FULL(strOutFolder, image.name), IDs, depthMap.size(), K, pose.R, pose.C, dMin, dMax, depthMap, normalMap, confMap)) + if (!ExportDepthDataRaw(strOutFolder+String::FormatString("depth%04u.dmap", image.ID), MAKE_PATH_FULL(strOutFolder, image.name), IDs, depthMap.size(), K, pose.R, pose.C, dMin, dMax, depthMap, normalMap, confMap, viewsMap)) return false; } } @@ -949,7 +949,7 @@ bool ExportScene(const String& strFolder, const Interface& scene) const uint32_t avgPointsPerSmallView(3000), avgPointsPerLargeView(12000); { images.resize(scene.images.size()); - cameras.resize(scene.images.size()); + cameras.resize((unsigned)scene.images.size()); for (uint32_t ID=0; ID<(uint32_t)scene.images.size(); ++ID) { const Interface::Image& image = scene.images[ID]; if (image.poseID == MVS::NO_ID) @@ -1114,8 +1114,8 @@ bool ExportScene(const String& strFolder, const Interface& scene) bool ExportIntrinsicsTxt(const String& fileName, const Interface& scene) { LOG_OUT() << "Writing intrinsics: " << fileName << std::endl; - size_t idxValidK(MVS::NO_ID); - FOREACH(ID, scene.images) { + uint32_t idxValidK(MVS::NO_ID); + for (uint32_t ID=0; ID<(uint32_t)scene.images.size(); ++ID) { const Interface::Image& image = scene.images[ID]; if (!image.IsValid()) continue; diff --git a/apps/InterfaceMVSNet/CMakeLists.txt b/apps/InterfaceMVSNet/CMakeLists.txt new file mode 100644 index 000000000..e38a82ced --- /dev/null +++ b/apps/InterfaceMVSNet/CMakeLists.txt @@ -0,0 +1,13 @@ +if(MSVC) + FILE(GLOB LIBRARY_FILES_C "*.cpp" "*.rc") +else() + FILE(GLOB LIBRARY_FILES_C "*.cpp") +endif() +FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") + +cxx_executable_with_flags(InterfaceMVSNet "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) + +# Install +INSTALL(TARGETS InterfaceMVSNet + EXPORT OpenMVSTargets + RUNTIME DESTINATION "${INSTALL_BIN_DIR}" COMPONENT bin) diff --git a/apps/InterfaceMVSNet/InterfaceMVSNet.cpp b/apps/InterfaceMVSNet/InterfaceMVSNet.cpp new file mode 100644 index 000000000..08d76a142 --- /dev/null +++ b/apps/InterfaceMVSNet/InterfaceMVSNet.cpp @@ -0,0 +1,364 @@ +/* + * InterfaceMVSNet.cpp + * + * Copyright (c) 2014-2021 SEACAVE + * + * Author(s): + * + * cDc + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional Terms: + * + * You are required to preserve legal notices and author attributions in + * that material or in the Appropriate Legal Notices displayed by works + * containing it. + */ + +#include "../../libs/MVS/Common.h" +#include "../../libs/MVS/Scene.h" +#include + +using namespace MVS; + + +// D E F I N E S /////////////////////////////////////////////////// + +#define APPNAME _T("InterfaceMVSNet") +#define MVS_EXT _T(".mvs") +#define MVSNET_IMAGES_FOLDER _T("images") +#define MVSNET_CAMERAS_FOLDER _T("cams") +#define MVSNET_IMAGES_EXT _T(".jpg") +#define MVSNET_CAMERAS_NAME _T("_cam.txt") + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace { + +namespace OPT { + String strInputFileName; + String strOutputFileName; + unsigned nArchiveType; + int nProcessPriority; + unsigned nMaxThreads; + String strConfigFileName; + boost::program_options::variables_map vm; +} // namespace OPT + +// initialize and parse the command line parameters +bool Initialize(size_t argc, LPCTSTR* argv) +{ + // initialize log and console + OPEN_LOG(); + OPEN_LOGCONSOLE(); + + // group of options allowed only on command line + boost::program_options::options_description generic("Generic options"); + generic.add_options() + ("help,h", "produce this help message") + ("working-folder,w", boost::program_options::value(&WORKING_FOLDER), "working directory (default current directory)") + ("config-file,c", boost::program_options::value(&OPT::strConfigFileName)->default_value(APPNAME _T(".cfg")), "file name containing program options") + ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(ARCHIVE_MVS), "project archive type: 0-text, 1-binary, 2-compressed binary") + ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") + ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") + #if TD_VERBOSE != TD_VERBOSE_OFF + ("verbosity,v", boost::program_options::value(&g_nVerbosityLevel)->default_value( + #if TD_VERBOSE == TD_VERBOSE_DEBUG + 3 + #else + 2 + #endif + ), "verbosity level") + #endif + ; + + // group of options allowed both on command line and in config file + boost::program_options::options_description config("Main options"); + config.add_options() + ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input filename containing camera poses and image list") + ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the mesh") + ; + + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic).add(config); + + boost::program_options::options_description config_file_options; + config_file_options.add(config); + + boost::program_options::positional_options_description p; + p.add("input-file", -1); + + try { + // parse command line options + boost::program_options::store(boost::program_options::command_line_parser((int)argc, argv).options(cmdline_options).positional(p).run(), OPT::vm); + boost::program_options::notify(OPT::vm); + INIT_WORKING_FOLDER; + // parse configuration file + std::ifstream ifs(MAKE_PATH_SAFE(OPT::strConfigFileName)); + if (ifs) { + boost::program_options::store(parse_config_file(ifs, config_file_options), OPT::vm); + boost::program_options::notify(OPT::vm); + } + } + catch (const std::exception& e) { + LOG(e.what()); + return false; + } + + // initialize the log file + OPEN_LOGFILE(MAKE_PATH(APPNAME _T("-") + Util::getUniqueName(0) + _T(".log")).c_str()); + + // print application details: version and command line + Util::LogBuild(); + LOG(_T("Command line: ") APPNAME _T("%s"), Util::CommandLineToString(argc, argv).c_str()); + + // validate input + Util::ensureValidPath(OPT::strInputFileName); + const String strInputFileNameExt(Util::getFileExt(OPT::strInputFileName).ToLower()); + const bool bInvalidCommand(OPT::strInputFileName.empty()); + if (OPT::vm.count("help") || bInvalidCommand) { + boost::program_options::options_description visible("Available options"); + visible.add(generic).add(config); + GET_LOG() << visible; + } + if (bInvalidCommand) + return false; + + // initialize optional options + Util::ensureValidPath(OPT::strOutputFileName); + Util::ensureUnifySlash(OPT::strOutputFileName); + if (OPT::strOutputFileName.IsEmpty()) + OPT::strOutputFileName = Util::getFileName(OPT::strInputFileName) + "scene" MVS_EXT; + + // initialize global options + Process::setCurrentProcessPriority((Process::Priority)OPT::nProcessPriority); + #ifdef _USE_OPENMP + if (OPT::nMaxThreads != 0) + omp_set_num_threads(OPT::nMaxThreads); + #endif + + #ifdef _USE_BREAKPAD + // start memory dumper + MiniDumper::Create(APPNAME, WORKING_FOLDER); + #endif + return true; +} + +// finalize application instance +void Finalize() +{ + #if TD_VERBOSE != TD_VERBOSE_OFF + // print memory statistics + Util::LogMemoryInfo(); + #endif + + CLOSE_LOGFILE(); + CLOSE_LOGCONSOLE(); + CLOSE_LOG(); +} + +void ImageListParse(const LPSTR* argv, Point3& C) +{ + // read position vector + C.x = String::FromString(argv[0]); + C.y = String::FromString(argv[1]); + C.z = String::FromString(argv[2]); +} + +void ImageListParse(const LPSTR* argv, Matrix3x3& R) +{ + // read rotation matrix + R(0, 0) = String::FromString(argv[0]); + R(0, 1) = String::FromString(argv[1]); + R(0, 2) = String::FromString(argv[2]); + R(1, 0) = String::FromString(argv[3]); + R(1, 1) = String::FromString(argv[4]); + R(1, 2) = String::FromString(argv[5]); + R(2, 0) = String::FromString(argv[6]); + R(2, 1) = String::FromString(argv[7]); + R(2, 2) = String::FromString(argv[8]); +} + +void ImageListParse(const LPSTR* argv, Matrix3x4& P) +{ + // read projection matrix + P(0, 0) = String::FromString(argv[0]); + P(0, 1) = String::FromString(argv[1]); + P(0, 2) = String::FromString(argv[2]); + P(0, 3) = String::FromString(argv[3]); + P(1, 0) = String::FromString(argv[4]); + P(1, 1) = String::FromString(argv[5]); + P(1, 2) = String::FromString(argv[6]); + P(1, 3) = String::FromString(argv[7]); + P(2, 0) = String::FromString(argv[8]); + P(2, 1) = String::FromString(argv[9]); + P(2, 2) = String::FromString(argv[10]); + P(2, 3) = String::FromString(argv[11]); +} + +// parse scene stored in MVSNet format composed of undistorted images and camera poses +// for example see GigaVision benchmark (http://gigamvs.net): +// |--sceneX +// |--images +// |--xxx.jpg +// |--xxx.jpg +// .... +// |--xxx.jpg +// |--cams +// |--xxx_cam.txt +// |--xxx_cam.txt +// .... +// |--xxx_cam.txt +// |--render_cams +// |--xxx_cam.txt +// |--xxx_cam.txt +// .... +// |--xxx_cam.txt +// +// where the camera parameter of one image stored in a cam.txt file contains the camera +// extrinsic E = [R|t], intrinsic K and the depth range: +// extrinsic +// E00 E01 E02 E03 +// E10 E11 E12 E13 +// E20 E21 E22 E23 +// E30 E31 E32 E33 +// +// intrinsic +// K00 K01 K02 +// K10 K11 K12 +// K20 K21 K22 +// +// DEPTH_MIN DEPTH_INTERVAL (DEPTH_NUM DEPTH_MAX) +bool ParseSceneMVSNet(Scene& scene) +{ + #if defined(_SUPPORT_CPP17) && (!defined(__GNUC__) || (__GNUC__ > 7)) + String strPath(MAKE_PATH_FULL(WORKING_FOLDER_FULL, OPT::strInputFileName)); + Util::ensureValidFolderPath(strPath); + const std::filesystem::path path(static_cast(strPath)); + for (const std::filesystem::directory_entry& entry : std::filesystem::directory_iterator(path / MVSNET_IMAGES_FOLDER)) { + if (entry.path().extension() != MVSNET_IMAGES_EXT) + continue; + // parse camera + const std::string strCamFileName((path / MVSNET_CAMERAS_FOLDER / entry.path().stem()).string() + MVSNET_CAMERAS_NAME); + std::ifstream fcam(strCamFileName); + if (!fcam) + continue; + String line; + do { + std::getline(fcam, line); + } while (line != "extrinsic" && fcam.good()); + String strP; + for (int i = 0; i < 3; ++i) { + std::getline(fcam, line); + strP += ' ' + line; + } + if (!fcam.good()) + continue; + size_t argc; + CAutoPtrArr argv; + argv = Util::CommandLineToArgvA(strP, argc); + if (argc != 12) + continue; + Matrix3x4 P; + ImageListParse(argv, P); + do { + std::getline(fcam, line); + } while (line != "intrinsic" && fcam.good()); + strP.clear(); + for (int i = 0; i < 3; ++i) { + std::getline(fcam, line); + strP += ' ' + line; + } + if (!fcam.good()) + continue; + argv = Util::CommandLineToArgvA(strP, argc); + if (argc != 9) + continue; + Matrix3x3 K; + ImageListParse(argv, K); + // setup camera + const IIndex platformID = scene.platforms.size(); + Platform& platform = scene.platforms.emplace_back(); + Platform::Camera& camera = platform.cameras.AddEmpty(); + camera.K = K; + camera.R = RMatrix::IDENTITY; + camera.C = CMatrix::ZERO; + // setup image + const IIndex ID = scene.images.size(); + Image& imageData = scene.images.emplace_back(); + imageData.platformID = platformID; + imageData.cameraID = 0; // only one camera per platform supported by this format + imageData.poseID = NO_ID; + imageData.ID = ID; + imageData.name = entry.path().string(); + Util::ensureUnifySlash(imageData.name); + imageData.name = MAKE_PATH_FULL(strPath, imageData.name); + // set image resolution + IMAGEPTR pimage = Image::ReadImageHeader(imageData.name); + imageData.width = pimage->GetWidth(); + imageData.height = pimage->GetHeight(); + imageData.scale = 1; + // set camera pose + imageData.poseID = platform.poses.size(); + Platform::Pose& pose = platform.poses.AddEmpty(); + DecomposeProjectionMatrix(P, pose.R, pose.C); + imageData.camera = platform.GetCamera(imageData.cameraID, imageData.poseID); + const float resolutionScale = Camera::GetNormalizationScale(imageData.width, imageData.height); + camera.K = camera.GetScaledK(REAL(1)/resolutionScale); + } + if (scene.images.size() < 2) + return false; + scene.nCalibratedImages = (unsigned)scene.images.size(); + return true; + #else + VERBOSE("error: C++17 is required to parse MVSNet format"); + return false; + #endif // _SUPPORT_CPP17 +} + +} // unnamed namespace + +int main(int argc, LPCTSTR* argv) +{ + #ifdef _DEBUGINFO + // set _crtBreakAlloc index to stop in at allocation + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);// | _CRTDBG_CHECK_ALWAYS_DF); + #endif + + if (!Initialize(argc, argv)) + return EXIT_FAILURE; + + TD_TIMER_START(); + + Scene scene(OPT::nMaxThreads); + + // convert data from MVSNet format to OpenMVS + if (!ParseSceneMVSNet(scene)) + return EXIT_FAILURE; + + // write OpenMVS input data + scene.Save(MAKE_PATH_SAFE(OPT::strOutputFileName), (ARCHIVE_TYPE)OPT::nArchiveType); + + VERBOSE("Imported data: %u platforms, %u images, %u vertices (%s)", + scene.platforms.size(), scene.images.size(), scene.pointcloud.GetSize(), + TD_TIMER_GET_FMT().c_str()); + + Finalize(); + return EXIT_SUCCESS; +} +/*----------------------------------------------------------------*/ diff --git a/apps/InterfaceMetashape/InterfaceMetashape.cpp b/apps/InterfaceMetashape/InterfaceMetashape.cpp index 1157bf29b..4886973ab 100644 --- a/apps/InterfaceMetashape/InterfaceMetashape.cpp +++ b/apps/InterfaceMetashape/InterfaceMetashape.cpp @@ -231,24 +231,11 @@ void ImageListParseP(const LPSTR* argv, Matrix3x4& P) // parse images list containing calibration and pose information // and load the corresponding 3D point-cloud -bool ParseImageListXML(Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& nCameras, size_t& nPoses) +bool ParseImageListXML(tinyxml2::XMLDocument& doc, Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& nCameras, size_t& nPoses) { - // open image list - nCameras = nPoses = 0; - const String strInputFileName(MAKE_PATH_SAFE(OPT::strInputFileName)); - ISTREAMPTR pStream(new File(strInputFileName, File::READ, File::OPEN)); - if (!((File*)(ISTREAM*)pStream)->isOpen()) { - LOG(_T("error: failed opening the input image list")); - return false; - } - - // parse camera list + String strInputFileName(MAKE_PATH_FULL(WORKING_FOLDER_FULL, OPT::strInputFileName)); + Util::ensureValidPath(strInputFileName); tinyxml2::XMLElement* elem; - const size_t nLen(pStream->getSize()); - String strCameras; strCameras.resize(nLen); - pStream->read(&strCameras[0], nLen); - tinyxml2::XMLDocument doc; - doc.Parse(strCameras.c_str(), nLen); if (doc.ErrorID() != tinyxml2::XML_SUCCESS) goto InvalidDocument; { @@ -362,7 +349,7 @@ bool ParseImageListXML(Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& { PMatrix P; size_t argc; - const String strPath(GET_PATH_FULL(strInputFileName)); + const String strPath(Util::getFilePath(strInputFileName)); for (tinyxml2::XMLElement* camera=cameras->FirstChildElement(); camera!=NULL; camera=camera->NextSiblingElement()) { unsigned ID; if (0 != _tcsicmp(camera->Value(), _T("camera")) || camera->QueryUnsignedAttribute(_T("id"), &ID) != tinyxml2::XML_SUCCESS) @@ -388,6 +375,7 @@ bool ParseImageListXML(Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& const cv::Size& resolution = resolutions[imageData.platformID]; imageData.width = resolution.width; imageData.height = resolution.height; + imageData.scale = 1; if (!bMetashapeFile && !camera->BoolAttribute(_T("enabled"))) { imageData.poseID = NO_ID; DEBUG_EXTRA("warning: uncalibrated image '%s'", name); @@ -415,6 +403,7 @@ bool ParseImageListXML(Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& ++nPoses; } } + scene.nCalibratedImages = (unsigned)nPoses; } // parse bounding-box @@ -466,6 +455,191 @@ bool ParseImageListXML(Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& return false; } +// parse scene stored in ContextCapture BlocksExchange format containing cameras, images and sparse point-cloud +bool ParseBlocksExchangeXML(tinyxml2::XMLDocument& doc, Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& nCameras, size_t& nPoses) { + String strInputFileName(MAKE_PATH_FULL(WORKING_FOLDER_FULL, OPT::strInputFileName)); + Util::ensureValidPath(strInputFileName); + const tinyxml2::XMLElement* blocksExchange; + const tinyxml2::XMLElement* document; + const tinyxml2::XMLElement* photogroups; + if (doc.ErrorID() != tinyxml2::XML_SUCCESS || + (blocksExchange=doc.FirstChildElement("BlocksExchange")) == NULL || + (document=blocksExchange->FirstChildElement("Block")) == NULL || + (photogroups=document->FirstChildElement("Photogroups")) == NULL) { + VERBOSE("error: invalid scene file"); + return false; + } + CLISTDEF0(cv::Size) resolutions; + std::unordered_map mapImageID; + const String strPath(Util::getFilePath(strInputFileName)); + const tinyxml2::XMLElement* elem; + for (const tinyxml2::XMLElement* photogroup=photogroups->FirstChildElement(); photogroup!=NULL; photogroup=photogroup->NextSiblingElement()) { + if ((elem=photogroup->FirstChildElement("CameraModelType")) == NULL || + std::strcmp(elem->GetText(), "Perspective") != 0) + continue; + if ((elem=photogroup->FirstChildElement("ImageDimensions")) == NULL) + continue; + const IIndex platformID = scene.platforms.size(); + Platform& platform = scene.platforms.AddEmpty(); + platform.name = photogroup->FirstChildElement("Name")->GetText(); + resolutions.emplace_back( + elem->FirstChildElement("Width")->UnsignedText(), + elem->FirstChildElement("Height")->UnsignedText() + ); + // parse camera + Platform::Camera& camera = platform.cameras.AddEmpty(); + camera.K = KMatrix::IDENTITY; + camera.R = RMatrix::IDENTITY; + camera.C = CMatrix::ZERO; + const float resolutionScale = Camera::GetNormalizationScale(resolutions.back().width, resolutions.back().height); + if ((elem=photogroup->FirstChildElement("FocalLengthPixels")) != NULL) { + camera.K(0,0) = camera.K(1,1) = photogroup->FirstChildElement("FocalLengthPixels")->DoubleText(); + } else { + camera.K(0,0) = camera.K(1,1) = photogroup->FirstChildElement("FocalLength")->DoubleText() * resolutionScale / photogroup->FirstChildElement("SensorSize")->DoubleText(); + } + if ((elem=photogroup->FirstChildElement("PrincipalPoint")) != NULL) { + camera.K(0,2) = elem->FirstChildElement("x")->DoubleText(); + camera.K(1,2) = elem->FirstChildElement("y")->DoubleText(); + } else { + camera.K(0,2) = resolutions.back().width*REAL(0.5); + camera.K(1,2) = resolutions.back().height*REAL(0.5); + } + if ((elem=photogroup->FirstChildElement("AspectRatio")) != NULL) + camera.K(1,1) *= elem->DoubleText(); + if ((elem=photogroup->FirstChildElement("Skew")) != NULL) + camera.K(0,1) = elem->DoubleText(); + camera.K = camera.GetScaledK(REAL(1)/resolutionScale); + // parse distortion parameters + DistCoeff& dc = pltDistCoeffs.AddEmpty().AddEmpty(); { + const tinyxml2::XMLElement* distortion=photogroup->FirstChildElement("Distortion"); + if ((elem=distortion->FirstChildElement("K1")) != NULL) + dc.k1 = elem->DoubleText(); + if ((elem=distortion->FirstChildElement("K2")) != NULL) + dc.k2 = elem->DoubleText(); + if ((elem=distortion->FirstChildElement("K3")) != NULL) + dc.k3 = elem->DoubleText(); + if ((elem=distortion->FirstChildElement("P1")) != NULL) + dc.p2 = elem->DoubleText(); + if ((elem=distortion->FirstChildElement("P2")) != NULL) + dc.p1 = elem->DoubleText(); + } + ++nCameras; + for (const tinyxml2::XMLElement* photo=photogroup->FirstChildElement("Photo"); photo!=NULL; photo=photo->NextSiblingElement()) { + Image imageData; + imageData.platformID = platformID; + imageData.cameraID = 0; // only one camera per platform supported by this format + imageData.poseID = NO_ID; + imageData.ID = photo->FirstChildElement("Id")->UnsignedText(); + imageData.name = photo->FirstChildElement("ImagePath")->GetText(); + Util::ensureUnifySlash(imageData.name); + imageData.name = MAKE_PATH_FULL(strPath, imageData.name); + // set image resolution + const cv::Size& resolution = resolutions[imageData.platformID]; + imageData.width = resolution.width; + imageData.height = resolution.height; + imageData.scale = 1; + // set camera pose + const tinyxml2::XMLElement* photoPose = photo->FirstChildElement("Pose"); + if (photoPose == NULL) + continue; + if ((elem=photoPose->FirstChildElement("Rotation")) == NULL) + continue; + imageData.poseID = platform.poses.size(); + Platform::Pose& pose = platform.poses.AddEmpty(); + pose.R = Matrix3x3( + elem->FirstChildElement("M_00")->DoubleText(), + elem->FirstChildElement("M_01")->DoubleText(), + elem->FirstChildElement("M_02")->DoubleText(), + elem->FirstChildElement("M_10")->DoubleText(), + elem->FirstChildElement("M_11")->DoubleText(), + elem->FirstChildElement("M_12")->DoubleText(), + elem->FirstChildElement("M_20")->DoubleText(), + elem->FirstChildElement("M_21")->DoubleText(), + elem->FirstChildElement("M_22")->DoubleText()); + if ((elem=photoPose->FirstChildElement("Center")) == NULL) + continue; + pose.C = Point3( + elem->FirstChildElement("x")->DoubleText(), + elem->FirstChildElement("y")->DoubleText(), + elem->FirstChildElement("z")->DoubleText()); + imageData.camera = platform.GetCamera(imageData.cameraID, imageData.poseID); + // set depth stats + if ((elem=photo->FirstChildElement("MedianDepth")) != NULL) + imageData.avgDepth = (float)elem->DoubleText(); + else if (photo->FirstChildElement("NearDepth") != NULL && photo->FirstChildElement("FarDepth") != NULL) + imageData.avgDepth = (float)((photo->FirstChildElement("NearDepth")->DoubleText() + photo->FirstChildElement("FarDepth")->DoubleText())/2); + mapImageID.emplace(imageData.ID, static_cast(scene.images.size())); + scene.images.emplace_back(std::move(imageData)); + ++nPoses; + } + } + if (scene.images.size() < 2) + return false; + scene.nCalibratedImages = (unsigned)nPoses; + // transform poses to a local coordinate system + const bool bLocalCoords(document->FirstChildElement("SRSId") == NULL || + ((elem=blocksExchange->FirstChildElement("SpatialReferenceSystems")) != NULL && (elem=elem->FirstChildElement("SRS")) != NULL && (elem=elem->FirstChildElement("Name")) != NULL && _tcsncmp(elem->GetText(), "Local Coordinates", 17) == 0)); + Point3 center = Point3::ZERO; + if (!bLocalCoords) { + for (const Image& imageData : scene.images) + center += imageData.camera.C; + center /= scene.images.size(); + for (Platform& platform : scene.platforms) + for (Platform::Pose& pose : platform.poses) + pose.C -= center; + } + // try to read also the sparse point-cloud + const tinyxml2::XMLElement* tiepoints = document->FirstChildElement("TiePoints"); + if (tiepoints == NULL) + return true; + for (const tinyxml2::XMLElement* tiepoint=tiepoints->FirstChildElement(); tiepoint!=NULL; tiepoint=tiepoint->NextSiblingElement()) { + if ((elem=tiepoint->FirstChildElement("Position")) == NULL) + continue; + scene.pointcloud.points.emplace_back( + (float)elem->FirstChildElement("x")->DoubleText(), + (float)elem->FirstChildElement("y")->DoubleText(), + (float)elem->FirstChildElement("z")->DoubleText()); + if (!bLocalCoords) + scene.pointcloud.points.back() -= Cast(center); + if ((elem=tiepoint->FirstChildElement("Color")) != NULL) + scene.pointcloud.colors.emplace_back( + (uint8_t)std::clamp(elem->FirstChildElement("Blue")->DoubleText()*255, 0.0, 255.0), + (uint8_t)std::clamp(elem->FirstChildElement("Green")->DoubleText()*255, 0.0, 255.0), + (uint8_t)std::clamp(elem->FirstChildElement("Red")->DoubleText()*255, 0.0, 255.0)); + PointCloud::ViewArr views; + for (const tinyxml2::XMLElement* view=tiepoint->FirstChildElement("Measurement"); view!=NULL; view=view->NextSiblingElement()) + views.emplace_back(mapImageID.at(view->FirstChildElement("PhotoId")->UnsignedText())); + scene.pointcloud.pointViews.emplace_back(std::move(views)); + } + return true; +} + +// parse scene stored either in Metashape images list format or ContextCapture BlocksExchange format +bool ParseSceneXML(Scene& scene, PlatformDistCoeffs& pltDistCoeffs, size_t& nCameras, size_t& nPoses) +{ + // parse XML file + const String strInputFileName(MAKE_PATH_SAFE(OPT::strInputFileName)); + tinyxml2::XMLDocument doc; { + ISTREAMPTR pStream(new File(strInputFileName, File::READ, File::OPEN)); + if (!((File*)(ISTREAM*)pStream)->isOpen()) { + VERBOSE("error: failed opening the input scene file"); + return false; + } + const size_t nLen(pStream->getSize()); + String str; str.resize(nLen); + pStream->read(&str[0], nLen); + doc.Parse(str.c_str(), nLen); + } + if (doc.ErrorID() != tinyxml2::XML_SUCCESS) { + VERBOSE("error: invalid XML file"); + return false; + } + // parse scene + if (doc.FirstChildElement("BlocksExchange") == NULL) + return ParseImageListXML(doc, scene, pltDistCoeffs, nCameras, nPoses); + return ParseBlocksExchangeXML(doc, scene, pltDistCoeffs, nCameras, nPoses); +} + // undistort image using Brown's model bool UndistortBrown(Image& imageData, uint32_t ID, const DistCoeff& dc, const String& pathData) { @@ -623,12 +797,13 @@ int main(int argc, LPCTSTR* argv) // convert data from Metashape format to OpenMVS PlatformDistCoeffs pltDistCoeffs; - size_t nCameras, nPoses; - if (!ParseImageListXML(scene, pltDistCoeffs, nCameras, nPoses)) + size_t nCameras(0), nPoses(0); + if (!ParseSceneXML(scene, pltDistCoeffs, nCameras, nPoses)) return EXIT_FAILURE; // undistort images const String pathData(MAKE_PATH_FULL(WORKING_FOLDER_FULL, OPT::strOutputImageFolder)); + const bool bAssignPoints(!scene.pointcloud.IsEmpty() && !scene.pointcloud.IsValid()); Util::Progress progress(_T("Processed images"), scene.images.size()); GET_LOGCONSOLE().Pause(); #ifdef _USE_OPENMP @@ -649,17 +824,17 @@ int main(int argc, LPCTSTR* argv) #pragma omp flush (bAbort) continue; #else - return false; + return EXIT_FAILURE; #endif } imageData.UpdateCamera(scene.platforms); - if (scene.pointcloud.IsValid()) + if (bAssignPoints) AssignPoints(imageData, ID, scene.pointcloud); } GET_LOGCONSOLE().Play(); #ifdef _USE_OPENMP if (bAbort) - return EXIT_SUCCESS; + return EXIT_FAILURE; #endif progress.close(); @@ -674,7 +849,7 @@ int main(int argc, LPCTSTR* argv) scene.Save(MAKE_PATH_SAFE(OPT::strOutputFileName), (ARCHIVE_TYPE)OPT::nArchiveType); VERBOSE("Exported data: %u platforms, %u cameras, %u poses, %u images, %u vertices (%s)", - scene.platforms.GetSize(), nCameras, nPoses, scene.images.GetSize(), scene.pointcloud.GetSize(), + scene.platforms.size(), nCameras, nPoses, scene.images.size(), scene.pointcloud.GetSize(), TD_TIMER_GET_FMT().c_str()); Finalize(); diff --git a/apps/ReconstructMesh/ReconstructMesh.cpp b/apps/ReconstructMesh/ReconstructMesh.cpp index de68513db..9e61d153b 100644 --- a/apps/ReconstructMesh/ReconstructMesh.cpp +++ b/apps/ReconstructMesh/ReconstructMesh.cpp @@ -68,6 +68,8 @@ bool bRemoveSpikes; unsigned nCloseHoles; unsigned nSmoothMesh; float fEdgeLength; +bool bCrop2ROI; +float fBorderROI; float fSplitMaxArea; unsigned nArchiveType; int nProcessPriority; @@ -130,6 +132,8 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("close-holes", boost::program_options::value(&OPT::nCloseHoles)->default_value(30), "try to close small holes in the reconstructed surface (0 - disabled)") ("smooth", boost::program_options::value(&OPT::nSmoothMesh)->default_value(2), "number of iterations to smooth the reconstructed surface (0 - disabled)") ("edge-length", boost::program_options::value(&OPT::fEdgeLength)->default_value(0.f), "remesh such that the average edge length is this size (0 - disabled)") + ("roi-border", boost::program_options::value(&OPT::fBorderROI)->default_value(0), "add a border to the region-of-interest when cropping the scene (0 - disabled, >0 - percentage, <0 - absolute)") + ("crop-to-roi", boost::program_options::value(&OPT::bCrop2ROI)->default_value(true), "crop scene using the region-of-interest") ; // hidden options, allowed both on command line and @@ -242,8 +246,7 @@ void Finalize() bool Export3DProjections(Scene& scene, const String& inputFileName) { SML smlPointList(_T("ImagePoints")); smlPointList.Load(inputFileName); - const LPSMLARR& arrSmlChild = smlPointList.GetArrChildren(); - ASSERT(arrSmlChild.size() <= 1); + ASSERT(smlPointList.GetArrChildren().size() <= 1); IDX idx(0); // read image name @@ -370,6 +373,11 @@ int main(int argc, LPCTSTR* argv) scene.ExportCamerasMLP(baseFileName+_T(".mlp"), fileName); #endif } else { + const OBB3f initialOBB(scene.obb); + if (OPT::fBorderROI > 0) + scene.obb.EnlargePercent(OPT::fBorderROI); + else if (OPT::fBorderROI < 0) + scene.obb.Enlarge(-OPT::fBorderROI); if (OPT::strMeshFileName.IsEmpty() && scene.mesh.IsEmpty()) { // reset image resolution to the original size and // make sure the image neighbors are initialized before deleting the point-cloud @@ -427,10 +435,19 @@ int main(int argc, LPCTSTR* argv) } // clean the mesh + if (OPT::bCrop2ROI && scene.IsBounded()) { + TD_TIMER_START(); + const size_t numVertices = scene.mesh.vertices.size(); + const size_t numFaces = scene.mesh.faces.size(); + scene.mesh.RemoveFacesOutside(scene.obb); + VERBOSE("Mesh trimmed to ROI: %u vertices and %u faces removed (%s)", + numVertices-scene.mesh.vertices.size(), numFaces-scene.mesh.faces.size(), TD_TIMER_GET_FMT().c_str()); + } const float fDecimate(OPT::nTargetFaceNum ? static_cast(OPT::nTargetFaceNum) / scene.mesh.faces.size() : OPT::fDecimateMesh); scene.mesh.Clean(fDecimate, OPT::fRemoveSpurious, OPT::bRemoveSpikes, OPT::nCloseHoles, OPT::nSmoothMesh, OPT::fEdgeLength, false); scene.mesh.Clean(1.f, 0.f, OPT::bRemoveSpikes, OPT::nCloseHoles, 0u, 0.f, false); // extra cleaning trying to close more holes scene.mesh.Clean(1.f, 0.f, false, 0u, 0u, 0.f, true); // extra cleaning to remove non-manifold problems created by closing holes + scene.obb = initialOBB; // save the final mesh scene.Save(baseFileName+_T(".mvs"), (ARCHIVE_TYPE)OPT::nArchiveType); diff --git a/apps/Tests/CMakeLists.txt b/apps/Tests/CMakeLists.txt new file mode 100644 index 000000000..192178b32 --- /dev/null +++ b/apps/Tests/CMakeLists.txt @@ -0,0 +1,15 @@ +if(MSVC) + FILE(GLOB LIBRARY_FILES_C "*.cpp" "*.rc") +else() + FILE(GLOB LIBRARY_FILES_C "*.cpp") +endif() +FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") + +ADD_DEFINITIONS(-D_DATA_PATH="${CMAKE_CURRENT_SOURCE_DIR}/data/") + +cxx_executable_with_flags(Tests "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) + +# Install +INSTALL(TARGETS Tests + EXPORT OpenMVSTargets + RUNTIME DESTINATION "${INSTALL_BIN_DIR}" COMPONENT bin) diff --git a/apps/Tests/Tests.cpp b/apps/Tests/Tests.cpp new file mode 100644 index 000000000..218437b7a --- /dev/null +++ b/apps/Tests/Tests.cpp @@ -0,0 +1,130 @@ +/* + * Tests.cpp + * + * Copyright (c) 2014-2021 SEACAVE + * + * Author(s): + * + * cDc + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional Terms: + * + * You are required to preserve legal notices and author attributions in + * that material or in the Appropriate Legal Notices displayed by works + * containing it. + */ + +#include "../../libs/MVS/Common.h" +#include "../../libs/MVS/Scene.h" + +using namespace MVS; + + +// D E F I N E S /////////////////////////////////////////////////// + + +// S T R U C T S /////////////////////////////////////////////////// + +// test various algorithms independently +bool UnitTests() +{ + TD_TIMER_START(); + if (!SEACAVE::cListTest(100)) { + VERBOSE("ERROR: cListTest failed!"); + return false; + } + if (!SEACAVE::OctreeTest(100)) { + VERBOSE("ERROR: OctreeTest failed!"); + return false; + } + if (!SEACAVE::OctreeTest(100)) { + VERBOSE("ERROR: OctreeTest failed!"); + return false; + } + if (!SEACAVE::TestRayTriangleIntersection(1000)) { + VERBOSE("ERROR: TestRayTriangleIntersection failed!"); + return false; + } + if (!SEACAVE::TestRayTriangleIntersection(1000)) { + VERBOSE("ERROR: TestRayTriangleIntersection failed!"); + return false; + } + VERBOSE("All unit tests passed (%s)", TD_TIMER_GET_FMT().c_str()); + return true; +} + + +// test MVS stages on a small sample dataset +bool PipelineTest(bool verbose=false) +{ + TD_TIMER_START(); + Scene scene; + if (!scene.Load(MAKE_PATH("scene.mvs"))) { + VERBOSE("ERROR: TestDataset failed loading the scene!"); + return false; + } + OPTDENSE::init(); + OPTDENSE::bRemoveDmaps = true; + if (!scene.DenseReconstruction() || scene.pointcloud.GetSize() < 200000u) { + VERBOSE("ERROR: TestDataset failed estimating dense point cloud!"); + return false; + } + if (verbose) + scene.pointcloud.Save(MAKE_PATH("scene_dense.ply")); + if (!scene.ReconstructMesh() || scene.mesh.faces.size() < 75000u) { + VERBOSE("ERROR: TestDataset failed reconstructing the mesh!"); + return false; + } + if (verbose) + scene.mesh.Save(MAKE_PATH("scene_dense_mesh.ply")); + constexpr float decimate = 0.5f; + scene.mesh.Clean(decimate); + if (!ISINSIDE(scene.mesh.faces.size(), 35000u, 45000u)) { + VERBOSE("ERROR: TestDataset failed cleaning the mesh!"); + return false; + } + if (!scene.TextureMesh(0, 0) || !scene.mesh.HasTexture()) { + VERBOSE("ERROR: TestDataset failed texturing the mesh!"); + return false; + } + if (verbose) + scene.mesh.Save(MAKE_PATH("scene_dense_mesh_texture.ply")); + VERBOSE("All pipeline stages passed (%s)", TD_TIMER_GET_FMT().c_str()); + return true; +} + +// test OpenMVS functionality +int main(int argc, LPCTSTR* argv) +{ + OPEN_LOG(); + OPEN_LOGCONSOLE(); + Util::Init(); + WORKING_FOLDER = _DATA_PATH; + INIT_WORKING_FOLDER; + if (argc < 2 || std::atoi(argv[1]) == 0) { + if (!UnitTests()) + return EXIT_FAILURE; + } else { + if (!PipelineTest()) + return EXIT_FAILURE; + } + CLOSE_LOGCONSOLE(); + CLOSE_LOG(); + return EXIT_SUCCESS; +} +/*----------------------------------------------------------------*/ diff --git a/apps/Tests/data/images/00000.jpg b/apps/Tests/data/images/00000.jpg new file mode 100644 index 000000000..2ec67cb9a Binary files /dev/null and b/apps/Tests/data/images/00000.jpg differ diff --git a/apps/Tests/data/images/00001.jpg b/apps/Tests/data/images/00001.jpg new file mode 100644 index 000000000..a5b6f9545 Binary files /dev/null and b/apps/Tests/data/images/00001.jpg differ diff --git a/apps/Tests/data/images/00002.jpg b/apps/Tests/data/images/00002.jpg new file mode 100644 index 000000000..d4caa0bd8 Binary files /dev/null and b/apps/Tests/data/images/00002.jpg differ diff --git a/apps/Tests/data/images/00003.jpg b/apps/Tests/data/images/00003.jpg new file mode 100644 index 000000000..504155048 Binary files /dev/null and b/apps/Tests/data/images/00003.jpg differ diff --git a/apps/Tests/data/scene.mvs b/apps/Tests/data/scene.mvs new file mode 100644 index 000000000..b6bd8f94b Binary files /dev/null and b/apps/Tests/data/scene.mvs differ diff --git a/apps/TextureMesh/TextureMesh.cpp b/apps/TextureMesh/TextureMesh.cpp index dcf397b10..e2b1dc1c4 100644 --- a/apps/TextureMesh/TextureMesh.cpp +++ b/apps/TextureMesh/TextureMesh.cpp @@ -54,6 +54,7 @@ float fDecimateMesh; unsigned nCloseHoles; unsigned nResolutionLevel; unsigned nMinResolution; +unsigned minCommonCameras; float fOutlierThreshold; float fRatioDataSmoothness; bool bGlobalSeamLeveling; @@ -61,6 +62,7 @@ bool bLocalSeamLeveling; unsigned nTextureSizeMultiple; unsigned nRectPackingHeuristic; uint32_t nColEmpty; +float fSharpnessWeight; unsigned nOrthoMapResolution; unsigned nArchiveType; int nProcessPriority; @@ -83,7 +85,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("help,h", "produce this help message") ("working-folder,w", boost::program_options::value(&WORKING_FOLDER), "working directory (default current directory)") ("config-file,c", boost::program_options::value(&OPT::strConfigFileName)->default_value(APPNAME _T(".cfg")), "file name containing program options") - ("export-type", boost::program_options::value(&OPT::strExportType)->default_value(_T("glb")), "file type used to export the 3D scene (ply, obj, glb or gltf)") + ("export-type", boost::program_options::value(&OPT::strExportType)->default_value(_T("ply")), "file type used to export the 3D scene (ply, obj, glb or gltf)") ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(ARCHIVE_DEFAULT), "project archive type: 0-text, 1-binary, 2-compressed binary") ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") @@ -112,11 +114,13 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("min-resolution", boost::program_options::value(&OPT::nMinResolution)->default_value(640), "do not scale images lower than this resolution") ("outlier-threshold", boost::program_options::value(&OPT::fOutlierThreshold)->default_value(6e-2f), "threshold used to find and remove outlier face textures (0 - disabled)") ("cost-smoothness-ratio", boost::program_options::value(&OPT::fRatioDataSmoothness)->default_value(0.1f), "ratio used to adjust the preference for more compact patches (1 - best quality/worst compactness, ~0 - worst quality/best compactness)") + ("virtual-face-images", boost::program_options::value(&OPT::minCommonCameras)->default_value(0), "generate texture patches using virtual faces composed of coplanar triangles sharing at least this number of views (0 - disabled, 3 - good value)") ("global-seam-leveling", boost::program_options::value(&OPT::bGlobalSeamLeveling)->default_value(true), "generate uniform texture patches using global seam leveling") ("local-seam-leveling", boost::program_options::value(&OPT::bLocalSeamLeveling)->default_value(true), "generate uniform texture patch borders using local seam leveling") ("texture-size-multiple", boost::program_options::value(&OPT::nTextureSizeMultiple)->default_value(0), "texture size should be a multiple of this value (0 - power of two)") ("patch-packing-heuristic", boost::program_options::value(&OPT::nRectPackingHeuristic)->default_value(3), "specify the heuristic used when deciding where to place a new patch (0 - best fit, 3 - good speed, 100 - best speed)") ("empty-color", boost::program_options::value(&OPT::nColEmpty)->default_value(0x00FF7F27), "color used for faces not covered by any image") + ("sharpness-weight", boost::program_options::value(&OPT::fSharpnessWeight)->default_value(0.5f), "amount of sharpness to be applied on the texture (0 - disabled)") ("orthographic-image-resolution", boost::program_options::value(&OPT::nOrthoMapResolution)->default_value(0), "orthographic image resolution to be generated from the textured mesh - the mesh is expected to be already geo-referenced or at least properly oriented (0 - disabled)") ; @@ -301,7 +305,7 @@ int main(int argc, LPCTSTR* argv) // compute mesh texture TD_TIMER_START(); - if (!scene.TextureMesh(OPT::nResolutionLevel, OPT::nMinResolution, OPT::fOutlierThreshold, OPT::fRatioDataSmoothness, OPT::bGlobalSeamLeveling, OPT::bLocalSeamLeveling, OPT::nTextureSizeMultiple, OPT::nRectPackingHeuristic, Pixel8U(OPT::nColEmpty), views)) + if (!scene.TextureMesh(OPT::nResolutionLevel, OPT::nMinResolution, OPT::minCommonCameras, OPT::fOutlierThreshold, OPT::fRatioDataSmoothness, OPT::bGlobalSeamLeveling, OPT::bLocalSeamLeveling, OPT::nTextureSizeMultiple, OPT::nRectPackingHeuristic, Pixel8U(OPT::nColEmpty), OPT::fSharpnessWeight, views)) return EXIT_FAILURE; VERBOSE("Mesh texturing completed: %u vertices, %u faces (%s)", scene.mesh.vertices.GetSize(), scene.mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str()); diff --git a/apps/TransformScene/CMakeLists.txt b/apps/TransformScene/CMakeLists.txt new file mode 100644 index 000000000..2ea8a7c97 --- /dev/null +++ b/apps/TransformScene/CMakeLists.txt @@ -0,0 +1,13 @@ +if(MSVC) + FILE(GLOB LIBRARY_FILES_C "*.cpp" "*.rc") +else() + FILE(GLOB LIBRARY_FILES_C "*.cpp") +endif() +FILE(GLOB LIBRARY_FILES_H "*.h" "*.inl") + +cxx_executable_with_flags(TransformScene "Apps" "${cxx_default}" "MVS;${OpenMVS_EXTRA_LIBS}" ${LIBRARY_FILES_C} ${LIBRARY_FILES_H}) + +# Install +INSTALL(TARGETS TransformScene + EXPORT OpenMVSTargets + RUNTIME DESTINATION "${INSTALL_BIN_DIR}" COMPONENT bin) diff --git a/apps/TransformScene/TransformScene.cpp b/apps/TransformScene/TransformScene.cpp new file mode 100644 index 000000000..580750fac --- /dev/null +++ b/apps/TransformScene/TransformScene.cpp @@ -0,0 +1,244 @@ +/* + * TransformScene.cpp + * + * Copyright (c) 2014-2021 SEACAVE + * + * Author(s): + * + * cDc + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional Terms: + * + * You are required to preserve legal notices and author attributions in + * that material or in the Appropriate Legal Notices displayed by works + * containing it. + */ + +#include "../../libs/MVS/Common.h" +#include "../../libs/MVS/Scene.h" +#include + +using namespace MVS; + + +// D E F I N E S /////////////////////////////////////////////////// + +#define APPNAME _T("TransformScene") +#define MVS_EXT _T(".mvs") + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace { + +namespace OPT { + String strInputFileName; + String strOutputFileName; + String strAlignFileName; + String strTransferTextureFileName; + bool bComputeVolume; + float fPlaneThreshold; + float fSampleMesh; + unsigned nUpAxis; + unsigned nArchiveType; + int nProcessPriority; + unsigned nMaxThreads; + String strConfigFileName; + boost::program_options::variables_map vm; +} // namespace OPT + +// initialize and parse the command line parameters +bool Initialize(size_t argc, LPCTSTR* argv) +{ + // initialize log and console + OPEN_LOG(); + OPEN_LOGCONSOLE(); + + // group of options allowed only on command line + boost::program_options::options_description generic("Generic options"); + generic.add_options() + ("help,h", "produce this help message") + ("working-folder,w", boost::program_options::value(&WORKING_FOLDER), "working directory (default current directory)") + ("config-file,c", boost::program_options::value(&OPT::strConfigFileName)->default_value(APPNAME _T(".cfg")), "file name containing program options") + ("archive-type", boost::program_options::value(&OPT::nArchiveType)->default_value(ARCHIVE_MVS), "project archive type: 0-text, 1-binary, 2-compressed binary") + ("process-priority", boost::program_options::value(&OPT::nProcessPriority)->default_value(-1), "process priority (below normal by default)") + ("max-threads", boost::program_options::value(&OPT::nMaxThreads)->default_value(0), "maximum number of threads (0 for using all available cores)") + #if TD_VERBOSE != TD_VERBOSE_OFF + ("verbosity,v", boost::program_options::value(&g_nVerbosityLevel)->default_value( + #if TD_VERBOSE == TD_VERBOSE_DEBUG + 3 + #else + 2 + #endif + ), "verbosity level") + #endif + ; + + // group of options allowed both on command line and in config file + boost::program_options::options_description config("Main options"); + config.add_options() + ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input scene filename") + ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the scene") + ("align-file,a", boost::program_options::value(&OPT::strAlignFileName), "input scene filename to which the scene will be cameras aligned") + ("transfer-texture-file,t", boost::program_options::value(&OPT::strTransferTextureFileName), "input mesh filename to which the texture of the scene's mesh will be transfered to (the two meshes should be aligned and the new mesh to have UV-map)") + ("compute-volume", boost::program_options::value(&OPT::bComputeVolume)->default_value(false), "compute the volume of the given watertight mesh, or else try to estimate the ground plane and assume the mesh is bounded by it") + ("plane-threshold", boost::program_options::value(&OPT::fPlaneThreshold)->default_value(0.f), "threshold used to estimate the ground plane (<0 - disabled, 0 - auto, >0 - desired threshold)") + ("sample-mesh", boost::program_options::value(&OPT::fSampleMesh)->default_value(-300000.f), "uniformly samples points on a mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit)") + ("up-axis", boost::program_options::value(&OPT::nUpAxis)->default_value(2), "scene axis considered to point upwards (0 - x, 1 - y, 2 - z)") + ; + + boost::program_options::options_description cmdline_options; + cmdline_options.add(generic).add(config); + + boost::program_options::options_description config_file_options; + config_file_options.add(config); + + boost::program_options::positional_options_description p; + p.add("input-file", -1); + + try { + // parse command line options + boost::program_options::store(boost::program_options::command_line_parser((int)argc, argv).options(cmdline_options).positional(p).run(), OPT::vm); + boost::program_options::notify(OPT::vm); + INIT_WORKING_FOLDER; + // parse configuration file + std::ifstream ifs(MAKE_PATH_SAFE(OPT::strConfigFileName)); + if (ifs) { + boost::program_options::store(parse_config_file(ifs, config_file_options), OPT::vm); + boost::program_options::notify(OPT::vm); + } + } + catch (const std::exception& e) { + LOG(e.what()); + return false; + } + + // initialize the log file + OPEN_LOGFILE(MAKE_PATH(APPNAME _T("-") + Util::getUniqueName(0) + _T(".log")).c_str()); + + // print application details: version and command line + Util::LogBuild(); + LOG(_T("Command line: ") APPNAME _T("%s"), Util::CommandLineToString(argc, argv).c_str()); + + // validate input + Util::ensureValidPath(OPT::strInputFileName); + Util::ensureValidPath(OPT::strAlignFileName); + Util::ensureValidPath(OPT::strTransferTextureFileName); + const String strInputFileNameExt(Util::getFileExt(OPT::strInputFileName).ToLower()); + const bool bInvalidCommand(OPT::strInputFileName.empty() || (OPT::strAlignFileName.empty() && OPT::strTransferTextureFileName.empty() && !OPT::bComputeVolume)); + if (OPT::vm.count("help") || bInvalidCommand) { + boost::program_options::options_description visible("Available options"); + visible.add(generic).add(config); + GET_LOG() << visible; + } + if (bInvalidCommand) + return false; + + // initialize optional options + Util::ensureValidPath(OPT::strOutputFileName); + Util::ensureUnifySlash(OPT::strOutputFileName); + if (OPT::strOutputFileName.IsEmpty()) + OPT::strOutputFileName = Util::getFileName(OPT::strInputFileName) + "_transformed" MVS_EXT; + + // initialize global options + Process::setCurrentProcessPriority((Process::Priority)OPT::nProcessPriority); + #ifdef _USE_OPENMP + if (OPT::nMaxThreads != 0) + omp_set_num_threads(OPT::nMaxThreads); + #endif + + #ifdef _USE_BREAKPAD + // start memory dumper + MiniDumper::Create(APPNAME, WORKING_FOLDER); + #endif + return true; +} + +// finalize application instance +void Finalize() +{ + #if TD_VERBOSE != TD_VERBOSE_OFF + // print memory statistics + Util::LogMemoryInfo(); + #endif + + CLOSE_LOGFILE(); + CLOSE_LOGCONSOLE(); + CLOSE_LOG(); +} + +} // unnamed namespace + +int main(int argc, LPCTSTR* argv) +{ + #ifdef _DEBUGINFO + // set _crtBreakAlloc index to stop in at allocation + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);// | _CRTDBG_CHECK_ALWAYS_DF); + #endif + + if (!Initialize(argc, argv)) + return EXIT_FAILURE; + + TD_TIMER_START(); + + Scene scene(OPT::nMaxThreads); + + // load given scene + if (!scene.Load(MAKE_PATH_SAFE(OPT::strInputFileName), !OPT::strTransferTextureFileName.empty() || OPT::bComputeVolume)) + return EXIT_FAILURE; + + if (!OPT::strAlignFileName.empty()) { + // transform this scene such that it best aligns with the given scene based on the camera positions + Scene sceneRef(OPT::nMaxThreads); + if (!sceneRef.Load(MAKE_PATH_SAFE(OPT::strAlignFileName))) + return EXIT_FAILURE; + if (!scene.AlignTo(sceneRef)) + return EXIT_FAILURE; + VERBOSE("Scene aligned to the given reference scene (%s)", TD_TIMER_GET_FMT().c_str()); + } + + if (!OPT::strTransferTextureFileName.empty()) { + // transfer the texture of the scene's mesh to the new mesh; + // the two meshes should be aligned and the new mesh to have UV-coordinates + Mesh newMesh; + if (!newMesh.Load(MAKE_PATH_SAFE(OPT::strTransferTextureFileName))) + return EXIT_FAILURE; + if (!scene.mesh.TransferTexture(newMesh)) + return EXIT_FAILURE; + newMesh.Save(Util::getFileFullName(MAKE_PATH_SAFE(OPT::strOutputFileName)) + _T(".ply")); + VERBOSE("Texture transfered (%s)", TD_TIMER_GET_FMT().c_str()); + return EXIT_SUCCESS; + } + + if (OPT::bComputeVolume && !scene.mesh.IsEmpty()) { + // compute the mesh volume + const REAL volume(scene.ComputeLeveledVolume(OPT::fPlaneThreshold, OPT::fSampleMesh, OPT::nUpAxis)); + VERBOSE("Mesh volume: %g (%s)", volume, TD_TIMER_GET_FMT().c_str()); + scene.mesh.Save(Util::getFileFullName(MAKE_PATH_SAFE(OPT::strOutputFileName)) + _T(".ply")); + if (scene.images.empty()) + return EXIT_SUCCESS; + OPT::nArchiveType = ARCHIVE_DEFAULT; + } + + // write transformed scene + scene.Save(MAKE_PATH_SAFE(OPT::strOutputFileName), (ARCHIVE_TYPE)OPT::nArchiveType); + + Finalize(); + return EXIT_SUCCESS; +} +/*----------------------------------------------------------------*/ diff --git a/apps/Viewer/Scene.cpp b/apps/Viewer/Scene.cpp index 0ccbb02c6..1ea0431a8 100644 --- a/apps/Viewer/Scene.cpp +++ b/apps/Viewer/Scene.cpp @@ -237,31 +237,37 @@ bool Scene::Open(LPCTSTR fileName, LPCTSTR meshFileName) // load given mesh scene.mesh.Load(meshFileName); } - if (scene.IsEmpty()) - return false; + if (!scene.pointcloud.IsEmpty()) + scene.pointcloud.PrintStatistics(scene.images.data(), &scene.obb); #if 1 // create octree structure used to accelerate selection functionality - events.AddEvent(new EVTComputeOctree(this)); + if (!scene.IsEmpty()) + events.AddEvent(new EVTComputeOctree(this)); #endif // init scene AABB3d bounds(true); - AABB3d imageBounds(true); Point3d center(Point3d::INF); - if (!scene.pointcloud.IsEmpty()) { - bounds = scene.pointcloud.GetAABB(MINF(3u,scene.nCalibratedImages)); - if (bounds.IsEmpty()) - bounds = scene.pointcloud.GetAABB(); - center = scene.pointcloud.GetCenter(); - } - if (!scene.mesh.IsEmpty()) { - scene.mesh.ComputeNormalFaces(); - bounds.Insert(scene.mesh.GetAABB()); - center = scene.mesh.GetCenter(); + if (scene.IsBounded()) { + bounds = AABB3d(scene.obb.GetAABB()); + center = bounds.GetCenter(); + } else { + if (!scene.pointcloud.IsEmpty()) { + bounds = scene.pointcloud.GetAABB(MINF(3u,scene.nCalibratedImages)); + if (bounds.IsEmpty()) + bounds = scene.pointcloud.GetAABB(); + center = scene.pointcloud.GetCenter(); + } + if (!scene.mesh.IsEmpty()) { + scene.mesh.ComputeNormalFaces(); + bounds.Insert(scene.mesh.GetAABB()); + center = scene.mesh.GetCenter(); + } } // init images + AABB3d imageBounds(true); images.Reserve(scene.images.size()); FOREACH(idxImage, scene.images) { const MVS::Image& imageData = scene.images[idxImage]; @@ -270,6 +276,10 @@ bool Scene::Open(LPCTSTR fileName, LPCTSTR meshFileName) images.emplace_back(idxImage); imageBounds.InsertFull(imageData.camera.C); } + if (imageBounds.IsEmpty()) + imageBounds.Enlarge(0.5); + if (bounds.IsEmpty()) + bounds = imageBounds; // init and load texture if (scene.mesh.HasTexture()) { @@ -309,7 +319,7 @@ bool Scene::Open(LPCTSTR fileName, LPCTSTR meshFileName) window.clbkTogleSceneBox = DELEGATEBINDCLASS(Window::ClbkTogleSceneBox, &Scene::TogleSceneBox, this); if (scene.IsBounded()) window.clbkCompileBounds = DELEGATEBINDCLASS(Window::ClbkCompileBounds, &Scene::CompileBounds, this); - if (!bounds.IsEmpty()) + if (!scene.IsEmpty()) window.clbkRayScene = DELEGATEBINDCLASS(Window::ClbkRayScene, &Scene::CastRay, this); window.Reset(!scene.pointcloud.IsEmpty()&&!scene.mesh.IsEmpty()?Window::SPR_NONE:Window::SPR_ALL, MINF(2u,images.size())); @@ -353,7 +363,7 @@ bool Scene::Save(LPCTSTR _fileName, bool bRescaleImages) } // export the scene -bool Scene::Export(LPCTSTR _fileName, LPCTSTR exportType, bool losslessTexture) const +bool Scene::Export(LPCTSTR _fileName, LPCTSTR exportType) const { if (!IsOpen()) return false; @@ -362,7 +372,7 @@ bool Scene::Export(LPCTSTR _fileName, LPCTSTR exportType, bool losslessTexture) const String fileName(_fileName != NULL ? String(_fileName) : sceneName); const String baseFileName(Util::getFileFullName(fileName)); const bool bPoints(scene.pointcloud.Save(lastFileName=(baseFileName+_T("_pointcloud.ply")))); - const bool bMesh(scene.mesh.Save(lastFileName=(baseFileName+_T("_mesh")+(exportType?exportType:(Util::getFileExt(fileName)==_T(".obj")?_T(".obj"):_T(".ply")))), true, losslessTexture)); + const bool bMesh(scene.mesh.Save(lastFileName=(baseFileName+_T("_mesh")+(exportType?exportType:(Util::getFileExt(fileName)==_T(".obj")?_T(".obj"):_T(".ply")))), cList(), true)); #if TD_VERBOSE != TD_VERBOSE_OFF if (VERBOSITY_LEVEL > 2 && (bPoints || bMesh)) scene.ExportCamerasMLP(Util::getFileFullName(lastFileName)+_T(".mlp"), lastFileName); @@ -422,6 +432,12 @@ void Scene::CompileMesh() if (scene.mesh.IsEmpty()) return; ReleaseMesh(); + if (scene.mesh.faceNormals.empty()) + scene.mesh.ComputeNormalFaces(); + // translate, normalize and flip Y axis of the texture coordinates + MVS::Mesh::TexCoordArr normFaceTexcoords; + if (scene.mesh.HasTexture()) + scene.mesh.FaceTexcoordsNormalize(normFaceTexcoords, true); listMesh = glGenLists(1); glNewList(listMesh, GL_COMPILE); // compile mesh @@ -435,8 +451,8 @@ void Scene::CompileMesh() const MVS::Mesh::Normal& n = scene.mesh.faceNormals[i]; glNormal3fv(n.ptr()); for (int j = 0; j < 3; ++j) { - if (!scene.mesh.faceTexcoords.IsEmpty() && window.bRenderTexture) { - const MVS::Mesh::TexCoord& t = scene.mesh.faceTexcoords[i * 3 + j]; + if (!normFaceTexcoords.empty() && window.bRenderTexture) { + const MVS::Mesh::TexCoord& t = normFaceTexcoords[i * 3 + j]; glTexCoord2fv(t.ptr()); } const MVS::Mesh::Vertex& p = scene.mesh.vertices[face[j]]; @@ -484,9 +500,9 @@ void Scene::Draw() if (listMesh) { glEnable(GL_DEPTH_TEST); glEnable(GL_CULL_FACE); - if (!scene.mesh.faceTexcoords.IsEmpty() && window.bRenderTexture) { + if (!scene.mesh.faceTexcoords.empty() && window.bRenderTexture) { glEnable(GL_TEXTURE_2D); - textures.First().Bind(); + textures.front().Bind(); glCallList(listMesh); glDisable(GL_TEXTURE_2D); } else { @@ -503,18 +519,8 @@ void Scene::Draw() Image& image = images[idx]; const MVS::Image& imageData = scene.images[image.idx]; const MVS::Camera& camera = imageData.camera; - // change coordinates system to the camera space - glPushMatrix(); - glMultMatrixd((GLdouble*)TransL2W((const Matrix3x3::EMat)camera.R, -(const Point3::EVec)camera.C).data()); - glPointSize(window.pointSize+1.f); - glDisable(GL_TEXTURE_2D); - // draw camera position and image center - const double scaleFocal(window.camera.scaleF); - glBegin(GL_POINTS); - glColor3f(1,0,0); glVertex3f(0,0,0); // camera position - glColor3f(0,1,0); glVertex3f(0,0,(float)scaleFocal); // image center - glEnd(); // cache image corner coordinates + const double scaleFocal(window.camera.scaleF); const Point2d pp(camera.GetPrincipalPoint()); const double focal(camera.GetFocalLength()/scaleFocal); const double cx(-pp.x/focal); @@ -525,6 +531,17 @@ void Scene::Draw() const Point3d ic2(cx, py, scaleFocal); const Point3d ic3(px, py, scaleFocal); const Point3d ic4(px, cy, scaleFocal); + // change coordinates system to the camera space + glPushMatrix(); + glMultMatrixd((GLdouble*)TransL2W((const Matrix3x3::EMat)camera.R, -(const Point3::EVec)camera.C).data()); + glPointSize(window.pointSize+1.f); + glDisable(GL_TEXTURE_2D); + // draw camera position and image center + glBegin(GL_POINTS); + glColor3f(1,0,0); glVertex3f(0,0,0); // camera position + glColor3f(0,1,0); glVertex3f(0,0,(float)scaleFocal); // image center + glColor3f(0,0,1); glVertex3d((0.5*imageData.width-pp.x)/focal, cy, scaleFocal); // image up + glEnd(); // draw image thumbnail const bool bSelectedImage(idx == window.camera.currentCamID); if (bSelectedImage) { @@ -678,10 +695,10 @@ void Scene::TogleSceneBox() }; if (scene.IsBounded()) scene.obb = OBB3f(true); - else if (!scene.pointcloud.IsEmpty()) - scene.obb.Set(EnlargeAABB(scene.pointcloud.GetAABB(window.minViews))); else if (!scene.mesh.IsEmpty()) scene.obb.Set(EnlargeAABB(scene.mesh.GetAABB())); + else if (!scene.pointcloud.IsEmpty()) + scene.obb.Set(EnlargeAABB(scene.pointcloud.GetAABB(window.minViews))); CompileBounds(); } @@ -717,7 +734,7 @@ void Scene::CastRay(const Ray3& ray, int action) window.selectionPoints[0] = scene.mesh.vertices[face[0]]; window.selectionPoints[1] = scene.mesh.vertices[face[1]]; window.selectionPoints[2] = scene.mesh.vertices[face[2]]; - window.selectionPoints[3] = (ray.m_pOrig + ray.m_vDir*intRay.pick.dist).cast(); + window.selectionPoints[3] = ray.GetPoint(intRay.pick.dist).cast(); window.selectionType = Window::SEL_TRIANGLE; window.selectionTime = now; window.selectionIdx = intRay.pick.idx; diff --git a/apps/Viewer/Scene.h b/apps/Viewer/Scene.h index 5997e3011..d1e360833 100644 --- a/apps/Viewer/Scene.h +++ b/apps/Viewer/Scene.h @@ -87,7 +87,7 @@ class Scene bool Init(const cv::Size&, LPCTSTR windowName, LPCTSTR fileName=NULL, LPCTSTR meshFileName=NULL); bool Open(LPCTSTR fileName, LPCTSTR meshFileName=NULL); bool Save(LPCTSTR fileName=NULL, bool bRescaleImages=false); - bool Export(LPCTSTR fileName, LPCTSTR exportType=NULL, bool losslessTexture=false) const; + bool Export(LPCTSTR fileName, LPCTSTR exportType=NULL) const; void CompilePointCloud(); void CompileMesh(); void CompileBounds(); diff --git a/apps/Viewer/Viewer.cpp b/apps/Viewer/Viewer.cpp index fe918998a..27aa73d93 100644 --- a/apps/Viewer/Viewer.cpp +++ b/apps/Viewer/Viewer.cpp @@ -50,7 +50,6 @@ namespace OPT { String strInputFileName; String strOutputFileName; String strMeshFileName; -bool bLosslessTexture; unsigned nArchiveType; int nProcessPriority; unsigned nMaxThreads; @@ -98,7 +97,6 @@ bool Initialize(size_t argc, LPCTSTR* argv) config.add_options() ("input-file,i", boost::program_options::value(&OPT::strInputFileName), "input project filename containing camera poses and scene (point-cloud/mesh)") ("output-file,o", boost::program_options::value(&OPT::strOutputFileName), "output filename for storing the mesh") - ("texture-lossless", boost::program_options::value(&OPT::bLosslessTexture)->default_value(false), "export texture using a lossless image format") ; // hidden options, allowed both on command line and @@ -230,7 +228,7 @@ int main(int argc, LPCTSTR* argv) return EXIT_FAILURE; if (viewer.IsOpen() && !OPT::strOutputFileName.IsEmpty()) { // export the scene - viewer.Export(MAKE_PATH_SAFE(OPT::strOutputFileName), OPT::strExportType.IsEmpty()?LPCTSTR(NULL):OPT::strExportType.c_str(), OPT::bLosslessTexture); + viewer.Export(MAKE_PATH_SAFE(OPT::strOutputFileName), OPT::strExportType.IsEmpty()?LPCTSTR(NULL):OPT::strExportType.c_str()); } // enter viewer loop viewer.Loop(); diff --git a/apps/Viewer/Window.cpp b/apps/Viewer/Window.cpp index 66ce6c669..63bbe1a19 100644 --- a/apps/Viewer/Window.cpp +++ b/apps/Viewer/Window.cpp @@ -291,7 +291,7 @@ void Window::Key(int k, int /*scancode*/, int action, int mod) break; case GLFW_KEY_E: if (action == GLFW_RELEASE && clbkExportScene != NULL) - clbkExportScene(NULL, NULL, false); + clbkExportScene(NULL, NULL); break; case GLFW_KEY_P: switch (sparseType) { diff --git a/apps/Viewer/Window.h b/apps/Viewer/Window.h index 4670ed12b..34195ae31 100644 --- a/apps/Viewer/Window.h +++ b/apps/Viewer/Window.h @@ -96,7 +96,7 @@ class Window ClbkOpenScene clbkOpenScene; typedef DELEGATE ClbkSaveScene; ClbkSaveScene clbkSaveScene; - typedef DELEGATE ClbkExportScene; + typedef DELEGATE ClbkExportScene; ClbkExportScene clbkExportScene; typedef DELEGATE ClbkCenterScene; ClbkCenterScene clbkCenterScene; diff --git a/build/Utils.cmake b/build/Utils.cmake index 2bfbc9822..0cb2d8e1a 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -451,7 +451,6 @@ macro(optimize_default_compiler_settings) add_extra_compiler_option(-Werror=sequence-point) add_extra_compiler_option(-Wformat) add_extra_compiler_option(-Werror=format-security -Wformat) - add_extra_compiler_option(-Wstrict-prototypes) add_extra_compiler_option(-Winit-self) add_extra_compiler_option(-Wsign-promo) add_extra_compiler_option(-Wreorder) diff --git a/docker/Dockerfile b/docker/Dockerfile index a528c34f8..8fb77b421 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,16 +1,12 @@ -FROM ubuntu:18.04 +FROM ubuntu:22.04 +ARG MASTER ARG USER_ID ARG GROUP_ID -# Initialize the environment -RUN apt update -RUN apt install -y cmake git vim - # Prepare and empty machine for building: RUN apt-get update -yq -RUN apt-get install -yq -RUN apt-get -y install git mercurial cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev +RUN apt-get -yq install build-essential git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev # Eigen RUN git clone https://gitlab.com/libeigen/eigen --branch 3.4 @@ -27,16 +23,13 @@ RUN apt-get -y install libboost-iostreams-dev libboost-program-options-dev libbo RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq libopencv-dev # CGAL -RUN apt-get -y install libcgal-dev libcgal-qt5-dev +RUN apt-get -yq install libcgal-dev libcgal-qt5-dev # VCGLib RUN git clone https://github.com/cdcseacave/VCG.git vcglib -# Build from stable openMVS release -RUN git clone https://github.com/cdcseacave/openMVS.git --branch master - -# Uncomment below (and comment above) to use the latest commit from the develop branch -#RUN git clone https://github.com/cdcseacave/openMVS.git --branch develop +# Build from stable openMVS release or the latest commit from the develop branch +RUN if [[ -n "$MASTER" ]] ; then git clone https://github.com/cdcseacave/openMVS.git --branch master ; else git clone https://github.com/cdcseacave/openMVS.git --branch develop ; fi RUN mkdir openMVS_build RUN cd openMVS_build &&\ @@ -48,8 +41,8 @@ RUN cd openMVS_build &&\ make install # Set permissions such that the output files can be accessed by the current user (optional) -RUN addgroup --gid $GROUP_ID user -RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user +RUN addgroup --gid $GROUP_ID user &&\ + adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user USER user # Add binaries to path diff --git a/docker/Dockerfile_CUDA b/docker/Dockerfile_CUDA index 86f4e8748..46170d2ef 100644 --- a/docker/Dockerfile_CUDA +++ b/docker/Dockerfile_CUDA @@ -1,18 +1,14 @@ -FROM nvidia/cuda:11.1-devel-ubuntu18.04 +FROM nvidia/cuda:11.7.1-devel-ubuntu22.04 +ARG MASTER ARG USER_ID ARG GROUP_ID -# Initialize the environment -RUN apt update -RUN apt install -y cmake git vim - # Prepare and empty machine for building: RUN apt-get update -yq -RUN apt-get install -yq -RUN apt-get -y install git mercurial cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev +RUN apt-get -yq install build-essential git cmake libpng-dev libjpeg-dev libtiff-dev libglu1-mesa-dev -ENV PATH=/usr/local/cuda-11.1/bin:$PATH +ENV PATH=/usr/local/cuda/bin:$PATH # Eigen RUN git clone https://gitlab.com/libeigen/eigen --branch 3.4 @@ -23,26 +19,23 @@ RUN cd eigen_build &&\ cd .. # Boost -RUN apt-get -y install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev +RUN apt-get -yq install libboost-iostreams-dev libboost-program-options-dev libboost-system-dev libboost-serialization-dev # OpenCV -RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq libopencv-dev +RUN DEBIAN_FRONTEND=noninteractive apt-get -yq install libopencv-dev # CGAL -RUN apt-get -y install libcgal-dev libcgal-qt5-dev +RUN apt-get -yq install libcgal-dev libcgal-qt5-dev # VCGLib RUN git clone https://github.com/cdcseacave/VCG.git vcglib -# Build from stable openMVS release -RUN git clone https://github.com/cdcseacave/openMVS.git --branch master - -# Uncomment below (and comment above) to use the latest commit from the develop branch -#RUN git clone https://github.com/cdcseacave/openMVS.git --branch develop +# Build from stable openMVS release or the latest commit from the develop branch +RUN if [[ -n "$MASTER" ]] ; then git clone https://github.com/cdcseacave/openMVS.git --branch master ; else git clone https://github.com/cdcseacave/openMVS.git --branch develop ; fi RUN mkdir openMVS_build RUN cd openMVS_build &&\ - cmake . ../openMVS -DCMAKE_BUILD_TYPE=Release -DVCG_ROOT=/vcglib -DOpenMVS_USE_CUDA=ON -DCMAKE_LIBRARY_PATH=/usr/local/cuda/lib64/stubs/ -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-11.1/ -DCUDA_INCLUDE_DIRS=/usr/local/cuda-11.1/include/ -DCUDA_CUDART_LIBRARY=/usr/local/cuda-11.1/lib64 -DCUDA_NVCC_EXECUTABLE=/usr/local/cuda-11.1/bin/ + cmake . ../openMVS -DCMAKE_BUILD_TYPE=Release -DVCG_ROOT=/vcglib -DOpenMVS_USE_CUDA=ON -DCMAKE_LIBRARY_PATH=/usr/local/cuda/lib64/stubs/ -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda/ -DCUDA_INCLUDE_DIRS=/usr/local/cuda/include/ -DCUDA_CUDART_LIBRARY=/usr/local/cuda/lib64 -DCUDA_NVCC_EXECUTABLE=/usr/local/cuda/bin/ # Install OpenMVS library RUN cd openMVS_build &&\ @@ -50,8 +43,8 @@ RUN cd openMVS_build &&\ make install # Set permissions such that the output files can be accessed by the current user (optional) -RUN addgroup --gid $GROUP_ID user -RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user +RUN addgroup --gid $GROUP_ID user &&\ + adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user USER user # Add binaries to path diff --git a/docker/buildFromScratch.sh b/docker/buildFromScratch.sh index d522ce5d6..c3b873d74 100755 --- a/docker/buildFromScratch.sh +++ b/docker/buildFromScratch.sh @@ -1,2 +1,2 @@ docker build --no-cache -t="openmvs-ubuntu" --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) .; -docker run -w /working -v $1:/working -it openmvs-ubuntu; \ No newline at end of file +docker run --entrypoint bash -w /work -v $1:/work -it openmvs-ubuntu; diff --git a/docker/buildFromScratch_CUDA.sh b/docker/buildFromScratch_CUDA.sh new file mode 100644 index 000000000..eb9149fd5 --- /dev/null +++ b/docker/buildFromScratch_CUDA.sh @@ -0,0 +1,2 @@ +docker build --no-cache -t="openmvs-ubuntu" --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) --file Dockerfile_CUDA .; +docker run --gpus=all --entrypoint bash -w /work -v $1:/work -it openmvs-ubuntu; diff --git a/libs/Common/AABB.h b/libs/Common/AABB.h index 25cba2d3d..da291f321 100644 --- a/libs/Common/AABB.h +++ b/libs/Common/AABB.h @@ -62,6 +62,7 @@ class TAABB void InsertFull(const POINT&); void Insert(const POINT&); void Insert(const TAABB&); + void BoundBy(const TAABB&); inline void Translate(const POINT&); inline void Transform(const MATRIX&); diff --git a/libs/Common/AABB.inl b/libs/Common/AABB.inl index d10543e38..b645ef12d 100644 --- a/libs/Common/AABB.inl +++ b/libs/Common/AABB.inl @@ -91,7 +91,7 @@ template inline bool TAABB::IsEmpty() const { for (int i=0; i ptMax[i]) + if (ptMin[i] >= ptMax[i]) return true; return false; } // IsEmpty @@ -108,8 +108,9 @@ inline TAABB& TAABB::Enlarge(TYPE x) template inline TAABB& TAABB::EnlargePercent(TYPE x) { - ptMin *= x; - ptMax *= x; + const POINT ptSizeDelta(GetSize() * (x - TYPE(1)) / TYPE(2)); + ptMin -= ptSizeDelta; + ptMax += ptSizeDelta; return *this; } // Enlarge /*----------------------------------------------------------------*/ @@ -242,7 +243,7 @@ void TAABB::Insert(const POINT& pt) } /*----------------------------------------------------------------*/ -// Update the box such that it contains the given point. +// Update the box such that it contains the given bounding box. template void TAABB::Insert(const TAABB& aabb) { @@ -267,6 +268,31 @@ void TAABB::Insert(const TAABB& aabb) } /*----------------------------------------------------------------*/ +// Update the box such that it does not exceed the given bounding box. +template +void TAABB::BoundBy(const TAABB& aabb) +{ + if (ptMin[0] < aabb.ptMin[0]) + ptMin[0] = aabb.ptMin[0]; + if (ptMax[0] > aabb.ptMax[0]) + ptMax[0] = aabb.ptMax[0]; + + if (DIMS > 1) { + if (ptMin[1] < aabb.ptMin[1]) + ptMin[1] = aabb.ptMin[1]; + if (ptMax[1] > aabb.ptMax[1]) + ptMax[1] = aabb.ptMax[1]; + } + + if (DIMS > 2) { + if (ptMin[2] < aabb.ptMin[2]) + ptMin[2] = aabb.ptMin[2]; + if (ptMax[2] > aabb.ptMax[2]) + ptMax[2] = aabb.ptMax[2]; + } +} +/*----------------------------------------------------------------*/ + // intersection between two AABBs template diff --git a/libs/Common/AutoEstimator.h b/libs/Common/AutoEstimator.h index 1cdb6bb1c..25d0cd487 100644 --- a/libs/Common/AutoEstimator.h +++ b/libs/Common/AutoEstimator.h @@ -88,7 +88,7 @@ struct ErrorIndex { #ifdef ACRANSAC_STD_VECTOR typedef std::vector ErrorIndexArr; #else -typedef SEACAVE::cList ErrorIndexArr; +typedef CLISTDEF0(ErrorIndex) ErrorIndexArr; #endif /// Find best NFA and its index wrt square error threshold in e. diff --git a/libs/Common/Config.h b/libs/Common/Config.h index b18df786a..cc37aab23 100644 --- a/libs/Common/Config.h +++ b/libs/Common/Config.h @@ -142,9 +142,14 @@ #if __cplusplus >= 201703L || __clang_major__ >= 5 #define _SUPPORT_CPP17 #endif +#if __cplusplus >= 202002L || __clang_major__ >= 10 +#define _SUPPORT_CPP20 +#endif -#if defined(__arm__) || defined (__arm64__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARMT) +#if defined(__powerpc__) +#define _PLATFORM_PPC 1 +#elif defined(__arm__) || defined (__arm64__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARMT) #define _PLATFORM_ARM 1 #else #define _PLATFORM_X86 1 @@ -166,7 +171,7 @@ # define RESTRICT __restrict //applied to a function parameter # define MEMALLOC __declspec(noalias) __declspec(restrict) # define DEPRECATED __declspec(deprecated) -# define NOWARNUNUSED +# define MAYBEUNUSED # define HOT # define COLD # define THREADLOCAL __declspec(thread) @@ -179,7 +184,7 @@ # define RESTRICT __restrict__ # define MEMALLOC __attribute__ ((__malloc__)) # define DEPRECATED __attribute__ ((__deprecated__)) -# define NOWARNUNUSED __attribute__ ((unused)) +# define MAYBEUNUSED __attribute__ ((unused)) # define HOT __attribute__((hot)) __attribute__((optimize("-O3"))) __attribute__((optimize("-ffast-math"))) //optimize for speed, even in debug # define COLD __attribute__((cold)) //optimize for size # define THREADLOCAL __thread @@ -192,7 +197,7 @@ # define RESTRICT # define MEMALLOC # define DEPRECATED -# define NOWARNUNUSED +# define MAYBEUNUSED # define HOT # define COLD # define THREADLOCAL __thread @@ -202,6 +207,10 @@ #ifndef _SUPPORT_CPP11 # define constexpr inline #endif +#ifdef _SUPPORT_CPP17 +# undef MAYBEUNUSED +# define MAYBEUNUSED [[maybe_unused]] +#endif #define SAFE_DELETE(p) { if (p!=NULL) { delete (p); (p)=NULL; } } #define SAFE_DELETE_ARR(p) { if (p!=NULL) { delete [] (p); (p)=NULL; } } @@ -258,13 +267,13 @@ namespace SEACAVE_ASSERT } #define STATIC_ASSERT(expression) \ - NOWARNUNUSED typedef char CTA##__LINE__[::SEACAVE_ASSERT::compile_time_assert<(bool)(expression)>::value] + MAYBEUNUSED typedef char CTA##__LINE__[::SEACAVE_ASSERT::compile_time_assert<(bool)(expression)>::value] #define ASSERT_ARE_SAME_TYPE(type1, type2) \ - NOWARNUNUSED typedef char AAST##__LINE__[::SEACAVE_ASSERT::assert_are_same_type::value] + MAYBEUNUSED typedef char AAST##__LINE__[::SEACAVE_ASSERT::assert_are_same_type::value] #define ASSERT_ARE_NOT_SAME_TYPE(type1, type2) \ - NOWARNUNUSED typedef char AANST##__LINE__[::SEACAVE_ASSERT::assert_are_not_same_type::value] + MAYBEUNUSED typedef char AANST##__LINE__[::SEACAVE_ASSERT::assert_are_not_same_type::value] /*----------------------------------------------------------------*/ #endif // __SEACAVE_CONFIG_H__ diff --git a/libs/Common/List.h b/libs/Common/List.h index 750aa90c4..2404c4832 100644 --- a/libs/Common/List.h +++ b/libs/Common/List.h @@ -142,6 +142,13 @@ class cList _vector = (TYPE*)(operator new[] (_vectorSize * sizeof(TYPE))); _ArrayCopyConstruct(_vector, rList._vector, _size); } + #ifdef _SUPPORT_CPP11 + // copy constructor: creates a move-copy of the given list + cList(cList&& rList) : _size(rList._size), _vectorSize(rList._vectorSize), _vector(rList._vector) + { + rList._Init(); + } + #endif // constructor a list from a raw data array explicit inline cList(TYPE* pDataBegin, TYPE* pDataEnd) : _size((IDX)(pDataEnd-pDataBegin)), _vectorSize(_size) @@ -624,13 +631,22 @@ class cList inline TYPE& GetNth(IDX index) { + ASSERT(index < _size); TYPE* const nth(Begin()+index); std::nth_element(Begin(), nth, End()); return *nth; } - inline TYPE& GetMedian() + template ::value,TYPE,REAL>::type> + inline RTYPE GetMedian() { - return GetNth(_size >> 1); + ASSERT(_size > 0); + if (_size%2) + return static_cast(GetNth(_size >> 1)); + TYPE* const nth(Begin() + (_size>>1)); + std::nth_element(Begin(), nth, End()); + TYPE* const nth1(nth-1); + std::nth_element(Begin(), nth1, nth); + return (static_cast(*nth1) + static_cast(*nth)) / RTYPE(2); } inline TYPE GetMean() @@ -1408,106 +1424,93 @@ inline bool ValidIDX(const IDX_TYPE& idx) { // some test functions -// run cListTest(99999); -#if 0 -inline bool cListTestIter(unsigned elems) { - std::vector arrR; - cList arr0; - cList arr1; - cList arr2; - cList arrC; - for (unsigned i=0; i +inline bool cListTest(unsigned iters) { + for (unsigned i=0; i arrR; + cList arr0; + cList arr1; + cList arr2; + cList arrC; + for (unsigned i=0; i= arrR.size()) - continue; - arrR.erase(arrR.begin()+nDel, arrR.begin()+nDel+nCount); - arr0.RemoveAtMove(nDel, nCount); - arr1.RemoveAtMove(nDel, nCount); - arr2.RemoveAtMove(nDel, nCount); - } - if (arrR.size() != arr0.GetSize() || - arrR.size() != arr1.GetSize() || - arrR.size() != arr2.GetSize()) - { - ASSERT("there is a problem" == NULL); - return false; - } - for (size_t i=0; i arrS(1+RAND()%(2*elems)); - for (size_t i=0; i<6; ++i) { - arrS.Insert(RAND()); - } - arrS.RemoveLast(RAND()%arrS.GetSize()); - arrS.CopyOf(&arrR[0], arrR.size()); - for (size_t i=0; i<6; ++i) { - arrS.Insert(RAND()); - } - arrS.RemoveLast(6); - for (size_t i=0; i= arrR.size()) + continue; + arrR.erase(arrR.begin()+nDel, arrR.begin()+nDel+nCount); + arr0.RemoveAtMove(nDel, nCount); + arr1.RemoveAtMove(nDel, nCount); + arr2.RemoveAtMove(nDel, nCount); + } + if (arrR.size() != arr0.GetSize() || + arrR.size() != arr1.GetSize() || + arrR.size() != arr2.GetSize()) { ASSERT("there is a problem" == NULL); return false; } + for (size_t i=0; i arrS(1+RAND()%(2*elems)); + for (size_t i=0; i<6; ++i) { + arrS.Insert(RAND()); + } + arrS.RemoveLast(RAND()%arrS.GetSize()); + arrS.CopyOf(&arrR[0], arrR.size()); + for (size_t i=0; i<6; ++i) { + arrS.Insert(RAND()); + } + arrS.RemoveLast(6); + for (size_t i=0; i + inline TOBB(const TOBB&); inline void Set(const AABB&); // build from AABB inline void Set(const MATRIX& rot, const POINT& ptMin, const POINT& ptMax); // build from rotation matrix from world to local, and local min/max corners @@ -69,8 +71,8 @@ class TOBB inline bool IsValid() const; - inline void Enlarge(TYPE); - inline void EnlargePercent(TYPE); + inline TOBB& Enlarge(TYPE); + inline TOBB& EnlargePercent(TYPE); inline void Translate(const POINT&); inline void Transform(const MATRIX&); diff --git a/libs/Common/OBB.inl b/libs/Common/OBB.inl index 9f1278b3c..1e2784d17 100644 --- a/libs/Common/OBB.inl +++ b/libs/Common/OBB.inl @@ -39,6 +39,15 @@ inline TOBB::TOBB(const POINT* pts, size_t n, const TRIANGLE* tris, s { Set(pts, n, tris, s); } // constructor +template +template +inline TOBB::TOBB(const TOBB& rhs) + : + m_rot(rhs.m_rot.template cast()), + m_pos(rhs.m_pos.template cast()), + m_ext(rhs.m_ext.template cast()) +{ +} // copy constructor /*----------------------------------------------------------------*/ @@ -268,14 +277,16 @@ inline bool TOBB::IsValid() const template -inline void TOBB::Enlarge(TYPE x) +inline TOBB& TOBB::Enlarge(TYPE x) { - m_ext.array() -= x; + m_ext.array() += x; + return *this; } template -inline void TOBB::EnlargePercent(TYPE x) +inline TOBB& TOBB::EnlargePercent(TYPE x) { m_ext *= x; + return *this; } // Enlarge /*----------------------------------------------------------------*/ diff --git a/libs/Common/Octree.h b/libs/Common/Octree.h index 70acd14c6..aabbd336d 100644 --- a/libs/Common/Octree.h +++ b/libs/Common/Octree.h @@ -57,7 +57,6 @@ class TOctree SIZE_TYPE size; // number of items contained by this cell in the global array DATA_TYPE data; // user data associated with this leaf } LEAF_TYPE; - enum { dataSize = (sizeof(NODE_TYPE)>sizeof(LEAF_TYPE) ? sizeof(NODE_TYPE) : sizeof(LEAF_TYPE)) }; enum { numChildren = (2<<(DIMS-1)) }; public: @@ -73,10 +72,10 @@ class TOctree inline bool IsLeaf() const { return (m_child==NULL); } inline const CELL_TYPE& GetChild(int i) const { ASSERT(!IsLeaf() && i(m_data); } - inline NODE_TYPE& Node() { ASSERT(!IsLeaf()); return *reinterpret_cast(m_data); } - inline const LEAF_TYPE& Leaf() const { ASSERT(IsLeaf()); return *reinterpret_cast(m_data); } - inline LEAF_TYPE& Leaf() { ASSERT(IsLeaf()); return *reinterpret_cast(m_data); } + inline const NODE_TYPE& Node() const { ASSERT(!IsLeaf()); return m_node; } + inline NODE_TYPE& Node() { ASSERT(!IsLeaf()); return m_node; } + inline const LEAF_TYPE& Leaf() const { ASSERT(IsLeaf()); return m_leaf; } + inline LEAF_TYPE& Leaf() { ASSERT(IsLeaf()); return m_leaf; } inline const POINT_TYPE& GetCenter() const { return Node().center; } inline AABB_TYPE GetAabb(TYPE radius) const { return AABB_TYPE(Node().center, radius); } inline AABB_TYPE GetChildAabb(unsigned idxChild, TYPE radius) const { const TYPE childRadius(radius / TYPE(2)); return AABB_TYPE(ComputeChildCenter(Node().center, childRadius, idxChild), childRadius); } @@ -89,7 +88,10 @@ class TOctree public: CELL_TYPE* m_child; // if not a leaf, 2^DIMS child objects - uint8_t m_data[dataSize]; // a LEAF_TYPE or NODE_TYPE object, if it is a leaf or not respectively + union { // a LEAF_TYPE or NODE_TYPE object, if it is a leaf or not respectively + NODE_TYPE m_node; + LEAF_TYPE m_leaf; + }; }; typedef SEACAVE::cList CELLPTRARR_TYPE; @@ -197,7 +199,6 @@ class TOctree CELL_TYPE m_root; // first cell of the tree (always of Node type) TYPE m_radius; // size of the sphere containing all cells -#ifndef _RELEASE public: typedef struct DEBUGINFO_TYPE { size_t memSize; // total memory used @@ -225,7 +226,6 @@ class TOctree protected: void _GetDebugInfo(const CELL_TYPE&, unsigned, DEBUGINFO&) const; -#endif }; // class TOctree /*----------------------------------------------------------------*/ diff --git a/libs/Common/Octree.inl b/libs/Common/Octree.inl index e0e0f298b..e51d1feb9 100644 --- a/libs/Common/Octree.inl +++ b/libs/Common/Octree.inl @@ -41,10 +41,10 @@ template inline void TOctree::CELL_TYPE::Swap(CELL_TYPE& rhs) { std::swap(m_child, rhs.m_child); - uint8_t tmpData[dataSize]; - memcpy(tmpData, m_data, dataSize); - memcpy(m_data, rhs.m_data, dataSize); - memcpy(rhs.m_data, tmpData, dataSize); + if (IsLeaf()) + std::swap(m_leaf, rhs.m_leaf); + else + std::swap(m_node, rhs.m_node); } // Swap /*----------------------------------------------------------------*/ @@ -167,6 +167,7 @@ void TOctree::_Insert(CELL_TYPE& cell, const P // if this child cell needs to be divided further if (bForceSplit || insertData.split(size, radius)) { // init node and proceed recursively + ASSERT(cell.m_child == NULL); cell.m_child = new CELL_TYPE[CELL_TYPE::numChildren]; cell.Node().center = center; struct ChildData { @@ -223,15 +224,14 @@ inline void TOctree::Insert(const ITEMARR_TYPE // create root as node, even if we do not need to divide m_indices.Reserve(items.size()); // divide cell - m_root.m_child = new CELL_TYPE[CELL_TYPE::numChildren]; - m_root.Node().center = aabb.GetCenter(); + const POINT_TYPE center = aabb.GetCenter(); m_radius = aabb.GetSize().maxCoeff()/Type(2); // single connected list of next item indices _InsertData insertData = {items.size(), split}; std::iota(insertData.successors.begin(), insertData.successors.end(), IDX_TYPE(1)); insertData.successors.back() = _InsertData::NO_INDEX; // setup each cell - _Insert(m_root, m_root.GetCenter(), m_radius, 0, items.size(), insertData); + _Insert(m_root, center, m_radius, 0, items.size(), insertData); } template template @@ -365,7 +365,8 @@ inline void TOctree::Collect(INSERTER& inserte template inline void TOctree::Collect(IDXARR_TYPE& indices, const AABB_TYPE& aabb) const { - _Collect(m_root, aabb, IndexInserter(indices)); + IndexInserter inserter(indices); + _Collect(m_root, aabb, inserter); } template @@ -377,7 +378,8 @@ inline void TOctree::Collect(INSERTER& inserte template inline void TOctree::Collect(IDXARR_TYPE& indices, const POINT_TYPE& center, TYPE radius) const { - _Collect(m_root, AABB_TYPE(center, radius), IndexInserter(indices)); + IndexInserter inserter(indices); + _Collect(m_root, AABB_TYPE(center, radius), inserter); } template @@ -390,7 +392,8 @@ template template inline void TOctree::Collect(IDXARR_TYPE& indices, const COLLECTOR& collector) const { - _Collect(m_root, m_radius, collector, IndexInserter(indices)); + IndexInserter inserter(indices); + _Collect(m_root, m_radius, collector, inserter); } template @@ -711,7 +714,8 @@ void TOctree::SplitVolume(float maxArea, AREAE { CELL_TYPE parent; parent.m_child = new CELL_TYPE[1]; - parent.m_child[0] = m_root; + parent.m_child[0].m_child = m_root.m_child; + parent.m_child[0].Node() = m_root.Node(); parent.Node().center = m_root.Node().center + POINT_TYPE::Constant(m_radius); _SplitVolume(parent, m_radius*TYPE(2), 0, maxArea, areaEstimator, chunkInserter); parent.m_child[0].m_child = NULL; @@ -719,7 +723,6 @@ void TOctree::SplitVolume(float maxArea, AREAE /*----------------------------------------------------------------*/ -#ifndef _RELEASE template void TOctree::_GetDebugInfo(const CELL_TYPE& cell, unsigned nDepth, DEBUGINFO& info) const { @@ -774,7 +777,7 @@ inline bool OctreeTest(unsigned iters, unsigned maxItems=1000, bool bRandom=true STATIC_ASSERT(DIMS > 0 && DIMS <= 3); srand(bRandom ? (unsigned)time(NULL) : 0); typedef Eigen::Matrix POINT_TYPE; - typedef SEACAVE::cList TestArr; + typedef CLISTDEF0(POINT_TYPE) TestArr; typedef TOctree TestTree; const TYPE ptMinData[] = {0,0,0}, ptMaxData[] = {640,480,240}; typename TestTree::AABB_TYPE aabb; @@ -793,12 +796,11 @@ inline bool OctreeTest(unsigned iters, unsigned maxItems=1000, bool bRandom=true TestArr items(elems); FOREACH(i, items) for (int j=0; j(RAND()%ROUND2INT(ptMaxData[j])); // random query point POINT_TYPE pt; - pt(0) = RAND()%ROUND2INT(ptMaxData[0]); - if (DIMS > 1) pt(1) = RAND()%ROUND2INT(ptMaxData[1]); - if (DIMS > 2) pt(2) = RAND()%ROUND2INT(ptMaxData[2]); + for (int j=0; j(RAND()%ROUND2INT(ptMaxData[j])); const TYPE radius(TYPE(3+RAND()%30)); // build octree and find interest items TestTree tree(items, aabb, [](typename TestTree::IDX_TYPE size, typename TestTree::Type radius) { @@ -832,8 +834,8 @@ inline bool OctreeTest(unsigned iters, unsigned maxItems=1000, bool bRandom=true } } nTotalMatches += nMatches; - nTotalMissed += trueIndices.size()-nMatches; - nTotalExtra += indices.size()-nMatches; + nTotalMissed += (unsigned)trueIndices.size()-nMatches; + nTotalExtra += (unsigned)indices.size()-nMatches; #ifndef _RELEASE // print stats typename TestTree::DEBUGINFO_TYPE info; @@ -847,4 +849,3 @@ inline bool OctreeTest(unsigned iters, unsigned maxItems=1000, bool bRandom=true #endif return (nTotalMissed == 0 && nTotalExtra == 0); } -#endif diff --git a/libs/Common/Plane.h b/libs/Common/Plane.h index 7cd439207..681d044c9 100644 --- a/libs/Common/Plane.h +++ b/libs/Common/Plane.h @@ -31,6 +31,8 @@ class TPlane typedef Eigen::Matrix POINT; typedef SEACAVE::TAABB AABB; typedef SEACAVE::TRay RAY; + enum { numScalar = DIMS+1 }; + enum { numParams = numScalar-1 }; VECTOR m_vN; // plane normal vector TYPE m_fD; // distance to origin @@ -47,12 +49,19 @@ class TPlane inline void Set(const VECTOR&, const POINT&); inline void Set(const POINT&, const POINT&, const POINT&); inline void Set(const TYPE p[DIMS+1]); + + inline void Invalidate(); + inline bool IsValid() const; + inline void Negate(); + inline TPlane Negated() const; inline TYPE Distance(const TPlane&) const; inline TYPE Distance(const POINT&) const; inline TYPE DistanceAbs(const POINT&) const; + inline POINT ProjectPoint(const POINT&) const; + inline GCLASS Classify(const POINT&) const; inline GCLASS Classify(const AABB&) const; @@ -62,8 +71,8 @@ class TPlane bool Intersects(const TPlane& plane, RAY& ray) const; bool Intersects(const AABB& aabb) const; - inline TYPE& operator [] (BYTE i) { ASSERT(i<=DIMS); return m_vN.data()[i]; } - inline TYPE operator [] (BYTE i) const { ASSERT(i<=DIMS); return m_vN.data()[i]; } + inline TYPE& operator [] (BYTE i) { ASSERT(i +struct FitPlaneOnline { + TYPEW sumX, sumSqX, sumXY, sumXZ; + TYPEW sumY, sumSqY, sumYZ; + TYPEW sumZ, sumSqZ; + size_t size; + FitPlaneOnline(); + void Update(const TPoint3& P); + TPoint3 GetPlane(TPoint3& avg, TPoint3& dir) const; + template TPoint3 GetPlane(TPlane& plane) const; +}; +/*----------------------------------------------------------------*/ + // Basic frustum class // (represented as 6 planes oriented toward outside the frustum volume) diff --git a/libs/Common/Plane.inl b/libs/Common/Plane.inl index 2fcbb76f1..0d15cc319 100644 --- a/libs/Common/Plane.inl +++ b/libs/Common/Plane.inl @@ -69,12 +69,30 @@ inline void TPlane::Set(const TYPE p[DIMS+1]) /*----------------------------------------------------------------*/ +template +inline void TPlane::Invalidate() +{ + m_fD = std::numeric_limits::max(); +} // Invalidate +template +inline bool TPlane::IsValid() const +{ + return m_fD != std::numeric_limits::max(); +} // IsValid +/*----------------------------------------------------------------*/ + + template inline void TPlane::Negate() { m_vN = -m_vN; m_fD = -m_fD; -} +} // Negate +template +inline TPlane TPlane::Negated() const +{ + return TPlane(-m_vN, -m_fD); +} // Negated /*----------------------------------------------------------------*/ @@ -99,6 +117,15 @@ inline TYPE TPlane::DistanceAbs(const POINT& p) const /*----------------------------------------------------------------*/ +// Calculate point's projection on this plane (closest point to this plane). +template +inline typename TPlane::POINT TPlane::ProjectPoint(const POINT& p) const +{ + return p - m_vN*Distance(p); +} +/*----------------------------------------------------------------*/ + + // Classify point to plane. template inline GCLASS TPlane::Classify(const POINT& p) const @@ -253,6 +280,92 @@ bool TPlane::Intersects(const AABB& aabb) const /*----------------------------------------------------------------*/ +// same as above, but online version +template +FitPlaneOnline::FitPlaneOnline() + : sumX(0), sumSqX(0), sumXY(0), sumXZ(0), sumY(0), sumSqY(0), sumYZ(0), sumZ(0), sumSqZ(0), size(0) +{ +} +template +void FitPlaneOnline::Update(const TPoint3& P) +{ + const TYPEW X((TYPEW)P.x), Y((TYPEW)P.y), Z((TYPEW)P.z); + sumX += X; sumSqX += X*X; sumXY += X*Y; sumXZ += X*Z; + sumY += Y; sumSqY += Y*Y; sumYZ += Y*Z; + sumZ += Z; sumSqZ += Z*Z; + ++size; +} +template +TPoint3 FitPlaneOnline::GetPlane(TPoint3& avg, TPoint3& dir) const +{ + const TYPEW avgX(sumX/(TYPEW)size), avgY(sumY/(TYPEW)size), avgZ(sumZ/(TYPEW)size); + // assemble covariance (lower-triangular) matrix + typedef Eigen::Matrix Mat3x3; + Mat3x3 A; + A(0,0) = sumSqX - TYPEW(2)*sumX*avgX + avgX*avgX*(TYPEW)size; + A(1,0) = sumXY - sumX*avgY - avgX*sumY + avgX*avgY*(TYPEW)size; + A(1,1) = sumSqY - TYPEW(2)*sumY*avgY + avgY*avgY*(TYPEW)size; + A(2,0) = sumXZ - sumX*avgZ - avgX*sumZ + avgX*avgZ*(TYPEW)size; + A(2,1) = sumYZ - sumY*avgZ - avgY*sumZ + avgY*avgZ*(TYPEW)size; + A(2,2) = sumSqZ - TYPEW(2)*sumZ*avgZ + avgZ*avgZ*(TYPEW)size; + // the plane normal is simply the eigenvector corresponding to least eigenvalue + const Eigen::SelfAdjointEigenSolver es(A); + ASSERT(ISEQUAL(es.eigenvectors().col(0).norm(), TYPEW(1))); + avg = TPoint3(avgX,avgY,avgZ); + dir = es.eigenvectors().col(0); + const TYPEW* const vals(es.eigenvalues().data()); + ASSERT(vals[0] <= vals[1] && vals[1] <= vals[2]); + return *reinterpret_cast*>(vals); +} +template +template +TPoint3 FitPlaneOnline::GetPlane(TPlane& plane) const +{ + TPoint3 avg, dir; + const TPoint3 quality(GetPlane(avg, dir)); + plane.Set(TPoint3(dir), TPoint3(avg)); + return TPoint3(quality); +} +/*----------------------------------------------------------------*/ + + +// Least squares fits a plane to a 3D point set. +// See http://www.geometrictools.com/Documentation/LeastSquaresFitting.pdf +// Returns a fitting quality (1 - lambda_min/lambda_max): +// 1 is best (zero variance orthogonally to the fitting line) +// 0 is worst (isotropic case, returns a plane with default direction) +template +TYPE FitPlane(const TPoint3* points, size_t size, TPlane& plane) { + // compute a point on the plane, which is shown to be the centroid of the points + const Eigen::Map< const Eigen::Matrix > vPoints((const TYPE*)points, size, 3); + const TPoint3 c(vPoints.colwise().mean()); + + // assemble covariance matrix; matrix numbering: + // 0 + // 1 2 + // 3 4 5 + Eigen::Matrix A(Eigen::Matrix::Zero()); + FOREACHRAWPTR(pPt, points, size) { + const TPoint3 X(*pPt - c); + A(0,0) += X.x*X.x; + A(1,0) += X.x*X.y; + A(1,1) += X.y*X.y; + A(2,0) += X.x*X.z; + A(2,1) += X.y*X.z; + A(2,2) += X.z*X.z; + } + + // the plane normal is simply the eigenvector corresponding to least eigenvalue + const Eigen::SelfAdjointEigenSolver< Eigen::Matrix > es(A); + ASSERT(ISEQUAL(es.eigenvectors().col(0).norm(), TYPE(1))); + plane.Set(es.eigenvectors().col(0), c); + const TYPE* const vals(es.eigenvalues().data()); + ASSERT(vals[0] <= vals[1] && vals[1] <= vals[2]); + return TYPE(1) - vals[0]/vals[1]; +} +/*----------------------------------------------------------------*/ + + // C L A S S ////////////////////////////////////////////////////// // Construct frustum given a projection matrix. diff --git a/libs/Common/Ray.h b/libs/Common/Ray.h index 96bf59a1a..8549a2ea7 100644 --- a/libs/Common/Ray.h +++ b/libs/Common/Ray.h @@ -41,6 +41,7 @@ class TTriangle inline void Set(const POINT&, const POINT&, const POINT&); + inline POINT GetCenter() const; inline AABB GetAABB() const; inline PLANE GetPlane() const; @@ -85,10 +86,6 @@ class TRay bool Intersects(const TRIANGLE&, TYPE *t) const; template bool Intersects(const TRIANGLE&, TYPE fL, TYPE *t) const; - bool Intersects(const POINT&, const POINT&, const POINT&, - bool bCull, TYPE *t) const; - bool Intersects(const POINT&, const POINT&, const POINT&, - bool bCull, TYPE fL, TYPE *t) const; bool Intersects(const PLANE& plane, bool bCull, TYPE *t, POINT* pPtHit) const; inline TYPE IntersectsDist(const PLANE& plane) const; diff --git a/libs/Common/Ray.inl b/libs/Common/Ray.inl index 7ec8f23b5..1078f3086 100644 --- a/libs/Common/Ray.inl +++ b/libs/Common/Ray.inl @@ -30,6 +30,12 @@ inline void TTriangle::Set(const POINT& p0, const POINT& p1, const PO /*----------------------------------------------------------------*/ +// get center +template +inline typename TTriangle::POINT TTriangle::GetCenter() const +{ + return (a + b + c) / TYPE(3); +} // get AABB template inline typename TTriangle::AABB TTriangle::GetAABB() const @@ -139,18 +145,18 @@ bool TRay::Intersects(const TRIANGLE& tri, TYPE *t) const // if close to 0 ray is parallel const VECTOR pvec(m_vDir.cross(edge2)); const TYPE det(edge1.dot(pvec)); - if ((bCull && (det < ZEROTOLERANCE())) || ISZERO(det)) + if ((bCull && det < ZEROTOLERANCE()) || (!bCull && ISZERO(det))) return false; // distance to plane, < 0 means beyond plane const VECTOR tvec(m_pOrig - tri.a); const TYPE u(tvec.dot(pvec)); - if (u < TYPE(0) || u > det) + if ((bCull && !ISINSIDE(u, -ZEROTOLERANCE(), det+ZEROTOLERANCE())) || (!bCull && (det > TYPE(0) ? (u < -ZEROTOLERANCE() || u > det+ZEROTOLERANCE()) : (u > ZEROTOLERANCE() || u < det-ZEROTOLERANCE())))) return false; const VECTOR qvec(tvec.cross(edge1)); const TYPE v(m_vDir.dot(qvec)); - if (v < TYPE(0) || u+v > det) + if ((bCull && (v < -ZEROTOLERANCE() || u+v > det+ZEROTOLERANCE())) || (!bCull && (det > TYPE(0) ? (v < -ZEROTOLERANCE() || u+v > det+ZEROTOLERANCE()) : (v > ZEROTOLERANCE() || u+v < det-ZEROTOLERANCE())))) return false; if (t) @@ -160,115 +166,17 @@ bool TRay::Intersects(const TRIANGLE& tri, TYPE *t) const } // Intersects(Tri) /*----------------------------------------------------------------*/ -// test for intersection with triangle at certain length (line segment), -// same as above but test distance to intersection vs segment length. +// test for intersection with triangle at certain length (line segment); +// same as above, but test distance to intersection vs segment length template template bool TRay::Intersects(const TRIANGLE& tri, TYPE fL, TYPE *t) const { - const VECTOR edge1(tri.b - tri.a); - const VECTOR edge2(tri.c - tri.a); - - // if close to 0 ray is parallel - const VECTOR pvec(m_vDir.cross(edge2)); - const TYPE det(edge1.dot(pvec)); - if ((bCull && (det < ZEROTOLERANCE())) || ISZERO(det)) - return false; - - // distance to plane, < 0 means beyond plane - const VECTOR tvec(m_pOrig - tri.a); - const TYPE u(tvec.dot(pvec)); - if (u < TYPE(0) || u > det) - return false; - - const VECTOR qvec(tvec.cross(edge1)); - const TYPE v(m_vDir.dot(qvec)); - if (v < TYPE(0) || u+v > det) - return false; - - if (t) { - *t = (edge2.dot(qvec)) / det; - // collision but not on segment? - if (*t > fL) return false; - } - else { - // collision but not on segment? - if ((edge2.dot(qvec)) / det > fL) return false; - } - - return true; -} // Intersects(Tri at length) -/*----------------------------------------------------------------*/ - - -// test for intersection with triangle -template -bool TRay::Intersects(const POINT& p0, const POINT& p1, const POINT& p2, bool bCull, TYPE *t) const -{ - const VECTOR edge1(p1 - p0); - const VECTOR edge2(p2 - p0); - - // if close to 0 ray is parallel - const VECTOR pvec = m_vDir.cross(edge2); - const TYPE det = edge1.dot(pvec); - if ((bCull && (det < ZEROTOLERANCE())) || ISZERO(det)) - return false; - - // distance to plane, < 0 means beyond plane - const VECTOR tvec = m_pOrig - p0; - const TYPE u = tvec * pvec; - if (u < TYPE(0) || u > det) - return false; - - const VECTOR qvec = tvec.cross(edge1); - const TYPE v = m_vDir.dot(qvec); - if (v < TYPE(0) || u+v > det) - return false; - - if (t) - *t = (edge2.dot(qvec)) / det; - - return true; -} // Intersects(Tri) -/*----------------------------------------------------------------*/ - - -// test for intersection with triangle at certain length (line segment), -// same as above but test distance to intersection vs segment length. -template -bool TRay::Intersects(const POINT& p0, const POINT& p1, const POINT& p2, bool bCull, TYPE fL, TYPE *t) const -{ - const VECTOR edge1(p1 - p0); - const VECTOR edge2(p2 - p0); - - // if close to 0 ray is parallel - const VECTOR pvec = m_vDir.cross(edge2); - const TYPE det = edge1.dot(pvec); - if ((bCull && (det < ZEROTOLERANCE())) || ISZERO(det)) - return false; - - // distance to plane, < 0 means beyond plane - const VECTOR tvec = m_pOrig - p0; - const TYPE u = tvec.dot(pvec); - if (u < 0.0f || u > det) - return false; - - const VECTOR qvec = tvec.cross(edge1); - const TYPE v = m_vDir.dot(qvec); - if (v < TYPE(0) || u+v > det) - return false; - - if (t) { - *t = (edge2.dot(qvec)) / det; - // collision but not on segment? - if (*t > fL) return false; - } - else { - // collision but not on segment? - if ((edge2.dot(qvec)) / det > fL) return false; - } - - return true; + TYPE _t; + TYPE* const pt(t ? t : &_t); + Intersects(tri, pt); + // collision but not on segment? + return *pt <= fL; } // Intersects(Tri at length) /*----------------------------------------------------------------*/ @@ -1007,3 +915,33 @@ GCLASS TConeIntersect::Classify(const POINT& p, TYPE& t) const return PLANAR; } // Classify /*----------------------------------------------------------------*/ + +template +bool TestRayTriangleIntersection(unsigned iters) { + typedef SEACAVE::TTriangle Triangle; + typedef SEACAVE::TRay Ray; + typedef typename Ray::POINT Point; + typedef typename Point::Scalar Type; + constexpr Type zeroEps(ZEROTOLERANCE() * 1000); + for (unsigned iter=0; iter(triangle, &t)) + return false; + const Point _center(rayCenter.GetPoint(t)); + if ((_center-center).norm() > zeroEps) + return false; + const BYTE o((BYTE)(RAND()%3)); + const Point side((triangle[o]+triangle[(o+1)%3]) / TYPE(2)); + const Ray raySide(rayCenter.m_pOrig, side, true); + if (!raySide.template Intersects(triangle, &t)) + return false; + const Point _side(raySide.GetPoint(t)); + if ((_side-side).norm() > zeroEps) + return false; + } + return true; +} diff --git a/libs/Common/Rotation.h b/libs/Common/Rotation.h index 585eefd93..ebc6b85f9 100644 --- a/libs/Common/Rotation.h +++ b/libs/Common/Rotation.h @@ -296,6 +296,9 @@ class TRMatrixBase : public TMatrix /** @brief Initialization with the rotation from roll/pitch/yaw (in rad) */ inline TRMatrixBase(TYPE roll, TYPE pitch, TYPE yaw); + /** @brief Initialization with the rotation from direction dir0 to direction dir1 */ + inline TRMatrixBase(const Vec& dir0, const Vec& dir1); + template inline TRMatrixBase& operator = (const cv::Matx& rhs) { BaseBase::operator = (rhs); return *this; } inline TRMatrixBase& operator = (const cv::Mat& rhs) { BaseBase::operator = (rhs); return *this; } #ifdef _USE_EIGEN @@ -459,6 +462,12 @@ class TRMatrixBase : public TMatrix @author jw */ void SetFromHV(const Vec& xh, const Vec& vy); + /** @brief Create rotation matrix that rotates from dir0 to dir1 + @param dir0 represents the first (reference) direction vector + @param dir1 represents the second (target) direction vector + @author cDc */ + TRMatrixBase& SetFromDir2Dir(const Vec& dir0, const Vec& dir1); + /** @brief Calculates quaternion representation for this rotation matrix @attention Scalar part of quaternion will always be non-negative for sake of uniqueness of the resulting quaternion! diff --git a/libs/Common/Rotation.inl b/libs/Common/Rotation.inl index f7868723e..81b2d0097 100644 --- a/libs/Common/Rotation.inl +++ b/libs/Common/Rotation.inl @@ -413,7 +413,7 @@ int TQuaternion::SetXYZ(TYPE radX, TYPE radY, TYPE radZ) q_x.Mult(q_y); q_x.Mult(q_z); - (*this) = q_x; + *this = q_x; return 0; } @@ -432,7 +432,7 @@ int TQuaternion::SetZYX(TYPE radX, TYPE radY, TYPE radZ) q_z.Mult(q_y); q_z.Mult(q_x); - (*this) = q_z; + *this = q_z; return 0; } @@ -551,6 +551,11 @@ inline TRMatrixBase::TRMatrixBase(TYPE roll, TYPE pitch, TYPE yaw) { SetXYZ(roll, pitch, yaw); } +template +inline TRMatrixBase::TRMatrixBase(const Vec& dir0, const Vec& dir1) +{ + SetFromDir2Dir(dir0, dir1); +} template @@ -580,7 +585,7 @@ void TRMatrixBase::SetXYZ(TYPE PhiX, TYPE PhiY, TYPE PhiZ) Rx(2,1) = -Rx(1,2); //sin(PhiX); Rx(2,2) = Rx(1,1); //cos(PhiX); - (*this) = Rx*Ry*Rz; + *this = Rx*Ry*Rz; #else const TYPE sin_x = sin(PhiX); const TYPE sin_y = sin(PhiY); @@ -627,7 +632,7 @@ void TRMatrixBase::SetZYX(TYPE PhiX, TYPE PhiY, TYPE PhiZ) Rx(2,1) = -Rx(1,2); //sin(PhiX); Rx(2,2) = Rx(1,1); //cos(PhiX); - (*this) = Rz*Ry*Rx; + *this = Rz*Ry*Rx; } @@ -657,7 +662,7 @@ void TRMatrixBase::SetYXZ(TYPE PhiY, TYPE PhiX, TYPE PhiZ) Rx(2,1) = -Rx(1,2); //sin(PhiX); Rx(2,2) = Rx(1,1); //cos(PhiX); - (*this) = Ry*Rx*Rz; + *this = Ry*Rx*Rz; } @@ -687,7 +692,7 @@ void TRMatrixBase::SetZXY(TYPE PhiX, TYPE PhiY, TYPE PhiZ) Rx(2,1) = -Rx(1,2); //sin(PhiX); Rx(2,2) = Rx(1,1); //cos(PhiX); - (*this) = Rz*Rx*Ry; + *this = Rz*Rx*Ry; } @@ -697,7 +702,7 @@ void TRMatrixBase::Set(const Vec& wa, TYPE phi) { // zero rotation results in identity matrix if (ISZERO(phi)) { - (*this) = Base::IDENTITY; + *this = Base::IDENTITY; return; } @@ -720,7 +725,7 @@ void TRMatrixBase::Set(const Vec& wa, TYPE phi) const Mat Sin_O(Omega * sin(phi)); const Mat Cos_O_O((Omega * Omega) * (TYPE(1)-cos(phi))); - (*this) = Base::IDENTITY + Sin_O + Cos_O_O; + *this = Base::IDENTITY + Sin_O + Cos_O_O; } @@ -1164,7 +1169,34 @@ void TRMatrixBase::SetFromHV(const Vec& xxx, const Vec& yyy) // compute y base orthogonal in rhs const Vec y0(normalized(cross(z0, x0))); - (*this).SetFromColumnVectors(x0, y0, z0); + SetFromColumnVectors(x0, y0, z0); +} + + +template +TRMatrixBase& TRMatrixBase::SetFromDir2Dir(const Vec& dir0, const Vec& dir1) +{ + ASSERT(ISEQUAL(norm(dir0), TYPE(1))); + ASSERT(ISEQUAL(norm(dir1), TYPE(1))); + const TYPE cos01(CLAMP(dir1.dot(dir0), TYPE(-1), TYPE(1))); + const TYPE sin01Sq(TYPE(1) - SQUARE(cos01)); + if (sin01Sq > EPSILONTOLERANCE()) { + const Vec v(cross(dir0, dir1)); + const Mat V(CreateCrossProductMatrix3T(v)); + *this = Mat::IDENTITY + V + (V*V)*((TYPE(1)-cos01)/sin01Sq); + } else { + *this = Mat::ZERO; + if (cos01 > TYPE(0)) { + Base::operator()(0,0) = TYPE(1); + Base::operator()(1,1) = TYPE(1); + Base::operator()(2,2) = TYPE(1); + } else { + Base::operator()(0,0) = TYPE(-1); + Base::operator()(1,1) = TYPE(-1); + Base::operator()(2,2) = TYPE(-1); + } + } + return *this; } @@ -1202,7 +1234,7 @@ inline void TRMatrixBase::SetRotationAxisAngle(const Vec& rot) { #if 0 // set rotation using Rodriguez formula - (*this) = Mat::IDENTITY; + *this = Mat::IDENTITY; const TYPE thetaSq = normSq(rot); if (thetaSq < TYPE(1e-12)) return; @@ -1254,7 +1286,7 @@ inline void TRMatrixBase::SetRotationAxisAngle(const Vec& rot) val[8] = TYPE(1); } #else - (*this) = Eigen::SO3(rot).get_matrix(); + *this = Eigen::SO3(rot).get_matrix(); #endif } @@ -1262,7 +1294,7 @@ template inline void TRMatrixBase::Apply(const Vec& delta) { const TRMatrixBase dR(delta); - (*this) = dR * (*this); + *this = dR * (*this); } template @@ -1492,4 +1524,40 @@ void TRMatrixBase::EnforceOrthogonality() } /*----------------------------------------------------------------*/ + +// if the matrix is a rotation, then the the transpose should be the inverse +template +inline bool IsRotationMatrix(const TMatrix& R) { + ASSERT(sizeof(TMatrix) == sizeof(TRMatrixBase)); + return ((const TRMatrixBase&)R).IsValid(); +} // IsRotationMatrix +template +inline bool IsRotationMatrix(const Eigen::Matrix& R) { + // the trace should be three and the determinant should be one + return (ISEQUAL(R.determinant(), TYPE(1)) && ISEQUAL((R*R.transpose()).trace(), TYPE(3))); +} // IsRotationMatrix +/*----------------------------------------------------------------*/ + +// enforce matrix orthogonality +template +inline void EnsureRotationMatrix(TMatrix& R) { + ASSERT(sizeof(TMatrix) == sizeof(TRMatrixBase)); + ((TRMatrixBase&)R).EnforceOrthogonality(); +} // EnsureRotationMatrix +/*----------------------------------------------------------------*/ + + +// compute the distance on SO(3) between the two given rotations +// using log(R) as in: "Efficient and Robust Large-Scale Rotation Averaging", 2013 +// same result as above, but returns directly the angle +template +inline TYPE ComputeAngleSO3(const TMatrix& I) { + return TYPE(norm(((const TRMatrixBase&)I).GetRotationAxisAngle())); +} // ComputeAngleSO3 +template +FORCEINLINE TYPE ComputeAngleSO3(const TMatrix& R1, const TMatrix& R2) { + return ComputeAngleSO3(TMatrix(R1*R2.t())); +} // ComputeAngleSO3 +/*----------------------------------------------------------------*/ + } // namespace SEACAVE diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 86316cdbb..d5dcb27e2 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -39,6 +39,11 @@ #else #include #endif +#ifdef _SUPPORT_CPP17 +#if !defined(__GNUC__) || (__GNUC__ > 7) +#include +#endif +#endif #include #include #include @@ -623,11 +628,11 @@ constexpr T& NEGATE(T& a) { } template constexpr T SQUARE(const T& a) { - return (a * a); + return a * a; } template constexpr T CUBE(const T& a) { - return (a * a * a); + return a * a * a; } template inline T SQRT(const T& a) { @@ -646,7 +651,7 @@ inline T LOG10(const T& a) { return T(log10(a)); } template -constexpr T powi(T base, int exp) { +constexpr T powi(T base, unsigned exp) { T result(1); while (exp) { if (exp & 1) @@ -656,15 +661,15 @@ constexpr T powi(T base, int exp) { } return result; } -constexpr int log2i(int val) { +constexpr int log2i(unsigned val) { int ret = -1; - while (val > 0) { + while (val) { val >>= 1; - ret++; + ++ret; } return ret; } -template constexpr inline int log2i() { return 1+log2i<(N>>1)>(); } +template constexpr inline int log2i() { return 1+log2i<(N>>1)>(); } template <> constexpr inline int log2i<0>() { return -1; } template <> constexpr inline int log2i<1>() { return 0; } template <> constexpr inline int log2i<2>() { return 1; } @@ -908,46 +913,52 @@ FORCEINLINE int CRound2Int(const double& x) { return c.i; } #endif -FORCEINLINE int Floor2Int(float x) { +template +FORCEINLINE INTTYPE Floor2Int(float x) { #ifdef _FAST_FLOAT2INT return CRound2Int(double(x)-_float2int_doublemagicroundeps); #else - return int(floor(x)); + return static_cast(floor(x)); #endif } -FORCEINLINE int Floor2Int(double x) { +template +FORCEINLINE INTTYPE Floor2Int(double x) { #ifdef _FAST_FLOAT2INT return CRound2Int(x-_float2int_doublemagicroundeps); #else - return int(floor(x)); + return static_cast(floor(x)); #endif } -FORCEINLINE int Ceil2Int(float x) { +template +FORCEINLINE INTTYPE Ceil2Int(float x) { #ifdef _FAST_FLOAT2INT return CRound2Int(double(x)+_float2int_doublemagicroundeps); #else - return int(ceil(x)); + return static_cast(ceil(x)); #endif } -FORCEINLINE int Ceil2Int(double x) { +template +FORCEINLINE INTTYPE Ceil2Int(double x) { #ifdef _FAST_FLOAT2INT return CRound2Int(x+_float2int_doublemagicroundeps); #else - return int(ceil(x)); + return static_cast(ceil(x)); #endif } -FORCEINLINE int Round2Int(float x) { +template +FORCEINLINE INTTYPE Round2Int(float x) { #ifdef _FAST_FLOAT2INT return CRound2Int(double(x)+_float2int_doublemagicdelta); #else - return int(floor(x+.5f)); + return static_cast(floor(x+.5f)); #endif } -FORCEINLINE int Round2Int(double x) { +template +FORCEINLINE INTTYPE Round2Int(double x) { #ifdef _FAST_FLOAT2INT return CRound2Int(x+_float2int_doublemagicdelta); #else - return int(floor(x+.5)); + return static_cast(floor(x+.5)); #endif } /*----------------------------------------------------------------*/ @@ -1179,12 +1190,14 @@ template inline bool ISFINITE(const _Tp* x, size_t n) { for (size_t i=0; i -inline bool ISINSIDE(_Tp v,_Tp l0,_Tp l1) { return (l0 <= v && v < l1); } +inline bool ISINSIDE(_Tp v,_Tp l0,_Tp l1) { ASSERT(l0 +inline bool ISINSIDES(_Tp v,_Tp l0,_Tp l1) { return l0 < l1 ? ISINSIDE(v, l0, l1) : ISINSIDE(v, l1, l0); } template -inline _Tp CLAMP(_Tp v, _Tp c0, _Tp c1) { ASSERT(c0<=c1); return MINF(MAXF(v, c0), c1); } +inline _Tp CLAMP(_Tp v, _Tp l0, _Tp l1) { ASSERT(l0<=l1); return MINF(MAXF(v, l0), l1); } template -inline _Tp CLAMPS(_Tp v, _Tp c0, _Tp c1) { if (c0 <= c1) return CLAMP(v, c0, c1); return CLAMP(v, c1, c0); } +inline _Tp CLAMPS(_Tp v, _Tp l0, _Tp l1) { return l0 <= l1 ? CLAMP(v, l0, l1) : CLAMP(v, l1, l0); } template inline _Tp SIGN(_Tp x) { if (x > _Tp(0)) return _Tp(1); if (x < _Tp(0)) return _Tp(-1); return _Tp(0); } @@ -1193,18 +1206,18 @@ template inline _Tp ABS(_Tp x) { return std::abs(x); } template -inline _Tp ZEROTOLERANCE() { return _Tp(0); } +constexpr _Tp ZEROTOLERANCE() { return _Tp(0); } template<> -inline float ZEROTOLERANCE() { return FZERO_TOLERANCE; } +constexpr float ZEROTOLERANCE() { return FZERO_TOLERANCE; } template<> -inline double ZEROTOLERANCE() { return ZERO_TOLERANCE; } +constexpr double ZEROTOLERANCE() { return ZERO_TOLERANCE; } template -inline _Tp EPSILONTOLERANCE() { return std::numeric_limits<_Tp>::epsilon(); } +constexpr _Tp EPSILONTOLERANCE() { return std::numeric_limits<_Tp>::epsilon(); } template<> -inline float EPSILONTOLERANCE() { return 0.00001f; } +constexpr float EPSILONTOLERANCE() { return 0.00001f; } template<> -inline double EPSILONTOLERANCE() { return 1e-10; } +constexpr double EPSILONTOLERANCE() { return 1e-10; } inline bool ISZERO(float x) { return ABS(x) < FZERO_TOLERANCE; } inline bool ISZERO(double x) { return ABS(x) < ZERO_TOLERANCE; } @@ -1384,8 +1397,8 @@ class TPoint3 : public cv::Point3_ inline TYPE* ptr() { return &x; } // 1D element access - inline const TYPE& operator [](size_t i) const { ASSERT(i>=0 && i<3); return ptr()[i]; } - inline TYPE& operator [](size_t i) { ASSERT(i>=0 && i<3); return ptr()[i]; } + inline const TYPE& operator [](BYTE i) const { ASSERT(i<3); return ptr()[i]; } + inline TYPE& operator [](BYTE i) { ASSERT(i<3); return ptr()[i]; } // Access point as vector equivalent inline operator const Vec& () const { return *((const Vec*)this); } @@ -1602,16 +1615,17 @@ class TDMatrix : public cv::Mat_ inline int area() const { ASSERT(dims == 2); return cols*rows; } /// Is this coordinate inside the 2D matrix? - inline bool isInside(const Size& pt) const { - return pt.width>=0 && pt.height>=0 && pt.width + static inline typename std::enable_if::value,bool>::type isInside(const cv::Point_& pt, const cv::Size& size) { + return pt.x>=T(0) && pt.y>=T(0) && pt.x - inline typename std::enable_if::value,bool>::type isInside(const cv::Point_& pt) const { - return pt.x>=0 && pt.y>=0 && pt.x::value,bool>::type isInside(const cv::Point_& pt, const cv::Size& size) { + return pt.x>=T(0) && pt.y>=T(0) && pt.x<=T(size.width) && pt.y<=T(size.height); } template - inline typename std::enable_if::value,bool>::type isInside(const cv::Point_& pt) const { - return pt.x>=T(0) && pt.y>=T(0) && pt.x<=T(Base::size().width) && pt.y<=T(Base::size().height); + inline bool isInside(const cv::Point_& pt) const { + return isInside(pt, Base::size()); } /// Is this coordinate inside the 2D matrix, and not too close to the edges? @@ -1721,8 +1735,8 @@ class TDMatrix : public cv::Mat_ } /// pointer to the beginning of the matrix data - inline const TYPE* getData() const { ASSERT(cv::Mat::isContinuous()); return (const TYPE*)data; } - inline TYPE* getData() { ASSERT(cv::Mat::isContinuous()); return (TYPE*)data; } + inline const TYPE* getData() const { ASSERT(cv::Mat::empty() || cv::Mat::isContinuous()); return (const TYPE*)data; } + inline TYPE* getData() { ASSERT(cv::Mat::empty() || cv::Mat::isContinuous()); return (TYPE*)data; } #ifdef _USE_EIGEN // Access point as Eigen::Map equivalent @@ -1746,8 +1760,8 @@ typedef TDMatrix DMatrix32S; typedef TDMatrix DMatrix32U; typedef TDMatrix DMatrix32F; typedef TDMatrix DMatrix64F; -typedef SEACAVE::cList DMatrixArr; -typedef SEACAVE::cList MatArr; +typedef CLISTDEF2(DMatrix) DMatrixArr; +typedef CLISTDEF2(cv::Mat) MatArr; /*----------------------------------------------------------------*/ @@ -1799,7 +1813,7 @@ typedef TDVector DVector32S; typedef TDVector DVector32U; typedef TDVector DVector32F; typedef TDVector DVector64F; -typedef SEACAVE::cList DVectorArr; +typedef CLISTDEF2(DVector) DVectorArr; /*----------------------------------------------------------------*/ @@ -2462,7 +2476,7 @@ struct TIndexScore { }; /*----------------------------------------------------------------*/ typedef TIndexScore IndexScore; -typedef SEACAVE::cList IndexScoreArr; +typedef CLISTDEF0(IndexScore) IndexScoreArr; /*----------------------------------------------------------------*/ @@ -2514,7 +2528,7 @@ struct PairIdx { inline bool operator==(const PairIdx& r) const { return (idx == r.idx); } }; /*----------------------------------------------------------------*/ -typedef SEACAVE::cList PairIdxArr; +typedef CLISTDEF0(PairIdx) PairIdxArr; inline PairIdx MakePairIdx(uint32_t idxImageA, uint32_t idxImageB) { return (idxImageA= 0 || -num >= (-denom)*maxIters ? maxIters : (INT_TYPE)ROUND2INT(num/denom)); + return (denom >= 0 || -num >= (-denom)*maxIters ? maxIters : ROUND2INT(num/denom)); } #endif @@ -571,58 +571,58 @@ FORCEINLINE bool ISEQUAL(const cv::Matx& v1, const cv::Matx& } // round -template -FORCEINLINE SEACAVE::TPoint2 Floor2Int(const cv::Point_& v) +template +FORCEINLINE SEACAVE::TPoint2 Floor2Int(const cv::Point_& v) { - return SEACAVE::TPoint2(FLOOR2INT(v.x), FLOOR2INT(v.y)); + return SEACAVE::TPoint2(FLOOR2INT(v.x), FLOOR2INT(v.y)); } -template -FORCEINLINE SEACAVE::TPoint2 Ceil2Int(const cv::Point_& v) +template +FORCEINLINE SEACAVE::TPoint2 Ceil2Int(const cv::Point_& v) { - return SEACAVE::TPoint2(CEIL2INT(v.x), CEIL2INT(v.y)); + return SEACAVE::TPoint2(CEIL2INT(v.x), CEIL2INT(v.y)); } -template -FORCEINLINE SEACAVE::TPoint2 Round2Int(const cv::Point_& v) +template +FORCEINLINE SEACAVE::TPoint2 Round2Int(const cv::Point_& v) { - return SEACAVE::TPoint2(ROUND2INT(v.x), ROUND2INT(v.y)); + return SEACAVE::TPoint2(ROUND2INT(v.x), ROUND2INT(v.y)); } -template -FORCEINLINE SEACAVE::TPoint3 Floor2Int(const cv::Point3_& v) +template +FORCEINLINE SEACAVE::TPoint3 Floor2Int(const cv::Point3_& v) { - return SEACAVE::TPoint3(FLOOR2INT(v.x), FLOOR2INT(v.y), FLOOR2INT(v.z)); + return SEACAVE::TPoint3(FLOOR2INT(v.x), FLOOR2INT(v.y), FLOOR2INT(v.z)); } -template -FORCEINLINE SEACAVE::TPoint3 Ceil2Int(const cv::Point3_& v) +template +FORCEINLINE SEACAVE::TPoint3 Ceil2Int(const cv::Point3_& v) { - return SEACAVE::TPoint3(CEIL2INT(v.x), CEIL2INT(v.y), CEIL2INT(v.z)); + return SEACAVE::TPoint3(CEIL2INT(v.x), CEIL2INT(v.y), CEIL2INT(v.z)); } -template -FORCEINLINE SEACAVE::TPoint3 Round2Int(const cv::Point3_& v) +template +FORCEINLINE SEACAVE::TPoint3 Round2Int(const cv::Point3_& v) { - return SEACAVE::TPoint3(ROUND2INT(v.x), ROUND2INT(v.y), ROUND2INT(v.z)); + return SEACAVE::TPoint3(ROUND2INT(v.x), ROUND2INT(v.y), ROUND2INT(v.z)); } -template -FORCEINLINE SEACAVE::TMatrix Floor2Int(const cv::Matx& v) +template +FORCEINLINE SEACAVE::TMatrix Floor2Int(const cv::Matx& v) { - SEACAVE::TMatrix nv; + SEACAVE::TMatrix nv; for (int i=0; i -FORCEINLINE SEACAVE::TMatrix Ceil2Int(const cv::Matx& v) +template +FORCEINLINE SEACAVE::TMatrix Ceil2Int(const cv::Matx& v) { - SEACAVE::TMatrix nv; + SEACAVE::TMatrix nv; for (int i=0; i -FORCEINLINE SEACAVE::TMatrix Round2Int(const cv::Matx& v) +template +FORCEINLINE SEACAVE::TMatrix Round2Int(const cv::Matx& v) { - SEACAVE::TMatrix nv; + SEACAVE::TMatrix nv; for (int i=0; i #endif -#ifdef _SUPPORT_CPP17 -#if !defined(__GNUC__) || (__GNUC__ > 7) -#include -#endif -#endif // _SUPPORT_CPP17 using namespace SEACAVE; diff --git a/libs/Common/Util.inl b/libs/Common/Util.inl index 67199026e..381975929 100644 --- a/libs/Common/Util.inl +++ b/libs/Common/Util.inl @@ -13,6 +13,23 @@ namespace SEACAVE { // I N L I N E ///////////////////////////////////////////////////// +// normalize inhomogeneous 2D point by the given camera intrinsics K +// K is assumed to be the [3,3] triangular matrix with: fx, fy, s, cx, cy and scale 1 +template +inline void NormalizeProjection(const TYPE1* K, const TYPE2* x, TYPE3* n) { + ASSERT(ISZERO(K[3]) && ISZERO(K[6]) && ISZERO(K[7]) && ISEQUAL(K[8], TYPE1(1))); + n[0] = TYPE3(K[0]*x[0] + K[1]*x[1] + K[2]); + n[1] = TYPE3( K[4]*x[1] + K[5]); +} // NormalizeProjection +// same as above, but using K inverse +template +inline void NormalizeProjectionInv(const TYPE1* K, const TYPE2* x, TYPE3* n) { + ASSERT(ISZERO(K[3]) && ISZERO(K[6]) && ISZERO(K[7]) && ISEQUAL(K[8], TYPE1(1))); + n[0] = TYPE3((K[4]*x[0] - K[1]*x[1] + (K[5]*K[1]-K[2]*K[4])) / (K[0]*K[4])); + n[1] = TYPE3((x[1] - K[5]) / K[4]); +} // NormalizeProjectionInv +/*----------------------------------------------------------------*/ + // compute relative pose of the given two cameras template inline void ComputeRelativeRotation(const TMatrix& Ri, const TMatrix& Rj, TMatrix& Rij) { @@ -25,6 +42,57 @@ inline void ComputeRelativePose(const TMatrix& Ri, const TPoint3 } // ComputeRelativePose /*----------------------------------------------------------------*/ +// Triangulate the position of a 3D point +// given two corresponding normalized projections and the pose of the second camera relative to the first one; +// returns the triangulated 3D point from the point of view of the first camera +template +bool TriangulatePoint3D( + const TMatrix& R, const TPoint3& C, + const TPoint2& pt1, const TPoint2& pt2, + TPoint3& X +) { + // convert image points to 3-vectors (of unit length) + // used to describe landmark observations/bearings in camera frames + const TPoint3 f1(pt1.x,pt1.y,1); + const TPoint3 f2(pt2.x,pt2.y,1); + const TPoint3 f2_unrotated = R.t() * f2; + const TPoint2 b(C.dot(f1), C.dot(f2_unrotated)); + // optimized inversion of A + const TYPE1 a = normSq(f1); + const TYPE1 c = f1.dot(f2_unrotated); + const TYPE1 d = -normSq(f2_unrotated); + const TYPE1 det = a*d+c*c; + if (ABS(det) < EPSILONTOLERANCE()) + return false; + const TYPE1 invDet = TYPE1(1)/det; + const TPoint2 lambda((d*b.x+c*b.y)*invDet, (a*b.y-c*b.x)*invDet); + const TPoint3 xm = lambda.x * f1; + const TPoint3 xn = C + lambda.y * f2_unrotated; + X = (xm + xn)*TYPE1(0.5); + return true; +} // TriangulatePoint3D +// same as above, but using the two camera poses; +// returns the 3D point in world coordinates +template +bool TriangulatePoint3D( + const TMatrix& K1, const TMatrix& K2, + const TMatrix& R1, const TMatrix& R2, + const TPoint3& C1, const TPoint3& C2, + const TPoint2& x1, const TPoint2& x2, + TPoint3& X +) { + TPoint2 pt1, pt2; + NormalizeProjectionInv(K1.val, x1.ptr(), pt1.ptr()); + NormalizeProjectionInv(K2.val, x2.ptr(), pt2.ptr()); + TMatrix R; TPoint3 C; + ComputeRelativePose(R1, C1, R2, C2, R, C); + if (!TriangulatePoint3D(R, C, pt1, pt2, X)) + return false; + X = R1.t() * X + C1; + return true; +} // TriangulatePoint3D +/*----------------------------------------------------------------*/ + // compute the homography matrix transforming points from image A to image B, // given the relative pose of B with respect to A, and the plane // (see R. Hartley, "Multiple View Geometry," 2004, pp. 234) @@ -109,28 +177,6 @@ inline TMatrix CreateF(const TMatrix& R, const TMatrix -inline bool IsRotationMatrix(const TMatrix& R) { - ASSERT(sizeof(TMatrix) == sizeof(TRMatrixBase)); - return ((const TRMatrixBase&)R).IsValid(); -} // IsRotationMatrix -template -inline bool IsRotationMatrix(const Eigen::Matrix& R) { - // the trace should be three and the determinant should be one - return (ISEQUAL(R.determinant(), TYPE(1)) && ISEQUAL((R*R.transpose()).trace(), TYPE(3))); -} // IsRotationMatrix -/*----------------------------------------------------------------*/ - -// enforce matrix orthogonality -template -inline void EnsureRotationMatrix(TMatrix& R) { - ASSERT(sizeof(TMatrix) == sizeof(TRMatrixBase)); - ((TRMatrixBase&)R).EnforceOrthogonality(); -} // EnsureRotationMatrix -/*----------------------------------------------------------------*/ - - // compute the angle between the two rotations given // as in: "Disambiguating Visual Relations Using Loop Constraints", 2010 // returns cos(angle) (same as cos(ComputeAngleSO3(I)) @@ -142,17 +188,6 @@ template FORCEINLINE TYPE ComputeAngle(const TMatrix& R1, const TMatrix& R2) { return ComputeAngle(TMatrix(R1*R2.t())); } // ComputeAngle -// compute the distance on SO(3) between the two given rotations -// using log(R) as in: "Efficient and Robust Large-Scale Rotation Averaging", 2013 -// same result as above, but returns directly the angle -template -inline TYPE ComputeAngleSO3(const TMatrix& I) { - return TYPE(norm(((const TRMatrixBase&)I).GetRotationAxisAngle())); -} // ComputeAngleSO3 -template -FORCEINLINE TYPE ComputeAngleSO3(const TMatrix& R1, const TMatrix& R2) { - return ComputeAngleSO3(TMatrix(R1*R2.t())); -} // ComputeAngleSO3 /*----------------------------------------------------------------*/ @@ -413,15 +448,19 @@ FORCEINLINE TYPE AngleFromRotationMatrix(const TMatrix& R) { // given two 3D vectors, // compute the angle between them // returns cos(angle) -template +template inline TYPE2 ComputeAngle(const TYPE1* V1, const TYPE1* V2) { - // compute the angle between the rays return CLAMP(TYPE2((V1[0]*V2[0]+V1[1]*V2[1]+V1[2]*V2[2])/SQRT((V1[0]*V1[0]+V1[1]*V1[1]+V1[2]*V1[2])*(V2[0]*V2[0]+V2[1]*V2[1]+V2[2]*V2[2]))), TYPE2(-1), TYPE2(1)); } // ComputeAngle +// same as above, but with the vectors normalized +template +inline TYPE2 ComputeAngleN(const TYPE1* V1, const TYPE1* V2) { + return CLAMP(TYPE2(V1[0]*V2[0]+V1[1]*V2[1]+V1[2]*V2[2]), TYPE2(-1), TYPE2(1)); +} // ComputeAngleN // given three 3D points, // compute the angle between the vectors formed by the first point with the other two // returns cos(angle) -template +template inline TYPE2 ComputeAngle(const TYPE1* B, const TYPE1* X1, const TYPE1* X2) { // create the two vectors const TYPE1 V1[] = {X1[0]-B[0], X1[1]-B[1], X1[2]-B[2]}; @@ -431,7 +470,7 @@ inline TYPE2 ComputeAngle(const TYPE1* B, const TYPE1* X1, const TYPE1* X2) { // given four 3D points, // compute the angle between the two vectors formed by the first and second pair of points // returns cos(angle) -template +template inline TYPE2 ComputeAngle(const TYPE1* X1, const TYPE1* C1, const TYPE1* X2, const TYPE1* C2) { // subtract out the camera center const TYPE1 V1[] = {X1[0]-C1[0], X1[1]-C1[1], X1[2]-C1[2]}; @@ -790,83 +829,73 @@ inline void ComputeMeanStdOffline(const TYPE* values, size_t size, TYPEW& mean, stddev = SQRT(variance); } // ComputeMeanStdOffline // same as above, but uses one pass only (online) -template -struct MeanStdOnline { - TYPEW mean; - TYPEW stddev; - size_t size; - MeanStdOnline() : mean(0), stddev(0), size(0) {} - MeanStdOnline(const TYPE* values, size_t _size) : mean(0), stddev(0), size(0) { - Compute(values, _size); - } - void Update(ARGTYPE v) { - const TYPEW val(v); - const TYPEW delta(val-mean); - mean += delta / (float)(++size); - stddev += delta * (val-mean); - } - void Compute(const TYPE* values, size_t _size) { - for (size_t i=0; i<_size; ++i) - Update(values[i]); - } - TYPEW GetMean() const { return mean; } - TYPEW GetVariance() const { return (stddev / (float)(size - 1)); } - TYPEW GetStdDev() const { return SQRT(GetVariance()); } -}; template inline void ComputeMeanStdOnline(const TYPE* values, size_t size, TYPEW& mean, TYPEW& stddev) { ASSERT(size > 0); - mean = TYPEW(0); - stddev = TYPEW(0); - for (size_t i=0; i -struct MeanStdOnlineFast { - TYPEW sum; - TYPEW sumSq; +} // ComputeMeanStdOnlineFast +#define ComputeMeanStd ComputeMeanStdOnline +// same as above, but an interactive version +template +struct MeanStd { + typedef TYPE Type; + typedef TYPEW TypeW; + typedef TYPER TypeR; + typedef ARGTYPE ArgType; + TYPEW sum, sumSq; size_t size; - MeanStdOnlineFast() : sum(0), sumSq(0), size(0) {} - MeanStdOnlineFast(const TYPE* values, size_t _size) : sum(0), sumSq(0), size(0) { - Compute(values, _size); - } - void Update(ARGTYPE v) { - const TYPEW val(v); + MeanStd() : sum(0), sumSq(0), size(0) {} + MeanStd(const Type* values, size_t _size) : MeanStd() { Compute(values, _size); } + void Update(ArgType v) { + const TYPEW val(static_cast(v)); sum += val; sumSq += SQUARE(val); ++size; } - void Compute(const TYPE* values, size_t _size) { + void Compute(const Type* values, size_t _size) { for (size_t i=0; i<_size; ++i) Update(values[i]); } - TYPEW GetMean() const { return sum / (float)size; } - TYPEW GetVariance() const { return (sumSq - SQUARE(sum) / (float)size) / (float)(size - 1); } + TYPEW GetSum() const { return sum; } + TYPEW GetMean() const { return static_cast(sum / static_cast(size)); } + TYPEW GetRMS() const { return static_cast(SQRT(sumSq / static_cast(size))); } + TYPEW GetVarianceN() const { return static_cast(sumSq - SQUARE(sum) / static_cast(size)); } + TYPEW GetVariance() const { return static_cast(GetVarianceN() / static_cast(size)); } TYPEW GetStdDev() const { return SQRT(GetVariance()); } + void Clear() { sum = sumSq = TYPEW(0); size = 0; } }; -template -inline void ComputeMeanStdOnlineFast(const TYPE* values, size_t size, TYPEW& mean, TYPEW& stddev) { - ASSERT(size > 0); - TYPEW sum(0), sumSq(0); - FOREACHRAWPTR(pVal, values, size) { - const TYPEW val(*pVal); - sum += val; - sumSq += SQUARE(val); +// same as above, but records also min/max values +template +struct MeanStdMinMax : MeanStd { + typedef MeanStd Base; + typedef TYPE Type; + typedef TYPEW TypeW; + typedef TYPER TypeR; + typedef ARGTYPE ArgType; + Type minVal, maxVal; + MeanStdMinMax() : minVal(std::numeric_limits::max()), maxVal(std::numeric_limits::lowest()) {} + MeanStdMinMax(const Type* values, size_t _size) : MeanStdMinMax() { Compute(values, _size); } + void Update(ArgType v) { + if (minVal > v) + minVal = v; + if (maxVal < v) + maxVal = v; + Base::Update(v); } - const TYPEW invSize(TYPEW(1)/(float)size); - mean = sum * invSize; - const TYPEW variance((sumSq - SQUARE(sum) * invSize) / (float)(size - 1)); - stddev = SQRT(variance); -} // ComputeMeanStdOnlineFast -#define ComputeMeanStd ComputeMeanStdOnlineFast -#define MeanStd MeanStdOnlineFast + void Compute(const Type* values, size_t _size) { + for (size_t i=0; i<_size; ++i) + Update(values[i]); + } +}; /*----------------------------------------------------------------*/ // given an array of values, compute the X84 threshold as in: @@ -1165,143 +1194,4 @@ void NormalizePoints(const CLISTDEF0(TPoint3)& pointsIn, CLISTDEF0(TPoint3 } /*----------------------------------------------------------------*/ - -// Least squares fits a plane to a 3D point set. -// See http://www.geometrictools.com/Documentation/LeastSquaresFitting.pdf -// Returns a fitting quality (1 - lambda_min/lambda_max): -// 1 is best (zero variance orthogonally to the fitting line) -// 0 is worst (isotropic case, returns a plane with default direction) -template -TYPE FitPlane(const TPoint3* points, size_t size, TPlane& plane) { - // compute a point on the plane, which is shown to be the centroid of the points - const Eigen::Map< const Eigen::Matrix > vPoints((const TYPE*)points, size, 3); - const TPoint3 c(vPoints.colwise().mean()); - - // assemble covariance matrix; matrix numbering: - // 0 - // 1 2 - // 3 4 5 - Eigen::Matrix A(Eigen::Matrix::Zero()); - FOREACHRAWPTR(pPt, points, size) { - const TPoint3 X(*pPt - c); - A(0,0) += X.x*X.x; - A(1,0) += X.x*X.y; - A(1,1) += X.y*X.y; - A(2,0) += X.x*X.z; - A(2,1) += X.y*X.z; - A(2,2) += X.z*X.z; - } - - // the plane normal is simply the eigenvector corresponding to least eigenvalue - const Eigen::SelfAdjointEigenSolver< Eigen::Matrix > es(A); - ASSERT(ISEQUAL(es.eigenvectors().col(0).norm(), TYPE(1))); - plane.Set(es.eigenvectors().col(0), c); - const TYPE* const vals(es.eigenvalues().data()); - ASSERT(vals[0] <= vals[1] && vals[1] <= vals[2]); - return TYPE(1) - vals[0]/vals[1]; -} -/*----------------------------------------------------------------*/ - - -template -struct CLeastSquares { - inline TYPE operator()(const TYPE) const { - return TYPE(1); - } -}; - -/* -template -struct CHuber_TorrMurray { - inline TYPE operator()(const TYPE d) const { - const TYPE b = 0.02; - - const TYPE d_abs = ABS(d); - if (d_abs < b) - return TYPE(1); - if (d_abs < 3 * b) - return b / d; //I think this is possibly wrong--should use 1/root(d) - return TYPE(0); //TODO, probably best to just return SIGMA/d; - } -}; -*/ - -template -struct CHuber { - inline CHuber(TYPE _threshold = 0.005) : threshold(_threshold) {} - inline TYPE operator()(const TYPE d) const { - const TYPE d_abs(ABS(d)); - if (d_abs < threshold) - return TYPE(1); - return SQRT(threshold * (TYPE(2) * d_abs - threshold)) / d_abs; - } - const TYPE threshold; -}; - -template -struct CBlakeZisserman { // Blake-Zisserman Gaussian + uniform - inline CBlakeZisserman(TYPE _threshold = 0.005) : threshold(_threshold) {} - inline TYPE operator()(const TYPE d) const { - const TYPE SD_INV = TYPE(1) / (0.5 * threshold); - const TYPE eps = exp(-SQUARE(threshold * SD_INV)); //Equally likely to be inlier or outlier at thresh - const TYPE d_abs = ABS(d) + 1e-12; - const TYPE zeroPoint = log(TYPE(1) + eps); //Needed for LS... - - const TYPE dCost_sq = zeroPoint - log(exp(-SQUARE(d * SD_INV)) + eps); - ASSERT(dCost_sq >= 0); // Cost computation failed? - - return SQRT(dCost_sq) / d_abs; - } - const TYPE threshold; -}; - -template -struct CPseudoHuber { - inline CPseudoHuber(TYPE _threshold = 0.005) : threshold(_threshold) {} - inline TYPE operator()(const TYPE d) const { - const TYPE b_sq = SQUARE(threshold); - const TYPE d_abs = ABS(d) + 1e-12; - - //C(delta) = 2*b^2*(sqrt(1+(delta/b)^2) - 1); - return SQRT(TYPE(2) * b_sq * (sqrt(1 + SQUARE(d * (1.0 / threshold))) - 1)) / d_abs; - } - const TYPE threshold; -}; - -template -struct CL1 { - inline TYPE operator()(const TYPE d) const { - return TYPE(1) / (SQRT(ABS(d)) + 1e-12); - } -}; - -typedef CBlakeZisserman CRobustNorm; -/*----------------------------------------------------------------*/ - - -// makes sure the inverse NCC score does not exceed 0.3 -#if 0 -template -inline T robustincc(const T x) { return x / (T(1) + T(3) * x); } -template -inline T robustinccg(const T x) { return T(1)/SQUARE(T(1) + T(3) * x); } -template -inline T unrobustincc(const T y) { return y / (T(1) - T(3) * y); } -#elif 0 -template -inline T robustincc(const T x) { return T(1)-EXP(x*x*T(-4)); } -template -inline T robustinccg(const T x) { return T(8)*x*EXP(x*x*T(-4)); } -template -inline T unrobustincc(const T y) { return SQRT(-LOGN(T(1) - y))/T(2); } -#else -template -inline T robustincc(const T x) { return x/SQRT(T(0.3)+x*x); } -template -inline T robustinccg(const T x) { return T(0.3)/((T(0.3)+x*x)*SQRT(T(0.3)+x*x)); } -template -inline T unrobustincc(const T y) { return (SQRT(30)*y)/(T(10)*SQRT(T(1) - y*y)); } -#endif -/*----------------------------------------------------------------*/ - } // namespace SEACAVE diff --git a/libs/IO/ImageTIFF.cpp b/libs/IO/ImageTIFF.cpp index 86252c0db..bfdd7e177 100644 --- a/libs/IO/ImageTIFF.cpp +++ b/libs/IO/ImageTIFF.cpp @@ -76,10 +76,10 @@ extern "C" { static tmsize_t _tiffisReadProc(thandle_t fd, void* buf, tmsize_t size); static tmsize_t _tiffosWriteProc(thandle_t fd, void* buf, tmsize_t size); static tmsize_t _tiffisWriteProc(thandle_t, void*, tmsize_t); - static uint64 _tiffosSeekProc(thandle_t fd, uint64 off, int whence); - static uint64 _tiffisSeekProc(thandle_t fd, uint64 off, int whence); - static uint64 _tiffosSizeProc(thandle_t fd); - static uint64 _tiffisSizeProc(thandle_t fd); + static uint64_t _tiffosSeekProc(thandle_t fd, uint64_t off, int whence); + static uint64_t _tiffisSeekProc(thandle_t fd, uint64_t off, int whence); + static uint64_t _tiffosSizeProc(thandle_t fd); + static uint64_t _tiffisSizeProc(thandle_t fd); static int _tiffosCloseProc(thandle_t fd); static int _tiffisCloseProc(thandle_t fd); static int _tiffDummyMapProc(thandle_t, void** base, toff_t* size); @@ -132,26 +132,26 @@ extern "C" { return 0; } - static uint64 _tiffosSeekProc(thandle_t fd, uint64 off, int whence) + static uint64_t _tiffosSeekProc(thandle_t fd, uint64_t off, int whence) { tiffos_data *data = reinterpret_cast(fd); OSTREAM* os = data->stream; // if the stream has already failed, don't do anything if (os == NULL) - return static_cast(-1); + return static_cast(-1); bool bSucceeded(true); switch (whence) { case SEEK_SET: { // Compute 64-bit offset - uint64 new_offset = static_cast(data->start_pos) + off; + uint64_t new_offset = static_cast(data->start_pos) + off; // Verify that value does not overflow size_f_t offset = static_cast(new_offset); - if (static_cast(offset) != new_offset) - return static_cast(-1); + if (static_cast(offset) != new_offset) + return static_cast(-1); bSucceeded = os->setPos(offset); break; @@ -160,8 +160,8 @@ extern "C" { { // Verify that value does not overflow size_f_t offset = static_cast(off); - if (static_cast(offset) != off) - return static_cast(-1); + if (static_cast(offset) != off) + return static_cast(-1); bSucceeded = os->setPos(os->getPos()+offset); break; @@ -170,8 +170,8 @@ extern "C" { { // Verify that value does not overflow size_f_t offset = static_cast(off); - if (static_cast(offset) != off) - return static_cast(-1); + if (static_cast(offset) != off) + return static_cast(-1); bSucceeded = os->setPos(os->getSize()-offset); break; @@ -199,23 +199,23 @@ extern "C" { } // only do something if desired seek position is valid - if ((static_cast(origin) + off) > static_cast(data->start_pos)) { - uint64 num_fill; + if ((static_cast(origin) + off) > static_cast(data->start_pos)) { + uint64_t num_fill; // extend the stream to the expected size os->setPos(os->getSize()); - num_fill = (static_cast(origin)) + off - os->getPos(); + num_fill = (static_cast(origin)) + off - os->getPos(); const char dummy = '\0'; - for (uint64 i = 0; i < num_fill; i++) + for (uint64_t i = 0; i < num_fill; i++) os->write(&dummy, 1); // retry the seek - os->setPos(static_cast(static_cast(origin) + off)); + os->setPos(static_cast(static_cast(origin) + off)); } } - return static_cast(os->getPos()); + return static_cast(os->getPos()); } - static uint64 _tiffisSeekProc(thandle_t fd, uint64 off, int whence) + static uint64_t _tiffisSeekProc(thandle_t fd, uint64_t off, int whence) { tiffis_data *data = reinterpret_cast(fd); ISTREAM* is = data->stream; @@ -224,12 +224,12 @@ extern "C" { case SEEK_SET: { // Compute 64-bit offset - uint64 new_offset = static_cast(data->start_pos) + off; + uint64_t new_offset = static_cast(data->start_pos) + off; // Verify that value does not overflow size_f_t offset = static_cast(new_offset); - if (static_cast(offset) != new_offset) - return static_cast(-1); + if (static_cast(offset) != new_offset) + return static_cast(-1); is->setPos(offset); break; @@ -238,8 +238,8 @@ extern "C" { { // Verify that value does not overflow size_f_t offset = static_cast(off); - if (static_cast(offset) != off) - return static_cast(-1); + if (static_cast(offset) != off) + return static_cast(-1); is->setPos(is->getPos()+offset); break; @@ -248,27 +248,27 @@ extern "C" { { // Verify that value does not overflow size_f_t offset = static_cast(off); - if (static_cast(offset) != off) - return static_cast(-1); + if (static_cast(offset) != off) + return static_cast(-1); is->setPos(is->getSize()-offset); break; } } - return (uint64)(is->getPos() - data->start_pos); + return (uint64_t)(is->getPos() - data->start_pos); } - static uint64 _tiffosSizeProc(thandle_t fd) + static uint64_t _tiffosSizeProc(thandle_t fd) { tiffos_data *data = reinterpret_cast(fd); - return (uint64)data->stream->getSize(); + return (uint64_t)data->stream->getSize(); } - static uint64 _tiffisSizeProc(thandle_t fd) + static uint64_t _tiffisSizeProc(thandle_t fd) { tiffis_data *data = reinterpret_cast(fd); - return (uint64)data->stream->getSize(); + return (uint64_t)data->stream->getSize(); } static int _tiffosCloseProc(thandle_t fd) @@ -448,7 +448,7 @@ HRESULT CImageTIFF::ReadData(void* pData, PIXELFORMAT dataFormat, Size nStride, { if (m_state && m_width && m_height) { TIFF* tif = (TIFF*)m_state; - uint32 tile_width0 = m_width, tile_height0 = 0; + uint32_t tile_width0 = m_width, tile_height0 = 0; int is_tiled = TIFFIsTiled(tif); uint16 photometric; TIFFGetField(tif, TIFFTAG_PHOTOMETRIC, &photometric); @@ -477,14 +477,14 @@ HRESULT CImageTIFF::ReadData(void* pData, PIXELFORMAT dataFormat, Size nStride, tile_width0 = m_width; if (tile_height0 <= 0 || - (!is_tiled && tile_height0 == std::numeric_limits::max())) + (!is_tiled && tile_height0 == std::numeric_limits::max())) tile_height0 = m_height; uint8_t* data = (uint8_t*)pData; if (!is_tiled && tile_height0 == 1 && dataFormat == m_format && nStride == m_stride) { // read image directly to the data buffer for (Size j=0; j m_height) tile_height = m_height - y; - for (uint32 x = 0; x < m_width; x += tile_width0) { - uint32 tile_width = tile_width0; + for (uint32_t x = 0; x < m_width; x += tile_width0) { + uint32_t tile_width = tile_width0; if (x + tile_width > m_width) tile_width = m_width - x; @@ -510,9 +510,9 @@ HRESULT CImageTIFF::ReadData(void* pData, PIXELFORMAT dataFormat, Size nStride, { uint8_t* bstart = buffer; if (!is_tiled) - ok = TIFFReadRGBAStrip(tif, y, (uint32*)buffer); + ok = TIFFReadRGBAStrip(tif, y, (uint32_t*)buffer); else { - ok = TIFFReadRGBATile(tif, x, y, (uint32*)buffer); + ok = TIFFReadRGBATile(tif, x, y, (uint32_t*)buffer); //Tiles fill the buffer from the bottom up bstart += (tile_height0 - tile_height) * tile_width0 * 4; } @@ -521,7 +521,7 @@ HRESULT CImageTIFF::ReadData(void* pData, PIXELFORMAT dataFormat, Size nStride, return _INVALIDFILE; } - for (uint32 i = 0; i < tile_height; ++i) { + for (uint32_t i = 0; i < tile_height; ++i) { uint8_t* dst = data + x*3 + lineWidth*(tile_height - i - 1); uint8_t* src = bstart + i*tile_width0*4; if (!FilterFormat(dst, dataFormat, nStride, src, m_format, m_stride, tile_width)) { diff --git a/libs/IO/OBJ.cpp b/libs/IO/OBJ.cpp index bf82ee43d..0c1f5309b 100644 --- a/libs/IO/OBJ.cpp +++ b/libs/IO/OBJ.cpp @@ -61,7 +61,8 @@ bool ObjModel::MaterialLib::Save(const String& prefix, bool texLossless) const for (int_t i = 0; i < (int_t)materials.size(); ++i) { const Material& mat = materials[i]; // save material description - out << "newmtl " << mat.name << "\n" + std::stringstream ss; + ss << "newmtl " << mat.name << "\n" << "Ka 1.000000 1.000000 1.000000" << "\n" << "Kd " << mat.Kd.r << " " << mat.Kd.g << " " << mat.Kd.b << "\n" << "Ks 0.000000 0.000000 0.000000" << "\n" @@ -69,11 +70,20 @@ bool ObjModel::MaterialLib::Save(const String& prefix, bool texLossless) const << "illum 1" << "\n" << "Ns 1.000000" << "\n"; // save material maps - if (mat.diffuse_map.empty()) + if (mat.diffuse_map.empty()) { + #ifdef OBJ_USE_OPENMP + #pragma omp critical + #endif + out << ss.str(); continue; + } if (mat.diffuse_name.IsEmpty()) const_cast(mat.diffuse_name) = name+"_"+mat.name+"_map_Kd."+(texLossless?"png":"jpg"); - out << "map_Kd " << mat.diffuse_name << "\n"; + ss << "map_Kd " << mat.diffuse_name << "\n"; + #ifdef OBJ_USE_OPENMP + #pragma omp critical + #endif + out << ss.str(); const bool bRet(mat.diffuse_map.Save(pathName+mat.diffuse_name)); #ifdef OBJ_USE_OPENMP #pragma omp critical diff --git a/libs/MVS/Camera.cpp b/libs/MVS/Camera.cpp index cdcdbc154..01464b713 100644 --- a/libs/MVS/Camera.cpp +++ b/libs/MVS/Camera.cpp @@ -135,9 +135,9 @@ void MVS::DecomposeProjectionMatrix(const PMatrix& P, KMatrix& K, RMatrix& R, CM { // extract camera center as the right null vector of P const Vec4 hC(P.RightNullVector()); - C = (const CMatrix&)hC * INVERT(hC[3]); + C = CMatrix(hC[0],hC[1],hC[2]) * INVERT(hC[3]); // perform RQ decomposition - const cv::Mat mP(3,4,cv::DataType::type,(void*)P.val); + const cv::Mat mP(3,4,cv::DataType::type,const_cast(P.val)); cv::RQDecomp3x3(mP(cv::Rect(0,0, 3,3)), K, R); // normalize calibration matrix K *= INVERT(K(2,2)); @@ -162,9 +162,9 @@ void MVS::DecomposeProjectionMatrix(const PMatrix& P, RMatrix& R, CMatrix& C) #endif // extract camera center as the right null vector of P const Vec4 hC(P.RightNullVector()); - C = (const CMatrix&)hC * INVERT(hC[3]); + C = CMatrix(hC[0],hC[1],hC[2]) * INVERT(hC[3]); // get rotation - const cv::Mat mP(3,4,cv::DataType::type,(void*)P.val); + const cv::Mat mP(3,4,cv::DataType::type,const_cast(P.val)); mP(cv::Rect(0,0, 3,3)).copyTo(R); ASSERT(R.IsValid()); } // DecomposeProjectionMatrix @@ -174,17 +174,15 @@ void MVS::DecomposeProjectionMatrix(const PMatrix& P, RMatrix& R, CMatrix& C) void MVS::AssembleProjectionMatrix(const KMatrix& K, const RMatrix& R, const CMatrix& C, PMatrix& P) { // compute temporary matrices - cv::Mat mP(3,4,cv::DataType::type,(void*)P.val); + cv::Mat mP(3,4,cv::DataType::type,const_cast(P.val)); cv::Mat M(mP, cv::Rect(0,0, 3,3)); cv::Mat(K * R).copyTo(M); //3x3 mP.col(3) = M * cv::Mat(-C); //3x1 } // AssembleProjectionMatrix void MVS::AssembleProjectionMatrix(const RMatrix& R, const CMatrix& C, PMatrix& P) { - Eigen::Map > eM(P.val); - eM = (const Matrix3x3::EMat)R; - Eigen::Map< Point3::EVec,0,Eigen::Stride<0,4> > eT(P.val+3); - eT = ((const Matrix3x3::EMat)R) * (-((const Point3::EVec)C)); //3x1 + Eigen::Map >(P.val) = (const Matrix3x3::EMat)R; + Eigen::Map >(P.val+3) = ((const Matrix3x3::EMat)R) * (-((const Point3::EVec)C)); } // AssembleProjectionMatrix /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Camera.h b/libs/MVS/Camera.h index 6ba938778..063ede568 100644 --- a/libs/MVS/Camera.h +++ b/libs/MVS/Camera.h @@ -203,16 +203,10 @@ class MVS_API CameraIntern // normalize inhomogeneous 2D point by the given camera intrinsics K // K is assumed to be the [3,3] triangular matrix with: fx, fy, s, cx, cy and scale 1 - template - static inline void NormalizeProjection(const TYPE1* K, const TYPE2* x, TYPE3* n) { - n[0] = TYPE3(K[0]*x[0] + K[1]*x[1] + K[2]); - n[1] = TYPE3( K[4]*x[1] + K[5]); - } template inline TPoint2 NormalizeProjection(const TPoint2& proj) const { TPoint2 pt; - const TMatrix invK(GetInvK()); - NormalizeProjection(invK.val, proj.ptr(), pt.ptr()); + NormalizeProjectionInv(GetInvK(), proj.ptr(), pt.ptr()); return pt; } @@ -422,7 +416,7 @@ class MVS_API Camera : public CameraIntern BOOST_SERIALIZATION_SPLIT_MEMBER() #endif }; -typedef SEACAVE::cList CameraArr; +typedef CLISTDEF0IDX(Camera,uint32_t) CameraArr; /*----------------------------------------------------------------*/ MVS_API void DecomposeProjectionMatrix(const PMatrix& P, KMatrix& K, RMatrix& R, CMatrix& C); diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 099148404..b72690de4 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -68,10 +68,10 @@ using namespace MVS; namespace MVS { DEFOPT_SPACE(OPTDENSE, _T("Dense")) - DEFVAR_OPTDENSE_uint32(nResolutionLevel, "Resolution Level", "How many times to scale down the images before dense reconstruction", "1") MDEFVAR_OPTDENSE_uint32(nMaxResolution, "Max Resolution", "Do not scale images lower than this resolution", "3200") MDEFVAR_OPTDENSE_uint32(nMinResolution, "Min Resolution", "Do not scale images lower than this resolution", "640") +DEFVAR_OPTDENSE_uint32(nSubResolutionLevels, "SubResolution levels", "Number of lower resolution levels to estimate the depth and normals", "2") DEFVAR_OPTDENSE_uint32(nMinViews, "Min Views", "minimum number of agreeing views to validate a depth", "2") MDEFVAR_OPTDENSE_uint32(nMaxViews, "Max Views", "maximum number of neighbor images used to compute the depth-map for the reference image", "12") DEFVAR_OPTDENSE_uint32(nMinViewsFuse, "Min Views Fuse", "minimum number of images that agrees with an estimate during fusion in order to consider it inlier (<2 - only merge depth-maps)", "2") @@ -82,13 +82,15 @@ MDEFVAR_OPTDENSE_uint32(nNumViews, "Num Views", "Number of views used for depth- MDEFVAR_OPTDENSE_uint32(nPointInsideROI, "Point Inside ROI", "consider a point shared only if inside ROI when estimating the neighbor views (0 - ignore ROI, 1 - weight more ROI points, 2 - consider only ROI points)", "1") MDEFVAR_OPTDENSE_bool(bFilterAdjust, "Filter Adjust", "adjust depth estimates during filtering", "1") MDEFVAR_OPTDENSE_bool(bAddCorners, "Add Corners", "add support points at image corners with nearest neighbor disparities", "0") +MDEFVAR_OPTDENSE_bool(bInitSparse, "Init Sparse", "init depth-map only with the sparse points (no interpolation)", "1") +MDEFVAR_OPTDENSE_bool(bRemoveDmaps, "Remove Dmaps", "remove depth-maps after fusion", "0") MDEFVAR_OPTDENSE_float(fViewMinScore, "View Min Score", "Min score to consider a neighbor images (0 - disabled)", "2.0") MDEFVAR_OPTDENSE_float(fViewMinScoreRatio, "View Min Score Ratio", "Min score ratio to consider a neighbor images", "0.03") MDEFVAR_OPTDENSE_float(fMinArea, "Min Area", "Min shared area for accepting the depth triangulation", "0.05") MDEFVAR_OPTDENSE_float(fMinAngle, "Min Angle", "Min angle for accepting the depth triangulation", "3.0") MDEFVAR_OPTDENSE_float(fOptimAngle, "Optim Angle", "Optimal angle for computing the depth triangulation", "12.0") MDEFVAR_OPTDENSE_float(fMaxAngle, "Max Angle", "Max angle for accepting the depth triangulation", "65.0") -MDEFVAR_OPTDENSE_float(fDescriptorMinMagnitudeThreshold, "Descriptor Min Magnitude Threshold", "minimum texture variance accepted when matching two patches (0 - disabled)", "0.01") +MDEFVAR_OPTDENSE_float(fDescriptorMinMagnitudeThreshold, "Descriptor Min Magnitude Threshold", "minimum patch texture variance accepted when matching two patches (0 - disabled)", "0.02") // 0.02: pixels with patch texture variance below 0.0004 (0.02^2) will be removed from depthmap; 0.12: patch texture variance below 0.02 (0.12^2) is considered texture-less MDEFVAR_OPTDENSE_float(fDepthDiffThreshold, "Depth Diff Threshold", "maximum variance allowed for the depths during refinement", "0.01") MDEFVAR_OPTDENSE_float(fNormalDiffThreshold, "Normal Diff Threshold", "maximum variance allowed for the normal during fusion (degrees)", "25") MDEFVAR_OPTDENSE_float(fPairwiseMul, "Pairwise Mul", "pairwise cost scale to match the unary cost", "0.3") @@ -97,13 +99,10 @@ MDEFVAR_OPTDENSE_int32(nOptimizerMaxIters, "Optimizer Max Iters", "MRF optimizer MDEFVAR_OPTDENSE_uint32(nSpeckleSize, "Speckle Size", "maximal size of a speckle (small speckles get removed)", "100") MDEFVAR_OPTDENSE_uint32(nIpolGapSize, "Interpolate Gap Size", "interpolate small gaps (left<->right, top<->bottom)", "7") MDEFVAR_OPTDENSE_int32(nIgnoreMaskLabel, "Ignore Mask Label", "label id used during ignore mask filter (<0 - disabled)", "-1") -MDEFVAR_OPTDENSE_uint32(nOptimize, "Optimize", "should we filter the extracted depth-maps?", "7") // see DepthFlags +DEFVAR_OPTDENSE_uint32(nOptimize, "Optimize", "should we filter the extracted depth-maps?", "7") // see DepthFlags MDEFVAR_OPTDENSE_uint32(nEstimateColors, "Estimate Colors", "should we estimate the colors for the dense point-cloud?", "2", "0", "1") MDEFVAR_OPTDENSE_uint32(nEstimateNormals, "Estimate Normals", "should we estimate the normals for the dense point-cloud?", "0", "1", "2") -MDEFVAR_OPTDENSE_float(fNCCThresholdKeep, "NCC Threshold Keep", "Maximum 1-NCC score accepted for a match", "0.55", "0.3") -#ifdef _USE_CUDA -MDEFVAR_OPTDENSE_float(fNCCThresholdKeepCUDA, "NCC Threshold Keep CUDA", "Maximum 1-NCC score accepted for a CUDA match (differs from the CPU version cause that has planarity score integrated)", "0.9", "0.6") -#endif // _USE_CUDA +MDEFVAR_OPTDENSE_float(fNCCThresholdKeep, "NCC Threshold Keep", "Maximum 1-NCC score accepted for a match", "0.9", "0.5") DEFVAR_OPTDENSE_uint32(nEstimationIters, "Estimation Iters", "Number of patch-match iterations", "3") DEFVAR_OPTDENSE_uint32(nEstimationGeometricIters, "Estimation Geometric Iters", "Number of geometric consistent patch-match iterations (0 - disabled)", "2") MDEFVAR_OPTDENSE_float(fEstimationGeometricWeight, "Estimation Geometric Weight", "pairwise geometric consistency cost weight", "0.1") @@ -121,6 +120,20 @@ MDEFVAR_OPTDENSE_float(fRandomSmoothBonus, "Random Smooth Bonus", "Score factor // S T R U C T S /////////////////////////////////////////////////// +//constructor from reference of DepthData +DepthData::DepthData(const DepthData& srcDepthData) : + images(srcDepthData.images), + neighbors(srcDepthData.neighbors), + points(srcDepthData.points), + mask(srcDepthData.mask), + depthMap(srcDepthData.depthMap), + normalMap(srcDepthData.normalMap), + confMap(srcDepthData.confMap), + dMin(srcDepthData.dMin), + dMax(srcDepthData.dMax), + references(srcDepthData.references) +{} + // return normal in world-space for the given pixel // the 3D points can be precomputed and passed here void DepthData::GetNormal(const ImageRef& ir, Point3f& N, const TImage* pPointMap) const @@ -229,7 +242,7 @@ bool DepthData::Save(const String& fileName) const for (const ViewData& image: images) IDs.push_back(image.GetID()); const ViewData& image0 = GetView(); - if (!ExportDepthDataRaw(fileNameTmp, image0.pImageData->name, IDs, depthMap.size(), image0.camera.K, image0.camera.R, image0.camera.C, dMin, dMax, depthMap, normalMap, confMap)) + if (!ExportDepthDataRaw(fileNameTmp, image0.pImageData->name, IDs, depthMap.size(), image0.camera.K, image0.camera.R, image0.camera.C, dMin, dMax, depthMap, normalMap, confMap, viewsMap)) return false; } if (!File::renameFile(fileNameTmp, fileName)) { @@ -246,7 +259,7 @@ bool DepthData::Load(const String& fileName, unsigned flags) IIndexArr IDs; cv::Size imageSize; Camera camera; - if (!ImportDepthDataRaw(fileName, imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap, flags)) + if (!ImportDepthDataRaw(fileName, imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap, viewsMap, flags)) return false; ASSERT(!IsValid() || (IDs.size() == images.size() && IDs.front() == GetView().GetID())); ASSERT(depthMap.size() == imageSize); @@ -339,7 +352,7 @@ void DepthEstimator::MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimat } } -// replace POWI(0.5f, (int)idxScaleRange): 0 1 2 3 4 5 6 7 8 9 10 11 +// replace POWI(0.5f, idxScaleRange): 0 1 2 3 4 5 6 7 8 9 10 11 const float DepthEstimator::scaleRanges[12] = {1.f, 0.5f, 0.25f, 0.125f, 0.0625f, 0.03125f, 0.015625f, 0.0078125f, 0.00390625f, 0.001953125f, 0.0009765625f, 0.00048828125f}; DepthEstimator::DepthEstimator( @@ -382,12 +395,12 @@ DepthEstimator::DepthEstimator( smoothSigmaDepth(-1.f/(2.f*SQUARE(OPTDENSE::fRandomSmoothDepth))), // used in exp(-x^2 / (2*(0.02^2))) smoothSigmaNormal(-1.f/(2.f*SQUARE(FD2R(OPTDENSE::fRandomSmoothNormal)))), // used in exp(-x^2 / (2*(0.22^2))) thMagnitudeSq(OPTDENSE::fDescriptorMinMagnitudeThreshold>0?SQUARE(OPTDENSE::fDescriptorMinMagnitudeThreshold):-1.f), - angle1Range(FD2R(OPTDENSE::fRandomAngle1Range)), - angle2Range(FD2R(OPTDENSE::fRandomAngle2Range)), - thConfSmall(OPTDENSE::fNCCThresholdKeep*0.2f), - thConfBig(OPTDENSE::fNCCThresholdKeep*0.4f), - thConfRand(OPTDENSE::fNCCThresholdKeep*0.9f), - thRobust(OPTDENSE::fNCCThresholdKeep*1.2f) + angle1Range(FD2R(OPTDENSE::fRandomAngle1Range)), //default 0.279252678=FD2R(20) + angle2Range(FD2R(OPTDENSE::fRandomAngle2Range)), //default 0.174532920=FD2R(16) + thConfSmall(OPTDENSE::fNCCThresholdKeep * 0.66f), // default 0.6 + thConfBig(OPTDENSE::fNCCThresholdKeep * 0.9f), // default 0.8 + thConfRand(OPTDENSE::fNCCThresholdKeep * 1.1f), // default 0.99 + thRobust(OPTDENSE::fNCCThresholdKeep * 4.f / 3.f) // default 1.2 #if DENSE_REFINE == DENSE_REFINE_EXACT , thPerturbation(1.f/POW(2.f,float(nIter+1))) #endif @@ -439,9 +452,9 @@ bool DepthEstimator::FillPixelPatch() } normSq0 = w.normSq0; #endif - if (normSq0 < thMagnitudeSq) + if (normSq0 < thMagnitudeSq && (lowResDepthMap.empty() || lowResDepthMap(x0) <= 0)) return false; - reinterpret_cast(X0) = image0.camera.TransformPointI2C(Cast(x0)); + X0 = image0.camera.TransformPointI2C(Cast(x0)); return true; } @@ -496,7 +509,7 @@ float DepthEstimator::ScorePixelImage(const DepthData::ViewData& image1, Depth d const float normSq1(normSqDelta(texels1.data(), sum/(float)nTexels)); #endif const float nrmSq(normSq0*normSq1); - if (nrmSq <= 0.f) + if (nrmSq <=1e-16f) return thRobust; #if DENSE_NCC == DENSE_NCC_DEFAULT const float num(texels0.dot(texels1)); @@ -506,19 +519,20 @@ float DepthEstimator::ScorePixelImage(const DepthData::ViewData& image1, Depth d #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA // encourage smoothness for (const NeighborEstimate& neighbor: neighborsClose) { + ASSERT(neighbor.depth > 0); #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE const float factorDepth(DENSE_EXP(SQUARE(plane.Distance(neighbor.X)/depth) * smoothSigmaDepth)); #else const float factorDepth(DENSE_EXP(SQUARE((depth-neighbor.depth)/depth) * smoothSigmaDepth)); #endif - const float factorNormal(DENSE_EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal)); + const float factorNormal(DENSE_EXP(SQUARE(ACOS(ComputeAngle(normal.ptr(), neighbor.normal.ptr()))) * smoothSigmaNormal)); score *= (1.f - smoothBonusDepth * factorDepth) * (1.f - smoothBonusNormal * factorNormal); } #endif if (!image1.depthMap.empty()) { ASSERT(OPTDENSE::fEstimationGeometricWeight > 0); float consistency(4.f); - const Point3f X1(image1.Tl*Point3f(float(X0(0))*depth,float(X0(1))*depth,depth)+image1.Tm); // Kj * Rj * (Ri.t() * X + Ci - Cj) + const Point3f X1(image1.Tl*Point3f(float(X0.x)*depth,float(X0.y)*depth,depth)+image1.Tm); // Kj * Rj * (Ri.t() * X + Ci - Cj) if (X1.z > 0) { const Point2f x1(X1); if (image1.depthMap.isInsideWithBorder(x1)) { @@ -532,14 +546,24 @@ float DepthEstimator::ScorePixelImage(const DepthData::ViewData& image1, Depth d } score += OPTDENSE::fEstimationGeometricWeight * consistency; } + // apply depth prior weight based on patch textureless + if (!lowResDepthMap.empty()) { + const Depth d0 = lowResDepthMap(x0); + if (d0 > 0) { + const float deltaDepth(MINF(DepthSimilarity(d0, depth), 0.5f)); + const float smoothSigmaDepth(-1.f / (1.f * 0.02f)); // 0.12: patch texture variance below 0.02 (0.12^2) is considered texture-less + const float factorDeltaDepth(DENSE_EXP(normSq0 * smoothSigmaDepth)); + score = (1.f-factorDeltaDepth)*score + factorDeltaDepth*deltaDepth; + } + } ASSERT(ISFINITE(score)); - return score; + return MIN(2.f, score); } // compute pixel's NCC score float DepthEstimator::ScorePixel(Depth depth, const Normal& normal) { - ASSERT(depth > 0 && normal.dot(Cast(static_cast(X0))) <= 0); + ASSERT(depth > 0 && normal.dot(Cast(X0)) <= 0); // compute score for this pixel as seen in each view ASSERT(scores.size() == images.size()); FOREACH(idxView, images) @@ -618,6 +642,7 @@ void DepthEstimator::ProcessPixel(IDX idx) const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighbors.emplace_back(nx); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE @@ -634,6 +659,7 @@ void DepthEstimator::ProcessPixel(IDX idx) const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighbors.emplace_back(nx); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE @@ -649,22 +675,26 @@ void DepthEstimator::ProcessPixel(IDX idx) if (x0.x < size.width-nSizeHalfWindow) { const ImageRef nx(x0.x+1, x0.y); const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) + if (ndepth > 0) { + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif }); + } } if (x0.y < size.height-nSizeHalfWindow) { const ImageRef nx(x0.x, x0.y+1); const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) + if (ndepth > 0) { + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif }); + } } #endif } else { @@ -675,6 +705,7 @@ void DepthEstimator::ProcessPixel(IDX idx) const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighbors.emplace_back(nx); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE @@ -691,6 +722,7 @@ void DepthEstimator::ProcessPixel(IDX idx) const Depth ndepth(depthMap0(nx)); if (ndepth > 0) { #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighbors.emplace_back(nx); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE @@ -706,29 +738,33 @@ void DepthEstimator::ProcessPixel(IDX idx) if (x0.x > nSizeHalfWindow) { const ImageRef nx(x0.x-1, x0.y); const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) + if (ndepth > 0) { + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif }); + } } if (x0.y > nSizeHalfWindow) { const ImageRef nx(x0.x, x0.y-1); const Depth ndepth(depthMap0(nx)); - if (ndepth > 0) + if (ndepth > 0) { + ASSERT(ISEQUAL(norm(normalMap0(nx)), 1.f)); neighborsClose.emplace_back(NeighborEstimate{ndepth,normalMap0(nx) #if DENSE_SMOOTHNESS == DENSE_SMOOTHNESS_PLANE , Cast(image0.camera.TransformPointI2C(Point3(nx, ndepth))) #endif }); + } } #endif } float& conf = confMap0(x0); Depth& depth = depthMap0(x0); Normal& normal = normalMap0(x0); - const Normal viewDir(Cast(reinterpret_cast(X0))); + const Normal viewDir(Cast(X0)); ASSERT(depth > 0 && normal.dot(viewDir) <= 0); #if DENSE_REFINE == DENSE_REFINE_ITER // check if any of the neighbor estimates are better then the current estimate @@ -742,7 +778,7 @@ void DepthEstimator::ProcessPixel(IDX idx) if (confMap0(nx) >= OPTDENSE::fNCCThresholdKeep) continue; #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA - NeighborEstimate& neighbor = neighborsClose[n]; + NeighborEstimate neighbor = neighborsClose[n]; #endif neighbor.depth = InterpolatePixel(nx, neighbor.depth, neighbor.normal); CorrectNormal(neighbor.normal); @@ -767,6 +803,9 @@ void DepthEstimator::ProcessPixel(IDX idx) idxScaleRange = 1; else if (conf >= thConfRand) { // try completely random values in order to find an initial estimate + #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA + neighborsClose.Empty(); + #endif for (unsigned iter=0; iter(X0))); + depthNew = (Depth)(planeD / planeN.dot(X0)); #endif } #endif @@ -921,10 +960,10 @@ Depth DepthEstimator::InterpolatePixel(const ImageRef& nx, Depth depth, const No void DepthEstimator::InitPlane(Depth depth, const Normal& normal) { #if 0 - plane.Set(reinterpret_cast(normal), Vec3f(depth*Cast(X0))); + plane.Set(normal, Normal(Cast(X0)*depth)); #else - plane.m_vN = reinterpret_cast(normal); - plane.m_fD = -depth*reinterpret_cast(normal).dot(Cast(X0)); + plane.m_vN = normal; + plane.m_fD = -depth*normal.dot(Cast(X0)); #endif } #endif @@ -940,7 +979,7 @@ DepthEstimator::PixelEstimate DepthEstimator::PerturbEstimate(const PixelEstimat ptbEst.depth = CLAMP(rnd.randomUniform(minDepth, maxDepth), dMin, dMax); // perturb normal - const Normal viewDir(Cast(static_cast(X0))); + const Normal viewDir(Cast(X0)); std::uniform_real_distribution urd(-1.f, 1.f); const int numMaxTrials = 3; int numTrials = 0; @@ -984,7 +1023,6 @@ std::pair TriangulatePointsDelaunay(const DepthData::ViewData& imag typedef CGAL::Delaunay_triangulation_2 Delaunay; typedef Delaunay::Face_circulator FaceCirculator; typedef Delaunay::Face_handle FaceHandle; - typedef Delaunay::Vertex_circulator VertexCirculator; typedef Delaunay::Vertex_handle VertexHandle; typedef kernel_t::Point_2 CPoint; @@ -1075,7 +1113,7 @@ std::pair TriangulatePointsDelaunay(const DepthData::ViewData& imag // and interpolating normal and depth for all pixels bool MVS::TriangulatePoints2DepthMap( const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, - DepthMap& depthMap, NormalMap& normalMap, Depth& dMin, Depth& dMax, bool bAddCorners) + DepthMap& depthMap, NormalMap& normalMap, Depth& dMin, Depth& dMax, bool bAddCorners, bool bSparseOnly) { ASSERT(image.pImageData != NULL); @@ -1091,50 +1129,68 @@ bool MVS::TriangulatePoints2DepthMap( mesh.ComputeNormalVertices(); depthMap.create(image.image.size()); normalMap.create(image.image.size()); - if (!bAddCorners) { + if (!bAddCorners || bSparseOnly) { depthMap.memset(0); normalMap.memset(0); } - struct RasterDepth : TRasterMeshBase { - typedef TRasterMeshBase Base; - using Base::camera; - using Base::depthMap; - using Base::ptc; - using Base::pti; - const Mesh::NormalArr& vertexNormals; - NormalMap& normalMap; - Mesh::Face face; - RasterDepth(const Mesh::NormalArr& _vertexNormals, const Camera& _camera, DepthMap& _depthMap, NormalMap& _normalMap) - : Base(_camera, _depthMap), vertexNormals(_vertexNormals), normalMap(_normalMap) {} - inline void operator()(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); - ASSERT(z > Depth(0)); - depthMap(pt) = z; - normalMap(pt) = normalized( - vertexNormals[face[0]] * pbary[0]+ - vertexNormals[face[1]] * pbary[1]+ - vertexNormals[face[2]] * pbary[2] - ); + if (bSparseOnly) { + // just project sparse pointcloud onto depthmap + FOREACH(i, mesh.vertices) { + const Point2f& x(projs[i]); + const Point2i ix(FLOOR2INT(x)); + const Depth z(mesh.vertices[i].z); + const Normal& normal(mesh.vertexNormals[i]); + for (const Point2i dx : {Point2i(0,0),Point2i(1,0),Point2i(0,1),Point2i(1,1)}) { + const Point2i ax(ix + dx); + if (!depthMap.isInside(ax)) + continue; + depthMap(ax) = z; + normalMap(ax) = normal; + } + } + } else { + // rasterize triangles onto depthmap + struct RasterDepth : TRasterMeshBase { + typedef TRasterMeshBase Base; + using Base::camera; + using Base::depthMap; + using Base::ptc; + using Base::pti; + const Mesh::NormalArr& vertexNormals; + NormalMap& normalMap; + Mesh::Face face; + RasterDepth(const Mesh::NormalArr& _vertexNormals, const Camera& _camera, DepthMap& _depthMap, NormalMap& _normalMap) + : Base(_camera, _depthMap), vertexNormals(_vertexNormals), normalMap(_normalMap) {} + inline void operator()(const ImageRef& pt, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); + const Depth z(ComputeDepth(pbary)); + ASSERT(z > Depth(0)); + depthMap(pt) = z; + normalMap(pt) = normalized( + vertexNormals[face[0]] * pbary[0]+ + vertexNormals[face[1]] * pbary[1]+ + vertexNormals[face[2]] * pbary[2] + ); + } + }; + RasterDepth rasterer = {mesh.vertexNormals, camera, depthMap, normalMap}; + for (const Mesh::Face& face : mesh.faces) { + rasterer.face = face; + rasterer.ptc[0].z = mesh.vertices[face[0]].z; + rasterer.ptc[1].z = mesh.vertices[face[1]].z; + rasterer.ptc[2].z = mesh.vertices[face[2]].z; + Image8U::RasterizeTriangleBary( + projs[face[0]], + projs[face[1]], + projs[face[2]], rasterer); } - }; - RasterDepth rasterer = {mesh.vertexNormals, camera, depthMap, normalMap}; - for (const Mesh::Face& face : mesh.faces) { - rasterer.face = face; - rasterer.ptc[0].z = mesh.vertices[face[0]].z; - rasterer.ptc[1].z = mesh.vertices[face[1]].z; - rasterer.ptc[2].z = mesh.vertices[face[2]].z; - Image8U::RasterizeTriangleBary( - projs[face[0]], - projs[face[1]], - projs[face[2]], rasterer); } return true; } // TriangulatePoints2DepthMap // same as above, but does not estimate the normal-map bool MVS::TriangulatePoints2DepthMap( const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, - DepthMap& depthMap, Depth& dMin, Depth& dMax, bool bAddCorners) + DepthMap& depthMap, Depth& dMin, Depth& dMax, bool bAddCorners, bool bSparseOnly) { ASSERT(image.pImageData != NULL); @@ -1148,29 +1204,45 @@ bool MVS::TriangulatePoints2DepthMap( // create rough depth-map by interpolating inside triangles const Camera& camera = image.camera; depthMap.create(image.image.size()); - if (!bAddCorners) + if (!bAddCorners || bSparseOnly) depthMap.memset(0); - struct RasterDepth : TRasterMeshBase { - typedef TRasterMeshBase Base; - using Base::depthMap; - RasterDepth(const Camera& _camera, DepthMap& _depthMap) - : Base(_camera, _depthMap) {} - inline void operator()(const ImageRef& pt, const Point3f& bary) { - const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); - const Depth z(ComputeDepth(pbary)); - ASSERT(z > Depth(0)); - depthMap(pt) = z; + if (bSparseOnly) { + // just project sparse pointcloud onto depthmap + FOREACH(i, mesh.vertices) { + const Point2f& x(projs[i]); + const Point2i ix(FLOOR2INT(x)); + const Depth z(mesh.vertices[i].z); + for (const Point2i dx : {Point2i(0,0),Point2i(1,0),Point2i(0,1),Point2i(1,1)}) { + const Point2i ax(ix + dx); + if (!depthMap.isInside(ax)) + continue; + depthMap(ax) = z; + } + } + } else { + // rasterize triangles onto depthmap + struct RasterDepth : TRasterMeshBase { + typedef TRasterMeshBase Base; + using Base::depthMap; + RasterDepth(const Camera& _camera, DepthMap& _depthMap) + : Base(_camera, _depthMap) {} + inline void operator()(const ImageRef& pt, const Point3f& bary) { + const Point3f pbary(PerspectiveCorrectBarycentricCoordinates(bary)); + const Depth z(ComputeDepth(pbary)); + ASSERT(z > Depth(0)); + depthMap(pt) = z; + } + }; + RasterDepth rasterer = {camera, depthMap}; + for (const Mesh::Face& face : mesh.faces) { + rasterer.ptc[0].z = mesh.vertices[face[0]].z; + rasterer.ptc[1].z = mesh.vertices[face[1]].z; + rasterer.ptc[2].z = mesh.vertices[face[2]].z; + Image8U::RasterizeTriangleBary( + projs[face[0]], + projs[face[1]], + projs[face[2]], rasterer); } - }; - RasterDepth rasterer = {camera, depthMap}; - for (const Mesh::Face& face : mesh.faces) { - rasterer.ptc[0].z = mesh.vertices[face[0]].z; - rasterer.ptc[1].z = mesh.vertices[face[1]].z; - rasterer.ptc[2].z = mesh.vertices[face[2]].z; - Image8U::RasterizeTriangleBary( - projs[face[0]], - projs[face[1]], - projs[face[2]], rasterer); } return true; } // TriangulatePoints2DepthMap @@ -1179,41 +1251,104 @@ bool MVS::TriangulatePoints2DepthMap( namespace MVS { -class PlaneSolverAdaptor +// least squares refinement of the given plane to the 3D point set +// (return the number of iterations) +template +int OptimizePlane(TPlane& plane, const Eigen::Matrix* points, size_t size, int maxIters, TYPE threshold) +{ + typedef TPlane PLANE; + typedef Eigen::Matrix POINT; + ASSERT(size >= PLANE::numParams); + struct OptimizationFunctor { + const POINT* points; + const size_t size; + const RobustNorm::GemanMcClure robust; + // construct with the data points + OptimizationFunctor(const POINT* _points, size_t _size, double _th) + : points(_points), size(_size), robust(_th) { ASSERT(size < (size_t)std::numeric_limits::max()); } + static void Residuals(const double* x, int nPoints, const void* pData, double* fvec, double* fjac, int* /*info*/) { + const OptimizationFunctor& data = *reinterpret_cast(pData); + ASSERT((size_t)nPoints == data.size && fvec != NULL && fjac == NULL); + TPlane plane; { + Point3d N; + plane.m_fD = x[0]; + Dir2Normal(reinterpret_cast(x[1]), N); + plane.m_vN = N; + } + for (size_t i=0; i())); + } + } functor(points, size, threshold); + double arrParams[PLANE::numParams]; { + arrParams[0] = (double)plane.m_fD; + const Point3d N(plane.m_vN.x(), plane.m_vN.y(), plane.m_vN.z()); + Normal2Dir(N, reinterpret_cast(arrParams[1])); + } + lm_control_struct control = {1.e-6, 1.e-7, 1.e-8, 1.e-7, 100.0, maxIters}; // lm_control_float; + lm_status_struct status; + lmmin(PLANE::numParams, arrParams, (int)size, &functor, OptimizationFunctor::Residuals, &control, &status); + switch (status.info) { + //case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + case 12: + DEBUG_ULTIMATE("error: refine plane: %s", lm_infmsg[status.info]); + return 0; + } + { + Point3d N; + plane.m_fD = (TYPE)arrParams[0]; + Dir2Normal(reinterpret_cast(arrParams[1]), N); + plane.m_vN = Cast(N); + } + return status.nfev; +} + +template +class TPlaneSolverAdaptor { public: enum { MINIMUM_SAMPLES = 3 }; enum { MAX_MODELS = 1 }; - typedef Plane Model; - typedef cList Models; + typedef TYPE Type; + typedef TPoint3 Point; + typedef CLISTDEF0(Point) Points; + typedef TPlane Model; + typedef CLISTDEF0(Model) Models; - PlaneSolverAdaptor(const Point3Arr& points) + TPlaneSolverAdaptor(const Points& points) : points_(points) { } - PlaneSolverAdaptor(const Point3Arr& points, float w, float h, float d) + TPlaneSolverAdaptor(const Points& points, float w, float h, float d) : points_(points) { // LogAlpha0 is used to make error data scale invariant // Ratio of containing diagonal image rectangle over image area const float D = SQRT(w*w + h*h + d*d); // diameter const float A = w*h*d+1.f; // volume - logalpha0_ = LOG10(2.0f*D/A*0.5f); + logalpha0_ = LOG10(2.f*D/A*0.5f); } inline bool Fit(const std::vector& samples, Models& models) const { - Point3 points[3]; - for (size_t i=0; i(points_.GetSize()); } + static double Error(const Model& plane, const Points& points) { + double e(0); + for (const Point& X: points) + e += plane.DistanceAbs(X); + return e/points.size(); + } + + inline size_t NumSamples() const { return static_cast(points_.size()); } inline double logalpha0() const { return logalpha0_; } inline double multError() const { return 0.5; } protected: - const Point3Arr& points_; // Normalized input data + const Points& points_; // Normalized input data double logalpha0_; // Alpha0 is used to make the error adaptive to the image size Model model2evaluate; // current model to be evaluated }; // Robustly estimate the plane that fits best the given points -template -unsigned TEstimatePlane(const Point3Arr& points, Plane& plane, double& maxThreshold, bool arrInliers[], size_t maxIters) +template +unsigned TEstimatePlane(const CLISTDEF0(TPoint3)& points, TPlane& plane, double& maxThreshold, bool arrInliers[], size_t maxIters) { - const unsigned nPoints = (unsigned)points.GetSize(); + typedef TPlaneSolverAdaptor PlaneSolverAdaptor; + + plane.Invalidate(); + + const unsigned nPoints = (unsigned)points.size(); if (nPoints < PlaneSolverAdaptor::MINIMUM_SAMPLES) { ASSERT("too few points" == NULL); return 0; } // normalize points - Matrix4x4 H; - Point3Arr normPoints; + TMatrix H; + typename PlaneSolverAdaptor::Points normPoints; NormalizePoints(points, normPoints, &H); // plane robust estimation std::vector vec_inliers; Sampler sampler; if (bFixThreshold) { + if (maxThreshold == 0) + maxThreshold = 0.35/H(0,0); PlaneSolverAdaptor kernel(normPoints); - RANSAC(kernel, sampler, vec_inliers, plane, maxThreshold!=0?maxThreshold*H(0,0):0.35, 0.99, maxIters); + RANSAC(kernel, sampler, vec_inliers, plane, maxThreshold*H(0,0), 0.99, maxIters); DEBUG_LEVEL(3, "Robust plane: %u/%u points", vec_inliers.size(), nPoints); } else { + if (maxThreshold != DBL_MAX) + maxThreshold *= H(0,0); PlaneSolverAdaptor kernel(normPoints, 1, 1, 1); const std::pair ACRansacOut(ACRANSAC(kernel, sampler, vec_inliers, plane, maxThreshold, 0.99, maxIters)); const double& thresholdSq = ACRansacOut.first; - maxThreshold = SQRT(thresholdSq); - DEBUG_LEVEL(3, "Auto-robust plane: %u/%u points (%g threshold)", vec_inliers.size(), nPoints, maxThreshold/H(0,0)); + maxThreshold = SQRT(thresholdSq)/H(0,0); + DEBUG_LEVEL(3, "Auto-robust plane: %u/%u points (%g threshold)", vec_inliers.size(), nPoints, maxThreshold); } - const unsigned inliers_count = (unsigned)vec_inliers.size(); + unsigned inliers_count = (unsigned)vec_inliers.size(); if (inliers_count < PlaneSolverAdaptor::MINIMUM_SAMPLES) return 0; // fit plane to all the inliers - Point3Arr normInliers(inliers_count); - for (uint32_t i=0; i fitPlane; + for (unsigned i=0; i(points, plane, maxThreshold, arrInliers, maxIters); + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); } // EstimatePlane // Robustly estimate the plane that fits best the given points, making sure the first point is part of the solution (if any) -unsigned MVS::EstimatePlaneLockFirstPoint(const Point3Arr& points, Plane& plane, double& maxThreshold, bool arrInliers[], size_t maxIters) +unsigned MVS::EstimatePlaneLockFirstPoint(const Point3dArr& points, Planed& plane, double& maxThreshold, bool arrInliers[], size_t maxIters) { - return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); } // EstimatePlaneLockFirstPoint // Robustly estimate the plane that fits best the given points using a known threshold -unsigned MVS::EstimatePlaneTh(const Point3Arr& points, Plane& plane, double maxThreshold, bool arrInliers[], size_t maxIters) +unsigned MVS::EstimatePlaneTh(const Point3dArr& points, Planed& plane, double maxThreshold, bool arrInliers[], size_t maxIters) { - return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); } // EstimatePlaneTh // Robustly estimate the plane that fits best the given points using a known threshold, making sure the first point is part of the solution (if any) -unsigned MVS::EstimatePlaneThLockFirstPoint(const Point3Arr& points, Plane& plane, double maxThreshold, bool arrInliers[], size_t maxIters) +unsigned MVS::EstimatePlaneThLockFirstPoint(const Point3dArr& points, Planed& plane, double maxThreshold, bool arrInliers[], size_t maxIters) { - return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); } // EstimatePlaneThLockFirstPoint +// least squares refinement of the given plane to the 3D point set +int MVS::OptimizePlane(Planed& plane, const Eigen::Vector3d* points, size_t size, int maxIters, double threshold) +{ + return OptimizePlane(plane, points, size, maxIters, threshold); +} // OptimizePlane +/*----------------------------------------------------------------*/ + +// Robustly estimate the plane that fits best the given points +unsigned MVS::EstimatePlane(const Point3fArr& points, Planef& plane, double& maxThreshold, bool arrInliers[], size_t maxIters) +{ + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); +} // EstimatePlane +// Robustly estimate the plane that fits best the given points, making sure the first point is part of the solution (if any) +unsigned MVS::EstimatePlaneLockFirstPoint(const Point3fArr& points, Planef& plane, double& maxThreshold, bool arrInliers[], size_t maxIters) +{ + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); +} // EstimatePlaneLockFirstPoint +// Robustly estimate the plane that fits best the given points using a known threshold +unsigned MVS::EstimatePlaneTh(const Point3fArr& points, Planef& plane, double maxThreshold, bool arrInliers[], size_t maxIters) +{ + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); +} // EstimatePlaneTh +// Robustly estimate the plane that fits best the given points using a known threshold, making sure the first point is part of the solution (if any) +unsigned MVS::EstimatePlaneThLockFirstPoint(const Point3fArr& points, Planef& plane, double maxThreshold, bool arrInliers[], size_t maxIters) +{ + return TEstimatePlane(points, plane, maxThreshold, arrInliers, maxIters); +} // EstimatePlaneThLockFirstPoint +// least squares refinement of the given plane to the 3D point set +int MVS::OptimizePlane(Planef& plane, const Eigen::Vector3f* points, size_t size, int maxIters, float threshold) +{ + return OptimizePlane(plane, points, size, maxIters, threshold); +} // OptimizePlane /*----------------------------------------------------------------*/ @@ -1360,7 +1542,7 @@ void MVS::EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, i // fetch the point set std::vector pointvectors(pointcloud.points.GetSize()); FOREACH(i, pointcloud.points) - (Point3d&)(pointvectors[i].first) = pointcloud.points[i]; + reinterpret_cast(pointvectors[i].first) = pointcloud.points[i]; // estimates normals direction; // Note: pca_estimate_normals() requires an iterator over points // as well as property maps to access each point's position and normal. @@ -1388,7 +1570,7 @@ void MVS::EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, i PointCloud::Normal& normal = pointcloud.normals[i]; const PointCloud::Point& point = pointcloud.points[i]; const PointCloud::ViewArr& views= pointcloud.pointViews[i]; - normal = (const Point3d&)(pointvectors[i].second); + normal = reinterpret_cast(pointvectors[i].second); // correct normal orientation ASSERT(!views.IsEmpty()); const Image& imageData = images[views.First()]; @@ -1751,18 +1933,20 @@ bool MVS::ExportPointCloud(const String& fileName, const Image& imageData, const } // ExportPointCloud /*----------------------------------------------------------------*/ - +// - IDs are the reference view ID and neighbor view IDs used to estimate the depth-map (global ID) bool MVS::ExportDepthDataRaw(const String& fileName, const String& imageFileName, const IIndexArr& IDs, const cv::Size& imageSize, const KMatrix& K, const RMatrix& R, const CMatrix& C, Depth dMin, Depth dMax, - const DepthMap& depthMap, const NormalMap& normalMap, const ConfidenceMap& confMap) + const DepthMap& depthMap, const NormalMap& normalMap, const ConfidenceMap& confMap, const ViewsMap& viewsMap) { + ASSERT(IDs.size() > 1 && IDs.size() < 256); ASSERT(!depthMap.empty()); ASSERT(confMap.empty() || depthMap.size() == confMap.size()); + ASSERT(viewsMap.empty() || depthMap.size() == viewsMap.size()); ASSERT(depthMap.width() <= imageSize.width && depthMap.height() <= imageSize.height); - FILE *f = fopen(fileName, "wb"); + FILE* f = fopen(fileName, "wb"); if (f == NULL) { DEBUG("error: opening file '%s' for writing depth-data", fileName.c_str()); return false; @@ -1782,6 +1966,8 @@ bool MVS::ExportDepthDataRaw(const String& fileName, const String& imageFileName header.type |= HeaderDepthDataRaw::HAS_NORMAL; if (!confMap.empty()) header.type |= HeaderDepthDataRaw::HAS_CONF; + if (!viewsMap.empty()) + header.type |= HeaderDepthDataRaw::HAS_VIEWS; fwrite(&header, sizeof(HeaderDepthDataRaw), 1, f); // write image file name @@ -1814,6 +2000,10 @@ bool MVS::ExportDepthDataRaw(const String& fileName, const String& imageFileName if ((header.type & HeaderDepthDataRaw::HAS_CONF) != 0) fwrite(confMap.getData(), sizeof(float), confMap.area(), f); + // write views-map + if ((header.type & HeaderDepthDataRaw::HAS_VIEWS) != 0) + fwrite(viewsMap.getData(), sizeof(uint8_t)*4, viewsMap.area(), f); + const bool bRet(ferror(f) == 0); fclose(f); return bRet; @@ -1823,9 +2013,9 @@ bool MVS::ImportDepthDataRaw(const String& fileName, String& imageFileName, IIndexArr& IDs, cv::Size& imageSize, KMatrix& K, RMatrix& R, CMatrix& C, Depth& dMin, Depth& dMax, - DepthMap& depthMap, NormalMap& normalMap, ConfidenceMap& confMap, unsigned flags) + DepthMap& depthMap, NormalMap& normalMap, ConfidenceMap& confMap, ViewsMap& viewsMap, unsigned flags) { - FILE *f = fopen(fileName, "rb"); + FILE* f = fopen(fileName, "rb"); if (f == NULL) { DEBUG("error: opening file '%s' for reading depth-data", fileName.c_str()); return false; @@ -1837,10 +2027,9 @@ bool MVS::ImportDepthDataRaw(const String& fileName, String& imageFileName, header.name != HeaderDepthDataRaw::HeaderDepthDataRawName() || (header.type & HeaderDepthDataRaw::HAS_DEPTH) == 0 || header.depthWidth <= 0 || header.depthHeight <= 0 || - header.imageWidth <= 0 || header.imageHeight <= 0) + header.imageWidth < header.depthWidth || header.imageHeight < header.depthHeight) { DEBUG("error: invalid depth-data file '%s'", fileName.c_str()); - fclose(f); return false; } @@ -1849,12 +2038,13 @@ bool MVS::ImportDepthDataRaw(const String& fileName, String& imageFileName, uint16_t nFileNameSize; fread(&nFileNameSize, sizeof(uint16_t), 1, f); imageFileName.resize(nFileNameSize); - fread(&imageFileName[0u], sizeof(char), nFileNameSize, f); + fread(imageFileName.data(), sizeof(char), nFileNameSize, f); // read neighbor IDs STATIC_ASSERT(sizeof(uint32_t) == sizeof(IIndex)); uint32_t nIDs; fread(&nIDs, sizeof(IIndex), 1, f); + ASSERT(nIDs > 0 && nIDs < 256); IDs.resize(nIDs); fread(IDs.data(), sizeof(IIndex), nIDs, f); @@ -1891,6 +2081,16 @@ bool MVS::ImportDepthDataRaw(const String& fileName, String& imageFileName, if ((flags & HeaderDepthDataRaw::HAS_CONF) != 0) { confMap.create(header.depthHeight, header.depthWidth); fread(confMap.getData(), sizeof(float), confMap.area(), f); + } else { + fseek(f, sizeof(float)*header.depthWidth*header.depthHeight, SEEK_CUR); + } + } + + // read visibility-map + if ((header.type & HeaderDepthDataRaw::HAS_VIEWS) != 0) { + if ((flags & HeaderDepthDataRaw::HAS_VIEWS) != 0) { + viewsMap.create(header.depthHeight, header.depthWidth); + fread(viewsMap.getData(), sizeof(uint8_t)*4, viewsMap.area(), f); } } diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 918376843..cda6e74b8 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -1,7 +1,7 @@ /* * DepthMap.h * -* Copyright (c) 2014-2015 SEACAVE +* Copyright (c) 2014-2022 SEACAVE * * Author(s): * @@ -35,7 +35,6 @@ // I N C L U D E S ///////////////////////////////////////////////// -#include "Image.h" #include "PointCloud.h" @@ -74,6 +73,10 @@ // S T R U C T S /////////////////////////////////////////////////// +namespace MVS { + typedef TMatrix ViewsID; +} +DEFINE_CVDATATYPE(MVS::ViewsID) namespace MVS { @@ -90,6 +93,7 @@ enum DepthFlags { extern unsigned nResolutionLevel; extern unsigned nMaxResolution; extern unsigned nMinResolution; +extern unsigned nSubResolutionLevels; extern unsigned nMinViews; extern unsigned nMaxViews; extern unsigned nMinViewsFuse; @@ -100,6 +104,8 @@ extern unsigned nNumViews; extern unsigned nPointInsideROI; extern bool bFilterAdjust; extern bool bAddCorners; +extern bool bInitSparse; +extern bool bRemoveDmaps; extern float fViewMinScore; extern float fViewMinScoreRatio; extern float fMinArea; @@ -119,9 +125,6 @@ extern unsigned nOptimize; extern unsigned nEstimateColors; extern unsigned nEstimateNormals; extern float fNCCThresholdKeep; -#ifdef _USE_CUDA -extern float fNCCThresholdKeepCUDA; -#endif // _USE_CUDA extern unsigned nEstimationIters; extern unsigned nEstimationGeometricIters; extern float fEstimationGeometricWeight; @@ -137,6 +140,8 @@ extern float fRandomSmoothBonus; /*----------------------------------------------------------------*/ +typedef TImage ViewsMap; + template struct WeightedPatchFix { struct Pixel { @@ -187,6 +192,7 @@ struct MVS_API DepthData { } static bool NeedScaleImage(float scale) { + ASSERT(scale > 0); return ABS(scale-1.f) >= 0.15f; } template @@ -206,11 +212,13 @@ struct MVS_API DepthData { DepthMap depthMap; // depth-map NormalMap normalMap; // normal-map in camera space ConfidenceMap confMap; // confidence-map + ViewsMap viewsMap; // view-IDs map (indexing images vector starting after first view) float dMin, dMax; // global depth range for this image unsigned references; // how many times this depth-map is referenced (on 0 can be safely unloaded) CriticalSection cs; // used to count references inline DepthData() : references(0) {} + DepthData(const DepthData&); inline void ReleaseImages() { for (ViewData& image: images) { @@ -222,6 +230,7 @@ struct MVS_API DepthData { depthMap.release(); normalMap.release(); confMap.release(); + viewsMap.release(); } inline bool IsValid() const { @@ -240,7 +249,7 @@ struct MVS_API DepthData { void ApplyIgnoreMask(const BitMatrix&); bool Save(const String& fileName) const; - bool Load(const String& fileName, unsigned flags=7); + bool Load(const String& fileName, unsigned flags=15); unsigned GetRef(); unsigned IncRef(const String& fileName); @@ -254,6 +263,7 @@ struct MVS_API DepthData { ar & depthMap; ar & normalMap; ar & confMap; + ar & viewsMap; ar & dMin; ar & dMax; } @@ -317,7 +327,7 @@ struct MVS_API DepthEstimator { #if DENSE_SMOOTHNESS != DENSE_SMOOTHNESS_NA CLISTDEF0IDX(NeighborEstimate,IIndex) neighborsClose; // close neighbor pixel depths to be used for smoothing #endif - Vec3 X0; // + Point3 X0; // ImageRef x0; // constants during one pixel loop float normSq0; // #if DENSE_NCC != DENSE_NCC_WEIGHTED @@ -340,6 +350,7 @@ struct MVS_API DepthEstimator { #if DENSE_NCC == DENSE_NCC_WEIGHTED WeightMap& weightMap0; #endif + DepthMap lowResDepthMap; const unsigned nIteration; // current PatchMatch iteration const DepthData::ViewDataArr images; // neighbor images used @@ -428,6 +439,7 @@ struct MVS_API DepthEstimator { inline Normal RandomNormal(const Point3f& viewRay) { Normal normal; Dir2Normal(Point2f(rnd.randomRange(FD2R(0.f),FD2R(180.f)), rnd.randomRange(FD2R(90.f),FD2R(180.f))), normal); + ASSERT(ISEQUAL(norm(normal), 1.f)); return normal.dot(viewRay) > 0 ? -normal : normal; } @@ -437,6 +449,7 @@ struct MVS_API DepthEstimator { const float cosAngLen(normal.dot(viewDir)); if (cosAngLen >= 0) normal = RMatrixBaseF(normal.cross(viewDir), MINF((ACOS(cosAngLen/norm(viewDir))-FD2R(90.f))*1.01f, -0.001f)) * normal; + ASSERT(ISEQUAL(norm(normal), 1.f)); } static bool ImportIgnoreMask(const Image&, const Image8U::Size&, BitMatrix&, uint16_t nIgnoreMaskLabel); @@ -459,15 +472,23 @@ struct MVS_API DepthEstimator { // Tools bool TriangulatePoints2DepthMap( const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, - DepthMap& depthMap, NormalMap& normalMap, Depth& dMin, Depth& dMax, bool bAddCorners); + DepthMap& depthMap, NormalMap& normalMap, Depth& dMin, Depth& dMax, bool bAddCorners, bool bSparseOnly=false); bool TriangulatePoints2DepthMap( const DepthData::ViewData& image, const PointCloud& pointcloud, const IndexArr& points, - DepthMap& depthMap, Depth& dMin, Depth& dMax, bool bAddCorners); + DepthMap& depthMap, Depth& dMin, Depth& dMax, bool bAddCorners, bool bSparseOnly=false); +// Robustly estimate the plane that fits best the given points MVS_API unsigned EstimatePlane(const Point3Arr&, Plane&, double& maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); MVS_API unsigned EstimatePlaneLockFirstPoint(const Point3Arr&, Plane&, double& maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); MVS_API unsigned EstimatePlaneTh(const Point3Arr&, Plane&, double maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); MVS_API unsigned EstimatePlaneThLockFirstPoint(const Point3Arr&, Plane&, double maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); +MATH_API int OptimizePlane(Planed& plane, const Eigen::Vector3d* points, size_t size, int maxIters, double threshold); +// same for float points +MATH_API unsigned EstimatePlane(const Point3fArr&, Planef&, double& maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); +MATH_API unsigned EstimatePlaneLockFirstPoint(const Point3fArr&, Planef&, double& maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); +MATH_API unsigned EstimatePlaneTh(const Point3fArr&, Planef&, double maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); +MATH_API unsigned EstimatePlaneThLockFirstPoint(const Point3fArr&, Planef&, double maxThreshold, bool arrInliers[]=NULL, size_t maxIters=0); +MATH_API int OptimizePlane(Planef& plane, const Eigen::Vector3f* points, size_t size, int maxIters, float threshold); MVS_API void EstimatePointColors(const ImageArr& images, PointCloud& pointcloud); MVS_API void EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, int numNeighbors=16/*K-nearest neighbors*/); @@ -491,12 +512,12 @@ MVS_API bool ExportDepthDataRaw(const String&, const String& imageFileName, const IIndexArr&, const cv::Size& imageSize, const KMatrix&, const RMatrix&, const CMatrix&, Depth dMin, Depth dMax, - const DepthMap&, const NormalMap&, const ConfidenceMap&); + const DepthMap&, const NormalMap&, const ConfidenceMap&, const ViewsMap&); MVS_API bool ImportDepthDataRaw(const String&, String& imageFileName, IIndexArr&, cv::Size& imageSize, KMatrix&, RMatrix&, CMatrix&, Depth& dMin, Depth& dMax, - DepthMap&, NormalMap&, ConfidenceMap&, unsigned flags=7); + DepthMap&, NormalMap&, ConfidenceMap&, ViewsMap&, unsigned flags=15); MVS_API void CompareDepthMaps(const DepthMap& depthMap, const DepthMap& depthMapGT, uint32_t idxImage, float threshold=0.01f); MVS_API void CompareNormalMaps(const NormalMap& normalMap, const NormalMap& normalMapGT, uint32_t idxImage); diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index a920f1e52..642767618 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -49,7 +49,7 @@ typedef uint32_t IIndex; typedef cList IIndexArr; struct MVS_API ViewInfo { - IIndex ID; // image ID + IIndex ID; // image local-ID (the index in the scene images list) uint32_t points; // number of 3D points shared with the reference image float scale; // image scale relative to the reference image float angle; // image angle relative to the reference image (radians) @@ -78,7 +78,7 @@ class MVS_API Image uint32_t platformID; // ID of the associated platform uint32_t cameraID; // ID of the associated camera on the associated platform uint32_t poseID; // ID of the pose of the associated platform - uint32_t ID; // global ID of the image + uint32_t ID; // global ID of the image (the ID given outside the current scene - ex. the index in the full list of image files) String name; // image file name (relative path) String maskName; // segmentation file name (optional) Camera camera; // view's pose @@ -135,7 +135,7 @@ class MVS_API Image ar & ID; const String relName(MAKE_PATH_REL(WORKING_FOLDER_FULL, name)); ar & relName; - const String relMaskName(MAKE_PATH_REL(WORKING_FOLDER_FULL, maskName)); + const String relMaskName(maskName.empty() ? String() : MAKE_PATH_REL(WORKING_FOLDER_FULL, maskName)); ar & relMaskName; ar & width & height; ar & neighbors; @@ -150,7 +150,7 @@ class MVS_API Image ar & name; name = MAKE_PATH_FULL(WORKING_FOLDER_FULL, name); ar & maskName; - maskName = MAKE_PATH_FULL(WORKING_FOLDER_FULL, maskName); + maskName = maskName.empty() ? String() : MAKE_PATH_FULL(WORKING_FOLDER_FULL, maskName); ar & width & height; ar & neighbors; ar & avgDepth; diff --git a/libs/MVS/Interface.h b/libs/MVS/Interface.h index fcbb20aed..666c588e9 100644 --- a/libs/MVS/Interface.h +++ b/libs/MVS/Interface.h @@ -643,6 +643,15 @@ struct Interface const Image& image = images[imageID]; return platforms[image.platformID].GetK(image.cameraID); } + Mat33d GetFullK(uint32_t imageID, uint32_t width, uint32_t height) const { + const Image& image = images[imageID]; + return platforms[image.platformID].GetFullK(image.cameraID, width, height); + } + + const Platform::Camera& GetCamera(uint32_t imageID) const { + const Image& image = images[imageID]; + return platforms[image.platformID].cameras[image.cameraID]; + } Platform::Pose GetPose(uint32_t imageID) const { const Image& image = images[imageID]; @@ -701,16 +710,18 @@ struct Interface // - depth-map-resolution, for now only the same resolution as the image is supported // - min/max-depth of the values in the depth-map // - image-file-name is the path to the reference color image -// - image-IDs are the reference view ID and neighbor view IDs used to estimate the depth-map +// - image-IDs are the reference view ID and neighbor view IDs used to estimate the depth-map (global ID) // - camera/rotation/position matrices (row-major) is the absolute pose corresponding to the reference view -// - depth-map represents the pixel depth -// - normal-map (optional) represents the 3D point normal in camera space; same resolution as the depth-map -// - confidence-map (optional) represents the 3D point confidence (usually a value in [0,1]); same resolution as the depth-map +// - depth-map: the pixels' depth +// - normal-map (optional): the 3D point normal in camera space; same resolution as the depth-map +// - confidence-map (optional): the 3D point confidence (usually a value in [0,1]); same resolution as the depth-map +// - views-map (optional): the pixels' views, indexing image-IDs starting after first view (up to 4); same resolution as the depth-map struct HeaderDepthDataRaw { enum { HAS_DEPTH = (1<<0), HAS_NORMAL = (1<<1), HAS_CONF = (1<<2), + HAS_VIEWS = (1<<3), }; uint16_t name; // file type uint8_t type; // content type diff --git a/libs/MVS/Mesh.cpp b/libs/MVS/Mesh.cpp index 3d6389a78..5bb7088d4 100644 --- a/libs/MVS/Mesh.cpp +++ b/libs/MVS/Mesh.cpp @@ -83,6 +83,21 @@ using namespace MVS; // D E F I N E S /////////////////////////////////////////////////// +// uncomment to enable multi-threading based on OpenMP +#ifdef _USE_OPENMP +#define MESH_USE_OPENMP +#endif + +// select fast ray-face intersection search method +#define USE_MESH_BF 0 // brute-force +#define USE_MESH_OCTREE 1 // octree (misses some triangles) +#define USE_MESH_BVH 2 // BVH (misses some triangles) +#define USE_MESH_INT USE_MESH_BF + +#if USE_MESH_INT == USE_MESH_BVH +#include +#endif + // S T R U C T S /////////////////////////////////////////////////// @@ -128,9 +143,42 @@ void Mesh::Swap(Mesh& rhs) faceTexcoords.Swap(rhs.faceTexcoords); std::swap(textureDiffuse, rhs.textureDiffuse); } // Swap +// combine this mesh with the given mesh, without removing duplicate vertices +void Mesh::Join(const Mesh& mesh) +{ + ASSERT(!HasTexture() && !mesh.HasTexture()); + vertexVertices.Release(); + vertexFaces.Release(); + vertexBoundary.Release(); + faceFaces.Release(); + if (IsEmpty()) { + *this = mesh; + return; + } + const VIndex offsetV(vertices.size()); + vertices.Join(mesh.vertices); + vertexNormals.Join(mesh.vertexNormals); + faces.ReserveExtra(mesh.faces.size()); + for (const Face& face: mesh.faces) + faces.emplace_back(face.x+offsetV, face.y+offsetV, face.z+offsetV); + faceNormals.Join(mesh.faceNormals); +} /*----------------------------------------------------------------*/ +bool Mesh::IsWatertight() +{ + if (vertexBoundary.empty()) { + if (vertexFaces.empty()) + ListIncidenteFaces(); + ListBoundaryVertices(); + } + for (const bool b : vertexBoundary) + if (b) + return false; + return true; +} + // compute the axis-aligned bounding-box of the mesh Mesh::Box Mesh::GetAABB() const { @@ -191,7 +239,7 @@ void Mesh::ListIncidenteVertices() void Mesh::ListIncidenteFaces() { vertexFaces.Empty(); - vertexFaces.Resize(vertices.GetSize()); + vertexFaces.resize(vertices.size()); FOREACH(i, faces) { const Face& face = faces[i]; for (int v=0; v<3; ++v) { @@ -306,7 +354,7 @@ void Mesh::ComputeNormalFaces() // computes the vertex normal as the area weighted face normals average void Mesh::ComputeNormalVertices() { - vertexNormals.Resize(vertices.GetSize()); + vertexNormals.resize(vertices.size()); vertexNormals.Memset(0); for (const Face& face: faces) { const Vertex& v0 = vertices[face[0]]; @@ -327,12 +375,6 @@ void Mesh::ComputeNormalVertices() // The normal of a vertex v computed as a weighted sum f the incident face normals. // The weight is simply the angle of the involved wedge. Described in: // G. Thurmer, C. A. Wuthrich "Computing vertex normals from polygonal facets", Journal of Graphics Tools, 1998 -inline float AngleN(const Mesh::Normal& p1, const Mesh::Normal& p2) { - float t(p1.dot(p2)); - if (t>1) t = 1; else - if (t<-1) t = -1; - return acosf(t); -} void Mesh::ComputeNormalVertices() { ASSERT(!faceNormals.IsEmpty()); @@ -347,14 +389,48 @@ void Mesh::ComputeNormalVertices() const Normal e0(normalized(v1-v0)); const Normal e1(normalized(v2-v1)); const Normal e2(normalized(v0-v2)); - vertexNormals[face[0]] += t*AngleN(e0, -e2); - vertexNormals[face[1]] += t*AngleN(-e0, e1); - vertexNormals[face[2]] += t*AngleN(-e1, e2); + vertexNormals[face[0]] += t*ACOS(-ComputeAngleN(e0.ptr(), e2.ptr())); + vertexNormals[face[1]] += t*ACOS(-ComputeAngleN(e0.ptr(), e1.ptr())); + vertexNormals[face[2]] += t*ACOS(-ComputeAngleN(e1.ptr(), e2.ptr())); } FOREACHPTR(pVertexNormal, vertexNormals) normalize(*pVertexNormal); } #endif + +// Smoothen the normals for each face +// - fMaxGradient: maximum angle (in degrees) difference between neighbor normals that is +// allowed to take into consideration; higher angles are ignored +// - fOriginalWeight: weight (0..1] to use for current normal value when averaging with neighbor normals +// - nIterations: number of times to repeat the smoothening process +void Mesh::SmoothNormalFaces(float fMaxGradient, float fOriginalWeight, unsigned nIterations) { + if (faceNormals.size() != faces.size()) + ComputeNormalFaces(); + if (vertexFaces.size() != vertices.size()) + ListIncidenteFaces(); + if (faceFaces.size() != faces.size()) + ListIncidenteFaceFaces(); + const float cosMaxGradient = COS(FD2R(fMaxGradient)); + for (unsigned rep = 0; rep < nIterations; ++rep) { + NormalArr newFaceNormals(faceNormals.size()); + FOREACH(idxFace, faces) { + const Normal& originalNormal = faceNormals[idxFace]; + Normal sumNeighborNormals = Normal::ZERO; + for (int i = 0; i < 3; ++i) { + const FIndex fIdx = faceFaces[idxFace][i]; + if (fIdx == NO_ID) + continue; + const Normal& neighborNormal = faceNormals[fIdx]; + if (ComputeAngleN(originalNormal.ptr(), neighborNormal.ptr()) >= cosMaxGradient) + sumNeighborNormals += neighborNormal; + } + const Normal avgNeighborsNormal = normalized(sumNeighborNormals); + const Normal newFaceNormal = normalized(originalNormal * fOriginalWeight + avgNeighborsNormal * (1.f - fOriginalWeight)); + newFaceNormals[idxFace] = newFaceNormal; + } + newFaceNormals.Swap(faceNormals); + } +} /*----------------------------------------------------------------*/ @@ -1338,6 +1414,55 @@ void Mesh::Clean(float fDecimate, float fSpurious, bool bRemoveSpikes, unsigned /*----------------------------------------------------------------*/ +// project vertices and compute bounding-box; +// account for diferences in pixel center convention: while OpenMVS uses the same convention as OpenCV and DirectX 9 where the center +// of a pixel is defined at integer coordinates, i.e. the center is at (0, 0) and the top left corner is at (-0.5, -0.5), +// DirectX 10+, OpenGL, and Vulkan convention is the center of a pixel is defined at half coordinates, i.e. the center is at (0.5, 0.5) +// and the top left corner is at (0, 0) +static const Mesh::TexCoord halfPixel(0.5f, 0.5f); + +// translate, normalize and flip Y axis of the texture coordinates +void Mesh::FaceTexcoordsNormalize(TexCoordArr& newFaceTexcoords, bool flipY) const +{ + ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); + const TexCoord invNorm(1.f/(float)textureDiffuse.cols, 1.f/(float)textureDiffuse.rows); + newFaceTexcoords.resize(faceTexcoords.size()); + if (flipY) { + FOREACH(i, faceTexcoords) { + const TexCoord& texcoord = faceTexcoords[i]; + newFaceTexcoords[i] = TexCoord( + (texcoord.x+halfPixel.x)*invNorm.x, + 1.f-(texcoord.y+halfPixel.y)*invNorm.y + ); + } + } else { + FOREACH(i, faceTexcoords) + newFaceTexcoords[i] = (faceTexcoords[i]+halfPixel)*invNorm; + } +} // FaceTexcoordsNormalize + +// flip Y axis, unnormalize and translate back texture coordinates +void Mesh::FaceTexcoordsUnnormalize(TexCoordArr& newFaceTexcoords, bool flipY) const +{ + ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); + const TexCoord scale((float)textureDiffuse.cols, (float)textureDiffuse.rows); + newFaceTexcoords.resize(faceTexcoords.size()); + if (flipY) { + FOREACH(i, faceTexcoords) { + const TexCoord& texcoord = faceTexcoords[i]; + newFaceTexcoords[i] = TexCoord( + texcoord.x*scale.x-halfPixel.x, + (1.f-texcoord.y)*scale.y-halfPixel.y + ); + } + } else { + FOREACH(i, faceTexcoords) + newFaceTexcoords[i] = faceTexcoords[i]*scale - halfPixel; + } +} // FaceTexcoordsUnnormalize +/*----------------------------------------------------------------*/ + + // define a PLY file format composed only of vertices and triangles namespace BasicPLY { // list of property information for a vertex @@ -1393,6 +1518,9 @@ bool Mesh::Load(const String& fileName) bool ret; if (ext == _T(".obj")) ret = LoadOBJ(fileName); + else + if (ext == _T(".gltf") || ext == _T(".glb")) + ret = LoadGLTF(fileName, ext == _T(".glb")); else ret = LoadPLY(fileName); if (!ret) @@ -1432,7 +1560,7 @@ bool Mesh::LoadPLY(const String& fileName) int elem_count; LPCSTR elem_name = ply.setup_element_read(i, &elem_count); if (PLY::equal_strings(BasicPLY::elem_names[0], elem_name)) { - ASSERT(vertices.GetSize() == (VIndex)elem_count); + ASSERT(vertices.size() == (VIndex)elem_count); ply.setup_property(BasicPLY::vert_props[0]); ply.setup_property(BasicPLY::vert_props[1]); ply.setup_property(BasicPLY::vert_props[2]); @@ -1440,7 +1568,7 @@ bool Mesh::LoadPLY(const String& fileName) ply.get_element(pVert); } else if (PLY::equal_strings(BasicPLY::elem_names[1], elem_name)) { - ASSERT(faces.GetSize() == (FIndex)elem_count); + ASSERT(faces.size() == (FIndex)elem_count); if (ply.find_property(ply.elems[i], BasicPLY::face_tex_props[1].name.c_str()) == -1) { // load vertex indices BasicPLY::Face face; @@ -1456,7 +1584,7 @@ bool Mesh::LoadPLY(const String& fileName) } } else { // load vertex indices and texture coordinates - faceTexcoords.Resize((FIndex)elem_count*3); + faceTexcoords.resize((FIndex)elem_count*3); BasicPLY::FaceTex face; ply.setup_property(BasicPLY::face_tex_props[0]); ply.setup_property(BasicPLY::face_tex_props[1]); @@ -1466,13 +1594,13 @@ bool Mesh::LoadPLY(const String& fileName) DEBUG_EXTRA("error: unsupported mesh file (face not triangle)"); return false; } - memcpy(faces.Begin()+f, face.face.pFace, sizeof(VIndex)*3); + memcpy(faces.data()+f, face.face.pFace, sizeof(VIndex)*3); delete[] face.face.pFace; if (face.tex.num != 6) { DEBUG_EXTRA("error: unsupported mesh file (texture coordinates not per face vertex)"); return false; } - memcpy(faceTexcoords.Begin()+f*3, face.tex.pTex, sizeof(TexCoord)*3); + memcpy(faceTexcoords.data()+f*3, face.tex.pTex, sizeof(TexCoord)*3); delete[] face.tex.pTex; } // load the texture @@ -1483,6 +1611,10 @@ bool Mesh::LoadPLY(const String& fileName) break; } } + // flip Y axis, unnormalize and translate back texture coordinates + TexCoordArr unnormFaceTexcoords; + FaceTexcoordsUnnormalize(unnormFaceTexcoords, true); + faceTexcoords.Swap(unnormFaceTexcoords); } } else { ply.get_other_element(); @@ -1526,13 +1658,13 @@ bool Mesh::LoadOBJ(const String& fileName) faces.Reserve((FIndex)group.faces.size()); for (const ObjModel::Face& f: group.faces) { ASSERT(f.vertices[0] != NO_ID); - faces.AddConstruct(f.vertices[0], f.vertices[1], f.vertices[2]); + faces.emplace_back(f.vertices[0], f.vertices[1], f.vertices[2]); if (f.texcoords[0] != NO_ID) { for (int i=0; i<3; ++i) - faceTexcoords.AddConstruct(model.get_texcoords()[f.texcoords[i]]); + faceTexcoords.emplace_back(model.get_texcoords()[f.texcoords[i]]); } if (f.normals[0] != NO_ID) { - Normal& n = faceNormals.AddConstruct(Normal::ZERO); + Normal& n = faceNormals.emplace_back(Normal::ZERO); for (int i=0; i<3; ++i) n += normalized(model.get_normals()[f.normals[i]]); normalize(n); @@ -1543,6 +1675,87 @@ bool Mesh::LoadOBJ(const String& fileName) ObjModel::MaterialLib::Material* pMaterial(model.GetMaterial(group.material_name)); if (pMaterial && pMaterial->LoadDiffuseMap()) cv::swap(textureDiffuse, pMaterial->diffuse_map); + + // flip Y axis, unnormalize and translate back texture coordinates + if (!faceTexcoords.empty()) { + TexCoordArr unnormFaceTexcoords; + FaceTexcoordsUnnormalize(unnormFaceTexcoords, true); + faceTexcoords.Swap(unnormFaceTexcoords); + } + return true; +} +// import the mesh as a GLTF file +bool Mesh::LoadGLTF(const String& fileName, bool bBinary) +{ + ASSERT(!fileName.IsEmpty()); + Release(); + + // load model + tinygltf::Model gltfModel; { + tinygltf::TinyGLTF loader; + std::string err, warn; + if (bBinary ? + !loader.LoadBinaryFromFile(&gltfModel, &err, &warn, fileName) : + !loader.LoadASCIIFromFile(&gltfModel, &err, &warn, fileName)) + return false; + if (!err.empty()) { + VERBOSE("error: %s", err.c_str()); + return false; + } + if (!warn.empty()) + DEBUG("warning: %s", warn.c_str()); + } + + // parse model + for (const tinygltf::Mesh& gltfMesh : gltfModel.meshes) { + for (const tinygltf::Primitive& gltfPrimitive : gltfMesh.primitives) { + if (gltfPrimitive.mode != TINYGLTF_MODE_TRIANGLES) + continue; + Mesh mesh; + // read vertices + { + const tinygltf::Accessor& gltfAccessor = gltfModel.accessors[gltfPrimitive.attributes.at("POSITION")]; + if (gltfAccessor.type != TINYGLTF_TYPE_VEC3) + continue; + const tinygltf::BufferView& gltfBufferView = gltfModel.bufferViews[gltfAccessor.bufferView]; + const tinygltf::Buffer& buffer = gltfModel.buffers[gltfBufferView.buffer]; + const uint8_t* pData = buffer.data.data() + gltfBufferView.byteOffset + gltfAccessor.byteOffset; + mesh.vertices.resize((VIndex)gltfAccessor.count); + if (gltfAccessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT) { + ASSERT(gltfBufferView.byteLength == sizeof(Vertex) * gltfAccessor.count); + memcpy(mesh.vertices.data(), pData, gltfBufferView.byteLength); + } + else if (gltfAccessor.componentType == TINYGLTF_COMPONENT_TYPE_DOUBLE) { + for (VIndex i = 0; i < gltfAccessor.count; ++i) + mesh.vertices[i] = ((const Point3d*)pData)[i]; + } + else { + VERBOSE("error: unsupported vertices (component type)"); + continue; + } + } + // read faces + { + const tinygltf::Accessor& gltfAccessor = gltfModel.accessors[gltfPrimitive.indices]; + if (gltfAccessor.type != TINYGLTF_TYPE_SCALAR) + continue; + const tinygltf::BufferView& gltfBufferView = gltfModel.bufferViews[gltfAccessor.bufferView]; + const tinygltf::Buffer& buffer = gltfModel.buffers[gltfBufferView.buffer]; + const uint8_t* pData = buffer.data.data() + gltfBufferView.byteOffset + gltfAccessor.byteOffset; + mesh.faces.resize((FIndex)(gltfAccessor.count/3)); + if (gltfAccessor.componentType == TINYGLTF_COMPONENT_TYPE_INT || + gltfAccessor.componentType == TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT) { + ASSERT(gltfBufferView.byteLength == sizeof(uint32_t) * gltfAccessor.count); + memcpy(mesh.faces.data(), pData, gltfBufferView.byteLength); + } + else { + VERBOSE("error: unsupported faces (component type)"); + continue; + } + } + Join(mesh); + } + } return true; } // Load /*----------------------------------------------------------------*/ @@ -1568,11 +1781,11 @@ bool Mesh::Save(const String& fileName, const cList& comments, bool bBin // export the mesh as a PLY file bool Mesh::SavePLY(const String& fileName, const cList& comments, bool bBinary) const { - ASSERT(!fileName.IsEmpty()); + ASSERT(!fileName.empty()); Util::ensureFolder(fileName); // create PLY object - const size_t bufferSize(vertices.GetSize()*(4*3/*pos*/+2/*eol*/) + faces.GetSize()*(1*1/*len*/+4*3/*idx*/+2/*eol*/) + 2048/*extra size*/); + const size_t bufferSize(vertices.size()*(4*3/*pos*/+2/*eol*/) + faces.size()*(1*1/*len*/+4*3/*idx*/+2/*eol*/) + 2048/*extra size*/); PLY ply; if (!ply.write(fileName, 2, BasicPLY::elem_names, bBinary?PLY::BINARY_LE:PLY::ASCII, bufferSize)) { DEBUG_EXTRA("error: can not create the mesh file"); @@ -1585,12 +1798,12 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b // export texture file name as comment if needed String textureFileName; - if (!faceTexcoords.IsEmpty() && !textureDiffuse.empty()) { + if (!faceTexcoords.empty() && !textureDiffuse.empty()) { textureFileName = Util::getFileFullName(fileName)+_T(".png"); ply.append_comment((_T("TextureFile ")+Util::getFileNameExt(textureFileName)).c_str()); } - if (vertexNormals.IsEmpty()) { + if (vertexNormals.empty()) { // describe what properties go into the vertex elements ply.describe_property(BasicPLY::elem_names[0], 3, BasicPLY::vert_props); @@ -1598,7 +1811,7 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b FOREACHPTR(pVert, vertices) ply.put_element(pVert); } else { - ASSERT(vertices.GetSize() == vertexNormals.GetSize()); + ASSERT(vertices.size() == vertexNormals.size()); // describe what properties go into the vertex elements ply.describe_property(BasicPLY::elem_names[0], 6, BasicPLY::vert_normal_props); @@ -1614,7 +1827,7 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b if (ply.get_current_element_count() == 0) return false; - if (faceTexcoords.IsEmpty()) { + if (faceTexcoords.empty()) { // describe what properties go into the vertex elements ply.describe_property(BasicPLY::elem_names[1], 1, BasicPLY::face_props); @@ -1625,7 +1838,11 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b ply.put_element(&face); } } else { - ASSERT(faceTexcoords.GetSize() == faces.GetSize()*3); + ASSERT(faceTexcoords.size() == faces.size()*3); + + // translate, normalize and flip Y axis of the texture coordinates + TexCoordArr normFaceTexcoords; + FaceTexcoordsNormalize(normFaceTexcoords, true); // describe what properties go into the vertex elements ply.describe_property(BasicPLY::elem_names[1], 2, BasicPLY::face_tex_props); @@ -1633,8 +1850,8 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b // export the array of faces BasicPLY::FaceTex face = {{3},{6}}; FOREACH(f, faces) { - face.face.pFace = faces.Begin()+f; - face.tex.pTex = faceTexcoords.Begin()+f*3; + face.face.pFace = faces.data()+f; + face.tex.pTex = normFaceTexcoords.data()+f*3; ply.put_element(&face); } @@ -1651,7 +1868,7 @@ bool Mesh::SavePLY(const String& fileName, const cList& comments, bool b // export the mesh as a OBJ file bool Mesh::SaveOBJ(const String& fileName) const { - ASSERT(!fileName.IsEmpty()); + ASSERT(!fileName.empty()); Util::ensureFolder(fileName); // create the OBJ model @@ -1659,35 +1876,38 @@ bool Mesh::SaveOBJ(const String& fileName) const // store vertices ASSERT(sizeof(ObjModel::Vertex) == sizeof(Vertex)); - model.get_vertices().insert(model.get_vertices().begin(), vertices.Begin(), vertices.End()); + model.get_vertices().insert(model.get_vertices().begin(), vertices.begin(), vertices.end()); // store vertex normals ASSERT(sizeof(ObjModel::Normal) == sizeof(Normal)); ASSERT(model.get_vertices().size() < std::numeric_limits::max()); - if (!vertexNormals.IsEmpty()) { - ASSERT(vertexNormals.GetSize() == vertices.GetSize()); - model.get_normals().insert(model.get_normals().begin(), vertexNormals.Begin(), vertexNormals.End()); + if (!vertexNormals.empty()) { + ASSERT(vertexNormals.size() == vertices.size()); + model.get_normals().insert(model.get_normals().begin(), vertexNormals.begin(), vertexNormals.end()); } // store face texture coordinates ASSERT(sizeof(ObjModel::TexCoord) == sizeof(TexCoord)); - if (!faceTexcoords.IsEmpty()) { - ASSERT(faceTexcoords.GetSize() == faces.GetSize()*3); - model.get_texcoords().insert(model.get_texcoords().begin(), faceTexcoords.Begin(), faceTexcoords.End()); + if (!faceTexcoords.empty()) { + // translate, normalize and flip Y axis of the texture coordinates + TexCoordArr normFaceTexcoords; + FaceTexcoordsNormalize(normFaceTexcoords, true); + ASSERT(normFaceTexcoords.size() == faces.size()*3); + model.get_texcoords().insert(model.get_texcoords().begin(), normFaceTexcoords.begin(), normFaceTexcoords.end()); } // store faces ObjModel::Group& group = model.AddGroup(_T("material_0")); - group.faces.reserve(faces.GetSize()); + group.faces.reserve(faces.size()); FOREACH(idxFace, faces) { const Face& face = faces[idxFace]; ObjModel::Face f; memset(&f, 0xFF, sizeof(ObjModel::Face)); for (int i=0; i<3; ++i) { f.vertices[i] = face[i]; - if (!faceTexcoords.IsEmpty()) + if (!faceTexcoords.empty()) f.texcoords[i] = idxFace*3+i; - if (!vertexNormals.IsEmpty()) + if (!vertexNormals.empty()) f.normals[i] = face[i]; } group.faces.push_back(f); @@ -1798,18 +2018,15 @@ bool Mesh::SaveGLTF(const String& fileName, bool bBinary) const vertexTexcoordAccessor.count = mesh.faceTexcoords.size(); vertexTexcoordAccessor.type = TINYGLTF_TYPE_VEC2; gltfModel.accessors.emplace_back(std::move(vertexTexcoordAccessor)); - // setup texture coordinates (flip Y) + // setup texture coordinates STATIC_ASSERT(2 * sizeof(TexCoord::Type) == sizeof(TexCoord)); // TexCoordArr should be continuous ASSERT(mesh.vertices.size() == mesh.faceTexcoords.size()); tinygltf::BufferView vertexTexcoordBufferView; vertexTexcoordBufferView.name = "vertexTexcoordBufferView"; vertexTexcoordBufferView.buffer = (int)gltfModel.buffers.size(); - TexCoordArr flipTexcoords(mesh.faceTexcoords.size()); - FOREACH(i, mesh.faceTexcoords) { - flipTexcoords[i].x = mesh.faceTexcoords[i].x; - flipTexcoords[i].y = 1.f - mesh.faceTexcoords[i].y; - } - ExtendBufferGLTF(flipTexcoords.data(), flipTexcoords.size(), gltfBuffer, + TexCoordArr normFaceTexcoords; + mesh.FaceTexcoordsNormalize(normFaceTexcoords, false); + ExtendBufferGLTF(normFaceTexcoords.data(), normFaceTexcoords.size(), gltfBuffer, vertexTexcoordBufferView.byteOffset, vertexTexcoordBufferView.byteLength); gltfModel.bufferViews.emplace_back(std::move(vertexTexcoordBufferView)); // setup texture @@ -3512,7 +3729,7 @@ void Mesh::CloseHoleQuality(VertexIdxArr& verts) const Normal n(mesh.FaceNormal(face)); // compute the angle between the two existing edges of the face // (the angle computation takes into account the case of reversed face) - angle = ACOS(ComputeAngle(mesh.vertices[face[1]].ptr(), mesh.vertices[face[0]].ptr(), mesh.vertices[face[2]].ptr())); + angle = ACOS(ComputeAngle(mesh.vertices[face[1]].ptr(), mesh.vertices[face[0]].ptr(), mesh.vertices[face[2]].ptr())); if (n.dot(mesh.VertexNormal(face[1])) < 0) angle = float(2*M_PI) - angle; // compute quality as a composition of dihedral angle and area/sum(edge^2); @@ -3541,10 +3758,10 @@ void Mesh::CloseHoleQuality(VertexIdxArr& verts) else { const Normal n0(mesh.FaceNormal(mesh.faces[indices[0]])); if (indices.GetSize() == 1) - dihedral = ACOS(ComputeAngle(n.ptr(), n0.ptr())); + dihedral = ACOS(ComputeAngle(n.ptr(), n0.ptr())); else { const Normal n1(mesh.FaceNormal(mesh.faces[indices[1]])); - dihedral = MAXF(ACOS(ComputeAngle(n.ptr(), n0.ptr())), ACOS(ComputeAngle(n.ptr(), n1.ptr()))); + dihedral = MAXF(ACOS(ComputeAngle(n.ptr(), n0.ptr())), ACOS(ComputeAngle(n.ptr(), n1.ptr()))); } } aspectRatio = ComputeTriangleQuality(mesh.vertices[face[0]], mesh.vertices[face[1]], mesh.vertices[face[2]]); @@ -3631,21 +3848,37 @@ void Mesh::CloseHoleQuality(VertexIdxArr& verts) } /*----------------------------------------------------------------*/ +// crop mesh such that none of its faces is touching or outside the given bounding-box +void Mesh::RemoveFacesOutside(const OBB3f& obb) { + ASSERT(obb.IsValid()); + VertexIdxArr vertexRemove; + FOREACH(i, vertices) + if (!obb.Intersects(vertices[i])) + vertexRemove.emplace_back(i); + if (!vertexRemove.empty()) { + if (vertices.size() != vertexFaces.size()) + ListIncidenteFaces(); + RemoveVertices(vertexRemove, true); + } +} + // remove the given list of faces void Mesh::RemoveFaces(FaceIdxArr& facesRemove, bool bUpdateLists) { facesRemove.Sort(); FIndex idxLast(FaceIdxArr::NO_INDEX); - if (!bUpdateLists || vertexFaces.IsEmpty()) { + if (!bUpdateLists || vertexFaces.empty()) { RFOREACHPTR(pIdxF, facesRemove) { const FIndex idxF(*pIdxF); if (idxLast == idxF) continue; faces.RemoveAt(idxF); + if (!faceTexcoords.empty()) + faceTexcoords.RemoveAt(idxF * 3, 3); idxLast = idxF; } } else { - ASSERT(vertices.GetSize() == vertexFaces.GetSize()); + ASSERT(vertices.size() == vertexFaces.size()); RFOREACHPTR(pIdxF, facesRemove) { const FIndex idxF(*pIdxF); if (idxLast == idxF) @@ -3674,16 +3907,18 @@ void Mesh::RemoveFaces(FaceIdxArr& facesRemove, bool bUpdateLists) } } faces.RemoveAt(idxF); + if (!faceTexcoords.empty()) + faceTexcoords.RemoveAt(idxF * 3, 3); idxLast = idxF; } } vertexVertices.Release(); } -// remove the given list of vertices +// remove the given list of vertices, together with all faces containing them void Mesh::RemoveVertices(VertexIdxArr& vertexRemove, bool bUpdateLists) { - ASSERT(vertices.GetSize() == vertexFaces.GetSize()); + ASSERT(vertices.size() == vertexFaces.size()); vertexRemove.Sort(); VIndex idxLast(VertexIdxArr::NO_INDEX); if (!bUpdateLists) { @@ -3691,7 +3926,7 @@ void Mesh::RemoveVertices(VertexIdxArr& vertexRemove, bool bUpdateLists) const VIndex idxV(*pIdxV); if (idxLast == idxV) continue; - const VIndex idxVM(vertices.GetSize()-1); + const VIndex idxVM(vertices.size()-1); if (idxV < idxVM) { // update all faces of the moved vertex const FaceIdxArr& vf(vertexFaces[idxVM]); @@ -3709,7 +3944,7 @@ void Mesh::RemoveVertices(VertexIdxArr& vertexRemove, bool bUpdateLists) const VIndex idxV(*pIdxV); if (idxLast == idxV) continue; - const VIndex idxVM(vertices.GetSize()-1); + const VIndex idxVM(vertices.size()-1); if (idxV < idxVM) { // update all faces of the moved vertex const FaceIdxArr& vf(vertexFaces[idxVM]); @@ -3725,7 +3960,8 @@ void Mesh::RemoveVertices(VertexIdxArr& vertexRemove, bool bUpdateLists) vertices.RemoveAt(idxV); idxLast = idxV; } - RemoveFaces(facesRemove); + if (!facesRemove.empty()) + RemoveFaces(facesRemove); } // remove all vertices that are not assigned to any face @@ -3794,6 +4030,27 @@ void Mesh::ConvertTexturePerVertex(Mesh& mesh) const /*----------------------------------------------------------------*/ +// estimate the ground-plane as the plane agreeing with most vertices +// - sampleMesh: uniformly samples points on the mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit) +// - planeThreshold: threshold used to estimate the ground plane (0 - auto) +Planef Mesh::EstimateGroundPlane(const ImageArr& images, float sampleMesh, float planeThreshold, const String& fileExportPlane) const +{ + ASSERT(!IsEmpty()); + PointCloud pointcloud; + if (sampleMesh != 0) { + // create the point cloud by sampling the mesh + if (sampleMesh > 0) + SamplePoints(sampleMesh, 0, pointcloud); + else + SamplePoints(ROUND2INT(-sampleMesh), pointcloud); + } else { + // create the point cloud containing all vertices + for (const Vertex& X: vertices) + pointcloud.points.emplace_back(X); + } + return pointcloud.EstimateGroundPlane(images, planeThreshold, fileExportPlane); +} +/*----------------------------------------------------------------*/ // computes the centroid of the given mesh face @@ -3911,7 +4168,7 @@ void Mesh::SamplePoints(REAL samplingDensity, unsigned mumPointsTheoretic, Point const TexCoord& TA = faceTexcoords[idxTexCoord+1]; const TexCoord& TB = faceTexcoords[idxTexCoord+2]; const TexCoord xt(TO + static_cast(x)*(TA - TO) + static_cast(y)*(TB - TO)); - pointcloud.colors.emplace_back(textureDiffuse.sampleSafe(Point2f(xt.x*textureDiffuse.width(), (1.f-xt.y)*textureDiffuse.height()))); + pointcloud.colors.emplace_back(textureDiffuse.sampleSafe(xt)); } } } @@ -3934,7 +4191,7 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap) const } void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) const { - ASSERT(!faceTexcoords.IsEmpty() && !textureDiffuse.empty()); + ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); struct RasterMesh : TRasterMesh { typedef TRasterMesh Base; const Mesh& mesh; @@ -3957,7 +4214,7 @@ void Mesh::Project(const Camera& camera, DepthMap& depthMap, Image8U3& image) co xt = mesh.faceTexcoords[idxFaceTex+0] * pbary[0]; xt += mesh.faceTexcoords[idxFaceTex+1] * pbary[1]; xt += mesh.faceTexcoords[idxFaceTex+2] * pbary[2]; - image(pt) = mesh.textureDiffuse.sampleSafe(Point2f(xt.x*mesh.textureDiffuse.width(), (1.f-xt.y)*mesh.textureDiffuse.height())); + image(pt) = mesh.textureDiffuse.sampleSafe(xt); } } }; @@ -4023,7 +4280,7 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap) const : Base(_vertices, _camera, _depthMap) {} inline bool ProjectVertex(const Mesh::Vertex& pt, int v) { return (ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && - depthMap.isInsideWithBorder(pti[v] = camera.TransformPointC2I(reinterpret_cast(ptc[v]))); + depthMap.isInsideWithBorder(pti[v] = camera.TransformPointC2I(Point2(ptc[v].x,ptc[v].y))); } void Raster(const ImageRef& pt, const Point3f& bary) { const Depth z(ComputeDepth(bary)); @@ -4040,7 +4297,7 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap) const } void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& image) const { - ASSERT(!faceTexcoords.IsEmpty() && !textureDiffuse.empty()); + ASSERT(!faceTexcoords.empty() && !textureDiffuse.empty()); struct RasterMesh : TRasterMesh { typedef TRasterMesh Base; const Mesh& mesh; @@ -4055,7 +4312,7 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& imag } inline bool ProjectVertex(const Mesh::Vertex& pt, int v) { return (ptc[v] = camera.TransformPointW2C(Cast(pt))).z > 0 && - depthMap.isInsideWithBorder(pti[v] = camera.TransformPointC2I(reinterpret_cast(ptc[v]))); + depthMap.isInsideWithBorder(pti[v] = camera.TransformPointC2I(Point2(ptc[v].x,ptc[v].y))); } void Raster(const ImageRef& pt, const Point3f& bary) { const Depth z(ComputeDepth(bary)); @@ -4066,7 +4323,7 @@ void Mesh::ProjectOrtho(const Camera& camera, DepthMap& depthMap, Image8U3& imag xt = mesh.faceTexcoords[idxFaceTex+0] * bary[0]; xt += mesh.faceTexcoords[idxFaceTex+1] * bary[1]; xt += mesh.faceTexcoords[idxFaceTex+2] * bary[2]; - image(pt) = mesh.textureDiffuse.sampleSafe(Point2f(xt.x*mesh.textureDiffuse.width(), (1.f-xt.y)*mesh.textureDiffuse.height())); + image(pt) = mesh.textureDiffuse.sampleSafe(xt); } } }; @@ -4193,6 +4450,162 @@ Mesh Mesh::SubMesh(const FaceIdxArr& chunk) const /*----------------------------------------------------------------*/ + +// transfer the texture of this mesh to the new mesh; +// the two meshes should be aligned and the new mesh to have UV-coordinates +#if USE_MESH_INT == USE_MESH_BVH +struct FaceBox { + Eigen::AlignedBox3f box; + Mesh::FIndex idxFace; +}; +inline Eigen::AlignedBox3f bounding_box(const FaceBox& faceBox) { + return faceBox.box; +} +#endif +bool Mesh::TransferTexture(Mesh& mesh, unsigned textureSize) +{ + ASSERT(HasTexture() && mesh.HasTexture()); + if (vertexFaces.size() != vertices.size()) + ListIncidenteFaces(); + if (mesh.vertexNormals.size() != mesh.vertices.size()) + mesh.ComputeNormalVertices(); + if (mesh.textureDiffuse.empty()) + mesh.textureDiffuse.create(textureSize, textureSize); + #if USE_MESH_INT == USE_MESH_BVH + std::vector boxes; + boxes.reserve(faces.size()); + FOREACH(idxFace, faces) + boxes.emplace_back([this](FIndex idxFace) { + const Face& face = faces[idxFace]; + Eigen::AlignedBox3f box; + box.extend(vertices[face[0]]); + box.extend(vertices[face[1]]); + box.extend(vertices[face[2]]); + return FaceBox{box, idxFace}; + } (idxFace)); + typedef Eigen::KdBVH BVH; + BVH tree(boxes.begin(), boxes.end()); + #endif + struct IntersectRayMesh { + const Mesh& mesh; + const Ray3f& ray; + IndexDist pick; + IntersectRayMesh(const Mesh& _mesh, const Ray3f& _ray) + : mesh(_mesh), ray(_ray) { + #if USE_MESH_INT == USE_MESH_BF + FOREACH(idxFace, mesh.faces) + IntersectsRayFace(idxFace); + #endif + } + inline void IntersectsRayFace(FIndex idxFace) { + const Face& face = mesh.faces[idxFace]; + Type dist; + if (ray.Intersects(Triangle3f(mesh.vertices[face[0]], mesh.vertices[face[1]], mesh.vertices[face[2]]), &dist)) { + ASSERT(dist >= 0); + if (pick.dist > dist) { + pick.dist = dist; + pick.idx = idxFace; + } + } + } + #if USE_MESH_INT == USE_MESH_BVH + inline bool intersectVolume(const BVH::Volume &volume) { + return ray.Intersects(AABB3f(volume.min(), volume.max())); + } + inline bool intersectObject(const BVH::Object &object) { + IntersectsRayFace(object.idxFace); + return false; + } + #endif + }; + #if USE_MESH_INT == USE_MESH_BF || USE_MESH_INT == USE_MESH_BVH + const float diagonal(GetAABB().GetSize().norm()); + #elif USE_MESH_INT == USE_MESH_OCTREE + const Octree octree(vertices, [](Octree::IDX_TYPE size, Octree::Type /*radius*/) { + return size > 8; + }); + const float diagonal(octree.GetAabb().GetSize().norm()); + struct OctreeIntersectRayMesh : IntersectRayMesh { + OctreeIntersectRayMesh(const Octree& octree, const Mesh& _mesh, const Ray3f& _ray) + : IntersectRayMesh(_mesh, _ray) { + octree.Collect(*this, *this); + } + inline bool Intersects(const Octree::POINT_TYPE& center, Octree::Type radius) const { + return ray.Intersects(AABB3f(center, radius)); + } + void operator () (const Octree::IDX_TYPE* idices, Octree::IDX_TYPE size) { + // store all contained faces only once + std::unordered_set set; + FOREACHRAWPTR(pIdx, idices, size) { + const VIndex idxVertex((VIndex)*pIdx); + const FaceIdxArr& faces = mesh.vertexFaces[idxVertex]; + set.insert(faces.begin(), faces.end()); + } + // test face intersection and keep the closest + for (FIndex idxFace : set) + IntersectsRayFace(idxFace); + } + }; + #endif + #ifdef MESH_USE_OPENMP + #pragma omp parallel for schedule(dynamic) + for (int_t i=0; i<(int_t)mesh.faces.size(); ++i) { + const FIndex idxFace((FIndex)i); + #else + FOREACH(idxFace, mesh.faces) { + #endif + struct RasterTraiangle { + #if USE_MESH_INT == USE_MESH_OCTREE + const Octree& octree; + #elif USE_MESH_INT == USE_MESH_BVH + BVH& tree; + #endif + const Mesh& meshRef; + Mesh& meshTrg; + const Face& face; + float diagonal; + inline cv::Size Size() const { return meshTrg.textureDiffuse.size(); } + inline void operator()(const ImageRef& pt, const Point3f& bary) { + ASSERT(meshTrg.textureDiffuse.isInside(pt)); + const Vertex X(meshTrg.vertices[face[0]]*bary.x + meshTrg.vertices[face[1]]*bary.y + meshTrg.vertices[face[2]]*bary.z); + const Normal N(normalized(meshTrg.vertexNormals[face[0]]*bary.x + meshTrg.vertexNormals[face[1]]*bary.y + meshTrg.vertexNormals[face[2]]*bary.z)); + const Ray3f ray(Vertex(X+N*diagonal), Normal(-N)); + #if USE_MESH_INT == USE_MESH_BF + const IntersectRayMesh intRay(meshRef, ray); + #elif USE_MESH_INT == USE_MESH_BVH + IntersectRayMesh intRay(meshRef, ray); + Eigen::BVIntersect(tree, intRay); + #else + const OctreeIntersectRayMesh intRay(octree, meshRef, ray); + #endif + if (intRay.pick.IsValid()) { + const FIndex refIdxFace((FIndex)intRay.pick.idx); + const Face& refFace = meshRef.faces[refIdxFace]; + const Vertex refX(ray.GetPoint((Type)intRay.pick.dist)); + const Vertex baryRef(CorrectBarycentricCoordinates(BarycentricCoordinatesUV(meshRef.vertices[refFace[0]], meshRef.vertices[refFace[1]], meshRef.vertices[refFace[2]], refX))); + const TexCoord* tri = meshRef.faceTexcoords.data()+refIdxFace*3; + const TexCoord x(tri[0]*baryRef.x + tri[1]*baryRef.y + tri[2]*baryRef.z); + const Pixel8U color(meshRef.textureDiffuse.sample(x)); + meshTrg.textureDiffuse(pt) = color; + } + } + #if USE_MESH_INT == USE_MESH_BF + } data{*this, mesh, mesh.faces[idxFace], diagonal}; + #elif USE_MESH_INT == USE_MESH_BVH + } data{tree, *this, mesh, mesh.faces[idxFace], diagonal}; + #else + } data{octree, *this, mesh, mesh.faces[idxFace], diagonal}; + #endif + // render triangle and for each pixel interpolate the color + // from the triangle corners using barycentric coordinates + const TexCoord* tri = mesh.faceTexcoords.data()+idxFace*3; + Image8U::RasterizeTriangleBary(tri[0], tri[1], tri[2], data); + } + return true; +} // TransferTexture +/*----------------------------------------------------------------*/ + + #ifdef _USE_CUDA CUDA::KernelRT Mesh::kernelComputeFaceNormal; diff --git a/libs/MVS/Mesh.h b/libs/MVS/Mesh.h index 5e8bd0e2a..0d0640363 100644 --- a/libs/MVS/Mesh.h +++ b/libs/MVS/Mesh.h @@ -146,8 +146,10 @@ class MVS_API Mesh void ReleaseExtra(); void EmptyExtra(); void Swap(Mesh&); - inline bool IsEmpty() const { return vertices.IsEmpty(); } - inline bool HasTexture() const { ASSERT(faceTexcoords.IsEmpty() == textureDiffuse.empty()); return !faceTexcoords.IsEmpty(); } + void Join(const Mesh&); + inline bool IsEmpty() const { return vertices.empty(); } + bool IsWatertight(); + inline bool HasTexture() const { ASSERT(faceTexcoords.empty() == textureDiffuse.empty()); return !faceTexcoords.empty(); } Box GetAABB() const; Box GetAABB(const Box& bound) const; @@ -160,6 +162,8 @@ class MVS_API Mesh void ComputeNormalFaces(); void ComputeNormalVertices(); + void SmoothNormalFaces(float fMaxGradient=25.f, float fOriginalWeight=0.5f, unsigned nIterations=3); + void GetEdgeFaces(VIndex, VIndex, FaceIdxArr&) const; void GetFaceFaces(FIndex, FaceIdxArr&) const; void GetEdgeVertices(FIndex, FIndex, uint32_t vs0[2], uint32_t vs1[2]) const; @@ -176,11 +180,15 @@ class MVS_API Mesh void Decimate(VertexIdxArr& verticesRemove); void CloseHole(VertexIdxArr& vertsLoop); void CloseHoleQuality(VertexIdxArr& vertsLoop); + void RemoveFacesOutside(const OBB3f&); void RemoveFaces(FaceIdxArr& facesRemove, bool bUpdateLists=false); void RemoveVertices(VertexIdxArr& vertexRemove, bool bUpdateLists=false); VIndex RemoveUnreferencedVertices(bool bUpdateLists=false); void ConvertTexturePerVertex(Mesh&) const; + void FaceTexcoordsNormalize(TexCoordArr& newFaceTexcoords, bool flipY=true) const; + void FaceTexcoordsUnnormalize(TexCoordArr& newFaceTexcoords, bool flipY=true) const; + inline Normal FaceNormal(const Face& f) const { return ComputeTriangleNormal(vertices[f[0]], vertices[f[1]], vertices[f[2]]); } @@ -194,6 +202,8 @@ class MVS_API Mesh return n; } + Planef EstimateGroundPlane(const ImageArr& images, float sampleMesh=0, float planeThreshold=0, const String& fileExportPlane="") const; + Vertex ComputeCentroid(FIndex) const; Type ComputeArea(FIndex) const; REAL ComputeArea() const; @@ -213,6 +223,8 @@ class MVS_API Mesh bool Split(FacesChunkArr&, float maxArea); Mesh SubMesh(const FaceIdxArr& faces) const; + bool TransferTexture(Mesh& mesh, unsigned textureSize=1024); + // file IO bool Load(const String& fileName); bool Save(const String& fileName, const cList& comments=cList(), bool bBinary=true) const; @@ -226,6 +238,7 @@ class MVS_API Mesh protected: bool LoadPLY(const String& fileName); bool LoadOBJ(const String& fileName); + bool LoadGLTF(const String& fileName, bool bBinary=true); bool SavePLY(const String& fileName, const cList& comments=cList(), bool bBinary=true) const; bool SaveOBJ(const String& fileName) const; diff --git a/libs/MVS/PatchMatchCUDA.cpp b/libs/MVS/PatchMatchCUDA.cpp index 9579a76f7..ac872df0b 100644 --- a/libs/MVS/PatchMatchCUDA.cpp +++ b/libs/MVS/PatchMatchCUDA.cpp @@ -88,17 +88,15 @@ void PatchMatchCUDA::ReleaseCUDA() cudaFree(cudaDepthNormalCosts); cudaFree(cudaRandStates); cudaFree(cudaSelectedViews); - if (params.bGeomConsistency) cudaFree(cudaTextureDepths); delete[] depthNormalEstimates; - delete[] depthNormalCosts; } -void PatchMatchCUDA::Init(bool geom_consistency) +void PatchMatchCUDA::Init(bool bGeomConsistency) { - if (geom_consistency) { + if (bGeomConsistency) { params.bGeomConsistency = true; params.nEstimationIters = 1; } else { @@ -119,11 +117,9 @@ void PatchMatchCUDA::AllocatePatchMatchCUDA(const cv::Mat1f& image) depthNormalEstimates = new Point4[size]; CUDA::checkCudaCall(cudaMalloc((void**)&cudaDepthNormalEstimates, sizeof(Point4) * size)); - depthNormalCosts = new float[size]; CUDA::checkCudaCall(cudaMalloc((void**)&cudaDepthNormalCosts, sizeof(float) * size)); - - CUDA::checkCudaCall(cudaMalloc((void**)&cudaRandStates, sizeof(curandState) * size)); CUDA::checkCudaCall(cudaMalloc((void**)&cudaSelectedViews, sizeof(unsigned) * size)); + CUDA::checkCudaCall(cudaMalloc((void**)&cudaRandStates, sizeof(curandState) * size)); } void PatchMatchCUDA::AllocateImageCUDA(size_t i, const cv::Mat1f& image, bool bInitImage, bool bInitDepthMap) @@ -180,7 +176,14 @@ void PatchMatchCUDA::EstimateDepthMap(DepthData& depthData) TD_TIMER_STARTD(); ASSERT(depthData.images.size() > 1); - const IIndex prevNumImages = (IIndex)images.size(); + + // multi-resolution + DepthData& fullResDepthData(depthData); + const unsigned totalScaleNumber(params.bGeomConsistency ? 0u : OPTDENSE::nSubResolutionLevels); + DepthMap lowResDepthMap; + NormalMap lowResNormalMap; + ViewsMap lowResViewsMap; + IIndex prevNumImages = (IIndex)images.size(); const IIndex numImages = depthData.images.size(); params.nNumViews = (int)numImages-1; params.nInitTopK = std::min(params.nInitTopK, params.nNumViews); @@ -192,131 +195,217 @@ void PatchMatchCUDA::EstimateDepthMap(DepthData& depthData) cudaImageArrays.resize(numImages); textureImages.resize(numImages); } - if (params.bGeomConsistency && cudaDepthArrays.size() < numImages-1) { - cudaDepthArrays.resize(numImages-1); - textureDepths.resize(numImages-1); + if (params.bGeomConsistency && cudaDepthArrays.size() < (size_t)params.nNumViews) { + cudaDepthArrays.resize(params.nNumViews); + textureDepths.resize(params.nNumViews); } - for (IIndex i = 0; i < numImages; ++i) { - const DepthData::ViewData& view = depthData.images[i]; - Image32F image = view.image; - Camera camera; - camera.K = Eigen::Map(view.camera.K.val).cast(); - camera.R = Eigen::Map(view.camera.R.val).cast(); - camera.C = Eigen::Map(view.camera.C.ptr()).cast(); - camera.height = image.rows; - camera.width = image.cols; - // store camera and image - if (i == 0 && (prevNumImages < numImages || images[0].size() != image.size())) { - // allocate/reallocate PatchMatch CUDA memory - if (prevNumImages > 0) - ReleaseCUDA(); - AllocatePatchMatchCUDA(image); + const int maxPixelViews(MINF(params.nNumViews, 4)); + for (unsigned scaleNumber = totalScaleNumber+1; scaleNumber-- > 0; ) { + // initialize + const float scale = 1.f / POWI(2, scaleNumber); + DepthData currentDepthData(DepthMapsData::ScaleDepthData(fullResDepthData, scale)); + DepthData& depthData(scaleNumber==0 ? fullResDepthData : currentDepthData); + const Image8U::Size size(depthData.images.front().image.size()); + params.bLowResProcessed = false; + if (scaleNumber != totalScaleNumber) { + // all resolutions, but the smallest one, if multi-resolution is enabled + params.bLowResProcessed = true; + cv::resize(lowResDepthMap, depthData.depthMap, size, 0, 0, cv::INTER_LINEAR); + cv::resize(lowResNormalMap, depthData.normalMap, size, 0, 0, cv::INTER_NEAREST); + cv::resize(lowResViewsMap, depthData.viewsMap, size, 0, 0, cv::INTER_NEAREST); + CUDA::checkCudaCall(cudaMalloc((void**)&cudaLowDepths, sizeof(float) * size.area())); + } else { + if (totalScaleNumber > 0) { + // smallest resolution, when multi-resolution is enabled + fullResDepthData.depthMap.release(); + fullResDepthData.normalMap.release(); + fullResDepthData.confMap.release(); + fullResDepthData.viewsMap.release(); + } + // smallest resolution if multi-resolution is enabled; highest otherwise + if (depthData.viewsMap.empty()) + depthData.viewsMap.create(size); } - if (i >= prevNumImages) { - // allocate image CUDA memory - AllocateImageCUDA(i, image, true, !view.depthMap.empty()); - } else - if (images[i].size() != image.size()) { - // reallocate image CUDA memory - cudaDestroyTextureObject(textureImages[i]); - cudaFreeArray(cudaImageArrays[i]); - if (params.bGeomConsistency && i > 0) { - cudaDestroyTextureObject(textureDepths[i-1]); - cudaFreeArray(cudaDepthArrays[i-1]); + if (scaleNumber == 0) { + // highest resolution + if (depthData.confMap.empty()) + depthData.confMap.create(size); + } + + // set keep threshold to: + params.fThresholdKeepCost = OPTDENSE::fNCCThresholdKeep; + if (totalScaleNumber) { + // multi-resolution enabled + if (scaleNumber > 0 && scaleNumber != totalScaleNumber) { + // all sub-resolutions, but the smallest and highest + params.fThresholdKeepCost = 0.f; // disable filtering + } else if (scaleNumber == totalScaleNumber || (!params.bGeomConsistency && OPTDENSE::nEstimationGeometricIters)) { + // smallest sub-resolution OR highest resolution and geometric consistency is not running but enabled + params.fThresholdKeepCost = OPTDENSE::fNCCThresholdKeep*1.2f; + } + } else { + // multi-resolution disabled + if (!params.bGeomConsistency && OPTDENSE::nEstimationGeometricIters) { + // geometric consistency is not running but enabled + params.fThresholdKeepCost = OPTDENSE::fNCCThresholdKeep*1.2f; + } + } + + for (IIndex i = 0; i < numImages; ++i) { + const DepthData::ViewData& view = depthData.images[i]; + Image32F image = view.image; + Camera camera; + camera.K = Eigen::Map(view.camera.K.val).cast(); + camera.R = Eigen::Map(view.camera.R.val).cast(); + camera.C = Eigen::Map(view.camera.C.ptr()).cast(); + camera.height = image.rows; + camera.width = image.cols; + // store camera and image + if (i == 0 && (prevNumImages < numImages || images[0].size() != image.size())) { + // allocate/reallocate PatchMatch CUDA memory + if (prevNumImages > 0) + ReleaseCUDA(); + AllocatePatchMatchCUDA(image); } - AllocateImageCUDA(i, image, true, !view.depthMap.empty()); - } else - if (params.bGeomConsistency && i > 0 && (view.depthMap.empty() != (cudaDepthArrays[i-1] == NULL))) { - // reallocate depth CUDA memory - if (cudaDepthArrays[i-1]) { + if (i >= prevNumImages) { + // allocate image CUDA memory + AllocateImageCUDA(i, image, true, !view.depthMap.empty()); + } else + if (images[i].size() != image.size()) { + // reallocate image CUDA memory + cudaDestroyTextureObject(textureImages[i]); + cudaFreeArray(cudaImageArrays[i]); + if (params.bGeomConsistency && i > 0) { + cudaDestroyTextureObject(textureDepths[i-1]); + cudaFreeArray(cudaDepthArrays[i-1]); + } + AllocateImageCUDA(i, image, true, !view.depthMap.empty()); + } else + if (params.bGeomConsistency && i > 0 && (view.depthMap.empty() != (cudaDepthArrays[i-1] == NULL))) { + // reallocate depth CUDA memory + if (cudaDepthArrays[i-1]) { + cudaDestroyTextureObject(textureDepths[i-1]); + cudaFreeArray(cudaDepthArrays[i-1]); + } + AllocateImageCUDA(i, image, false, !view.depthMap.empty()); + } + CUDA::checkCudaCall(cudaMemcpy2DToArray(cudaImageArrays[i], 0, 0, image.ptr(), image.step[0], image.cols * sizeof(float), image.rows, cudaMemcpyHostToDevice)); + if (params.bGeomConsistency && i > 0 && !view.depthMap.empty()) { + // set previously computed depth-map + DepthMap depthMap(view.depthMap); + if (depthMap.size() != image.size()) + cv::resize(depthMap, depthMap, image.size(), 0, 0, cv::INTER_LINEAR); + CUDA::checkCudaCall(cudaMemcpy2DToArray(cudaDepthArrays[i-1], 0, 0, depthMap.ptr(), depthMap.step[0], sizeof(float) * depthMap.cols, depthMap.rows, cudaMemcpyHostToDevice)); + } + images[i] = std::move(image); + cameras[i] = std::move(camera); + } + if (params.bGeomConsistency && cudaDepthArrays.size() > numImages - 1) { + for (IIndex i = numImages; i < prevNumImages; ++i) { + // free image CUDA memory cudaDestroyTextureObject(textureDepths[i-1]); cudaFreeArray(cudaDepthArrays[i-1]); } - AllocateImageCUDA(i, image, false, !view.depthMap.empty()); - } - CUDA::checkCudaCall(cudaMemcpy2DToArray(cudaImageArrays[i], 0, 0, image.ptr(), image.step[0], image.cols*sizeof(float), image.rows, cudaMemcpyHostToDevice)); - if (params.bGeomConsistency && i > 0 && !view.depthMap.empty()) { - // set previously computed depth-map - DepthMap depthMap(view.depthMap); - if (depthMap.size() != image.size()) - cv::resize(depthMap, depthMap, image.size(), 0, 0, cv::INTER_LINEAR); - CUDA::checkCudaCall(cudaMemcpy2DToArray(cudaDepthArrays[i-1], 0, 0, depthMap.ptr(), depthMap.step[0], sizeof(float)*depthMap.cols, depthMap.rows, cudaMemcpyHostToDevice)); + cudaDepthArrays.resize(params.nNumViews); + textureDepths.resize(params.nNumViews); } - images[i] = std::move(image); - cameras[i] = std::move(camera); - } - if (params.bGeomConsistency && cudaDepthArrays.size() > numImages-1) { - for (IIndex i = numImages; i < prevNumImages; ++i) { - // free image CUDA memory - cudaDestroyTextureObject(textureDepths[i-1]); - cudaFreeArray(cudaDepthArrays[i-1]); + if (prevNumImages > numImages) { + for (IIndex i = numImages; i < prevNumImages; ++i) { + // free image CUDA memory + cudaDestroyTextureObject(textureImages[i]); + cudaFreeArray(cudaImageArrays[i]); + } + images.resize(numImages); + cameras.resize(numImages); + cudaImageArrays.resize(numImages); + textureImages.resize(numImages); } - cudaDepthArrays.resize(numImages-1); - textureDepths.resize(numImages-1); - } - if (prevNumImages > numImages) { - for (IIndex i = numImages; i < prevNumImages; ++i) { - // free image CUDA memory - cudaDestroyTextureObject(textureImages[i]); - cudaFreeArray(cudaImageArrays[i]); + prevNumImages = numImages; + + // setup CUDA memory + CUDA::checkCudaCall(cudaMemcpy(cudaTextureImages, textureImages.data(), sizeof(cudaTextureObject_t) * numImages, cudaMemcpyHostToDevice)); + CUDA::checkCudaCall(cudaMemcpy(cudaCameras, cameras.data(), sizeof(Camera) * numImages, cudaMemcpyHostToDevice)); + if (params.bGeomConsistency) { + // set previously computed depth-maps + ASSERT(depthData.depthMap.size() == depthData.GetView().image.size()); + CUDA::checkCudaCall(cudaMemcpy(cudaTextureDepths, textureDepths.data(), sizeof(cudaTextureObject_t) * params.nNumViews, cudaMemcpyHostToDevice)); } - images.resize(numImages); - cameras.resize(numImages); - cudaImageArrays.resize(numImages); - textureImages.resize(numImages); - } - // setup CUDA memory - CUDA::checkCudaCall(cudaMemcpy(cudaTextureImages, textureImages.data(), sizeof(cudaTextureObject_t)*numImages, cudaMemcpyHostToDevice)); - CUDA::checkCudaCall(cudaMemcpy(cudaCameras, cameras.data(), sizeof(Camera)*numImages, cudaMemcpyHostToDevice)); - if (params.bGeomConsistency) { - // set previously computed depth-maps - ASSERT(depthData.depthMap.size() == depthData.GetView().image.size()); - CUDA::checkCudaCall(cudaMemcpy(cudaTextureDepths, textureDepths.data(), sizeof(cudaTextureObject_t)*(numImages-1), cudaMemcpyHostToDevice)); - } + // load depth-map and normal-map into CUDA memory + for (int r = 0; r < depthData.depthMap.rows; ++r) { + const int baseIndex = r * depthData.depthMap.cols; + for (int c = 0; c < depthData.depthMap.cols; ++c) { + const Normal& n = depthData.normalMap(r, c); + const int index = baseIndex + c; + Point4& depthNormal = depthNormalEstimates[index]; + depthNormal.topLeftCorner<3, 1>() = Eigen::Map(n.ptr()); + depthNormal.w() = depthData.depthMap(r, c); + } + } + CUDA::checkCudaCall(cudaMemcpy(cudaDepthNormalEstimates, depthNormalEstimates, sizeof(Point4) * depthData.depthMap.size().area(), cudaMemcpyHostToDevice)); - // load depth-map and normal-map into CUDA memory - for (int r = 0; r < depthData.depthMap.rows; ++r) { - for (int c = 0; c < depthData.depthMap.cols; ++c) { - const Normal& n = depthData.normalMap(r,c); - const int index = r * depthData.depthMap.cols + c; - Point4& depthNormal = depthNormalEstimates[index]; - depthNormal.topLeftCorner<3,1>() = Eigen::Map(n.ptr()); - depthNormal.w() = depthData.depthMap(r,c); + // load low resolution depth-map into CUDA memory + if (params.bLowResProcessed) { + ASSERT(depthData.depthMap.isContinuous()); + CUDA::checkCudaCall(cudaMemcpy(cudaLowDepths, depthData.depthMap.ptr(), sizeof(float) * depthData.depthMap.size().area(), cudaMemcpyHostToDevice)); } - } - CUDA::checkCudaCall(cudaMemcpy(cudaDepthNormalEstimates, depthNormalEstimates, sizeof(Point4) * depthData.depthMap.size().area(), cudaMemcpyHostToDevice)); - - // run CUDA patch-match - RunCUDA(); - CUDA::checkCudaCall(cudaGetLastError()); - - // load depth-map, normal-map and confidence-map from CUDA memory - const float fNCCThresholdKeep(!params.bGeomConsistency && OPTDENSE::nEstimationGeometricIters ? - OPTDENSE::fNCCThresholdKeepCUDA * 1.5f : OPTDENSE::fNCCThresholdKeepCUDA); - if (depthData.confMap.empty()) - depthData.confMap.create(depthData.depthMap.size()); - for (int r = 0; r < depthData.depthMap.rows; ++r) { - for (int c = 0; c < depthData.depthMap.cols; ++c) { - const int index = r * depthData.depthMap.cols + c; - const Point4& depthNormal = depthNormalEstimates[index]; - ASSERT(std::isfinite(depthNormal.w())); - // check if the score is good enough - // and that the cross-estimates is close enough to the current estimate - float& conf = depthData.confMap(r,c); - conf = depthNormalCosts[index]; - ASSERT(std::isfinite(conf)); - if (depthNormal.w() <= 0 || conf >= fNCCThresholdKeep) { - conf = 0; - depthData.depthMap(r,c) = 0; - depthData.normalMap(r,c) = Normal::ZERO; - } else { - depthData.depthMap(r,c) = depthNormal.w(); - depthData.normalMap(r,c) = depthNormal.topLeftCorner<3,1>(); - // converted ZNCC [0-2] score, where 0 is best, to [0-1] confidence, where 1 is best - conf = conf>=1.f ? 0.f : 1.f-conf; + + // run CUDA patch-match + ASSERT(!depthData.viewsMap.empty()); + RunCUDA(depthData.confMap.getData(), (uint32_t*)depthData.viewsMap.getData()); + CUDA::checkCudaCall(cudaGetLastError()); + if (params.bLowResProcessed) + CUDA::checkCudaCall(cudaFree(cudaLowDepths)); + + // load depth-map, normal-map and confidence-map from CUDA memory + for (int r = 0; r < depthData.depthMap.rows; ++r) { + for (int c = 0; c < depthData.depthMap.cols; ++c) { + const int index = r * depthData.depthMap.cols + c; + const Point4& depthNormal = depthNormalEstimates[index]; + const Depth depth = depthNormal.w(); + ASSERT(std::isfinite(depth)); + depthData.depthMap(r, c) = depth; + depthData.normalMap(r, c) = depthNormal.topLeftCorner<3, 1>(); + if (scaleNumber == 0) { + // converted ZNCC [0-2] score, where 0 is best, to [0-1] confidence, where 1 is best + ASSERT(!depthData.confMap.empty()); + float& conf = depthData.confMap(r, c); + conf = conf >= 1.f ? 0.f : 1.f - conf; + // map pixel views from bit-mask to index + ASSERT(!depthData.viewsMap.empty()); + ViewsID& views = depthData.viewsMap(r, c); + if (depth > 0) { + const uint32_t bitviews(*reinterpret_cast(views.val)); + int j = 0; + for (int i = 0; i < 32; ++i) { + if (bitviews & (1 << i)) { + views[j] = i; + if (++j == maxPixelViews) + break; + } + } + while (j < 4) + views[j++] = 255; + } else + views = ViewsID(255, 255, 255, 255); + } } } + + // remember sub-resolution estimates for next iteration + if (scaleNumber > 0) { + lowResDepthMap = depthData.depthMap; + lowResNormalMap = depthData.normalMap; + lowResViewsMap = depthData.viewsMap; + } + } + + // apply ignore mask + if (OPTDENSE::nIgnoreMaskLabel >= 0) { + const DepthData::ViewData& view = depthData.GetView(); + BitMatrix mask; + if (DepthEstimator::ImportIgnoreMask(*view.pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel)) + depthData.ApplyIgnoreMask(mask); } DEBUG_EXTRA("Depth-map for image %3u %s: %dx%d (%s)", depthData.images.front().GetID(), diff --git a/libs/MVS/PatchMatchCUDA.cu b/libs/MVS/PatchMatchCUDA.cu index 7fe1d4e1d..26a9c15d6 100644 --- a/libs/MVS/PatchMatchCUDA.cu +++ b/libs/MVS/PatchMatchCUDA.cu @@ -296,10 +296,10 @@ __device__ inline float GeometricConsistencyWeight(const ImagePixels depthImage, } // compute photometric score using weighted ZNCC -__device__ float ScorePlane(const ImagePixels refImage, const PatchMatchCUDA::Camera& refCamera, const ImagePixels trgImage, const PatchMatchCUDA::Camera& trgCamera, const Point2i& p, const Point4& plane, const PatchMatchCUDA::Params& params) +__device__ float ScorePlane(const ImagePixels refImage, const PatchMatchCUDA::Camera& refCamera, const ImagePixels trgImage, const PatchMatchCUDA::Camera& trgCamera, const Point2i& p, const Point4& plane, const float lowDepth, const PatchMatchCUDA::Params& params) { constexpr float maxCost = 1.2f; - + Matrix3 H = ComputeHomography(refCamera, trgCamera, p.cast(), plane); const Point2 pt = (H * p.cast().homogeneous()).hnormalized(); if (pt.x() >= trgCamera.width || pt.x() < 0.f || pt.y() >= trgCamera.height || pt.y() < 0.f) @@ -337,31 +337,40 @@ __device__ float ScorePlane(const ImagePixels refImage, const PatchMatchCUDA::Ca } const float varRef = sumRefRef * bilateralWeightSum - sumRef * sumRef; + if (lowDepth <= 0 && varRef < 1e-8f) + return maxCost; const float varTrg = sumTrgTrg * bilateralWeightSum - sumTrg * sumTrg; - - constexpr float kMinVar = 1e-5f; - if (varRef < kMinVar || varTrg < kMinVar) + const float varRefTrg = varRef * varTrg; + if (varRefTrg < 1e-16f) return maxCost; const float covarTrgRef = sumRefTrg * bilateralWeightSum - sumRef * sumTrg; - const float varRefTrg = sqrt(varRef * varTrg); - const float ncc = 1.f - covarTrgRef / varRefTrg; - return max(0.f, min(maxCost, ncc)); + float ncc = 1.f - covarTrgRef / sqrt(varRefTrg); + + // apply depth prior weight based on patch textureless + if (lowDepth > 0) { + const float depth(plane.w()); + const float deltaDepth(MIN((abs(lowDepth-depth) / lowDepth), 0.5f)); + constexpr float smoothSigmaDepth(-1.f / (1.f * 0.02f)); // 0.12: patch texture variance below 0.02 (0.12^2) is considered texture-less + const float factorDeltaDepth(exp(varRef * smoothSigmaDepth)); + ncc = (1.f-factorDeltaDepth)*ncc + factorDeltaDepth*deltaDepth; + } + return max(0.f, min(2.f, ncc)); } // compute photometric score for all neighbor images -__device__ inline void MultiViewScorePlane(const ImagePixels *images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, const Point2i& p, const Point4& plane, float* costVector, const PatchMatchCUDA::Params& params) +__device__ inline void MultiViewScorePlane(const ImagePixels *images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, const Point2i& p, const Point4& plane, const float lowDepth, float* costVector, const PatchMatchCUDA::Params& params) { for (int imgId = 1; imgId <= params.nNumViews; ++imgId) - costVector[imgId-1] = ScorePlane(images[0], cameras[0], images[imgId], cameras[imgId], p, plane, params); + costVector[imgId-1] = ScorePlane(images[0], cameras[0], images[imgId], cameras[imgId], p, plane, lowDepth, params); if (params.bGeomConsistency) for (int imgId = 0; imgId < params.nNumViews; ++imgId) costVector[imgId] += 0.1f * GeometricConsistencyWeight(depthImages[imgId], cameras[0], cameras[imgId+1], plane, p); } // same as above, but interpolate the plane to current pixel position -__device__ inline float MultiViewScoreNeighborPlane(const ImagePixels* images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, const Point2i& p, const Point2i& np, Point4 plane, float* costVector, const PatchMatchCUDA::Params& params) +__device__ inline float MultiViewScoreNeighborPlane(const ImagePixels* images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, const Point2i& p, const Point2i& np, Point4 plane, const float lowDepth, float* costVector, const PatchMatchCUDA::Params& params) { plane.w() = InterpolatePixel(cameras[0], p, np, plane.w(), plane.topLeftCorner<3,1>(), params); - MultiViewScorePlane(images, depthImages, cameras, p, plane, costVector, params); + MultiViewScorePlane(images, depthImages, cameras, p, plane, lowDepth, costVector, params); return plane.w(); } @@ -372,12 +381,12 @@ __device__ inline float AggregateMultiViewScores(const unsigned* viewWeights, co for (int imgId = 0; imgId < numViews; ++imgId) if (viewWeights[imgId]) cost += viewWeights[imgId] * costVector[imgId]; - return cost; + return cost / NUM_SAMPLES; } // propagate and refine the plane estimate for the current pixel employing the asymmetric approach described in: // "Multi-View Stereo with Asymmetric Checkerboard Propagation and Multi-Hypothesis Joint View Selection", 2018 -__device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, Point4* planes, float* costs, RandState* randStates, unsigned* selectedViews, const Point2i& p, const PatchMatchCUDA::Params& params, const int iter) +__device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, Point4* planes, const float* lowDepths, float* costs, RandState* randStates, unsigned* selectedViews, const Point2i& p, const PatchMatchCUDA::Params& params, const int iter) { int width = cameras[0].width; int height = cameras[0].height; @@ -385,6 +394,9 @@ __device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depth return; const int idx = Point2Idx(p, width); RandState* randState = &randStates[idx]; + float lowDepth = 0; + if (params.bLowResProcessed) + lowDepth = lowDepths[idx]; // adaptive sampling: 0 up-near, 1 down-near, 2 left-near, 3 right-near, 4 up-far, 5 down-far, 6 left-far, 7 right-far static constexpr int2 dirs[8][11] = { @@ -427,7 +439,7 @@ __device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depth if (bestConf < FLT_MAX) { valid[posId] = true; positions[posId] = Point2Idx(bestNx, width); - neighborDepths[posId] = MultiViewScoreNeighborPlane(images, depthImages, cameras, p, bestNx, planes[positions[posId]], costArray[posId], params); + neighborDepths[posId] = MultiViewScoreNeighborPlane(images, depthImages, cameras, p, bestNx, planes[positions[posId]], lowDepth, costArray[posId], params); } } @@ -486,11 +498,11 @@ __device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depth SetBit(newSelectedViews, imgId); float finalCosts[8]; for (int posId = 0; posId < 8; ++posId) - finalCosts[posId] = AggregateMultiViewScores(viewWeights, costArray[posId], params.nNumViews) / NUM_SAMPLES; + finalCosts[posId] = AggregateMultiViewScores(viewWeights, costArray[posId], params.nNumViews); const int minCostIdx = FindMinIndex(finalCosts, 8); float costVector[MAX_VIEWS]; - MultiViewScorePlane(images, depthImages, cameras, p, plane, costVector, params); - cost = AggregateMultiViewScores(viewWeights, costVector, params.nNumViews) / NUM_SAMPLES; + MultiViewScorePlane(images, depthImages, cameras, p, plane, lowDepth, costVector, params); + cost = AggregateMultiViewScores(viewWeights, costVector, params.nNumViews); if (finalCosts[minCostIdx] < cost && valid[minCostIdx]) { plane = planes[positions[minCostIdx]]; plane.w() = neighborDepths[minCostIdx]; @@ -518,15 +530,15 @@ __device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depth surfaceNormal = ComputeDepthGradient(cameras[0].K, depth, p, ndepths); numValidPlanes = 4; } - const int numPlanes = 4; + constexpr int numPlanes = 4; const float depths[numPlanes] = {depthPerturbed, depth, depth, depth}; const Point3 normals[numPlanes] = {plane.topLeftCorner<3,1>(), perturbedNormal, normalRand, surfaceNormal}; for (int i = 0; i < numValidPlanes; ++i) { Point4 newPlane; newPlane.topLeftCorner<3,1>() = normals[i]; newPlane.w() = depths[i]; - MultiViewScorePlane(images, depthImages, cameras, p, newPlane, costVector, params); - const float costPlane = AggregateMultiViewScores(viewWeights, costVector, params.nNumViews) / NUM_SAMPLES; + MultiViewScorePlane(images, depthImages, cameras, p, newPlane, lowDepth, costVector, params); + const float costPlane = AggregateMultiViewScores(viewWeights, costVector, params.nNumViews); if (cost > costPlane) { cost = costPlane; plane = newPlane; @@ -535,13 +547,16 @@ __device__ void ProcessPixel(const ImagePixels* images, const ImagePixels* depth } // compute the score of the current plane estimate -__device__ void InitializePixelScore(const ImagePixels *images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, Point4* planes, float* costs, RandState* randStates, unsigned* selectedViews, const Point2i& p, const PatchMatchCUDA::Params params) +__device__ void InitializePixelScore(const ImagePixels *images, const ImagePixels* depthImages, const PatchMatchCUDA::Camera* cameras, Point4* planes, const float* lowDepths, float* costs, RandState* randStates, unsigned* selectedViews, const Point2i& p, const PatchMatchCUDA::Params params) { const int width = cameras[0].width; const int height = cameras[0].height; if (p.x() >= width || p.y() >= height) return; const int idx = Point2Idx(p, width); + float lowDepth = 0; + if (params.bLowResProcessed) + lowDepth = lowDepths[idx]; // initialize estimate randomly if not set RandState* randState = &randStates[idx]; curand_init(1234/*threadIdx.x*/, p.y(), p.x(), randState); @@ -557,7 +572,7 @@ __device__ void InitializePixelScore(const ImagePixels *images, const ImagePixel } // compute costs float costVector[MAX_VIEWS]; - MultiViewScorePlane(images, depthImages, cameras, p, plane, costVector, params); + MultiViewScorePlane(images, depthImages, cameras, p, plane, lowDepth, costVector, params); // select best views float costVectorSorted[MAX_VIEWS]; Sort(costVector, costVectorSorted, params.nNumViews); @@ -572,29 +587,46 @@ __device__ void InitializePixelScore(const ImagePixels *images, const ImagePixel SetBit(selectedView, imgId); costs[idx] = cost / params.nInitTopK; } -__global__ void InitializeScore(const cudaTextureObject_t* textureImages, const cudaTextureObject_t* textureDepths, const PatchMatchCUDA::Camera* cameras, Point4* planes, float* costs, curandState* randStates, unsigned* selectedViews, const PatchMatchCUDA::Params params) +__global__ void InitializeScore(const cudaTextureObject_t* textureImages, const cudaTextureObject_t* textureDepths, const PatchMatchCUDA::Camera* cameras, Point4* planes, const float* lowDepths, float* costs, curandState* randStates, unsigned* selectedViews, const PatchMatchCUDA::Params params) { const Point2i p = Point2i(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); - InitializePixelScore((const ImagePixels*)textureImages, (const ImagePixels*)textureDepths, cameras, planes, costs, (RandState*)randStates, selectedViews, p, params); + InitializePixelScore((const ImagePixels*)textureImages, (const ImagePixels*)textureDepths, cameras, planes, lowDepths, costs, (RandState*)randStates, selectedViews, p, params); } // traverse image in a back/red checkerboard pattern -__global__ void BlackPixelProcess(const cudaTextureObject_t* textureImages, const cudaTextureObject_t* textureDepths, const PatchMatchCUDA::Camera* cameras, Point4* planes, float* costs, curandState* randStates, unsigned* selectedViews, const PatchMatchCUDA::Params params, const int iter) +__global__ void BlackPixelProcess(const cudaTextureObject_t* textureImages, const cudaTextureObject_t* textureDepths, const PatchMatchCUDA::Camera* cameras, Point4* planes, const float* lowDepths, float* costs, curandState* randStates, unsigned* selectedViews, const PatchMatchCUDA::Params params, const int iter) { Point2i p = Point2i(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); p.y() = p.y() * 2 + (threadIdx.x % 2 == 0 ? 0 : 1); - ProcessPixel((const ImagePixels*)textureImages, (const ImagePixels*)textureDepths, cameras, planes, costs, (RandState*)randStates, selectedViews, p, params, iter); + ProcessPixel((const ImagePixels*)textureImages, (const ImagePixels*)textureDepths, cameras, planes, lowDepths, costs, (RandState*)randStates, selectedViews, p, params, iter); } -__global__ void RedPixelProcess(const cudaTextureObject_t* textureImages, const cudaTextureObject_t* textureDepths, const PatchMatchCUDA::Camera* cameras, Point4* planes, float* costs, curandState* randStates, unsigned* selectedViews, const PatchMatchCUDA::Params params, const int iter) +__global__ void RedPixelProcess(const cudaTextureObject_t* textureImages, const cudaTextureObject_t* textureDepths, const PatchMatchCUDA::Camera* cameras, Point4* planes, const float* lowDepths, float* costs, curandState* randStates, unsigned* selectedViews, const PatchMatchCUDA::Params params, const int iter) { Point2i p = Point2i(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); p.y() = p.y() * 2 + (threadIdx.x % 2 == 0 ? 1 : 0); - ProcessPixel((const ImagePixels*)textureImages, (const ImagePixels*)textureDepths, cameras, planes, costs, (RandState*)randStates, selectedViews, p, params, iter); + ProcessPixel((const ImagePixels*)textureImages, (const ImagePixels*)textureDepths, cameras, planes, lowDepths, costs, (RandState*)randStates, selectedViews, p, params, iter); +} + +// filter depth/normals +__global__ void FilterPlanes(Point4* planes, float* costs, unsigned* selectedViews, int width, int height, const PatchMatchCUDA::Params params) +{ + const Point2i p = Point2i(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); + if (p.x() >= width || p.y() >= height) + return; + const int idx = Point2Idx(p, width); + // filter estimates if the score is not good enough + Point4& plane = planes[idx]; + float conf = costs[idx]; + if (plane.w() <= 0 || conf >= params.fThresholdKeepCost) { + conf = 0; + plane = Point4::Zero(); + selectedViews[idx] = 0; + } } /*----------------------------------------------------------------*/ -__host__ void PatchMatchCUDA::RunCUDA() +__host__ void PatchMatchCUDA::RunCUDA(float* ptrCostMap, uint32_t* ptrViewsMap) { const unsigned width = cameras[0].width; const unsigned height = cameras[0].height; @@ -603,21 +635,28 @@ __host__ void PatchMatchCUDA::RunCUDA() constexpr unsigned BLOCK_H = (BLOCK_W / 2); const dim3 blockSize(BLOCK_W, BLOCK_H, 1); - const dim3 gridSizeInitialize((width + BLOCK_H - 1) / BLOCK_H, (height + BLOCK_H - 1) / BLOCK_H, 1); + const dim3 gridSizeFull((width + BLOCK_H - 1) / BLOCK_H, (height + BLOCK_H - 1) / BLOCK_H, 1); const dim3 gridSizeCheckerboard((width + BLOCK_W - 1) / BLOCK_W, ((height / 2) + BLOCK_H - 1) / BLOCK_H, 1); - InitializeScore<<>>(cudaTextureImages, cudaTextureDepths, cudaCameras, cudaDepthNormalEstimates, cudaDepthNormalCosts, cudaRandStates, cudaSelectedViews, params); + InitializeScore<<>>(cudaTextureImages, cudaTextureDepths, cudaCameras, cudaDepthNormalEstimates, cudaLowDepths, cudaDepthNormalCosts, cudaRandStates, cudaSelectedViews, params); cudaDeviceSynchronize(); for (int iter = 0; iter < params.nEstimationIters; ++iter) { - BlackPixelProcess<<>>(cudaTextureImages, cudaTextureDepths, cudaCameras, cudaDepthNormalEstimates, cudaDepthNormalCosts, cudaRandStates, cudaSelectedViews, params, iter); + BlackPixelProcess<<>>(cudaTextureImages, cudaTextureDepths, cudaCameras, cudaDepthNormalEstimates, cudaLowDepths, cudaDepthNormalCosts, cudaRandStates, cudaSelectedViews, params, iter); cudaDeviceSynchronize(); - RedPixelProcess<<>>(cudaTextureImages, cudaTextureDepths, cudaCameras, cudaDepthNormalEstimates, cudaDepthNormalCosts, cudaRandStates, cudaSelectedViews, params, iter); + RedPixelProcess<<>>(cudaTextureImages, cudaTextureDepths, cudaCameras, cudaDepthNormalEstimates, cudaLowDepths, cudaDepthNormalCosts, cudaRandStates, cudaSelectedViews, params, iter); cudaDeviceSynchronize(); } + if (params.fThresholdKeepCost > 0) + FilterPlanes<<>>(cudaDepthNormalEstimates, cudaDepthNormalCosts, cudaSelectedViews, width, height, params); + cudaMemcpy(depthNormalEstimates, cudaDepthNormalEstimates, sizeof(Point4) * width * height, cudaMemcpyDeviceToHost); - cudaMemcpy(depthNormalCosts, cudaDepthNormalCosts, sizeof(float) * width * height, cudaMemcpyDeviceToHost); + if (ptrCostMap) + cudaMemcpy(ptrCostMap, cudaDepthNormalCosts, sizeof(float) * width * height, cudaMemcpyDeviceToHost); + if (ptrViewsMap) + cudaMemcpy(ptrViewsMap, cudaSelectedViews, sizeof(uint32_t) * width * height, cudaMemcpyDeviceToHost); + cudaDeviceSynchronize(); } /*----------------------------------------------------------------*/ diff --git a/libs/MVS/PatchMatchCUDA.h b/libs/MVS/PatchMatchCUDA.h index 5049ff18b..22680c90d 100644 --- a/libs/MVS/PatchMatchCUDA.h +++ b/libs/MVS/PatchMatchCUDA.h @@ -37,7 +37,7 @@ // I N C L U D E S ///////////////////////////////////////////////// -#include "DepthMap.h" +#include "SceneDensify.h" #pragma push_macro("EIGEN_DEFAULT_DENSE_INDEX_TYPE") #undef EIGEN_DEFAULT_DENSE_INDEX_TYPE #include "PatchMatchCUDA.inl" diff --git a/libs/MVS/PatchMatchCUDA.inl b/libs/MVS/PatchMatchCUDA.inl index 5dbc4be1d..349cc33d7 100644 --- a/libs/MVS/PatchMatchCUDA.inl +++ b/libs/MVS/PatchMatchCUDA.inl @@ -86,6 +86,8 @@ public: float fDepthMax = 100.f; int nInitTopK = 3; bool bGeomConsistency = false; + bool bLowResProcessed = false; + float fThresholdKeepCost = 0; }; struct Camera { @@ -100,7 +102,7 @@ public: PatchMatchCUDA(int device=0); ~PatchMatchCUDA(); - void Init(bool geom_consistency); + void Init(bool bGeomConsistency); void Release(); void EstimateDepthMap(DepthData&); @@ -112,7 +114,7 @@ private: void ReleaseCUDA(); void AllocatePatchMatchCUDA(const cv::Mat1f& image); void AllocateImageCUDA(size_t i, const cv::Mat1f& image, bool bInitImage, bool bInitDepthMap); - void RunCUDA(); + void RunCUDA(float* ptrCostMap=NULL, uint32_t* ptrViewsMap=NULL); public: Params params; @@ -122,7 +124,6 @@ public: std::vector textureImages; std::vector textureDepths; Point4* depthNormalEstimates; - float* depthNormalCosts; Camera *cudaCameras; std::vector cudaImageArrays; @@ -130,9 +131,10 @@ public: cudaTextureObject_t* cudaTextureImages; cudaTextureObject_t* cudaTextureDepths; Point4* cudaDepthNormalEstimates; + float* cudaLowDepths; float* cudaDepthNormalCosts; curandState* cudaRandStates; - unsigned* cudaSelectedViews; + uint32_t* cudaSelectedViews; }; /*----------------------------------------------------------------*/ diff --git a/libs/MVS/PointCloud.cpp b/libs/MVS/PointCloud.cpp index 870664c43..6c6bf04f8 100644 --- a/libs/MVS/PointCloud.cpp +++ b/libs/MVS/PointCloud.cpp @@ -31,6 +31,7 @@ #include "Common.h" #include "PointCloud.h" +#include "DepthMap.h" using namespace MVS; @@ -73,6 +74,12 @@ void PointCloud::RemovePointsOutside(const OBB3f& obb) { if (!obb.Intersects(points[i])) RemovePoint(i); } +void PointCloud::RemoveMinViews(uint32_t thMinViews) { + ASSERT(!pointViews.empty()); + RFOREACH(i, points) + if (pointViews[i].size() < thMinViews) + RemovePoint(i); +} /*----------------------------------------------------------------*/ @@ -126,6 +133,86 @@ PointCloud::Point PointCloud::GetCenter() const /*----------------------------------------------------------------*/ +// estimate the ground-plane as the plane agreeing with most points +// - planeThreshold: threshold used to estimate the ground plane (0 - auto) +Planef PointCloud::EstimateGroundPlane(const ImageArr& images, float planeThreshold, const String& fileExportPlane) const +{ + ASSERT(!IsEmpty()); + + // remove some random points to speed up plane fitting + const unsigned randMinPoints(150000); + PointArr workPoints; + const PointArr* pPoints; + if (GetSize() > randMinPoints) { + #ifndef _RELEASE + SEACAVE::Random rnd(SEACAVE::Random::default_seed); + #else + SEACAVE::Random rnd; + #endif + const REAL randPointsRatio(MAXF(REAL(1e-4),(REAL)randMinPoints/GetSize())); + const SEACAVE::Random::result_type randPointsTh(CEIL2INT(randPointsRatio*SEACAVE::Random::max())); + workPoints.reserve(CEIL2INT(randPointsRatio*GetSize())); + for (const Point& X: points) + if (rnd() <= randPointsTh) + workPoints.emplace_back(X); + pPoints = &workPoints; + } else { + pPoints = &points; + } + + // fit plane to the point-cloud + Planef plane; + const float minInliersRatio(0.05f); + double threshold(planeThreshold>0 ? (double)planeThreshold : DBL_MAX); + const unsigned numInliers(planeThreshold > 0 ? EstimatePlaneTh(*pPoints, plane, threshold) : EstimatePlane(*pPoints, plane, threshold)); + if (numInliers < MINF(ROUND2INT(minInliersRatio*pPoints->size()), 1000u)) { + plane.Invalidate(); + return plane; + } + if (planeThreshold <= 0) + DEBUG("Ground plane estimated threshold: %g", threshold); + + // refine plane to inliers + CLISTDEF0(Planef::POINT) inliers; + const float maxThreshold(static_cast(threshold * 2)); + for (const Point& X: *pPoints) + if (plane.DistanceAbs(X) < maxThreshold) + inliers.emplace_back(X); + OptimizePlane(plane, inliers.data(), inliers.size(), 100, static_cast(threshold)); + + // make sure the plane is well oriented, negate plane normal if it faces same direction as cameras on average + if (!images.empty()) { + FloatArr cosView(0, images.size()); + for (const Image& imageData: images) { + if (!imageData.IsValid()) + continue; + cosView.push_back(plane.m_vN.dot((const Point3f::EVecMap&)Cast(imageData.camera.Direction()))); + } + if (cosView.GetMedian() > 0) + plane.Negate(); + } + + // export points on the found plane if requested + if (!fileExportPlane.empty()) { + PointCloud pc; + const Point orig(Point(plane.m_vN)*-plane.m_fD); + pc.colors.emplace_back(Color::RED); + pc.points.emplace_back(orig); + for (const Point& X: *pPoints) { + const float dist(plane.DistanceAbs(X)); + if (dist < threshold) { + pc.points.emplace_back(X); + const uint8_t color((uint8_t)(255.f*(1.f-dist/threshold))); + pc.colors.emplace_back(color, color, color); + } + } + pc.Save(fileExportPlane); + } + return plane; +} +/*----------------------------------------------------------------*/ + + // define a PLY file format composed only of vertices namespace BasicPLY { typedef PointCloud::Point Point; @@ -321,3 +408,159 @@ bool PointCloud::SaveNViews(const String& fileName, uint32_t minViews, bool bLeg return true; } // SaveNViews /*----------------------------------------------------------------*/ + + +// print various statistics about the point cloud +void PointCloud::PrintStatistics(const Image* pImages, const OBB3f* pObb) const +{ + String strPoints; + if (pObb && pObb->IsValid()) { + // print points distribution + size_t nInsidePoints(0); + MeanStdMinMax accInside, accOutside; + FOREACH(idx, points) { + const bool bInsideROI(pObb->Intersects(points[idx])); + if (bInsideROI) + ++nInsidePoints; + if (!pointViews.empty()) { + if (bInsideROI) + accInside.Update(pointViews[idx].size()); + else + accOutside.Update(pointViews[idx].size()); + } + } + strPoints = String::FormatString( + "\n - points info:" + "\n\t%u points inside ROI (%.2f%%)", + nInsidePoints, 100.0*nInsidePoints/GetSize() + ); + if (!pointViews.empty()) { + strPoints += String::FormatString( + "\n\t inside ROI track length: %g min / %g mean (%g std) / %g max" + "\n\toutside ROI track length: %g min / %g mean (%g std) / %g max", + accInside.minVal, accInside.GetMean(), accInside.GetStdDev(), accInside.maxVal, + accOutside.minVal, accOutside.GetMean(), accOutside.GetStdDev(), accOutside.maxVal + ); + } + } + String strViews; + if (!pointViews.empty()) { + // print views distribution + size_t nViews(0); + size_t nPoints1m(0), nPoints2(0), nPoints3(0), nPoints4p(0); + size_t nPointsOpposedViews(0); + MeanStdMinMax acc; + FOREACH(idx, points) { + const PointCloud::ViewArr& views = pointViews[idx]; + nViews += views.size(); + switch (views.size()) { + case 0: + case 1: + ++nPoints1m; + break; + case 2: + ++nPoints2; + break; + case 3: + ++nPoints3; + break; + default: + ++nPoints4p; + } + acc.Update(views.size()); + } + strViews = String::FormatString( + "\n - visibility info (%u views - %.2f views/point)%s:" + "\n\t% 9u points with 1- views (%.2f%%)" + "\n\t% 9u points with 2 views (%.2f%%)" + "\n\t% 9u points with 3 views (%.2f%%)" + "\n\t% 9u points with 4+ views (%.2f%%)" + "\n\t%g min / %g mean (%g std) / %g max", + nViews, (REAL)nViews/GetSize(), + nPointsOpposedViews ? String::FormatString(" (%u (%.2f%%) points with opposed views)", nPointsOpposedViews, 100.f*nPointsOpposedViews/GetSize()).c_str() : "", + nPoints1m, 100.f*nPoints1m/GetSize(), nPoints2, 100.f*nPoints2/GetSize(), nPoints3, 100.f*nPoints3/GetSize(), nPoints4p, 100.f*nPoints4p/GetSize(), + acc.minVal, acc.GetMean(), acc.GetStdDev(), acc.maxVal + ); + } + String strNormals; + if (!normals.empty()) { + if (!pointViews.empty() && pImages != NULL) { + // print normal/views angle distribution + size_t nViews(0); + size_t nPointsm(0), nPoints3(0), nPoints10(0), nPoints25(0), nPoints40(0), nPoints60(0), nPoints90p(0); + const REAL thCosAngle3(COS(D2R(3.f))); + const REAL thCosAngle10(COS(D2R(10.f))); + const REAL thCosAngle25(COS(D2R(25.f))); + const REAL thCosAngle40(COS(D2R(40.f))); + const REAL thCosAngle60(COS(D2R(60.f))); + const REAL thCosAngle90(COS(D2R(90.f))); + FOREACH(idx, points) { + const PointCloud::Point& X = points[idx]; + const PointCloud::Normal& N = normals[idx]; + const PointCloud::ViewArr& views = pointViews[idx]; + nViews += views.size(); + for (IIndex idxImage: views) { + const Point3f X2Cam(Cast(pImages[idxImage].camera.C)-X); + const REAL cosAngle(ComputeAngle(X2Cam.ptr(), N.ptr())); + if (cosAngle <= thCosAngle90) + ++nPoints90p; + else if (cosAngle <= thCosAngle60) + ++nPoints60; + else if (cosAngle <= thCosAngle40) + ++nPoints40; + else if (cosAngle <= thCosAngle25) + ++nPoints25; + else if (cosAngle <= thCosAngle10) + ++nPoints10; + else if (cosAngle <= thCosAngle3) + ++nPoints3; + else + ++nPointsm; + } + } + strNormals = String::FormatString( + "\n - normals visibility info:" + "\n\t% 9u points with 3- degrees (%.2f%%)" + "\n\t% 9u points with 10 degrees (%.2f%%)" + "\n\t% 9u points with 25 degrees (%.2f%%)" + "\n\t% 9u points with 40 degrees (%.2f%%)" + "\n\t% 9u points with 60 degrees (%.2f%%)" + "\n\t% 9u points with 90+ degrees (%.2f%%)", + nPointsm, 100.f*nPointsm/nViews, nPoints3, 100.f*nPoints3/nViews, nPoints10, 100.f*nPoints10/nViews, + nPoints40, 100.f*nPoints40/nViews, nPoints60, 100.f*nPoints60/nViews, nPoints90p, 100.f*nPoints90p/nViews + ); + } else { + strNormals = "\n - normals info"; + } + } + String strWeights; + if (!pointWeights.empty()) { + // print weights statistics + MeanStdMinMax acc; + for (const PointCloud::WeightArr& weights: pointWeights) { + float avgWeight(0); + for (PointCloud::Weight w: weights) + avgWeight += w; + acc.Update(avgWeight/weights.size()); + } + strWeights = String::FormatString( + "\n - weights info:" + "\n\t%g min / %g mean (%g std) / %g max", + acc.minVal, acc.GetMean(), acc.GetStdDev(), acc.maxVal + ); + } + String strColors; + if (!colors.empty()) { + // print colors statistics + strColors = "\n - colors"; + } + VERBOSE("Point-cloud composed of %u points with:%s%s%s%s", + GetSize(), + strPoints.c_str(), + strViews.c_str(), + strNormals.c_str(), + strWeights.c_str(), + strColors.c_str() + ); +} // PrintStatistics +/*----------------------------------------------------------------*/ diff --git a/libs/MVS/PointCloud.h b/libs/MVS/PointCloud.h index c0ebf5ad7..901823757 100644 --- a/libs/MVS/PointCloud.h +++ b/libs/MVS/PointCloud.h @@ -35,6 +35,8 @@ // I N C L U D E S ///////////////////////////////////////////////// +#include "Image.h" + // D E F I N E S /////////////////////////////////////////////////// @@ -52,21 +54,21 @@ class MVS_API PointCloud typedef IDX Index; typedef TPoint3 Point; - typedef SEACAVE::cList PointArr; + typedef CLISTDEF0IDX(Point,Index) PointArr; typedef uint32_t View; typedef SEACAVE::cList ViewArr; - typedef SEACAVE::cList PointViewArr; + typedef CLISTDEFIDX(ViewArr,Index) PointViewArr; typedef float Weight; typedef SEACAVE::cList WeightArr; - typedef SEACAVE::cList PointWeightArr; + typedef CLISTDEFIDX(WeightArr,Index) PointWeightArr; typedef TPoint3 Normal; - typedef CLISTDEF0(Normal) NormalArr; + typedef CLISTDEF0IDX(Normal,Index) NormalArr; typedef Pixel8U Color; - typedef CLISTDEF0(Color) ColorArr; + typedef CLISTDEF0IDX(Color,Index) ColorArr; typedef AABB3f Box; @@ -90,16 +92,21 @@ class MVS_API PointCloud void RemovePoint(IDX); void RemovePointsOutside(const OBB3f&); + void RemoveMinViews(uint32_t thMinViews); Box GetAABB() const; Box GetAABB(const Box& bound) const; Box GetAABB(unsigned minViews) const; Point GetCenter() const; + Planef EstimateGroundPlane(const ImageArr& images, float planeThreshold=0, const String& fileExportPlane="") const; + bool Load(const String& fileName); bool Save(const String& fileName, bool bLegacyTypes=false) const; bool SaveNViews(const String& fileName, uint32_t minViews, bool bLegacyTypes=false) const; + void PrintStatistics(const Image* pImages = NULL, const OBB3f* pObb = NULL) const; + #ifdef _USE_BOOST // implement BOOST serialization template diff --git a/libs/MVS/RectsBinPack.cpp b/libs/MVS/RectsBinPack.cpp index e9472cb61..3a376fd8f 100644 --- a/libs/MVS/RectsBinPack.cpp +++ b/libs/MVS/RectsBinPack.cpp @@ -522,7 +522,7 @@ int MaxRectsBinPack::ComputeTextureSize(const RectArr& rects, int mult) return ((sizeTex+mult-1)/mult)*mult; } // ... as power of two - return POWI((int)2, CEIL2INT(LOGN((float)sizeTex) / LOGN(2.f))); + return POWI(2, CEIL2INT(LOGN((float)sizeTex) / LOGN(2.f))); } /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 6f642ea83..3b8023207 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -33,6 +33,7 @@ #include "Scene.h" #define _USE_OPENCV #include "Interface.h" +#include "../Math/SimilarityTransform.h" using namespace MVS; @@ -321,7 +322,8 @@ bool Scene::LoadDMAP(const String& fileName) DepthMap depthMap; NormalMap normalMap; ConfidenceMap confMap; - if (!ImportDepthDataRaw(fileName, imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap)) + ViewsMap viewsMap; + if (!ImportDepthDataRaw(fileName, imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, dMin, dMax, depthMap, normalMap, confMap, viewsMap)) return false; // create image @@ -417,8 +419,7 @@ bool Scene::LoadViewNeighbors(const String& fileName) // parse image list SML smlImages(_T("ImageNeighbors")); smlImages.Load(fileName); - const LPSMLARR& arrSmlChild = smlImages.GetArrChildren(); - ASSERT(arrSmlChild.size() <= 1); + ASSERT(smlImages.GetArrChildren().size() <= 1); // fetch image IDs list size_t argc; @@ -451,7 +452,7 @@ bool Scene::SaveViewNeighbors(const String& fileName) const ASSERT(ImagesHaveNeighbors()); TD_TIMER_STARTD(); - + File file(fileName, File::WRITE, File::CREATE | File::TRUNCATE); if (!file.isOpen()) { VERBOSE("error: unable to write file '%s'", fileName.c_str()); @@ -479,8 +480,8 @@ bool Scene::Import(const String& fileName) Release(); return LoadDMAP(fileName); } - if (ext == _T(".obj")) { - // import mesh from obj file + if (ext == _T(".obj") || ext == _T(".gltf") || ext == _T(".glb")) { + // import mesh from obj/gltf file Release(); return mesh.Load(fileName); } @@ -644,7 +645,7 @@ void Scene::SampleMeshWithVisibility(unsigned maxResolution) const Point3f xz(camera.TransformPointW2I3(Cast(mesh.vertices[idxVertex]))); if (xz.z <= 0) continue; - const Point2f& x(reinterpret_cast(xz)); + const Point2f x(xz.x, xz.y); if (depthMap.isInsideWithBorder(x) && xz.z * thFrontDepth < depthMap(ROUND2INT(x))) { #ifdef SCENE_USE_OPENMP #pragma omp critical @@ -654,10 +655,12 @@ void Scene::SampleMeshWithVisibility(unsigned maxResolution) } } RFOREACH(idx, pointcloud.points) { - if (pointcloud.pointViews[idx].size() < 2) + if (pointcloud.pointViews[idx].size() < 2) { pointcloud.RemovePoint(idx); - else - pointcloud.points[idx] = mesh.vertices[(Mesh::VIndex)idx]; + continue; + } + pointcloud.points[idx] = mesh.vertices[(Mesh::VIndex)idx]; + pointcloud.pointViews[idx].Sort(); } } // SampleMeshWithVisibility /*----------------------------------------------------------------*/ @@ -699,7 +702,7 @@ bool Scene::ExportMeshToDepthMaps(const String& baseName) IDs.push_back(idxImage); for (const ViewScore& neighbor: image.neighbors) IDs.push_back(neighbor.idx.ID); - return ExportDepthDataRaw(fileName, image.name, IDs, image.GetSize(), image.camera.K, image.camera.R, image.camera.C, 0.001f, FLT_MAX, depthMap, normalMap, ConfidenceMap()); + return ExportDepthDataRaw(fileName, image.name, IDs, image.GetSize(), image.camera.K, image.camera.R, image.camera.C, 0.001f, FLT_MAX, depthMap, normalMap, ConfidenceMap(), ViewsMap()); } ()) || (nType == 1 && !depthMap.Save(fileName)) || (nType == 0 && !ExportDepthMap(fileName, depthMap))) @@ -722,6 +725,62 @@ bool Scene::ExportMeshToDepthMaps(const String& baseName) /*----------------------------------------------------------------*/ +// create a virtual point-cloud to be used to initialize the neighbor view +// from image pair points at the intersection of the viewing directions +bool Scene::EstimateNeighborViewsPointCloud(unsigned maxResolution) +{ + constexpr Depth minPercentDepthPerturb(0.3f); + constexpr Depth maxPercentDepthPerturb(1.3f); + const auto ProjectGridToImage = [&](IIndex idI, IIndex idJ, Depth depth) { + const Depth minDepthPerturb(depth * minPercentDepthPerturb); + const Depth maxDepthPerturb(depth * maxPercentDepthPerturb); + const Image& imageData = images[idI]; + const Image& imageData2 = images[idJ]; + const float stepW((float)imageData.width / maxResolution); + const float stepH((float)imageData.height / maxResolution); + for (unsigned r = 0; r < maxResolution; ++r) { + for (unsigned c = 0; c < maxResolution; ++c) { + const Point2f x(c*stepW + stepW/2, r*stepH + stepH/2); + const Depth depthPerturb(randomRange(minDepthPerturb, maxDepthPerturb)); + const Point3 X(imageData.camera.TransformPointI2W(Point3(x.x, x.y, depthPerturb))); + const Point3 X2(imageData2.camera.TransformPointW2C(X)); + if (X2.z < 0) + continue; + const Point2f x2(imageData2.camera.TransformPointC2I(X2)); + if (!Image8U::isInside(x2, imageData2.GetSize())) + continue; + pointcloud.points.emplace_back(X); + pointcloud.pointViews.emplace_back(idI < idJ ? PointCloud::ViewArr{idI, idJ} : PointCloud::ViewArr{idJ, idI}); + } + } + }; + pointcloud.Release(); + FOREACH(i, images) { + const Image& imageData = images[i]; + if (!imageData.IsValid()) + continue; + FOREACH(j, images) { + if (i == j) + continue; + const Image& imageData2 = images[j]; + Point3 X; + TriangulatePoint3D( + imageData.camera.K, imageData2.camera.K, + imageData.camera.R, imageData2.camera.R, + imageData.camera.C, imageData2.camera.C, + Point2::ZERO, Point2::ZERO, X); + const Depth depth((Depth)imageData.camera.PointDepth(X)); + const Depth depth2((Depth)imageData2.camera.PointDepth(X)); + if (depth <= 0 || depth2 <= 0) + continue; + ProjectGridToImage(i, j, depth); + ProjectGridToImage(j, i, depth2); + } + } + return true; +} // EstimateNeighborViewsPointCloud +/*----------------------------------------------------------------*/ + inline float Footprint(const Camera& camera, const Point3f& X) { #if 0 const REAL fSphereRadius(1); @@ -739,20 +798,20 @@ inline float Footprint(const Camera& camera, const Point3f& X) { // - nInsideROI: 0 - ignore ROI, 1 - weight more ROI points, 2 - consider only ROI points bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinViews, unsigned nMinPointViews, float fOptimAngle, unsigned nInsideROI) { - ASSERT(points.IsEmpty()); + ASSERT(points.empty()); // extract the estimated 3D points and the corresponding 2D projections for the reference image Image& imageData = images[ID]; ASSERT(imageData.IsValid()); ViewScoreArr& neighbors = imageData.neighbors; - ASSERT(neighbors.IsEmpty()); + ASSERT(neighbors.empty()); struct Score { float score; float avgScale; float avgAngle; uint32_t points; }; - CLISTDEF0(Score) scores(images.GetSize()); + CLISTDEF0(Score) scores(images.size()); scores.Memset(0); if (nMinPointViews > nCalibratedImages) nMinPointViews = nCalibratedImages; @@ -773,10 +832,14 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView continue; wROI = 0.7f; } + const Depth depth((float)imageData.camera.PointDepth(point)); + ASSERT(depth > 0); + if (depth <= 0) + continue; // store this point - if (views.GetSize() >= nMinPointViews) - points.Insert((uint32_t)idx); - imageData.avgDepth += (float)imageData.camera.PointDepth(point); + if (views.size() >= nMinPointViews) + points.push_back((uint32_t)idx); + imageData.avgDepth += depth; ++nPoints; // score shared views const Point3f V1(imageData.camera.C - Cast(point)); @@ -786,7 +849,7 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView continue; const Image& imageData2 = images[view]; const Point3f V2(imageData2.camera.C - Cast(point)); - const float fAngle(ACOS(ComputeAngle(V1.ptr(), V2.ptr()))); + const float fAngle(ACOS(ComputeAngle(V1.ptr(), V2.ptr()))); const float wAngle(EXP(SQUARE(fAngle-fOptimAngle)*(fAngle 3); // select best neighborViews - Point2fArr projs(0, points.GetSize()); + Point2fArr projs(0, points.size()); FOREACH(IDB, images) { const Image& imageDataB = images[IDB]; if (!imageDataB.IsValid()) @@ -820,7 +883,7 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView // compute how well the matched features are spread out (image covered area) const Point2f boundsA(imageData.GetSize()); const Point2f boundsB(imageDataB.GetSize()); - ASSERT(projs.IsEmpty()); + ASSERT(projs.empty()); for (uint32_t idx: points) { const PointCloud::ViewArr& views = pointcloud.pointViews[idx]; ASSERT(views.IsSorted()); @@ -828,15 +891,15 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView if (views.FindFirst(IDB) == PointCloud::ViewArr::NO_INDEX) continue; const PointCloud::Point& point = pointcloud.points[idx]; - Point2f& ptA = projs.AddConstruct(imageData.camera.ProjectPointP(point)); + Point2f& ptA = projs.emplace_back(imageData.camera.ProjectPointP(point)); Point2f ptB = imageDataB.camera.ProjectPointP(point); if (!imageData.camera.IsInside(ptA, boundsA) || !imageDataB.camera.IsInside(ptB, boundsB)) projs.RemoveLast(); } - ASSERT(projs.GetSize() <= score.points); - if (projs.IsEmpty()) + ASSERT(projs.size() <= score.points); + if (projs.empty()) continue; - const float area(ComputeCoveredArea((const float*)projs.Begin(), projs.GetSize(), boundsA.ptr())); + const float area(ComputeCoveredArea((const float*)projs.data(), projs.size(), boundsA.ptr())); projs.Empty(); // store image score ViewScore& neighbor = neighbors.AddEmpty(); @@ -854,10 +917,10 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView String msg; FOREACH(n, neighbors) msg += String::FormatString(" %3u(%upts,%.2fscl)", neighbors[n].idx.ID, neighbors[n].idx.points, neighbors[n].idx.scale); - VERBOSE("Reference image %3u sees %u views:%s (%u shared points)", ID, neighbors.GetSize(), msg.c_str(), nPoints); + VERBOSE("Reference image %3u sees %u views:%s (%u shared points)", ID, neighbors.size(), msg.c_str(), nPoints); } #endif - if (points.GetSize() <= 3 || neighbors.GetSize() < MINF(nMinViews,nCalibratedImages-1)) { + if (points.size() <= 3 || neighbors.size() < MINF(nMinViews,nCalibratedImages-1)) { DEBUG_EXTRA("error: reference image %3u has not enough images in view", ID); return false; } @@ -869,16 +932,18 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView bool Scene::FilterNeighborViews(ViewScoreArr& neighbors, float fMinArea, float fMinScale, float fMaxScale, float fMinAngle, float fMaxAngle, unsigned nMaxViews) { // remove invalid neighbor views + const unsigned nMinViews(MAXF(4u, nMaxViews*3/4)); RFOREACH(n, neighbors) { const ViewScore& neighbor = neighbors[n]; - if (neighbor.idx.area < fMinArea || - !ISINSIDE(neighbor.idx.scale, fMinScale, fMaxScale) || - !ISINSIDE(neighbor.idx.angle, fMinAngle, fMaxAngle)) + if (neighbors.size() > nMinViews && + (neighbor.idx.area < fMinArea || + !ISINSIDE(neighbor.idx.scale, fMinScale, fMaxScale) || + !ISINSIDE(neighbor.idx.angle, fMinAngle, fMaxAngle))) neighbors.RemoveAtMove(n); } - if (neighbors.GetSize() > nMaxViews) - neighbors.Resize(nMaxViews); - return !neighbors.IsEmpty(); + if (neighbors.size() > nMaxViews) + neighbors.resize(nMaxViews); + return !neighbors.empty(); } // FilterNeighborViews /*----------------------------------------------------------------*/ @@ -1166,6 +1231,18 @@ unsigned Scene::Split(ImagesChunkArr& chunks, float maxArea, int depthMapStep) c chunks.RemoveAt(cSmall); } #endif + if (IsBounded()) { + // make sure the chunks bounding box do not exceed the scene bounding box + const AABB3f aabb(obb.GetAABB()); + RFOREACH(c, chunks) { + ImagesChunk& chunk = chunks[c]; + chunk.aabb.BoundBy(aabb); + if (chunk.aabb.IsEmpty()) { + DEBUG_ULTIMATE("warning: chunk bounding box is empty"); + chunks.RemoveAt(c); + } + } + } DEBUG_EXTRA("Scene split (%g max-area): %u chunks (%s)", maxArea, chunks.size(), TD_TIMER_GET_FMT().c_str()); #if 0 || defined(_DEBUG) // dump chunks for visualization @@ -1316,7 +1393,7 @@ bool Scene::Scale(const REAL* pScale) scale = REAL(1)/mesh.GetAABB().GetSize().maxCoeff(); else return false; - const float scalef(scale); + const float scalef(static_cast(scale)); if (IsBounded()) obb.Transform(OBB3f::MATRIX::Identity() * scalef); for (Platform& platform: platforms) @@ -1361,4 +1438,190 @@ bool Scene::ScaleImages(unsigned nMaxResolution, REAL scale, const String& folde } return true; } // ScaleImages + +// apply similarity transform +void Scene::Transform(const Matrix3x3& rotation, const Point3& translation, REAL scale) { + const Matrix3x3 rotationScale(rotation * scale); + for (Platform& platform : platforms) { + for (Platform::Pose& pose : platform.poses) { + pose.R = pose.R * rotation.t(); + pose.C = rotationScale * pose.C + translation; + } + } + for (Image& image : images) { + image.UpdateCamera(platforms); + } + FOREACH(i, pointcloud.points) { + pointcloud.points[i] = rotationScale * Cast(pointcloud.points[i]) + translation; + if (!pointcloud.normals.empty()) + pointcloud.normals[i] = rotation * Cast(pointcloud.normals[i]); + } + FOREACH(i, mesh.vertices) { + mesh.vertices[i] = rotationScale * Cast(mesh.vertices[i]) + translation; + if (!mesh.vertexNormals.empty()) + mesh.vertexNormals[i] = rotation * Cast(mesh.vertexNormals[i]); + } + FOREACH(i, mesh.faceNormals) { + mesh.faceNormals[i] = rotation * Cast(mesh.faceNormals[i]); + } + if (obb.IsValid()) { + obb.Transform(Cast(rotationScale)); + obb.Translate(Cast(translation)); + } +} + +// transform this scene such that it best aligns with the given scene based on the camera positions +bool Scene::AlignTo(const Scene& scene) +{ + if (images.size() < 3) { + DEBUG("error: insufficient number of cameras to perform a similarity transform alignment"); + return false; + } + if (images.size() != scene.images.size()) { + DEBUG("error: the two scenes differ in number of cameras"); + return false; + } + CLISTDEF0(Point3) points, pointsRef; + FOREACH(idx, images) { + const Image& image = images[idx]; + if (!image.IsValid()) + continue; + const Image& imageRef = scene.images[idx]; + if (!imageRef.IsValid()) + continue; + points.emplace_back(image.camera.C); + pointsRef.emplace_back(imageRef.camera.C); + } + Matrix4x4 transform; + SimilarityTransform(points, pointsRef, transform); + Matrix3x3 rotation; Point3 translation; REAL scale; + DecomposeSimilarityTransform(transform, rotation, translation, scale); + Transform(rotation, translation, scale); + return true; +} // AlignTo + +// estimate ground plane, transform scene such that it is positioned at origin, and compute the volume of the mesh; +// - planeThreshold: threshold used to estimate the ground plane (0 - auto) +// - sampleMesh: uniformly samples points on the mesh (0 - disabled, <0 - number of points, >0 - sample density per square unit) +// returns <0 if an error occurred +REAL Scene::ComputeLeveledVolume(float planeThreshold, float sampleMesh, unsigned upAxis, bool verbose) +{ + ASSERT(!mesh.IsEmpty()); + if (planeThreshold >= 0 && !mesh.IsWatertight()) { + // assume the mesh is opened only at the contact with the ground plane; + // move mesh such that the ground plane is at the origin so that the volume can be computed + TD_TIMER_START(); + Planef groundPlane(mesh.EstimateGroundPlane(images, sampleMesh, planeThreshold, verbose?MAKE_PATH("ground_plane.ply"):String())); + if (!groundPlane.IsValid()) { + VERBOSE("error: can not estimate the ground plane"); + return -1; + } + const Point3f up(upAxis==0?1.f:0.f, upAxis==1?1.f:0.f, upAxis==2?1.f:0.f); + if (groundPlane.m_vN.dot(Point3f::EVec(up)) < 0.f) + groundPlane.Negate(); + VERBOSE("Ground plane estimated at: (%.2f,%.2f,%.2f) %.2f (%s)", + groundPlane.m_vN.x(), groundPlane.m_vN.y(), groundPlane.m_vN.z(), groundPlane.m_fD, TD_TIMER_GET_FMT().c_str()); + // transform the scene such that the up vector aligns with ground plane normal, + // and the mesh center projected on the ground plane is at the origin + const Matrix3x3 rotation(RMatrix(Cast(up), Cast(Point3f(groundPlane.m_vN))).t()); + const Point3 translation(rotation*-Cast(Point3f(groundPlane.ProjectPoint(mesh.GetCenter())))); + const REAL scale(1); + Transform(rotation, translation, scale); + } + return mesh.ComputeVolume(); +} +/*----------------------------------------------------------------*/ + + +// estimate region-of-interest based on camera positions, directions and sparse points +// scale specifies the ratio of the ROI's diameter +bool Scene::EstimateROI(int nEstimateROI, float scale) +{ + ASSERT(nEstimateROI >= 0 && nEstimateROI <= 2 && scale > 0); + if (nEstimateROI == 0) { + DEBUG_ULTIMATE("The scene will be considered as unbounded (no ROI)"); + return false; + } + if (!pointcloud.IsValid()) { + VERBOSE("error: no valid point-cloud for the ROI estimation"); + return false; + } + CameraArr cameras; + FOREACH(i, images) { + const Image& imageData = images[i]; + if (!imageData.IsValid()) + continue; + cameras.emplace_back(imageData.camera); + } + const unsigned nCameras = cameras.size(); + if (nCameras < 3) { + VERBOSE("warning: not enough valid views for the ROI estimation"); + return false; + } + // compute the camera center and the direction median + FloatArr x(nCameras), y(nCameras), z(nCameras), nx(nCameras), ny(nCameras), nz(nCameras); + FOREACH(i, cameras) { + const Point3f camC(cameras[i].C); + x[i] = camC.x; + y[i] = camC.y; + z[i] = camC.z; + const Point3f camDirect(cameras[i].Direction()); + nx[i] = camDirect.x; + ny[i] = camDirect.y; + nz[i] = camDirect.z; + } + const CMatrix camCenter(x.GetMedian(), y.GetMedian(), z.GetMedian()); + CMatrix camDirectMean(nx.GetMean(), ny.GetMean(), nz.GetMean()); + const float camDirectMeanLen = (float)norm(camDirectMean); + if (!ISZERO(camDirectMeanLen)) + camDirectMean /= camDirectMeanLen; + if (camDirectMeanLen > FSQRT_2 / 2.f && nEstimateROI == 2) { + VERBOSE("The camera directions mean is unbalanced; the scene will be considered unbounded (no ROI)"); + return false; + } + DEBUG_ULTIMATE("The camera positions median is (%f,%f,%f), directions mean and norm are (%f,%f,%f), %f", + camCenter.x, camCenter.y, camCenter.z, camDirectMean.x, camDirectMean.y, camDirectMean.z, camDirectMeanLen); + FloatArr cameraDistances(nCameras); + FOREACH(i, cameras) + cameraDistances[i] = (float)cameras[i].Distance(camCenter); + // estimate scene center and radius + const float camDistMed = cameraDistances.GetMedian(); + const float camShiftCoeff = TAN(ASIN(CLAMP(camDirectMeanLen, 0.f, 0.999f))); + const CMatrix sceneCenter = camCenter + camShiftCoeff * camDistMed * camDirectMean; + FOREACH(i, cameras) { + if (cameras[i].PointDepth(sceneCenter) <= 0 && nEstimateROI == 2) { + VERBOSE("Found a camera not pointing towards the scene center; the scene will be considered unbounded (no ROI)"); + return false; + } + cameraDistances[i] = (float)cameras[i].Distance(sceneCenter); + } + const float sceneRadius = cameraDistances.GetMax(); + DEBUG_ULTIMATE("The estimated scene center is (%f,%f,%f), radius is %f", + sceneCenter.x, sceneCenter.y, sceneCenter.z, sceneRadius); + Point3fArr ptsInROI; + FOREACH(i, pointcloud.points) { + const PointCloud::Point& point = pointcloud.points[i]; + const PointCloud::ViewArr& views = pointcloud.pointViews[i]; + FOREACH(j, views) { + const Image& imageData = images[views[j]]; + if (!imageData.IsValid()) + continue; + const Camera& camera = imageData.camera; + if (camera.PointDepth(point) < sceneRadius * 2.0f) { + ptsInROI.emplace_back(point); + break; + } + } + } + obb.Set(AABB3f(ptsInROI.begin(), ptsInROI.size()).EnlargePercent(scale)); + #if TD_VERBOSE != TD_VERBOSE_OFF + if (VERBOSITY_LEVEL > 2) { + VERBOSE("Set the ROI with the AABB of position (%f,%f,%f) and extent (%f,%f,%f)", + obb.m_pos[0], obb.m_pos[1], obb.m_pos[2], obb.m_ext[0], obb.m_ext[1], obb.m_ext[2]); + } else { + VERBOSE("Set the ROI by the estimated core points"); + } + #endif + return true; +} // EstimateROI /*----------------------------------------------------------------*/ diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 4eb8b575e..c3807e966 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -1,7 +1,7 @@ /* * Scene.h * -* Copyright (c) 2014-2015 SEACAVE +* Copyright (c) 2014-2022 SEACAVE * * Author(s): * @@ -82,6 +82,7 @@ class MVS_API Scene bool Load(const String& fileName, bool bImport=false); bool Save(const String& fileName, ARCHIVE_TYPE type=ARCHIVE_DEFAULT) const; + bool EstimateNeighborViewsPointCloud(unsigned maxResolution=16); void SampleMeshWithVisibility(unsigned maxResolution=320); bool ExportMeshToDepthMaps(const String& baseName); @@ -103,9 +104,15 @@ class MVS_API Scene bool Center(const Point3* pCenter = NULL); bool Scale(const REAL* pScale = NULL); bool ScaleImages(unsigned nMaxResolution = 0, REAL scale = 0, const String& folderName = String()); + void Transform(const Matrix3x3& rotation, const Point3& translation, REAL scale); + bool AlignTo(const Scene&); + REAL ComputeLeveledVolume(float planeThreshold=0, float sampleMesh=-100000, unsigned upAxis=2, bool verbose=true); + + // Estimate and set region-of-interest + bool EstimateROI(int nEstimateROI=0, float scale=1.f); // Dense reconstruction - bool DenseReconstruction(int nFusionMode=0); + bool DenseReconstruction(int nFusionMode=0, bool bCrop2ROI=true, float fBorderROI=0); bool ComputeDepthMaps(DenseDepthMapData& data); void DenseReconstructionEstimate(void*); void DenseReconstructionFilter(void*); @@ -118,13 +125,18 @@ class MVS_API Scene float kInf=(float)(INT_MAX/8)); // Mesh refinement - bool RefineMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned nMaxViews, float fDecimateMesh, unsigned nCloseHoles, unsigned nEnsureEdgeSize, unsigned nMaxFaceArea, unsigned nScales, float fScaleStep, unsigned nReduceMemory, unsigned nAlternatePair, float fRegularityWeight, float fRatioRigidityElasticity, float fThPlanarVertex, float fGradientStep); + bool RefineMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned nMaxViews, float fDecimateMesh, unsigned nCloseHoles, unsigned nEnsureEdgeSize, + unsigned nMaxFaceArea, unsigned nScales, float fScaleStep, unsigned nReduceMemory, unsigned nAlternatePair, float fRegularityWeight, float fRatioRigidityElasticity, + float fThPlanarVertex, float fGradientStep); #ifdef _USE_CUDA - bool RefineMeshCUDA(unsigned nResolutionLevel, unsigned nMinResolution, unsigned nMaxViews, float fDecimateMesh, unsigned nCloseHoles, unsigned nEnsureEdgeSize, unsigned nMaxFaceArea, unsigned nScales, float fScaleStep, unsigned nAlternatePair, float fRegularityWeight, float fRatioRigidityElasticity, float fGradientStep); + bool RefineMeshCUDA(unsigned nResolutionLevel, unsigned nMinResolution, unsigned nMaxViews, float fDecimateMesh, unsigned nCloseHoles, unsigned nEnsureEdgeSize, + unsigned nMaxFaceArea, unsigned nScales, float fScaleStep, unsigned nAlternatePair, float fRegularityWeight, float fRatioRigidityElasticity, float fGradientStep); #endif // Mesh texturing - bool TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, float fOutlierThreshold=0.f, float fRatioDataSmoothness=0.3f, bool bGlobalSeamLeveling=true, bool bLocalSeamLeveling=true, unsigned nTextureSizeMultiple=0, unsigned nRectPackingHeuristic=3, Pixel8U colEmpty=Pixel8U(255,127,39), const IIndexArr& views=IIndexArr()); + bool TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras=0, float fOutlierThreshold=0.f, float fRatioDataSmoothness=0.3f, + bool bGlobalSeamLeveling=true, bool bLocalSeamLeveling=true, unsigned nTextureSizeMultiple=0, unsigned nRectPackingHeuristic=3, Pixel8U colEmpty=Pixel8U(255,127,39), + float fSharpnessWeight=0.5f, const IIndexArr& views=IIndexArr()); #ifdef _USE_BOOST // implement BOOST serialization diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index f749932d1..ee7929612 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -182,7 +182,7 @@ bool DepthMapsData::SelectViews(IIndexArr& images, IIndexArr& imagesMap, IIndexA typedef MRFEnergy MRFEnergyType; CAutoPtr energy(new MRFEnergyType(TypeGeneral::GlobalSize())); CAutoPtrArr nodes(new MRFEnergyType::NodeId[_num_nodes]); - typedef SEACAVE::cList EnergyCostArr; + typedef SEACAVE::cList EnergyCostArr; // unary costs: inverse proportional to the image pair score EnergyCostArr arrUnary(_num_labels); for (IIndex n=0; n<_num_nodes; ++n) { @@ -385,9 +385,10 @@ bool DepthMapsData::InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex n Depth dMin, dMax; NormalMap normalMap; ConfidenceMap confMap; + ViewsMap viewsMap; ImportDepthDataRaw(ComposeDepthFilePath(view.GetID(), "dmap"), imageFileName, IDs, imageSize, view.cameraDepthMap.K, view.cameraDepthMap.R, view.cameraDepthMap.C, - dMin, dMax, view.depthMap, normalMap, confMap, 1); + dMin, dMax, view.depthMap, normalMap, confMap, viewsMap, 1); } view.Init(viewRef.camera); } @@ -399,9 +400,10 @@ bool DepthMapsData::InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex n cv::Size imageSize; Camera camera; ConfidenceMap confMap; + ViewsMap viewsMap; if (!ImportDepthDataRaw(ComposeDepthFilePath(viewRef.GetID(), "dmap"), imageFileName, IDs, imageSize, camera.K, camera.R, camera.C, depthData.dMin, depthData.dMax, - depthData.depthMap, depthData.normalMap, confMap, 3)) + depthData.depthMap, depthData.normalMap, confMap, viewsMap, 3)) return false; ASSERT(viewRef.image.size() == depthData.depthMap.size()); } else if (loadDepthMaps == 0) { @@ -459,7 +461,7 @@ bool DepthMapsData::InitDepthMap(DepthData& depthData) ASSERT(depthData.images.GetSize() > 1 && !depthData.points.IsEmpty()); const DepthData::ViewData& image(depthData.GetView()); - TriangulatePoints2DepthMap(image, scene.pointcloud, depthData.points, depthData.depthMap, depthData.normalMap, depthData.dMin, depthData.dMax, OPTDENSE::bAddCorners); + TriangulatePoints2DepthMap(image, scene.pointcloud, depthData.points, depthData.depthMap, depthData.normalMap, depthData.dMin, depthData.dMax, OPTDENSE::bAddCorners, OPTDENSE::bInitSparse); depthData.dMin *= 0.9f; depthData.dMax *= 1.1f; @@ -502,6 +504,7 @@ void* STCALL DepthMapsData::ScoreDepthMapTmp(void* arg) // replace invalid normal with random values normal = estimator.RandomNormal(viewDir); } + ASSERT(ISEQUAL(norm(normal), 1.f)); estimator.confMap0(x) = estimator.ScorePixel(depth, normal); } return NULL; @@ -520,7 +523,7 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) { DepthEstimator& estimator = *((DepthEstimator*)arg); IDX idx; - const float fOptimAngle(FD2R(OPTDENSE::fOptimAngle)); + MAYBEUNUSED const float fOptimAngle(FD2R(OPTDENSE::fOptimAngle)); while ((idx=(IDX)Thread::safeInc(estimator.idxPixel)) < estimator.coords.GetSize()) { const ImageRef& x = estimator.coords[idx]; ASSERT(estimator.depthMap0(x) >= 0); @@ -566,6 +569,31 @@ void* STCALL DepthMapsData::EndDepthMapTmp(void* arg) return NULL; } +DepthData DepthMapsData::ScaleDepthData(const DepthData& inputDeptData, float scale) { + ASSERT(scale <= 1); + if (scale == 1) + return inputDeptData; + DepthData rescaledDepthData(inputDeptData); + FOREACH (idxView, rescaledDepthData.images) { + DepthData::ViewData& viewData = rescaledDepthData.images[idxView]; + ASSERT(viewData.depthMap.empty() || viewData.image.size() == viewData.depthMap.size()); + cv::resize(viewData.image, viewData.image, cv::Size(), scale, scale, cv::INTER_AREA); + viewData.camera = viewData.pImageData->camera; + viewData.camera.K = viewData.camera.GetScaledK(viewData.pImageData->GetSize(), viewData.image.size()); + if (!viewData.depthMap.empty()) { + cv::resize(viewData.depthMap, viewData.depthMap, viewData.image.size(), 0, 0, cv::INTER_AREA); + viewData.cameraDepthMap = viewData.pImageData->camera; + viewData.cameraDepthMap.K = viewData.cameraDepthMap.GetScaledK(viewData.pImageData->GetSize(), viewData.image.size()); + } + viewData.Init(rescaledDepthData.images[0].camera); + } + if (!rescaledDepthData.depthMap.empty()) + cv::resize(rescaledDepthData.depthMap, rescaledDepthData.depthMap, cv::Size(), scale, scale, cv::INTER_NEAREST); + if (!rescaledDepthData.normalMap.empty()) + cv::resize(rescaledDepthData.normalMap, rescaledDepthData.normalMap, cv::Size(), scale, scale, cv::INTER_NEAREST); + return rescaledDepthData; +} + // estimate depth-map using propagation and random refinement with NCC score // as in: "Accurate Multiple View 3D Reconstruction Using Patch-Based Stereo for Large-Scale Scenes", S. Shen, 2013 // The implementations follows closely the paper, although there are some changes/additions. @@ -590,147 +618,183 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage, int nGeometricIter) TD_TIMER_STARTD(); - // initialize - DepthData& depthData(arrDepthData[idxImage]); - ASSERT(depthData.images.size() > 1); - const DepthData::ViewData& image(depthData.images.First()); - ASSERT(!image.image.empty() && !depthData.images[1].image.empty()); - const Image8U::Size size(image.image.size()); - depthData.confMap.create(size); const unsigned nMaxThreads(scene.nMaxThreads); const unsigned iterBegin(nGeometricIter < 0 ? 0u : OPTDENSE::nEstimationIters+(unsigned)nGeometricIter); const unsigned iterEnd(nGeometricIter < 0 ? OPTDENSE::nEstimationIters : iterBegin+1); - // init integral images and index to image-ref map for the reference data - #if DENSE_NCC == DENSE_NCC_WEIGHTED - DepthEstimator::WeightMap weightMap0(size.area()-(size.width+1)*DepthEstimator::nSizeHalfWindow); - #else - Image64F imageSum0; - cv::integral(image.image, imageSum0, CV_64F); - #endif - if (prevDepthMapSize != size) { - BitMatrix mask; - if (OPTDENSE::nIgnoreMaskLabel >= 0 && DepthEstimator::ImportIgnoreMask(*depthData.GetView().pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel)) - depthData.ApplyIgnoreMask(mask); - DepthEstimator::MapMatrix2ZigzagIdx(size, coords, mask, MAXF(64,(int)nMaxThreads*8)); - #if 0 - // show pixels to be processed - Image8U cmask(size); - cmask.memset(0); - for (const DepthEstimator::MapRef& x: coords) - cmask(x.y, x.x) = 255; - cmask.Show("cmask"); - #endif - if (mask.empty()) - prevDepthMapSize = size; - } - // init threads ASSERT(nMaxThreads > 0); cList estimators; - estimators.Reserve(nMaxThreads); + estimators.reserve(nMaxThreads); cList threads; if (nMaxThreads > 1) - threads.Resize(nMaxThreads-1); // current thread is also used + threads.resize(nMaxThreads-1); // current thread is also used volatile Thread::safe_t idxPixel; - // initialize the reference confidence map (NCC score map) with the score of the current estimates - { - // create working threads - idxPixel = -1; - ASSERT(estimators.IsEmpty()); - while (estimators.GetSize() < nMaxThreads) - estimators.AddConstruct(iterBegin, depthData, idxPixel, - #if DENSE_NCC == DENSE_NCC_WEIGHTED - weightMap0, - #else - imageSum0, - #endif - coords); - ASSERT(estimators.GetSize() == threads.GetSize()+1); - FOREACH(i, threads) - threads[i].start(ScoreDepthMapTmp, &estimators[i]); - ScoreDepthMapTmp(&estimators.Last()); - // wait for the working threads to close - FOREACHPTR(pThread, threads) - pThread->join(); - estimators.Release(); - #if TD_VERBOSE != TD_VERBOSE_OFF - // save rough depth map as image - if (g_nVerbosityLevel > 4 && nGeometricIter < 0) { - ExportDepthMap(ComposeDepthFilePath(image.GetID(), "rough.png"), depthData.depthMap); - ExportNormalMap(ComposeDepthFilePath(image.GetID(), "rough.normal.png"), depthData.normalMap); - ExportPointCloud(ComposeDepthFilePath(image.GetID(), "rough.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + // Multi-Resolution : + DepthData& fullResDepthData(arrDepthData[idxImage]); + const unsigned totalScaleNumber(nGeometricIter < 0 ? OPTDENSE::nSubResolutionLevels : 0u); + DepthMap lowResDepthMap; + NormalMap lowResNormalMap; + #if DENSE_NCC == DENSE_NCC_WEIGHTED + DepthEstimator::WeightMap weightMap0; + #else + Image64F imageSum0; + #endif + DepthMap currentSizeResDepthMap; + for (unsigned scaleNumber = totalScaleNumber+1; scaleNumber-- > 0; ) { + // initialize + float scale = 1.f / POWI(2, scaleNumber); + DepthData currentDepthData(ScaleDepthData(fullResDepthData, scale)); + DepthData& depthData(scaleNumber==0 ? fullResDepthData : currentDepthData); + ASSERT(depthData.images.size() > 1); + const DepthData::ViewData& image(depthData.images.front()); + ASSERT(!image.image.empty() && !depthData.images[1].image.empty()); + const Image8U::Size size(image.image.size()); + if (scaleNumber != totalScaleNumber) { + cv::resize(lowResDepthMap, depthData.depthMap, size, 0, 0, OPTDENSE::nIgnoreMaskLabel >= 0 ? cv::INTER_NEAREST : cv::INTER_LINEAR); + cv::resize(lowResNormalMap, depthData.normalMap, size, 0, 0, cv::INTER_NEAREST); + depthData.depthMap.copyTo(currentSizeResDepthMap); + } + else if (totalScaleNumber > 0) { + fullResDepthData.depthMap.release(); + fullResDepthData.normalMap.release(); + fullResDepthData.confMap.release(); } + depthData.confMap.create(size); + + // init integral images and index to image-ref map for the reference data + #if DENSE_NCC == DENSE_NCC_WEIGHTED + weightMap0.clear(); + weightMap0.resize(size.area()-(size.width+1)*DepthEstimator::nSizeHalfWindow); + #else + cv::integral(image.image, imageSum0, CV_64F); #endif - } + if (prevDepthMapSize != size || OPTDENSE::nIgnoreMaskLabel >= 0) { + BitMatrix mask; + if (OPTDENSE::nIgnoreMaskLabel >= 0 && DepthEstimator::ImportIgnoreMask(*image.pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel)) + depthData.ApplyIgnoreMask(mask); + DepthEstimator::MapMatrix2ZigzagIdx(size, coords, mask, MAXF(64,(int)nMaxThreads*8)); + #if 0 + // show pixels to be processed + Image8U cmask(size); + cmask.memset(0); + for (const DepthEstimator::MapRef& x: coords) + cmask(x.y, x.x) = 255; + cmask.Show("cmask"); + #endif + prevDepthMapSize = size; + } - // run propagation and random refinement cycles on the reference data - for (unsigned iter=iterBegin; iterjoin(); - estimators.Release(); - #if 1 && TD_VERBOSE != TD_VERBOSE_OFF - // save intermediate depth map as image - if (g_nVerbosityLevel > 4) { - String path(ComposeDepthFilePath(image.GetID(), "iter")+String::ToString(iter)); - if (nGeometricIter >= 0) - path += String::FormatString(".geo%d", nGeometricIter); - ExportDepthMap(path+".png", depthData.depthMap); - ExportNormalMap(path+".normal.png", depthData.normalMap); - ExportPointCloud(path+".ply", *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + // initialize the reference confidence map (NCC score map) with the score of the current estimates + { + // create working threads + idxPixel = -1; + ASSERT(estimators.empty()); + while (estimators.size() < nMaxThreads) { + estimators.emplace_back(iterBegin, depthData, idxPixel, + #if DENSE_NCC == DENSE_NCC_WEIGHTED + weightMap0, + #else + imageSum0, + #endif + coords); + estimators.Last().lowResDepthMap = currentSizeResDepthMap; + } + ASSERT(estimators.size() == threads.size()+1); + FOREACH(i, threads) + threads[i].start(ScoreDepthMapTmp, &estimators[i]); + ScoreDepthMapTmp(&estimators.back()); + // wait for the working threads to close + FOREACHPTR(pThread, threads) + pThread->join(); + estimators.clear(); + #if TD_VERBOSE != TD_VERBOSE_OFF + // save rough depth map as image + if (g_nVerbosityLevel > 4 && nGeometricIter < 0) { + ExportDepthMap(ComposeDepthFilePath(image.GetID(), "rough.png"), depthData.depthMap); + ExportNormalMap(ComposeDepthFilePath(image.GetID(), "rough.normal.png"), depthData.normalMap); + ExportPointCloud(ComposeDepthFilePath(image.GetID(), "rough.ply"), *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + } + #endif + } + + // run propagation and random refinement cycles on the reference data + for (unsigned iter=iterBegin; iterjoin(); + estimators.clear(); + #if 1 && TD_VERBOSE != TD_VERBOSE_OFF + // save intermediate depth map as image + if (g_nVerbosityLevel > 4) { + String path(ComposeDepthFilePath(image.GetID(), "iter")+String::ToString(iter)); + if (nGeometricIter >= 0) + path += String::FormatString(".geo%d", nGeometricIter); + ExportDepthMap(path+".png", depthData.depthMap); + ExportNormalMap(path+".normal.png", depthData.normalMap); + ExportPointCloud(path+".ply", *depthData.images.First().pImageData, depthData.depthMap, depthData.normalMap); + } + #endif + } + + // remember sub-resolution estimates for next iteration + if (scaleNumber > 0) { + lowResDepthMap = depthData.depthMap; + lowResNormalMap = depthData.normalMap; } - #endif } + DepthData& depthData(fullResDepthData); // remove all estimates with too big score and invert confidence map { const float fNCCThresholdKeep(OPTDENSE::fNCCThresholdKeep); if (nGeometricIter < 0 && OPTDENSE::nEstimationGeometricIters) - OPTDENSE::fNCCThresholdKeep *= 1.5f; + OPTDENSE::fNCCThresholdKeep *= 1.333f; // create working threads idxPixel = -1; - ASSERT(estimators.IsEmpty()); - while (estimators.GetSize() < nMaxThreads) - estimators.AddConstruct(0, depthData, idxPixel, + ASSERT(estimators.empty()); + while (estimators.size() < nMaxThreads) + estimators.emplace_back(0, depthData, idxPixel, #if DENSE_NCC == DENSE_NCC_WEIGHTED weightMap0, #else imageSum0, #endif coords); - ASSERT(estimators.GetSize() == threads.GetSize()+1); + ASSERT(estimators.size() == threads.size()+1); FOREACH(i, threads) threads[i].start(EndDepthMapTmp, &estimators[i]); - EndDepthMapTmp(&estimators.Last()); + EndDepthMapTmp(&estimators.back()); // wait for the working threads to close FOREACHPTR(pThread, threads) pThread->join(); - estimators.Release(); + estimators.clear(); OPTDENSE::fNCCThresholdKeep = fNCCThresholdKeep; } - DEBUG_EXTRA("Depth-map for image %3u %s: %dx%d (%s)", image.GetID(), - depthData.images.GetSize() > 2 ? - String::FormatString("estimated using %2u images", depthData.images.GetSize()-1).c_str() : + DEBUG_EXTRA("Depth-map for image %3u %s: %dx%d (%s)", depthData.images.front().GetID(), + depthData.images.size() > 2 ? + String::FormatString("estimated using %2u images", depthData.images.size()-1).c_str() : String::FormatString("with image %3u estimated", depthData.images[1].GetID()).c_str(), - size.width, size.height, TD_TIMER_GET_FMT().c_str()); + depthData.depthMap.cols, depthData.depthMap.rows, TD_TIMER_GET_FMT().c_str()); return true; } // EstimateDepthMap /*----------------------------------------------------------------*/ @@ -1401,7 +1465,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b PointCloud::ViewArr& views = pointcloud.pointViews.AddEmpty(); views.Insert(idxImage); PointCloud::WeightArr& weights = pointcloud.pointWeights.AddEmpty(); - REAL confidence(weights.emplace_back(Conf2Weight(depthData.confMap(x),depth))); + REAL confidence(weights.emplace_back(Conf2Weight(depthData.confMap.empty() ? 1.f : depthData.confMap(x),depth))); ProjArr& pointProjs = projs.AddEmpty(); pointProjs.Insert(Proj(x)); const PointCloud::Normal normal(bNormalMap ? Cast(imageData.camera.R.t()*Cast(depthData.normalMap(x))) : Normal(0,0,-1)); @@ -1437,7 +1501,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b if (normal.dot(normalB) > normalError) { // add view to the 3D point ASSERT(views.FindFirst(idxImageB) == PointCloud::ViewArr::NO_INDEX); - const float confidenceB(Conf2Weight(depthDataB.confMap(xB),depthB)); + const float confidenceB(Conf2Weight(depthDataB.confMap.empty() ? 1.f : depthDataB.confMap(xB),depthB)); const IIndex idx(views.InsertSort(idxImageB)); weights.InsertAt(idx, confidenceB); pointProjs.InsertAt(idx, Proj(xB)); @@ -1561,7 +1625,7 @@ void DenseDepthMapData::SignalCompleteDepthmapFilter() static void* DenseReconstructionEstimateTmp(void*); static void* DenseReconstructionFilterTmp(void*); -bool Scene::DenseReconstruction(int nFusionMode) +bool Scene::DenseReconstruction(int nFusionMode, bool bCrop2ROI, float fBorderROI) { DenseDepthMapData data(*this, nFusionMode); @@ -1603,10 +1667,11 @@ bool Scene::DenseReconstruction(int nFusionMode) #endif if (!pointcloud.IsEmpty()) { - if (IsBounded()) { + if (bCrop2ROI && IsBounded()) { TD_TIMER_START(); const size_t numPoints = pointcloud.GetSize(); - pointcloud.RemovePointsOutside(obb); + const OBB3f ROI(fBorderROI == 0 ? obb : (fBorderROI > 0 ? OBB3f(obb).EnlargePercent(fBorderROI) : OBB3f(obb).Enlarge(-fBorderROI))); + pointcloud.RemovePointsOutside(ROI); VERBOSE("Point-cloud trimmed to ROI: %u points removed (%s)", numPoints-pointcloud.GetSize(), TD_TIMER_GET_FMT().c_str()); } @@ -1615,6 +1680,16 @@ bool Scene::DenseReconstruction(int nFusionMode) if (pointcloud.normals.IsEmpty() && OPTDENSE::nEstimateNormals == 1) EstimatePointNormals(images, pointcloud); } + + if (OPTDENSE::bRemoveDmaps) { + // delete all depth-map files + FOREACH(i, images) { + const DepthData& depthData = data.depthMaps.arrDepthData[i]; + if (!depthData.IsValid()) + continue; + File::deleteFile(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); + } + } return true; } // DenseReconstruction /*----------------------------------------------------------------*/ @@ -1624,10 +1699,16 @@ bool Scene::DenseReconstruction(int nFusionMode) bool Scene::ComputeDepthMaps(DenseDepthMapData& data) { // compute point-cloud from the existing mesh - if (pointcloud.IsEmpty() && !mesh.IsEmpty() && !ImagesHaveNeighbors()) { + if (!mesh.IsEmpty() && !ImagesHaveNeighbors()) { SampleMeshWithVisibility(); mesh.Release(); } + + // compute point-cloud from the existing mesh + if (IsEmpty() && !ImagesHaveNeighbors()) { + VERBOSE("warning: empty point-cloud, rough neighbor views selection based on image pairs baseline"); + EstimateNeighborViewsPointCloud(); + } { // maps global view indices to our list of views to be processed diff --git a/libs/MVS/SceneDensify.h b/libs/MVS/SceneDensify.h index d8a14b70e..34078658f 100644 --- a/libs/MVS/SceneDensify.h +++ b/libs/MVS/SceneDensify.h @@ -68,6 +68,8 @@ class MVS_API DepthMapsData void MergeDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal); void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal); + static DepthData ScaleDepthData(const DepthData& inputDeptData, float scale); + protected: static void* STCALL ScoreDepthMapTmp(void*); static void* STCALL EstimateDepthMapTmp(void*); diff --git a/libs/MVS/SceneReconstruct.cpp b/libs/MVS/SceneReconstruct.cpp index b3fb7d376..a684d3b77 100644 --- a/libs/MVS/SceneReconstruct.cpp +++ b/libs/MVS/SceneReconstruct.cpp @@ -1100,10 +1100,11 @@ bool Scene::ReconstructMesh(float distInsert, bool bUseFreeSpaceSupport, bool bU // create graph MaxFlow graph(delaunay.number_of_cells()); // set weights + constexpr edge_cap_t maxCap(FLT_MAX*0.0001f); for (delaunay_t::All_cells_iterator ci=delaunay.all_cells_begin(), ce=delaunay.all_cells_end(); ci!=ce; ++ci) { const cell_size_t ciID(ci->info()); const cell_info_t& ciInfo(infoCells[ciID]); - graph.AddNode(ciID, ciInfo.s, ciInfo.t); + graph.AddNode(ciID, ciInfo.s, MINF(ciInfo.t, maxCap)); for (int i=0; i<4; ++i) { const cell_handle_t cj(ci->neighbor(i)); const cell_size_t cjID(cj->info()); diff --git a/libs/MVS/SceneRefine.cpp b/libs/MVS/SceneRefine.cpp index 65f73828d..bf1201d14 100644 --- a/libs/MVS/SceneRefine.cpp +++ b/libs/MVS/SceneRefine.cpp @@ -96,7 +96,7 @@ class MeshRefine { DepthMap depthMap; // depth-map BaryMap baryMap; // barycentric coordinates }; - typedef SEACAVE::cList ViewsArr; + typedef CLISTDEF2(View) ViewsArr; // used to render a mesh for optimization struct RasterMesh : TRasterMesh { @@ -411,7 +411,7 @@ void MeshRefine::ListVertexFacesPost() void MeshRefine::ListCameraFaces() { // extract array of faces viewed by each camera - typedef SEACAVE::cList CameraFacesArr; + typedef CLISTDEF2(Mesh::FaceIdxArr) CameraFacesArr; CameraFacesArr arrCameraFaces(images.GetSize()); { Mesh::Octree octree; Mesh::FacesInserter::CreateOctree(octree, scene.mesh); @@ -934,7 +934,7 @@ void MeshRefine::ComputePhotometricGradient( const Point3 X(rayA*REAL(depthA)+cameraA.C); // project point in second image and // projection Jacobian matrix in the second image of the 3D point on the surface - const float depthB(ProjectVertex(cameraB.P.val, X.ptr(), xB.ptr(), xJac.val)); + MAYBEUNUSED const float depthB(ProjectVertex(cameraB.P.val, X.ptr(), xB.ptr(), xJac.val)); ASSERT(depthB > 0); // compute gradient in image B const TMatrix gB(viewB.imageGrad.sample(sampler, xB)); @@ -1292,8 +1292,8 @@ bool Scene::RefineMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsig // run the mesh optimization on multiple scales (coarse to fine) for (unsigned nScale=0; nScale CameraFacesArr; + typedef CLISTDEF2(CameraFaces) CameraFacesArr; // store necessary data about a view struct View { @@ -1958,7 +1958,7 @@ class MeshRefineCUDA { inline View() {} inline View(View&) {} }; - typedef SEACAVE::cList ViewsArr; + typedef CLISTDEF2(View) ViewsArr; struct CameraCUDA { Matrix3x4f P; @@ -2799,8 +2799,8 @@ bool Scene::RefineMeshCUDA(unsigned nResolutionLevel, unsigned nMinResolution, u // run the mesh optimization on multiple scales (coarse to fine) for (unsigned nScale=0; nScale -#include #include using namespace MVS; @@ -126,6 +125,7 @@ struct TRWSInference { } #endif + // S T R U C T S /////////////////////////////////////////////////// typedef Mesh::Vertex Vertex; @@ -175,7 +175,7 @@ struct MeshTexture { // used to store info about a face (view, quality) struct FaceData { - VIndex idxView;// the view seeing this face + IIndex idxView;// the view seeing this face float quality; // how well the face is seen by this view #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA Color color; // additionally store mean color (used to remove outliers) @@ -184,6 +184,8 @@ struct MeshTexture { typedef cList FaceDataArr; // store information about one face seen from several views typedef cList FaceDataViewArr; // store data for all the faces of the mesh + typedef cList VirtualFaceIdxsArr; // store face indices for each virtual face + // used to assign a view to a face typedef uint32_t Label; typedef cList LabelArr; @@ -234,7 +236,7 @@ struct MeshTexture { Patch& GetPatch(uint32_t idxPatch) { const uint32_t idx(patches.Find(idxPatch)); if (idx == NO_ID) - return patches.AddConstruct(idxPatch); + return patches.emplace_back(idxPatch); return patches[idx]; } inline void SortByPatchIndex(IndexArr& indices) const { @@ -313,55 +315,6 @@ struct MeshTexture { // used to interpolate adjustments color over the whole texture patch typedef TImage ColorMap; - struct RasterPatchColorData { - const TexCoord* tri; - Color colors[3]; - ColorMap& image; - inline RasterPatchColorData(ColorMap& _image) : image(_image) {} - inline cv::Size Size() const { return image.size(); } - inline void operator()(const ImageRef& pt, const Point3f& bary) { - ASSERT(image.isInside(pt)); - image(pt) = colors[0]*bary.x + colors[1]*bary.y + colors[2]*bary.z; - } - }; - - // used to compute the coverage of a texture patch - struct RasterPatchCoverageData { - const TexCoord* tri; - Image8U& image; - inline RasterPatchCoverageData(Image8U& _image) : image(_image) {} - inline void operator()(const ImageRef& pt) { - ASSERT(image.isInside(pt)); - image(pt) = interior; - } - }; - - // used to draw the average edge color of a texture patch - struct RasterPatchMeanEdgeData { - Image32F3& image; - Image8U& mask; - const Image32F3& image0; - const Image8U3& image1; - const TexCoord p0, p0Dir; - const TexCoord p1, p1Dir; - const float length; - const Sampler sampler; - inline RasterPatchMeanEdgeData(Image32F3& _image, Image8U& _mask, const Image32F3& _image0, const Image8U3& _image1, - const TexCoord& _p0, const TexCoord& _p0Adj, const TexCoord& _p1, const TexCoord& _p1Adj) - : image(_image), mask(_mask), image0(_image0), image1(_image1), - p0(_p0), p0Dir(_p0Adj-_p0), p1(_p1), p1Dir(_p1Adj-_p1), length((float)norm(p0Dir)), sampler() {} - inline void operator()(const ImageRef& pt) { - const float l((float)norm(TexCoord(pt)-p0)/length); - // compute mean color - const TexCoord samplePos0(p0 + p0Dir * l); - AccumColor accumColor(image0.sample(sampler, samplePos0), 1.f); - const TexCoord samplePos1(p1 + p1Dir * l); - accumColor.Add(image1.sample(sampler, samplePos1)/255.f, 1.f); - image(pt) = accumColor.Normalized(); - // set mask edge also - mask(pt) = border; - } - }; public: @@ -375,13 +328,16 @@ struct MeshTexture { #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA bool FaceOutlierDetection(FaceDataArr& faceDatas, float fOutlierThreshold) const; #endif + + void CreateVirtualFaces(const FaceDataViewArr& facesDatas, FaceDataViewArr& virtualFacesDatas, VirtualFaceIdxsArr& virtualFaces, unsigned minCommonCameras=2, float thMaxNormalDeviation=25.f) const; + IIndexArr SelectBestView(const FaceDataArr& faceDatas, FIndex fid, unsigned minCommonCameras, float ratioAngleToQuality) const; - bool FaceViewSelection(float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views); - + bool FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views); + void CreateSeamVertices(); void GlobalSeamLeveling(); void LocalSeamLeveling(); - void GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty); + void GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight); template static inline PIXEL RGB2YCBCR(const PIXEL& v) { @@ -541,6 +497,8 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr cv::filter2D(imageGradMag, grad[1], cv::DataType::type, kernel.t()); #endif (TImage::EMatMap)imageGradMag = (mGrad[0].cwiseAbs2()+mGrad[1].cwiseAbs2()).cwiseSqrt(); + // apply some blur on the gradient to lower noise/glossiness effects onto face-quality score + cv::GaussianBlur(imageGradMag, imageGradMag, cv::Size(15, 15), 0, 0, cv::BORDER_DEFAULT); // select faces inside view frustum Mesh::FaceIdxArr cameraFaces; Mesh::FacesInserter inserter(cameraFaces); @@ -566,6 +524,11 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr #pragma omp critical #endif { + // faceQuality is influenced by : + // + area: the higher the area the more gradient scores will be added to the face quality + // + sharpness: sharper image or image resolution or how close is to the face will result in higher gradient on the same face + // ON GLOSS IMAGES it happens to have a high volatile sharpness depending on how the light reflects under different angles + // + angle: low angle increases the surface area for (int j=0; j(imageData.camera.C) - faceCenter); + const Normal& faceNormal = scene.mesh.faceNormals[idxFace]; + const float cosFaceCam(MAXF(0.001f, ComputeAngle(camDir.ptr(), faceNormal.ptr()))); + faceDatas.back().quality *= SQUARE(cosFaceCam); + } #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA FOREACH(idxFace, areas) { const uint32_t& area = areas[idxFace]; if (area > 0) { - Color& color = facesDatas[idxFace].Last().color; + Color& color = facesDatas[idxFace].back().color; color = RGB2YCBCR(Color(color * (1.f/(float)area))); } } @@ -620,13 +596,206 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr if (fOutlierThreshold > 0) { // try to detect outlier views for each face // (views for which the face is occluded by a dynamic object in the scene, ex. pedestrians) - FOREACHPTR(pFaceDatas, facesDatas) - FaceOutlierDetection(*pFaceDatas, fOutlierThreshold); + for (FaceDataArr& faceDatas: facesDatas) + FaceOutlierDetection(faceDatas, fOutlierThreshold); } #endif return true; } +// order the camera view scores with highest score first and return the list of first cameras +// ratioAngleToQuality represents the ratio in witch we combine normal angle to quality for a face to obtain the selection score +// - a ratio of 1 means only angle is considered +// - a ratio of 0.5 means angle and quality are equally important +// - a ratio of 0 means only camera quality is considered when sorting +IIndexArr MeshTexture::SelectBestView(const FaceDataArr& faceDatas, FIndex fid, unsigned minCommonCameras, float ratioAngleToQuality) const +{ + ASSERT(!faceDatas.empty()); + #if 1 + + // compute scores based on the view quality and its angle to the face normal + float maxQuality = 0; + for (const FaceData& faceData: faceDatas) + maxQuality = MAXF(maxQuality, faceData.quality); + const Face& f = faces[fid]; + const Vertex faceCenter((vertices[f[0]] + vertices[f[1]] + vertices[f[2]]) / 3.f); + CLISTDEF0IDX(float,IIndex) scores(faceDatas.size()); + FOREACH(idxFaceData, faceDatas) { + const FaceData& faceData = faceDatas[idxFaceData]; + const Image& imageData = images[faceData.idxView]; + const Point3f camDir(Cast(imageData.camera.C) - faceCenter); + const Normal& faceNormal = scene.mesh.faceNormals[fid]; + const float cosFaceCam(ComputeAngle(camDir.ptr(), faceNormal.ptr())); + scores[idxFaceData] = ratioAngleToQuality*cosFaceCam + (1.f-ratioAngleToQuality)*faceData.quality/maxQuality; + } + // and sort the scores from to highest to smallest to get the best overall cameras + IIndexArr scorePodium(faceDatas.size()); + std::iota(scorePodium.begin(), scorePodium.end(), 0); + scorePodium.Sort([&scores](IIndex i, IIndex j) { + return scores[i] > scores[j]; + }); + + #else + + // sort qualityPodium in relation to faceDatas[index].quality decreasing + IIndexArr qualityPodium(faceDatas.size()); + std::iota(qualityPodium.begin(), qualityPodium.end(), 0); + qualityPodium.Sort([&faceDatas](IIndex i, IIndex j) { + return faceDatas[i].quality > faceDatas[j].quality; + }); + + // sort anglePodium in relation to face angle to camera increasing + const Face& f = faces[fid]; + const Vertex faceCenter((vertices[f[0]] + vertices[f[1]] + vertices[f[2]]) / 3.f); + CLISTDEF0IDX(float,IIndex) cameraAngles(0, faceDatas.size()); + for (const FaceData& faceData: faceDatas) { + const Image& imageData = images[faceData.idxView]; + const Point3f camDir(Cast(imageData.camera.C) - faceCenter); + const Normal& faceNormal = scene.mesh.faceNormals[fid]; + const float cosFaceCam(ComputeAngle(camDir.ptr(), faceNormal.ptr())); + cameraAngles.emplace_back(cosFaceCam); + } + IIndexArr anglePodium(faceDatas.size()); + std::iota(anglePodium.begin(), anglePodium.end(), 0); + anglePodium.Sort([&cameraAngles](IIndex i, IIndex j) { + return cameraAngles[i] > cameraAngles[j]; + }); + + // combine podium scores to get overall podium + // and sort the scores in smallest to highest to get the best overall camera for current virtual face + CLISTDEF0IDX(float,IIndex) scores(faceDatas.size()); + scores.Memset(0); + FOREACH(sIdx, faceDatas) { + scores[anglePodium[sIdx]] += ratioAngleToQuality * (sIdx+1); + scores[qualityPodium[sIdx]] += (1.f - ratioAngleToQuality) * (sIdx+1); + } + IIndexArr scorePodium(faceDatas.size()); + std::iota(scorePodium.begin(), scorePodium.end(), 0); + scorePodium.Sort([&scores](IIndex i, IIndex j) { + return scores[i] < scores[j]; + }); + + #endif + IIndexArr cameras(MIN(minCommonCameras, faceDatas.size())); + FOREACH(i, cameras) + cameras[i] = faceDatas[scorePodium[i]].idxView; + return cameras; +} + +static bool IsFaceVisible(const MeshTexture::FaceDataArr& faceDatas, const IIndexArr& cameraList) { + size_t camFoundCounter(0); + for (const MeshTexture::FaceData& faceData : faceDatas) { + const IIndex cfCam = faceData.idxView; + for (IIndex camId : cameraList) { + if (cfCam == camId) { + if (++camFoundCounter == cameraList.size()) + return true; + break; + } + } + } + return camFoundCounter == cameraList.size(); +} + +// build virtual faces with: +// - similar normal +// - high percentage of common images that see them +void MeshTexture::CreateVirtualFaces(const FaceDataViewArr& facesDatas, FaceDataViewArr& virtualFacesDatas, VirtualFaceIdxsArr& virtualFaces, unsigned minCommonCameras, float thMaxNormalDeviation) const +{ + const float ratioAngleToQuality(0.67f); + const float cosMaxNormalDeviation(COS(FD2R(thMaxNormalDeviation))); + Mesh::FaceIdxArr remainingFaces(faces.size()); + std::iota(remainingFaces.begin(), remainingFaces.end(), 0); + std::vector selectedFaces(faces.size(), false); + cQueue currentVirtualFaceQueue; + std::unordered_set queuedFaces; + do { + const FIndex startPos = RAND() % remainingFaces.size(); + const FIndex virtualFaceCenterFaceID = remainingFaces[startPos]; + ASSERT(currentVirtualFaceQueue.IsEmpty()); + const Normal& normalCenter = scene.mesh.faceNormals[virtualFaceCenterFaceID]; + const FaceDataArr& centerFaceDatas = facesDatas[virtualFaceCenterFaceID]; + // select the common cameras + Mesh::FaceIdxArr virtualFace; + FaceDataArr virtualFaceDatas; + if (centerFaceDatas.empty()) { + virtualFace.emplace_back(virtualFaceCenterFaceID); + selectedFaces[virtualFaceCenterFaceID] = true; + const auto posToErase = remainingFaces.FindFirst(virtualFaceCenterFaceID); + ASSERT(posToErase != Mesh::FaceIdxArr::NO_INDEX); + remainingFaces.RemoveAtMove(posToErase); + } else { + const IIndexArr selectedCams = SelectBestView(centerFaceDatas, virtualFaceCenterFaceID, minCommonCameras, ratioAngleToQuality); + currentVirtualFaceQueue.AddTail(virtualFaceCenterFaceID); + queuedFaces.clear(); + do { + const FIndex currentFaceId = currentVirtualFaceQueue.GetHead(); + currentVirtualFaceQueue.PopHead(); + // check for condition to add in current virtual face + // normal angle smaller than thMaxNormalDeviation degrees + const Normal& faceNormal = scene.mesh.faceNormals[currentFaceId]; + const float cosFaceToCenter(ComputeAngleN(normalCenter.ptr(), faceNormal.ptr())); + if (cosFaceToCenter < cosMaxNormalDeviation) + continue; + // check if current face is seen by all cameras in selectedCams + ASSERT(!selectedCams.empty()); + if (!IsFaceVisible(facesDatas[currentFaceId], selectedCams)) + continue; + // remove it from remaining faces and add it to the virtual face + { + const auto posToErase = remainingFaces.FindFirst(currentFaceId); + ASSERT(posToErase != Mesh::FaceIdxArr::NO_INDEX); + remainingFaces.RemoveAtMove(posToErase); + selectedFaces[currentFaceId] = true; + virtualFace.push_back(currentFaceId); + } + // add all new neighbors to the queue + const Mesh::FaceFaces& ffaces = faceFaces[currentFaceId]; + for (int i = 0; i < 3; ++i) { + const FIndex fIdx = ffaces[i]; + if (fIdx == NO_ID) + continue; + if (!selectedFaces[fIdx] && queuedFaces.find(fIdx) == queuedFaces.end()) { + currentVirtualFaceQueue.AddTail(fIdx); + queuedFaces.emplace(fIdx); + } + } + } while (!currentVirtualFaceQueue.IsEmpty()); + // compute virtual face quality and create virtual face + for (IIndex idxView: selectedCams) { + FaceData& virtualFaceData = virtualFaceDatas.AddEmpty(); + virtualFaceData.quality = 0; + virtualFaceData.idxView = idxView; + #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA + virtualFaceData.color = Point3f::ZERO; + #endif + unsigned processedFaces(0); + for (FIndex fid : virtualFace) { + const FaceDataArr& faceDatas = facesDatas[fid]; + for (FaceData& faceData: faceDatas) { + if (faceData.idxView == idxView) { + virtualFaceData.quality += faceData.quality; + #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA + virtualFaceData.color += faceData.color; + #endif + ++processedFaces; + break; + } + } + } + ASSERT(processedFaces > 0); + virtualFaceData.quality /= processedFaces; + #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA + virtualFaceData.color /= processedFaces; + #endif + } + ASSERT(!virtualFaceDatas.empty()); + } + virtualFacesDatas.emplace_back(std::move(virtualFaceDatas)); + virtualFaces.emplace_back(std::move(virtualFace)); + } while (!remainingFaces.empty()); +} + #if TEXOPT_FACEOUTLIER == TEXOPT_FACEOUTLIER_MEDIAN // decrease the quality of / remove all views in which the face's projection @@ -805,13 +974,16 @@ bool MeshTexture::FaceOutlierDetection(FaceDataArr& faceDatas, float thOutlier) } #endif -bool MeshTexture::FaceViewSelection(float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views) +bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views) { // extract array of triangles incident to each vertex ListVertexFaces(); // create texture patches { + // compute face normals and smoothen them + scene.mesh.SmoothNormalFaces(); + // list all views for each face FaceDataViewArr facesDatas; if (!ListCameraFaces(facesDatas, fOutlierThreshold, views)) @@ -822,181 +994,310 @@ bool MeshTexture::FaceViewSelection(float fOutlierThreshold, float fRatioDataSmo typedef boost::graph_traits::edge_iterator EdgeIter; typedef boost::graph_traits::out_edge_iterator EdgeOutIter; Graph graph; - { - FOREACH(idxFace, faces) { - const Mesh::FIndex idx((Mesh::FIndex)boost::add_vertex(graph)); - ASSERT(idx == idxFace); + LabelArr labels; + + // construct and use virtual faces for patch creation instead of actual mesh faces; + // the virtual faces are composed of coplanar triangles sharing same views + const bool bUseVirtualFaces(minCommonCameras > 0); + if (bUseVirtualFaces) { + // 1) create FaceToVirtualFaceMap + FaceDataViewArr virtualFacesDatas; + VirtualFaceIdxsArr virtualFaces; // stores each virtual face as an array of mesh face ID + CreateVirtualFaces(facesDatas, virtualFacesDatas, virtualFaces, minCommonCameras); + Mesh::FaceIdxArr mapFaceToVirtualFace(faces.size()); // for each mesh face ID, store the virtual face ID witch contains it + size_t controlCounter(0); + FOREACH(idxVF, virtualFaces) { + const Mesh::FaceIdxArr& vf = virtualFaces[idxVF]; + for (FIndex idxFace : vf) { + mapFaceToVirtualFace[idxFace] = idxVF; + ++controlCounter; + } } - Mesh::FaceIdxArr afaces; - FOREACH(idxFace, faces) { - const Mesh::FaceFaces& afaces = faceFaces[idxFace]; - for (int v=0; v<3; ++v) { - const FIndex idxFaceAdj = afaces[v]; - if (idxFaceAdj == NO_ID || idxFace >= idxFaceAdj) - continue; - const bool bInvisibleFace(facesDatas[idxFace].IsEmpty()); - const bool bInvisibleFaceAdj(facesDatas[idxFaceAdj].IsEmpty()); - if (bInvisibleFace || bInvisibleFaceAdj) { - if (bInvisibleFace != bInvisibleFaceAdj) - seamEdges.AddConstruct(idxFace, idxFaceAdj); - continue; + ASSERT(controlCounter == faces.size()); + // 2) create function to find virtual faces neighbors + VirtualFaceIdxsArr virtualFaceNeighbors; { // for each virtual face, the list of virtual faces with at least one vertex in common + virtualFaceNeighbors.resize(virtualFaces.size()); + FOREACH(idxVF, virtualFaces) { + const Mesh::FaceIdxArr& vf = virtualFaces[idxVF]; + Mesh::FaceIdxArr& vfNeighbors = virtualFaceNeighbors[idxVF]; + for (FIndex idxFace : vf) { + const Mesh::FaceFaces& adjFaces = faceFaces[idxFace]; + for (int i = 0; i < 3; ++i) { + const FIndex fAdj(adjFaces[i]); + if (fAdj == NO_ID) + continue; + if (mapFaceToVirtualFace[fAdj] == idxVF) + continue; + if (fAdj != idxFace && vfNeighbors.Find(mapFaceToVirtualFace[fAdj]) == Mesh::FaceIdxArr::NO_INDEX) { + vfNeighbors.emplace_back(mapFaceToVirtualFace[fAdj]); + } + } } - boost::add_edge(idxFace, idxFaceAdj, graph); } } - faceFaces.Release(); - ASSERT((Mesh::FIndex)boost::num_vertices(graph) == faces.size()); - } - - // assign the best view to each face - LabelArr labels(faces.size()); - components.Resize(faces.size()); - { - // normalize quality values - float maxQuality(0); - for (const FaceDataArr& faceDatas: facesDatas) { - for (const FaceData& faceData: faceDatas) - if (maxQuality < faceData.quality) - maxQuality = faceData.quality; + // 3) use virtual faces to build the graph + // 4) assign images to virtual faces + // 5) spread image ID to each mesh face from virtual face + FOREACH(idxFace, virtualFaces) { + MAYBEUNUSED const Mesh::FIndex idx((Mesh::FIndex)boost::add_vertex(graph)); + ASSERT(idx == idxFace); } - Histogram32F hist(std::make_pair(0.f, maxQuality), 1000); - for (const FaceDataArr& faceDatas: facesDatas) { - for (const FaceData& faceData: faceDatas) - hist.Add(faceData.quality); + FOREACH(idxVirtualFace, virtualFaces) { + const Mesh::FaceIdxArr& afaces = virtualFaceNeighbors[idxVirtualFace]; + for (FIndex idxVirtualFaceAdj: afaces) { + if (idxVirtualFace >= idxVirtualFaceAdj) + continue; + const bool bInvisibleFace(virtualFacesDatas[idxVirtualFace].empty()); + const bool bInvisibleFaceAdj(virtualFacesDatas[idxVirtualFaceAdj].empty()); + if (bInvisibleFace || bInvisibleFaceAdj) + continue; + boost::add_edge(idxVirtualFace, idxVirtualFaceAdj, graph); + } } - const float normQuality(hist.GetApproximatePermille(0.95f)); - - #if TEXOPT_INFERENCE == TEXOPT_INFERENCE_LBP - // initialize inference structures - const LBPInference::EnergyType MaxEnergy(fRatioDataSmoothness*LBPInference::MaxEnergy); - LBPInference inference; { - inference.SetNumNodes(faces.size()); - inference.SetSmoothCost(SmoothnessPotts); - EdgeOutIter ei, eie; - FOREACH(f, faces) { - for (boost::tie(ei, eie) = boost::out_edges(f, graph); ei != eie; ++ei) { - ASSERT(f == (FIndex)ei->m_source); - const FIndex fAdj((FIndex)ei->m_target); - ASSERT(components.empty() || components[f] == components[fAdj]); - if (f < fAdj) // add edges only once - inference.SetNeighbors(f, fAdj); + ASSERT((Mesh::FIndex)boost::num_vertices(graph) == virtualFaces.size()); + // assign the best view to each face + labels.resize(faces.size()); + components.resize(faces.size()); { + // normalize quality values + float maxQuality(0); + for (const FaceDataArr& faceDatas: virtualFacesDatas) { + for (const FaceData& faceData: faceDatas) + if (maxQuality < faceData.quality) + maxQuality = faceData.quality; + } + Histogram32F hist(std::make_pair(0.f, maxQuality), 1000); + for (const FaceDataArr& faceDatas: virtualFacesDatas) { + for (const FaceData& faceData: faceDatas) + hist.Add(faceData.quality); + } + const float normQuality(hist.GetApproximatePermille(0.95f)); + + #if TEXOPT_INFERENCE == TEXOPT_INFERENCE_LBP + // initialize inference structures + const LBPInference::EnergyType MaxEnergy(fRatioDataSmoothness*LBPInference::MaxEnergy); + LBPInference inference; { + inference.SetNumNodes(virtualFaces.size()); + inference.SetSmoothCost(SmoothnessPotts); + EdgeOutIter ei, eie; + FOREACH(f, virtualFaces) { + for (boost::tie(ei, eie) = boost::out_edges(f, graph); ei != eie; ++ei) { + ASSERT(f == (FIndex)ei->m_source); + const FIndex fAdj((FIndex)ei->m_target); + ASSERT(components.empty() || components[f] == components[fAdj]); + if (f < fAdj) // add edges only once + inference.SetNeighbors(f, fAdj); + } + // set costs for label 0 (undefined) + inference.SetDataCost((Label)0, f, MaxEnergy); } - // set costs for label 0 (undefined) - inference.SetDataCost((Label)0, f, MaxEnergy); } - } - // set data costs for all labels (except label 0 - undefined) - FOREACH(f, facesDatas) { - const FaceDataArr& faceDatas = facesDatas[f]; - for (const FaceData& faceData: faceDatas) { - const Label label((Label)faceData.idxView+1); - const float normalizedQuality(faceData.quality>=normQuality ? 1.f : faceData.quality/normQuality); - const float dataCost((1.f-normalizedQuality)*MaxEnergy); - inference.SetDataCost(label, f, dataCost); + // set data costs for all labels (except label 0 - undefined) + FOREACH(f, virtualFacesDatas) { + const FaceDataArr& faceDatas = virtualFacesDatas[f]; + for (const FaceData& faceData: faceDatas) { + const Label label((Label)faceData.idxView+1); + const float normalizedQuality(faceData.quality>=normQuality ? 1.f : faceData.quality/normQuality); + const float dataCost((1.f-normalizedQuality)*MaxEnergy); + inference.SetDataCost(label, f, dataCost); + } } - } - // assign the optimal view (label) to each face - // (label 0 is reserved as undefined) - inference.Optimize(); - - // extract resulting labeling - labels.Memset(0xFF); - FOREACH(l, labels) { - const Label label(inference.GetLabel(l)); - ASSERT(label < images.GetSize()+1); - if (label > 0) - labels[l] = label-1; - } - #endif + // assign the optimal view (label) to each face + // (label 0 is reserved as undefined) + inference.Optimize(); - #if TEXOPT_INFERENCE == TEXOPT_INFERENCE_TRWS - // find connected components - const FIndex nComponents(boost::connected_components(graph, components.data())); + // extract resulting labeling + LabelArr virtualLabels(virtualFaces.size()); + virtualLabels.Memset(0xFF); + FOREACH(l, virtualLabels) { + const Label label(inference.GetLabel(l)); + ASSERT(label < images.GetSize()+1); + if (label > 0) + virtualLabels[l] = label-1; + } + FOREACH(l, labels) { + labels[l] = virtualLabels[mapFaceToVirtualFace[l]]; + } + #endif + } - // map face ID from global to component space - typedef cList NodeIDs; - NodeIDs nodeIDs(faces.GetSize()); - NodeIDs sizes(nComponents); - sizes.Memset(0); - FOREACH(c, components) - nodeIDs[c] = sizes[components[c]]++; - - // initialize inference structures - const LabelID numLabels(images.GetSize()+1); - CLISTDEFIDX(TRWSInference, FIndex) inferences(nComponents); - FOREACH(s, sizes) { - const NodeID numNodes(sizes[s]); - ASSERT(numNodes > 0); - if (numNodes <= 1) + graph.clear(); + } + + // create the graph of faces: each vertex is a face and the edges are the edges shared by the faces + FOREACH(idxFace, faces) { + MAYBEUNUSED const Mesh::FIndex idx((Mesh::FIndex)boost::add_vertex(graph)); + ASSERT(idx == idxFace); + } + FOREACH(idxFace, faces) { + const Mesh::FaceFaces& afaces = faceFaces[idxFace]; + for (int v=0; v<3; ++v) { + const FIndex idxFaceAdj = afaces[v]; + if (idxFaceAdj == NO_ID || idxFace >= idxFaceAdj) continue; - TRWSInference& inference = inferences[s]; - inference.Init(numNodes, numLabels); + const bool bInvisibleFace(facesDatas[idxFace].IsEmpty()); + const bool bInvisibleFaceAdj(facesDatas[idxFaceAdj].IsEmpty()); + if (bInvisibleFace || bInvisibleFaceAdj) { + if (bInvisibleFace != bInvisibleFaceAdj) + seamEdges.emplace_back(idxFace, idxFaceAdj); + continue; + } + boost::add_edge(idxFace, idxFaceAdj, graph); } - - // set data costs + } + faceFaces.Release(); + ASSERT((Mesh::FIndex)boost::num_vertices(graph) == faces.size()); + + // start patch creation starting directly from individual faces + if (!bUseVirtualFaces) { + // assign the best view to each face + labels.resize(faces.size()); + components.resize(faces.size()); { - // add nodes - CLISTDEF0(EnergyType) D(numLabels); + // normalize quality values + float maxQuality(0); + for (const FaceDataArr& faceDatas: facesDatas) { + for (const FaceData& faceData: faceDatas) + if (maxQuality < faceData.quality) + maxQuality = faceData.quality; + } + Histogram32F hist(std::make_pair(0.f, maxQuality), 1000); + for (const FaceDataArr& faceDatas: facesDatas) { + for (const FaceData& faceData: faceDatas) + hist.Add(faceData.quality); + } + const float normQuality(hist.GetApproximatePermille(0.95f)); + + #if TEXOPT_INFERENCE == TEXOPT_INFERENCE_LBP + // initialize inference structures + const LBPInference::EnergyType MaxEnergy(fRatioDataSmoothness*LBPInference::MaxEnergy); + LBPInference inference; { + inference.SetNumNodes(faces.size()); + inference.SetSmoothCost(SmoothnessPotts); + EdgeOutIter ei, eie; + FOREACH(f, faces) { + for (boost::tie(ei, eie) = boost::out_edges(f, graph); ei != eie; ++ei) { + ASSERT(f == (FIndex)ei->m_source); + const FIndex fAdj((FIndex)ei->m_target); + ASSERT(components.empty() || components[f] == components[fAdj]); + if (f < fAdj) // add edges only once + inference.SetNeighbors(f, fAdj); + } + // set costs for label 0 (undefined) + inference.SetDataCost((Label)0, f, MaxEnergy); + } + } + + // set data costs for all labels (except label 0 - undefined) FOREACH(f, facesDatas) { - TRWSInference& inference = inferences[components[f]]; - if (inference.IsEmpty()) - continue; - // invisible faces are given a data cost 3 times the edge costs - // 3 as each face can have at most 3 edges - D.MemsetValue(MaxEnergy * 3); const FaceDataArr& faceDatas = facesDatas[f]; - FOREACHPTR(pFaceData, faceDatas) { - const FaceData& faceData = *pFaceData; - const Label label((Label)faceData.idxView); + for (const FaceData& faceData: faceDatas) { + const Label label((Label)faceData.idxView+1); const float normalizedQuality(faceData.quality>=normQuality ? 1.f : faceData.quality/normQuality); - const EnergyType dataCost(MaxEnergy*(1.f-normalizedQuality)); - D[label] = dataCost; + const float dataCost((1.f-normalizedQuality)*MaxEnergy); + inference.SetDataCost(label, f, dataCost); } - const NodeID nodeID(nodeIDs[f]); - inference.AddNode(nodeID, D.Begin()); } - // add edges - EdgeOutIter ei, eie; - FOREACH(f, faces) { - TRWSInference& inference = inferences[components[f]]; - if (inference.IsEmpty()) + + // assign the optimal view (label) to each face + // (label 0 is reserved as undefined) + inference.Optimize(); + + // extract resulting labeling + labels.Memset(0xFF); + FOREACH(l, labels) { + const Label label(inference.GetLabel(l)); + ASSERT(label < images.size()+1); + if (label > 0) + labels[l] = label-1; + } + #endif + + #if TEXOPT_INFERENCE == TEXOPT_INFERENCE_TRWS + // find connected components + const FIndex nComponents(boost::connected_components(graph, components.data())); + + // map face ID from global to component space + typedef cList NodeIDs; + NodeIDs nodeIDs(faces.GetSize()); + NodeIDs sizes(nComponents); + sizes.Memset(0); + FOREACH(c, components) + nodeIDs[c] = sizes[components[c]]++; + + // initialize inference structures + const LabelID numLabels(images.GetSize()+1); + CLISTDEFIDX(TRWSInference, FIndex) inferences(nComponents); + FOREACH(s, sizes) { + const NodeID numNodes(sizes[s]); + ASSERT(numNodes > 0); + if (numNodes <= 1) continue; - for (boost::tie(ei, eie) = boost::out_edges(f, graph); ei != eie; ++ei) { - ASSERT(f == (FIndex)ei->m_source); - const FIndex fAdj((FIndex)ei->m_target); - ASSERT(components[f] == components[fAdj]); - if (f < fAdj) // add edges only once - inference.AddEdge(nodeIDs[f], nodeIDs[fAdj]); + TRWSInference& inference = inferences[s]; + inference.Init(numNodes, numLabels); + } + + // set data costs + { + // add nodes + CLISTDEF0(EnergyType) D(numLabels); + FOREACH(f, facesDatas) { + TRWSInference& inference = inferences[components[f]]; + if (inference.IsEmpty()) + continue; + D.MemsetValue(MaxEnergy); + const FaceDataArr& faceDatas = facesDatas[f]; + for (const FaceData& faceData: faceDatas) { + const Label label((Label)faceData.idxView); + const float normalizedQuality(faceData.quality>=normQuality ? 1.f : faceData.quality/normQuality); + const EnergyType dataCost(MaxEnergy*(1.f-normalizedQuality)); + D[label] = dataCost; + } + const NodeID nodeID(nodeIDs[f]); + inference.AddNode(nodeID, D.Begin()); + } + // add edges + EdgeOutIter ei, eie; + FOREACH(f, faces) { + TRWSInference& inference = inferences[components[f]]; + if (inference.IsEmpty()) + continue; + for (boost::tie(ei, eie) = boost::out_edges(f, graph); ei != eie; ++ei) { + ASSERT(f == (FIndex)ei->m_source); + const FIndex fAdj((FIndex)ei->m_target); + ASSERT(components[f] == components[fAdj]); + if (f < fAdj) // add edges only once + inference.AddEdge(nodeIDs[f], nodeIDs[fAdj]); + } } } - } - // assign the optimal view (label) to each face - #ifdef TEXOPT_USE_OPENMP - #pragma omp parallel for schedule(dynamic) - for (int i=0; i<(int)inferences.GetSize(); ++i) { - #else - FOREACH(i, inferences) { - #endif - TRWSInference& inference = inferences[i]; - if (inference.IsEmpty()) - continue; - inference.Optimize(); - } - // extract resulting labeling - labels.Memset(0xFF); - FOREACH(l, labels) { - TRWSInference& inference = inferences[components[l]]; - if (inference.IsEmpty()) - continue; - const Label label(inference.GetLabel(nodeIDs[l])); - ASSERT(label >= 0 && label < numLabels); - if (label < images.GetSize()) - labels[l] = label; + // assign the optimal view (label) to each face + #ifdef TEXOPT_USE_OPENMP + #pragma omp parallel for schedule(dynamic) + for (int i=0; i<(int)inferences.GetSize(); ++i) { + #else + FOREACH(i, inferences) { + #endif + TRWSInference& inference = inferences[i]; + if (inference.IsEmpty()) + continue; + inference.Optimize(); + } + // extract resulting labeling + labels.Memset(0xFF); + FOREACH(l, labels) { + TRWSInference& inference = inferences[components[l]]; + if (inference.IsEmpty()) + continue; + const Label label(inference.GetLabel(nodeIDs[l])); + ASSERT(label >= 0 && label < numLabels); + if (label < images.GetSize()) + labels[l] = label; + } + #endif } - #endif } // create texture patches @@ -1009,7 +1310,7 @@ bool MeshTexture::FaceViewSelection(float fOutlierThreshold, float fRatioDataSmo const FIndex fTarget((FIndex)ei->m_target); ASSERT(components[fSource] == components[fTarget]); if (labels[fSource] != labels[fTarget]) - seamEdges.AddConstruct(fSource, fTarget); + seamEdges.emplace_back(fSource, fTarget); } for (const PairIdx *pEdge=seamEdges.Begin()+startLabelSeamEdges, *pEdgeEnd=seamEdges.End(); pEdge!=pEdgeEnd; ++pEdge) boost::remove_edge(pEdge->i, pEdge->j, graph); @@ -1078,29 +1379,29 @@ void MeshTexture::CreateSeamVertices() uint32_t vs0[2], vs1[2]; std::unordered_map mapVertexSeam; const unsigned numPatches(texturePatches.GetSize()-1); - FOREACHPTR(pEdge, seamEdges) { + for (const PairIdx& edge: seamEdges) { // store edge for the later seam optimization - ASSERT(pEdge->i < pEdge->j); - const uint32_t idxPatch0(mapIdxPatch[components[pEdge->i]]); - const uint32_t idxPatch1(mapIdxPatch[components[pEdge->j]]); + ASSERT(edge.i < edge.j); + const uint32_t idxPatch0(mapIdxPatch[components[edge.i]]); + const uint32_t idxPatch1(mapIdxPatch[components[edge.j]]); ASSERT(idxPatch0 != idxPatch1 || idxPatch0 == numPatches); if (idxPatch0 == idxPatch1) continue; seamVertices.ReserveExtra(2); - scene.mesh.GetEdgeVertices(pEdge->i, pEdge->j, vs0, vs1); - ASSERT(faces[pEdge->i][vs0[0]] == faces[pEdge->j][vs1[0]]); - ASSERT(faces[pEdge->i][vs0[1]] == faces[pEdge->j][vs1[1]]); - vs[0] = faces[pEdge->i][vs0[0]]; - vs[1] = faces[pEdge->i][vs0[1]]; + scene.mesh.GetEdgeVertices(edge.i, edge.j, vs0, vs1); + ASSERT(faces[edge.i][vs0[0]] == faces[edge.j][vs1[0]]); + ASSERT(faces[edge.i][vs0[1]] == faces[edge.j][vs1[1]]); + vs[0] = faces[edge.i][vs0[0]]; + vs[1] = faces[edge.i][vs0[1]]; const auto itSeamVertex0(mapVertexSeam.emplace(std::make_pair(vs[0], seamVertices.GetSize()))); if (itSeamVertex0.second) - seamVertices.AddConstruct(vs[0]); + seamVertices.emplace_back(vs[0]); SeamVertex& seamVertex0 = seamVertices[itSeamVertex0.first->second]; const auto itSeamVertex1(mapVertexSeam.emplace(std::make_pair(vs[1], seamVertices.GetSize()))); if (itSeamVertex1.second) - seamVertices.AddConstruct(vs[1]); + seamVertices.emplace_back(vs[1]); SeamVertex& seamVertex1 = seamVertices[itSeamVertex1.first->second]; if (idxPatch0 < numPatches) { @@ -1108,22 +1409,22 @@ void MeshTexture::CreateSeamVertices() SeamVertex::Patch& patch00 = seamVertex0.GetPatch(idxPatch0); SeamVertex::Patch& patch10 = seamVertex1.GetPatch(idxPatch0); ASSERT(patch00.edges.Find(itSeamVertex1.first->second) == NO_ID); - patch00.edges.AddConstruct(itSeamVertex1.first->second).idxFace = pEdge->i; - patch00.proj = faceTexcoords[pEdge->i*3+vs0[0]]+offset0; + patch00.edges.emplace_back(itSeamVertex1.first->second).idxFace = edge.i; + patch00.proj = faceTexcoords[edge.i*3+vs0[0]]+offset0; ASSERT(patch10.edges.Find(itSeamVertex0.first->second) == NO_ID); - patch10.edges.AddConstruct(itSeamVertex0.first->second).idxFace = pEdge->i; - patch10.proj = faceTexcoords[pEdge->i*3+vs0[1]]+offset0; + patch10.edges.emplace_back(itSeamVertex0.first->second).idxFace = edge.i; + patch10.proj = faceTexcoords[edge.i*3+vs0[1]]+offset0; } if (idxPatch1 < numPatches) { const TexCoord offset1(texturePatches[idxPatch1].rect.tl()); SeamVertex::Patch& patch01 = seamVertex0.GetPatch(idxPatch1); SeamVertex::Patch& patch11 = seamVertex1.GetPatch(idxPatch1); ASSERT(patch01.edges.Find(itSeamVertex1.first->second) == NO_ID); - patch01.edges.AddConstruct(itSeamVertex1.first->second).idxFace = pEdge->j; - patch01.proj = faceTexcoords[pEdge->j*3+vs1[0]]+offset1; + patch01.edges.emplace_back(itSeamVertex1.first->second).idxFace = edge.j; + patch01.proj = faceTexcoords[edge.j*3+vs1[0]]+offset1; ASSERT(patch11.edges.Find(itSeamVertex0.first->second) == NO_ID); - patch11.edges.AddConstruct(itSeamVertex0.first->second).idxFace = pEdge->j; - patch11.proj = faceTexcoords[pEdge->j*3+vs1[1]]+offset1; + patch11.edges.emplace_back(itSeamVertex0.first->second).idxFace = edge.j; + patch11.proj = faceTexcoords[edge.j*3+vs1[1]]+offset1; } } seamEdges.Release(); @@ -1163,9 +1464,9 @@ void MeshTexture::GlobalSeamLeveling() // vertex is part of multiple patches const SeamVertex& seamVertex = seamVertices[patchIndex.idxSeamVertex]; ASSERT(seamVertex.idxVertex == i); - FOREACHPTR(pPatch, seamVertex.patches) { - ASSERT(pPatch->idxPatch != numPatches); - vertpatch2row[pPatch->idxPatch] = rowsX++; + for (const SeamVertex::Patch& patch: seamVertex.patches) { + ASSERT(patch.idxPatch != numPatches); + vertpatch2row[patch.idxPatch] = rowsX++; } } else if (patchIndex.idxPatch < numPatches) { @@ -1188,8 +1489,7 @@ void MeshTexture::GlobalSeamLeveling() if (idxPatch == numPatches) continue; const MatIdx col(vertpatch2rows[v].at(idxPatch)); - FOREACHPTR(pAdjVert, adjVerts) { - const VIndex vAdj(*pAdjVert); + for (const VIndex vAdj: adjVerts) { if (v >= vAdj) continue; VertexPatchIterator itVAdj(patchIndices[vAdj], seamVertices); @@ -1197,8 +1497,8 @@ void MeshTexture::GlobalSeamLeveling() const uint32_t idxPatchAdj(itVAdj); if (idxPatch == idxPatchAdj) { const MatIdx colAdj(vertpatch2rows[vAdj].at(idxPatchAdj)); - rows.AddConstruct(rowsGamma, col, lambda); - rows.AddConstruct(rowsGamma, colAdj, -lambda); + rows.emplace_back(rowsGamma, col, lambda); + rows.emplace_back(rowsGamma, colAdj, -lambda); ++rowsGamma; } } @@ -1215,8 +1515,7 @@ void MeshTexture::GlobalSeamLeveling() IndexArr indices; Colors vertexColors; Colors coeffB; - FOREACHPTR(pSeamVertex, seamVertices) { - const SeamVertex& seamVertex = *pSeamVertex; + for (const SeamVertex& seamVertex: seamVertices) { if (seamVertex.patches.GetSize() < 2) continue; seamVertex.SortByPatchIndex(indices); @@ -1225,8 +1524,8 @@ void MeshTexture::GlobalSeamLeveling() const SeamVertex::Patch& patch0 = seamVertex.patches[indices[i]]; ASSERT(patch0.idxPatch < numPatches); SampleImage sampler(images[texturePatches[patch0.idxPatch].label].image); - FOREACHPTR(pEdge, patch0.edges) { - const SeamVertex& seamVertex1 = seamVertices[pEdge->idxSeamVertex]; + for (const SeamVertex::Patch::Edge& edge: patch0.edges) { + const SeamVertex& seamVertex1 = seamVertices[edge.idxSeamVertex]; const SeamVertex::Patches::IDX idxPatch1(seamVertex1.patches.Find(patch0.idxPatch)); ASSERT(idxPatch1 != SeamVertex::Patches::NO_INDEX); const SeamVertex::Patch& patch1 = seamVertex1.patches[idxPatch1]; @@ -1247,8 +1546,8 @@ void MeshTexture::GlobalSeamLeveling() const MatIdx rowA((MatIdx)coeffB.GetSize()); coeffB.Insert(color1 - color0); ASSERT(ISFINITE(coeffB.Last())); - rows.AddConstruct(rowA, col0, 1.f); - rows.AddConstruct(rowA, col1, -1.f); + rows.emplace_back(rowA, col0, 1.f); + rows.emplace_back(rowA, col1, -1.f); } } } @@ -1279,7 +1578,7 @@ void MeshTexture::GlobalSeamLeveling() #endif for (int channel=0; channel<3; ++channel) { // init right hand side vector - const Eigen::Map< Eigen::VectorXf, Eigen::Unaligned, Eigen::Stride<0,3> > b(coeffB.Begin()->ptr()+channel, rowsA); + const Eigen::Map< Eigen::VectorXf, Eigen::Unaligned, Eigen::Stride<0,3> > b(coeffB.front().ptr()+channel, rowsA); const Eigen::VectorXf Rhs(SparseMat(A.transpose()) * b); // solve for x const Eigen::VectorXf x(solver.solve(Rhs)); @@ -1303,9 +1602,18 @@ void MeshTexture::GlobalSeamLeveling() ColorMap imageAdj(texturePatch.rect.size()); imageAdj.memset(0); // interpolate color adjustments over the whole patch - RasterPatchColorData data(imageAdj); - FOREACHPTR(pIdxFace, texturePatch.faces) { - const FIndex idxFace(*pIdxFace); + struct RasterPatch { + const TexCoord* tri; + Color colors[3]; + ColorMap& image; + inline RasterPatch(ColorMap& _image) : image(_image) {} + inline cv::Size Size() const { return image.size(); } + inline void operator()(const ImageRef& pt, const Point3f& bary) { + ASSERT(image.isInside(pt)); + image(pt) = colors[0]*bary.x + colors[1]*bary.y + colors[2]*bary.z; + } + } data(imageAdj); + for (const FIndex idxFace: texturePatch.faces) { const Face& face = faces[idxFace]; data.tri = faceTexcoords.Begin()+idxFace*3; for (int v=0; v<3; ++v) @@ -1569,7 +1877,7 @@ void MeshTexture::PoissonBlending(const Image32F3& src, Image32F3& dst, const Im case border: { const MatIdx idx(indices(i)); ASSERT(idx != -1); - coeffA.AddConstruct(idx, idx, 1.f); + coeffA.emplace_back(idx, idx, 1.f); coeffB[idx] = (const Color&)dst(i); } break; case interior: { @@ -1580,11 +1888,11 @@ void MeshTexture::PoissonBlending(const Image32F3& src, Image32F3& dst, const Im const MatIdx idxDown(indices(i + width)); // all indices should be either border conditions or part of the optimization ASSERT(idxUp != -1 && idxLeft != -1 && idxCenter != -1 && idxRight != -1 && idxDown != -1); - coeffA.AddConstruct(idxCenter, idxUp, 1.f); - coeffA.AddConstruct(idxCenter, idxLeft, 1.f); - coeffA.AddConstruct(idxCenter, idxCenter,-4.f); - coeffA.AddConstruct(idxCenter, idxRight, 1.f); - coeffA.AddConstruct(idxCenter, idxDown, 1.f); + coeffA.emplace_back(idxCenter, idxUp, 1.f); + coeffA.emplace_back(idxCenter, idxLeft, 1.f); + coeffA.emplace_back(idxCenter, idxCenter,-4.f); + coeffA.emplace_back(idxCenter, idxRight, 1.f); + coeffA.emplace_back(idxCenter, idxDown, 1.f); // set target coefficient coeffB[idxCenter] = (bias == 1.f ? ColorLaplacian(src,i) : @@ -1607,7 +1915,7 @@ void MeshTexture::PoissonBlending(const Image32F3& src, Image32F3& dst, const Im #endif ASSERT(solver.info() == Eigen::Success); for (int channel=0; channel<3; ++channel) { - const Eigen::Map< Eigen::VectorXf, Eigen::Unaligned, Eigen::Stride<0,3> > b(coeffB.Begin()->ptr()+channel, nnz); + const Eigen::Map< Eigen::VectorXf, Eigen::Unaligned, Eigen::Stride<0,3> > b(coeffB.front().ptr()+channel, nnz); const Eigen::VectorXf x(solver.solve(b)); ASSERT(solver.info() == Eigen::Success); for (int i = 0; i < n; ++i) { @@ -1620,8 +1928,8 @@ void MeshTexture::PoissonBlending(const Image32F3& src, Image32F3& dst, const Im void MeshTexture::LocalSeamLeveling() { - ASSERT(!seamVertices.IsEmpty()); - const unsigned numPatches(texturePatches.GetSize()-1); + ASSERT(!seamVertices.empty()); + const unsigned numPatches(texturePatches.size()-1); // adjust texture patches locally, so that the border continues smoothly inside the patch #ifdef TEXOPT_USE_OPENMP @@ -1631,27 +1939,32 @@ void MeshTexture::LocalSeamLeveling() for (unsigned i=0; iidxSeamVertex]; + const SeamVertex& seamVertex1 = seamVertices[edge0.idxSeamVertex]; const uint32_t idxVertPatch0Adj(seamVertex1.patches.Find(idxPatch)); ASSERT(idxVertPatch0Adj != SeamVertex::Patches::NO_INDEX); const SeamVertex::Patch& patch0Adj = seamVertex1.patches[idxVertPatch0Adj]; @@ -1671,7 +1984,7 @@ void MeshTexture::LocalSeamLeveling() if (idxVertPatch1 == idxVertPatch0) continue; const SeamVertex::Patch& patch1 = seamVertex0.patches[idxVertPatch1]; - const uint32_t idxEdge1(patch1.edges.Find(pEdge0->idxSeamVertex)); + const uint32_t idxEdge1(patch1.edges.Find(edge0.idxSeamVertex)); if (idxEdge1 == SeamVertex::Patch::Edges::NO_INDEX) continue; const TexCoord& p1(patch1.proj); @@ -1683,33 +1996,46 @@ void MeshTexture::LocalSeamLeveling() // this is an edge separating two (valid) patches; // draw it on this patch as the mean color of the two patches const Image8U3& image1(images[texturePatches[patch1.idxPatch].label].image); - RasterPatchMeanEdgeData data(image, mask, imageOrg, image1, p0, p0Adj, p1, p1Adj); + struct RasterPatch { + Image32F3& image; + Image8U& mask; + const Image32F3& image0; + const Image8U3& image1; + const TexCoord p0, p0Dir; + const TexCoord p1, p1Dir; + const float length; + const Sampler sampler; + inline RasterPatch(Image32F3& _image, Image8U& _mask, const Image32F3& _image0, const Image8U3& _image1, + const TexCoord& _p0, const TexCoord& _p0Adj, const TexCoord& _p1, const TexCoord& _p1Adj) + : image(_image), mask(_mask), image0(_image0), image1(_image1), + p0(_p0), p0Dir(_p0Adj-_p0), p1(_p1), p1Dir(_p1Adj-_p1), length((float)norm(p0Dir)), sampler() {} + inline void operator()(const ImageRef& pt) { + const float l((float)norm(TexCoord(pt)-p0)/length); + // compute mean color + const TexCoord samplePos0(p0 + p0Dir * l); + const Color color0(image0.sample(sampler, samplePos0)); + const TexCoord samplePos1(p1 + p1Dir * l); + const Color color1(image1.sample(sampler, samplePos1)/255.f); + image(pt) = Color((color0 + color1) * 0.5f); + // set mask edge also + mask(pt) = border; + } + } data(image, mask, imageOrg, image1, p0, p0Adj, p1, p1Adj); Image32F3::DrawLine(p0, p0Adj, data); // skip remaining patches, // as a manifold edge is shared by maximum two face (one in each patch), which we found already break; } } - } - // render the vertices at the patch border meeting neighbor patches - const Sampler sampler; - FOREACHPTR(pSeamVertex, seamVertices) { - const SeamVertex& seamVertex = *pSeamVertex; - if (seamVertex.patches.GetSize() < 2) - continue; - const uint32_t idxVertPatch(seamVertex.patches.Find(idxPatch)); - if (idxVertPatch == SeamVertex::Patches::NO_INDEX) - continue; + // render the vertex at the patch border meeting neighbor patches AccumColor accumColor; // for each patch... - FOREACHPTR(pPatch, seamVertex.patches) { - const SeamVertex::Patch& patch = *pPatch; + for (const SeamVertex::Patch& patch: seamVertex0.patches) { // add its view to the vertex mean color const Image8U3& img(images[texturePatches[patch.idxPatch].label].image); accumColor.Add(img.sample(sampler, patch.proj)/255.f, 1.f); } - const SeamVertex::Patch& thisPatch = seamVertex.patches[idxVertPatch]; - const ImageRef pt(ROUND2INT(thisPatch.proj-offset)); + const ImageRef pt(ROUND2INT(patch0.proj-offset)); image(pt) = accumColor.Normalized(); mask(pt) = border; } @@ -1719,7 +2045,7 @@ void MeshTexture::LocalSeamLeveling() // compute texture patch blending PoissonBlending(imageOrg, image, mask); // apply color correction to the patch image - cv::Mat imagePatch(images[texturePatch.label].image(texturePatch.rect)); + cv::Mat imagePatch(image0(texturePatch.rect)); for (int r=0; r 0) { + constexpr double sigma = 1.5; + Image8U3 blurryTextureDiffuse; + cv::GaussianBlur(textureDiffuse, blurryTextureDiffuse, cv::Size(), sigma); + cv::addWeighted(textureDiffuse, 1+fSharpnessWeight, blurryTextureDiffuse, -fSharpnessWeight, 0, textureDiffuse); + } } } // texture mesh -bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, float fOutlierThreshold, float fRatioDataSmoothness, bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, const IIndexArr& views) +// - minCommonCameras: generate texture patches using virtual faces composed of coplanar triangles sharing at least this number of views (0 - disabled, 3 - good value) +// - fSharpnessWeight: sharpness weight to be applied on the texture (0 - disabled, 0.5 - good value) +bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, + bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight, + const IIndexArr& views) { MeshTexture texture(*this, nResolutionLevel, nMinResolution); // assign the best view to each face { TD_TIMER_STARTD(); - if (!texture.FaceViewSelection(fOutlierThreshold, fRatioDataSmoothness, views)) + if (!texture.FaceViewSelection(minCommonCameras, fOutlierThreshold, fRatioDataSmoothness, views)) return false; DEBUG_EXTRA("Assigning the best view to each face completed: %u faces (%s)", mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str()); } @@ -1938,7 +2262,7 @@ bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, floa // generate the texture image and atlas { TD_TIMER_STARTD(); - texture.GenerateTexture(bGlobalSeamLeveling, bLocalSeamLeveling, nTextureSizeMultiple, nRectPackingHeuristic, colEmpty); + texture.GenerateTexture(bGlobalSeamLeveling, bLocalSeamLeveling, nTextureSizeMultiple, nRectPackingHeuristic, colEmpty, fSharpnessWeight); DEBUG_EXTRA("Generating texture atlas and image completed: %u patches, %u image size (%s)", texture.texturePatches.GetSize(), mesh.textureDiffuse.width(), TD_TIMER_GET_FMT().c_str()); } diff --git a/libs/MVS/SemiGlobalMatcher.cpp b/libs/MVS/SemiGlobalMatcher.cpp index ed6986074..89bcd0e47 100644 --- a/libs/MVS/SemiGlobalMatcher.cpp +++ b/libs/MVS/SemiGlobalMatcher.cpp @@ -596,7 +596,7 @@ void SemiGlobalMatcher::Match(const Scene& scene, IIndex idxImage, IIndex numNei const REAL _scale(scale*0.5); Matrix3x3 _H(H); Matrix4x4 _Q(Q); Image::ScaleStereoRectification(_H, _Q, _scale); - ExportDisparityDataRawFull(String::FormatString("%s_%d.dimap", pairName.c_str(), log2i(ROUND2INT(REAL(1)/_scale))), leftDisparityMap, costMap, Image8U::computeResize(leftImage.GetSize(), _scale), H, _Q, 1); + ExportDisparityDataRawFull(String::FormatString("%s_%d.dimap", pairName.c_str(), LOG2I(ROUND2INT(REAL(1)/_scale))), leftDisparityMap, costMap, Image8U::computeResize(leftImage.GetSize(), _scale), H, _Q, 1); } #endif // initialize @@ -1389,7 +1389,7 @@ SemiGlobalMatcher::Index SemiGlobalMatcher::Disparity2RangeMap(const DisparityMa range.minDisp = -range.maxDisp; numDisp = range.numDisp(); } else { - const Disparity disp(disps.GetMedian()*2); + const Disparity disp(disps.GetMedian()*2); const auto minmax(disps.GetMinMax()); numDisp = (minmax.second-minmax.first)*2; if (numDisp < minNumDisp) { diff --git a/libs/Math/Common.h b/libs/Math/Common.h index 173bd09d3..62fd0b287 100644 --- a/libs/Math/Common.h +++ b/libs/Math/Common.h @@ -23,6 +23,16 @@ #ifndef MATH_TPL #define MATH_TPL GENERAL_TPL #endif -/*----------------------------------------------------------------*/ + +#include "LMFit/lmmin.h" +#include "RobustNorms.h" + + +// D E F I N E S /////////////////////////////////////////////////// + + +// P R O T O T Y P E S ///////////////////////////////////////////// + +using namespace SEACAVE; #endif // __MATH_COMMON_H__ diff --git a/libs/Math/RobustNorms.h b/libs/Math/RobustNorms.h new file mode 100644 index 000000000..e31d05fe4 --- /dev/null +++ b/libs/Math/RobustNorms.h @@ -0,0 +1,169 @@ +/* +* RobustNorms.h +* +* Copyright (c) 2014-2022 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#ifndef _SEACAVE_ROBUSTNORMS_H_ +#define _SEACAVE_ROBUSTNORMS_H_ + + +// I N C L U D E S ///////////////////////////////////////////////// + + +// D E F I N E S /////////////////////////////////////////////////// + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace SEACAVE { + +// M-estimators to be used during nonlinear refinement for increased robustness to outlier residuals: +// new_residual = robustNorm(residual) +namespace RobustNorm { + +template +struct L1 { + inline TYPE operator()(const TYPE& r) const { + return r / (std::sqrt(std::abs(r)) + TYPE(1e-12)); + } +}; + +template +struct Exp { + inline Exp(const TYPE& _threshold) + : sigma(TYPE(-1) / (TYPE(2) * SQUARE(_threshold))) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + return r * std::exp(SQUARE(r)*sigma); + } + const TYPE sigma; +}; + +template +struct BlakeZisserman { + inline BlakeZisserman(const TYPE& _threshold) + : threshold(_threshold), invThreshold(TYPE(1)/threshold), eps(std::exp(-SQUARE(threshold))) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + return -log(std::exp(-SQUARE(r) * invThreshold) + eps); + } + const TYPE threshold; + const TYPE invThreshold; + const TYPE eps; +}; + +template +struct Huber { + inline Huber(const TYPE& _threshold) + : threshold(_threshold) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + const TYPE dAbs(std::abs(r)); + if (dAbs <= threshold) + return r; + return std::copysign(std::sqrt(threshold * (TYPE(2) * dAbs - threshold)), r); + } + const TYPE threshold; +}; + +template +struct PseudoHuber { + inline PseudoHuber(const TYPE& _threshold) + : threshold(_threshold), thresholdSq(SQUARE(_threshold)), invThreshold(TYPE(1)/threshold) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + return std::sqrt(TYPE(2) * thresholdSq * (std::sqrt(TYPE(1) + SQUARE(r * invThreshold)) - TYPE(1))); + } + const TYPE threshold; + const TYPE thresholdSq; + const TYPE invThreshold; +}; + +template +struct GemanMcClure { + inline GemanMcClure(const TYPE& _threshold) + : thresholdSq(SQUARE(_threshold)) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + return r * std::sqrt(thresholdSq/(thresholdSq+SQUARE(r))); + } + const TYPE thresholdSq; +}; + +template +struct Cauchy { + inline Cauchy(const TYPE& _threshold) + : thresholdSq(SQUARE(_threshold)), invThresholdSq(TYPE(1)/thresholdSq) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + return std::sqrt(thresholdSq*log(TYPE(1)+SQUARE(r)*invThresholdSq)); + } + const TYPE thresholdSq; + const TYPE invThresholdSq; +}; + +template +struct Tukey { + inline Tukey(const TYPE& _threshold) + : threshold(_threshold), thresholdCb6(CUBE(_threshold)/TYPE(6)), invThreshold(TYPE(1)/threshold), sqrtThresholdCb6(std::sqrt(thresholdCb6)) { ASSERT(_threshold > 0); } + inline TYPE operator()(const TYPE& r) const { + return std::abs(r) < threshold ? std::sqrt(thresholdCb6*(TYPE(1)-CUBE(TYPE(1)-SQUARE(r*invThreshold)))) : sqrtThresholdCb6; + } + const TYPE threshold; + const TYPE thresholdCb6; + const TYPE invThreshold; + const TYPE sqrtThresholdCb6; +}; + +} // namespace RobustNorm +/*----------------------------------------------------------------*/ + + +// makes sure the inverse NCC score does not exceed 0.3 +#if 0 +template +inline T robustincc(const T x) { return x / (T(1) + T(3) * x); } +template +inline T robustinccg(const T x) { return T(1)/SQUARE(T(1) + T(3) * x); } +template +inline T unrobustincc(const T y) { return y / (T(1) - T(3) * y); } +#elif 0 +template +inline T robustincc(const T x) { return T(1)-EXP(x*x*T(-4)); } +template +inline T robustinccg(const T x) { return T(8)*x*EXP(x*x*T(-4)); } +template +inline T unrobustincc(const T y) { return SQRT(-LOGN(T(1) - y))/T(2); } +#else +template +inline T robustincc(const T x) { return x/SQRT(T(0.3)+x*x); } +template +inline T robustinccg(const T x) { return T(0.3)/((T(0.3)+x*x)*SQRT(T(0.3)+x*x)); } +template +inline T unrobustincc(const T y) { return (SQRT(30)*y)/(T(10)*SQRT(T(1) - y*y)); } +#endif +/*----------------------------------------------------------------*/ + +} // namespace SEACAVE + +#endif // _SEACAVE_ROBUSTNORMS_H_ diff --git a/libs/Math/RobustNorms.png b/libs/Math/RobustNorms.png new file mode 100644 index 000000000..740fcfbea Binary files /dev/null and b/libs/Math/RobustNorms.png differ diff --git a/libs/Math/SimilarityTransform.cpp b/libs/Math/SimilarityTransform.cpp new file mode 100644 index 000000000..1b5685d95 --- /dev/null +++ b/libs/Math/SimilarityTransform.cpp @@ -0,0 +1,65 @@ +/* +* SimilarityTransform.cpp +* +* Copyright (c) 2014-2022 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#include "Common.h" +#include "SimilarityTransform.h" + + +// D E F I N E S /////////////////////////////////////////////////// + + +// S T R U C T S /////////////////////////////////////////////////// + +// find the similarity transform that best aligns the given two sets of corresponding 3D points +bool SEACAVE::SimilarityTransform(const CLISTDEF0(Point3)& points, const CLISTDEF0(Point3)& pointsRef, Matrix4x4& transform) +{ + ASSERT(points.size() == pointsRef.size()); + typedef Eigen::Matrix PointsVec; + PointsVec p(3, points.size()); + PointsVec pRef(3, pointsRef.size()); + FOREACH(i, points) { + p.col(i) = static_cast(points[i]); + pRef.col(i) = static_cast(pointsRef[i]); + } + transform = Eigen::umeyama(p, pRef); + return true; +} // SimilarityTransform +/*----------------------------------------------------------------*/ + +void SEACAVE::DecomposeSimilarityTransform(const Matrix4x4& transform, Matrix3x3& R, Point3& t, REAL& s) +{ + const Eigen::Transform T(static_cast(transform)); + Eigen::Matrix scaling; + T.computeRotationScaling(&static_cast(R), &scaling); + t = T.translation(); + s = scaling.diagonal().mean(); +} // DecomposeSimilarityTransform +/*----------------------------------------------------------------*/ diff --git a/libs/Math/SimilarityTransform.h b/libs/Math/SimilarityTransform.h new file mode 100644 index 000000000..0c69bfa56 --- /dev/null +++ b/libs/Math/SimilarityTransform.h @@ -0,0 +1,55 @@ +/* +* SimilarityTransform.h +* +* Copyright (c) 2014-2022 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#ifndef _SEACAVE_SIMILARITY_TRANSFORM_H_ +#define _SEACAVE_SIMILARITY_TRANSFORM_H_ + + +// I N C L U D E S ///////////////////////////////////////////////// + + +// D E F I N E S /////////////////////////////////////////////////// + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace SEACAVE { + +// find the similarity transform that best aligns the given two sets of corresponding 3D points +bool SimilarityTransform(const CLISTDEF0(Point3)& points, const CLISTDEF0(Point3)& pointsRef, Matrix4x4& transform); + +// decompose similarity transform into rotation, translation and scale +void DecomposeSimilarityTransform(const Matrix4x4& transform, Matrix3x3& R, Point3& t, REAL& s); +/*----------------------------------------------------------------*/ + +} // namespace SEACAVE + +#endif // _SEACAVE_SIMILARITY_TRANSFORM_H_ diff --git a/libs/Math/TRWS/typeGeneral.h b/libs/Math/TRWS/typeGeneral.h index 4d1d7cd4d..3afd59459 100644 --- a/libs/Math/TRWS/typeGeneral.h +++ b/libs/Math/TRWS/typeGeneral.h @@ -480,20 +480,17 @@ inline TypeGeneral::REAL TypeGeneral::Edge::UpdateMessage(GlobalSize Kglobal, Lo { assert(Ksource.m_K == Kdest.m_K); - int k, kMin; + int k; m_message->m_data[0] = gamma*source->m_data[0] - m_message->m_data[0]; - kMin = 0; vMin = m_message->m_data[0]; for (k=1; km_data[k] = gamma*source->m_data[k] - m_message->m_data[k]; - kMin = 0; vMin = buf->m_data[0]; if (vMin > m_message->m_data[k]) { - kMin = k; vMin = m_message->m_data[k]; } } diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 000000000..17eae3888 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,37 @@ +{ + "name": "openmvs", + "version": "2.1.0", + "description": "OpenMVS: open Multi-View Stereo reconstruction library", + "homepage": "https://cdcseacave.github.io/openMVS", + "dependencies": [ + "boost-iostreams", + "boost-program-options", + "boost-serialization", + "boost-system", + "boost-throw-exception", + { + "name": "cgal", + "default-features": false + }, + "eigen3", + "glew", + "glfw3", + "libpng", + "opencv", + "opengl", + "tiff", + "vcglib", + "zlib" + ], + "features": { + "cuda": { + "description": "CUDA support for OpenMVS", + "dependencies": [ + "cuda" + ] + }, + "openmp": { + "description": "OpenMP support for OpenMVS" + } + } +}