Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Arrow: Refactor auto options #23163

Merged
merged 14 commits into from
Apr 9, 2024
Merged

Arrow: Refactor auto options #23163

Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
7efb057
Arrow: Refactor auto options
danimtb Mar 18, 2024
f13504d
add todo
danimtb Mar 18, 2024
527172d
make boost compulsory in msvc
danimtb Mar 19, 2024
796cc75
enable parquet and thrift by default
danimtb Mar 19, 2024
e368865
bump boost version to avoid conflicts with thrift's
danimtb Mar 19, 2024
79b9991
try with re2
danimtb Mar 19, 2024
7848b47
bool compute
danimtb Mar 19, 2024
596e566
back to previous default values
danimtb Mar 20, 2024
d4ec621
avoid config
danimtb Mar 21, 2024
c71fe29
Merge branch 'master' into feature/arrow_options_refactor
danimtb Mar 21, 2024
c1a716d
fix
danimtb Mar 22, 2024
926e8a0
Arrow: remove dead code (#6)
ericLemanissier Apr 1, 2024
c9d0621
Update recipes/arrow/all/conanfile.py
danimtb Apr 1, 2024
b25d3f5
remove warnings
danimtb Apr 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 43 additions & 172 deletions recipes/arrow/all/conanfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from conan import ConanFile
from conan.errors import ConanInvalidConfiguration
from conan.errors import ConanInvalidConfiguration, ConanException
from conan.tools.build import check_min_cppstd, cross_building
from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, rmdir
Expand Down Expand Up @@ -71,13 +71,13 @@ class ArrowConan(ConanFile):
"shared": False,
"fPIC": True,
"gandiva": False,
"parquet": "auto",
"parquet": False,
danimtb marked this conversation as resolved.
Show resolved Hide resolved
"skyhook": False,
"substrait": False,
"acero": False,
"cli": False,
"compute": "auto",
"dataset_modules": "auto",
"compute": False,
"dataset_modules": False,
"deprecated": True,
"encryption": False,
"filesystem_layer": False,
Expand All @@ -86,29 +86,29 @@ class ArrowConan(ConanFile):
"simd_level": "default",
"runtime_simd_level": "max",
"with_backtrace": False,
"with_boost": "auto",
"with_boost": False,
"with_brotli": False,
"with_bz2": False,
"with_csv": False,
"with_cuda": False,
"with_flight_rpc": "auto",
"with_flight_rpc": False,
"with_flight_sql": False,
"with_gcs": False,
"with_gflags": "auto",
"with_jemalloc": "auto",
"with_gflags": False,
"with_jemalloc": False,
"with_mimalloc": False,
"with_glog": "auto",
"with_grpc": "auto",
"with_glog": False,
"with_grpc": False,
"with_json": False,
"with_thrift": "auto",
"with_llvm": "auto",
"with_openssl": "auto",
"with_thrift": False,
"with_llvm": False,
"with_openssl": False,
"with_opentelemetry": False,
"with_orc": False,
"with_protobuf": "auto",
"with_re2": "auto",
"with_protobuf": False,
"with_re2": False,
"with_s3": False,
"with_utf8proc": "auto",
"with_utf8proc": False,
"with_lz4": False,
"with_snappy": False,
"with_zlib": False,
Expand Down Expand Up @@ -144,35 +144,10 @@ def export_sources(self):
def config_options(self):
if self.settings.os == "Windows":
del self.options.fPIC
if Version(self.version) < "2.0.0":
del self.options.simd_level
del self.options.runtime_simd_level
elif Version(self.version) < "6.0.0":
self.options.simd_level = "sse4_2"
if Version(self.version) < "6.0.0":
del self.options.with_gcs
if Version(self.version) < "7.0.0":
del self.options.skyhook
del self.options.with_flight_sql
del self.options.with_opentelemetry
if Version(self.version) < "8.0.0":
del self.options.substrait

self.options.parquet = self._parquet()
self.options.compute = self._compute()
self.options.dataset_modules = self._dataset_modules()
self.options.with_boost = self._with_boost()
self.options.with_flight_rpc = self._with_flight_rpc()
self.options.with_gflags = self._with_gflags()
self.options.with_glog = self._with_glog()
self.options.with_grpc = self._with_grpc()
self.options.with_jemalloc = self._with_jemalloc()
self.options.with_thrift = self._with_thrift()
self.options.with_llvm = self._with_llvm()
self.options.with_openssl = self._with_openssl()
self.options.with_protobuf = self._with_protobuf()
self.options.with_re2 = self._with_re2()
self.options.with_utf8proc = self._with_utf8proc()
if is_msvc(self):
self.options.with_boost = True

danimtb marked this conversation as resolved.
Show resolved Hide resolved
def configure(self):
if self.options.shared:
Expand All @@ -181,115 +156,8 @@ def configure(self):
def layout(self):
cmake_layout(self, src_folder="src")

def _compute(self):
if self.options.compute == "auto":
return bool(self._parquet() or self._dataset_modules()) or bool(self.options.get_safe("substrait", False))
else:
return bool(self.options.compute)

def _parquet(self):
if self.options.parquet == "auto":
return bool(self.options.get_safe("substrait", False))
else:
return bool(self.options.parquet)

def _dataset_modules(self):
if self.options.dataset_modules == "auto":
return bool(self.options.get_safe("substrait", False))
else:
return bool(self.options.dataset_modules)

def _with_jemalloc(self):
if self.options.with_jemalloc == "auto":
return bool("BSD" in str(self.settings.os))
else:
return bool(self.options.with_jemalloc)

def _with_re2(self):
if self.options.with_re2 == "auto":
if self.options.gandiva or self.options.parquet:
return True
if Version(self) >= "7.0.0" and (self._compute() or self._dataset_modules()):
return True
return False
else:
return bool(self.options.with_re2)

def _with_protobuf(self):
if self.options.with_protobuf == "auto":
return bool(self.options.gandiva or self._with_flight_rpc() or self.options.with_orc or self.options.get_safe("substrait", False))
else:
return bool(self.options.with_protobuf)

def _with_flight_rpc(self):
if self.options.with_flight_rpc == "auto":
return bool(self.options.get_safe("with_flight_sql", False))
else:
return bool(self.options.with_flight_rpc)

def _with_gflags(self):
if self.options.with_gflags == "auto":
return bool(self._with_glog() or self._with_grpc())
else:
return bool(self.options.with_gflags)

def _with_glog(self):
if self.options.with_glog == "auto":
return False
else:
return bool(self.options.with_glog)

def _with_grpc(self):
if self.options.with_grpc == "auto":
return self._with_flight_rpc()
else:
return bool(self.options.with_grpc)

def _with_boost(self):
if self.options.with_boost == "auto":
if self.options.gandiva:
return True
version = Version(self.version)
if version.major == "1":
if self._parquet() and self.settings.compiler == "gcc" and self.settings.compiler.version < Version("4.9"):
return True
elif version.major >= "2":
if is_msvc(self):
return True
return False
else:
return bool(self.options.with_boost)

def _with_thrift(self):
if self.options.with_thrift == "auto":
return bool(self._parquet())
else:
return bool(self.options.with_thrift)

def _with_utf8proc(self):
if self.options.with_utf8proc == "auto":
return bool(self._compute() or self.options.gandiva)
else:
return bool(self.options.with_utf8proc)

def _with_llvm(self):
if self.options.with_llvm == "auto":
return bool(self.options.gandiva)
else:
return bool(self.options.with_llvm)

def _with_openssl(self):
if self.options.with_openssl == "auto":
return bool(self.options.encryption or self._with_flight_rpc() or self.options.with_s3)
else:
return bool(self.options.with_openssl)

def _requires_rapidjson(self):
if self.options.with_json:
return True
if Version(self.version) >= "7.0.0" and self.options.encryption:
return True
return False
return self.options.with_json or self.options.encryption

def requirements(self):
if self.options.with_thrift:
Expand All @@ -301,7 +169,7 @@ def requirements(self):
if self.options.with_mimalloc:
self.requires("mimalloc/1.7.6")
if self.options.with_boost:
self.requires("boost/1.83.0")
self.requires("boost/1.84.0")
if self.options.with_gflags:
self.requires("gflags/2.2.2")
if self.options.with_glog:
Expand Down Expand Up @@ -332,8 +200,7 @@ def requirements(self):
self.requires("lz4/1.9.4")
if self.options.with_snappy:
self.requires("snappy/1.1.9")
if Version(self.version) >= "6.0.0" and \
self.options.get_safe("simd_level") != None or \
if self.options.get_safe("simd_level") != None or \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if self.options.get_safe("simd_level") != None or \
if self.options.get_safe("simd_level") is not None or \

self.options.get_safe("runtime_simd_level") != None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
self.options.get_safe("runtime_simd_level") != None:
self.options.get_safe("runtime_simd_level") is not None:

self.requires("xsimd/9.0.1")
if self.options.with_zlib:
Expand All @@ -348,11 +215,21 @@ def requirements(self):
self.requires("libbacktrace/cci.20210118")

def validate(self):
# validate options with 'auto' as default value
auto_options = ["parquet", "compute", "dataset_modules", "with_boost", "with_flight_rpc", "with_gflags", "with_glog",
"with_grpc", "with_jemalloc", "with_thrift", "with_llvm", "with_openssl", "with_protobuf", "with_re2", "with_utf8proc"]
for option in auto_options:
assert "auto" not in str(self.options.get_safe(option)), f"Option '{option}' contains 'auto' value, wich is not allowed. Generally the final value should be True/False"
# Do not allow options with 'auto' value
# TODO: Remove "auto" from the possible values for these options
auto_options = [option for option, value in self.options.items() if value == "auto"]
if auto_options:
raise ConanException("Options with value 'auto' are deprecated. Please set them true/false or use its default value."
f" Please change the following options: {auto_options}")

# From https://github.com/conan-io/conan-center-index/pull/23163#issuecomment-2039808851
if self.options.gandiva:
if not self.options.with_re2:
raise ConanException("'with_re2' option should be True when'gandiva=True'")
if not self.options.with_boost:
raise ConanException("'with_boost' option should be True when'gandiva=True'")
if not self.options.with_utf8proc:
raise ConanException("'with_utf8proc' option should be True when'gandiva=True'")

if self.settings.compiler.get_safe("cppstd"):
check_min_cppstd(self, self._min_cppstd)
Expand All @@ -376,8 +253,6 @@ def validate(self):
if self.dependencies["jemalloc"].options.enable_cxx:
raise ConanInvalidConfiguration("jemmalloc.enable_cxx of a static jemalloc must be disabled")

if Version(self.version) < "6.0.0" and self.options.get_safe("simd_level") == "default":
raise ConanInvalidConfiguration(f"In {self.ref}, simd_level options is not supported `default` value.")

def build_requirements(self):
if Version(self.version) >= "13.0.0":
Expand Down Expand Up @@ -414,7 +289,7 @@ def generate(self):
tc.variables["ARROW_NO_DEPRECATED_API"] = not bool(self.options.deprecated)
tc.variables["ARROW_FLIGHT"] = self.options.with_flight_rpc
tc.variables["ARROW_FLIGHT_SQL"] = bool(self.options.get_safe("with_flight_sql", False))
tc.variables["ARROW_COMPUTE"] = self.options.compute
tc.variables["ARROW_COMPUTE"] = bool(self.options.compute)
tc.variables["ARROW_CSV"] = bool(self.options.with_csv)
tc.variables["ARROW_CUDA"] = bool(self.options.with_cuda)
tc.variables["ARROW_JEMALLOC"] = self.options.with_jemalloc
Expand Down Expand Up @@ -459,12 +334,9 @@ def generate(self):
tc.variables["ZLIB_SOURCE"] = "SYSTEM"
tc.variables["xsimd_SOURCE"] = "SYSTEM"
tc.variables["ARROW_WITH_ZSTD"] = bool(self.options.with_zstd)
if Version(self.version) >= "2.0":
tc.variables["zstd_SOURCE"] = "SYSTEM"
tc.variables["ARROW_SIMD_LEVEL"] = str(self.options.simd_level).upper()
tc.variables["ARROW_RUNTIME_SIMD_LEVEL"] = str(self.options.runtime_simd_level).upper()
else:
tc.variables["ZSTD_SOURCE"] = "SYSTEM"
tc.variables["zstd_SOURCE"] = "SYSTEM"
tc.variables["ARROW_SIMD_LEVEL"] = str(self.options.simd_level).upper()
tc.variables["ARROW_RUNTIME_SIMD_LEVEL"] = str(self.options.runtime_simd_level).upper()
if self.options.with_zstd:
tc.variables["ARROW_ZSTD_USE_SHARED"] = bool(self.dependencies["zstd"].options.shared)
tc.variables["ORC_SOURCE"] = "SYSTEM"
Expand Down Expand Up @@ -509,7 +381,7 @@ def generate(self):

def _patch_sources(self):
apply_conandata_patches(self)
if "7.0.0" <= Version(self.version) < "10.0.0":
if Version(self.version) < "10.0.0":
for filename in glob.glob(os.path.join(self.source_folder, "cpp", "cmake_modules", "Find*.cmake")):
if os.path.basename(filename) not in [
"FindArrow.cmake",
Expand Down Expand Up @@ -613,9 +485,8 @@ def package_info(self):
self.cpp_info.components["libgandiva"].requires.append("boost::boost")
if self.options.parquet and self.settings.compiler == "gcc" and self.settings.compiler.version < Version("4.9"):
self.cpp_info.components["libparquet"].requires.append("boost::boost")
if Version(self.version) >= "2.0":
# FIXME: only headers components is used
self.cpp_info.components["libarrow"].requires.append("boost::boost")
# FIXME: only headers components is used
self.cpp_info.components["libarrow"].requires.append("boost::boost")
Comment on lines +488 to +489
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# FIXME: only headers components is used
self.cpp_info.components["libarrow"].requires.append("boost::boost")
self.cpp_info.components["libarrow"].requires.append("boost::headers")

if self.options.with_openssl:
self.cpp_info.components["libarrow"].requires.append("openssl::openssl")
if self.options.with_gflags:
Expand Down
Loading