diff --git a/cpp/src/arrow/filesystem/s3_test_cert.h b/cpp/src/arrow/filesystem/s3_test_cert.h new file mode 100644 index 0000000000000..689c0447880fd --- /dev/null +++ b/cpp/src/arrow/filesystem/s3_test_cert.h @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace arrow { +namespace fs { + +const char* kMinioPrivateKey = R"(-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCqwKYHsTSciGqP +uU3qkTWpnXIi3iC0eeW7JSzJHGFs880WdR5JdK4WufPK+1xzgiYjMEPfAcuSWz3b +qYyCI61q+a9Iu2nj7cFTW9bfZrmWlnI0YOLJc+q0AAdAjF1lvRKenH8tbjz/2jyl +i/cYQ+I5Tg4nngrX8OmOfluNzwD/nwGLq6/DVbzDUdPI9q1XtVT/0Vf7qwbDG1HD +NkIzKT5B+YdSLaOCRYNK3x7RPsfazKIBrTmRy1v454wKe8TjTmTB7+m5wKqfCJcq +lI253WHcK0lsw6zCNtX/kahPAvm/8mniPolW4qxoD6xwebgMVkrNTs3ztcPIG9O4 +pmCbATijAgMBAAECggEACL5swiAU7Z8etdVrZAOjl9f0LEzrp9JGLVst++50Hrwt +WGUO8/wBnjBPh6lvhoq3oT2rfBP/dLMva7w28cMZ8kxu6W6PcZiPOdGOI0qDXm69 +0mjTtDU3Y5hMxsVpUvhnp6+j45Otk/x89o1ATgHL59tTZjv1mjFABIf78DsVdgF9 +CMi2q6Lv7NLftieyWmz1K3p109z9+xkDNSOkVrv1JFChviKqWgIS0rdFjySvTgoy +rHYT+TweDliKJrZCeoUJmNB0uVW/dM9lXhcvkvkJZKPPurylx1oH5a7K/sWFPf7A +Ed1vjvZQFlaXu/bOUUSOZtkErAir/oCxrUDsHxGsAQKBgQDZghyy7jNGNdjZe1Xs +On1ZVgIS3Nt+OLGCVH7tTsfZsCOb+SkrhB1RQva3YzPMfgoZScI9+bN/pRVf49Pj +qGEHkW/wozutUve7UMzeTOm1aWxUuaKSrmYST7muvAnlYEtO7agd0wrcusYXlMoG +KQwghkufO9I7wXcrudMKXZalIwKBgQDI+FaUwhgfThkgq6bRbdMEeosgohrCM9Wm +E5JMePQq4VaGcgGveWUoNOgT8kvJa0qQwQOqLZj7kUIdj+SCRt0u+Wu3p5IMqdOq +6tMnLNQ3wzUC2KGFLSfISR3L/bo5Bo6Jqz4hVtjMk3PV9bu50MNTNaofYb2xlf/f +/WgiEG0WgQKBgAr8RVLMMQ7EvXUOg6Jwuc//Rg+J1BQl7OE2P0rhBbr66HGCPhAS +liB6j1dnzT/wxbXNQeA7clNqFRBIw3TmFjB5qfuvYt44KIbvZ8l6fPtKncwRrCJY +aJNYL3qhyKYrHOKZojoPZKcNT9/1BdcVz6T842jhbpbSCKDOu9f0Lh2dAoGATZeM +Hh0eISAPFY0QeDV1znnds3jC6g4HQ/q0dnAQnWmo9XmY6v3sr2xV2jWnSxnwjRjo +aFD4itBXfYBr0ly30wYbr6mz+s2q2oeVhL+LJAhrNDEdk4SOooaQSY0p1BCTAdYq +w8Z7J+kaRRZ+J0zRzROgHkOncKQgSYPWK6i55YECgYAC+ECrHhUlPsfusjKpFsEe +stW1HCt3wXtKQn6SJ6IAesbxwALZS6Da/ZC2x1mdBHS3GwWvtGLc0BPnPVfJjr9V +m82qkgJ+p5d7qp7pRA7SFD+5809yVqRnEF3rSLafgGet9ah0ZjZvQ3fwnYZNnNH9 +t9pJcv2E5xY7/nFNIorpKg== +-----END PRIVATE KEY----- +)"; +const char* kMinioPublicCert = R"(-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIUXbHZ6FAhKSXg4WSGUQySlSyE4U0wDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMQ4wDAYDVQQHDAVBcnJvdzEO +MAwGA1UECgwFQXJyb3cxDjAMBgNVBAsMBUFycm93MRMwEQYDVQQDDApBcnJyb3dU +ZXN0MB4XDTI0MDkyNDA5MzUxNloXDTM0MDkyMjA5MzUxNlowXzELMAkGA1UEBhMC +VVMxCzAJBgNVBAgMAlZBMQ4wDAYDVQQHDAVBcnJvdzEOMAwGA1UECgwFQXJyb3cx +DjAMBgNVBAsMBUFycm93MRMwEQYDVQQDDApBcnJyb3dUZXN0MIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqsCmB7E0nIhqj7lN6pE1qZ1yIt4gtHnluyUs +yRxhbPPNFnUeSXSuFrnzyvtcc4ImIzBD3wHLkls926mMgiOtavmvSLtp4+3BU1vW +32a5lpZyNGDiyXPqtAAHQIxdZb0Snpx/LW48/9o8pYv3GEPiOU4OJ54K1/Dpjn5b +jc8A/58Bi6uvw1W8w1HTyPatV7VU/9FX+6sGwxtRwzZCMyk+QfmHUi2jgkWDSt8e +0T7H2syiAa05kctb+OeMCnvE405kwe/pucCqnwiXKpSNud1h3CtJbMOswjbV/5Go +TwL5v/Jp4j6JVuKsaA+scHm4DFZKzU7N87XDyBvTuKZgmwE4owIDAQABoz0wOzAa +BgNVHREEEzARhwR/AAABgglsb2NhbGhvc3QwHQYDVR0OBBYEFOUNqUSfROf1dz3o +hAVBhgd3UIvKMA0GCSqGSIb3DQEBCwUAA4IBAQBSwWJ2dSw3jlHU0l2V3ozqthTt +XFo07AyWGw8AWNCM6mQ+GKBf0JJ1d7e4lyTf2lCobknS94EgGPORWeiucKYAoCjS +dh1eKGsSevz1rNbp7wsO7DoiRPciK+S95DbsPowloGI6fvOeE12Cf1udeNIpEYWs +OBFwN0HxfYqdPALCtw7l0icpTrJ2Us06UfL9kbkdZwQhXvOscG7JDRtNjBxl9XNm +TFeMNKROmrEPCWaYr6MJ+ItHtb5Cawapea4THz9GCjR9eLq2CbMqLezZ8xBHPzc4 +ixI2l0uCfg7ZUSA+90yaScc7bhEQ8CMiPtJgNKaKIqB58DpY7028xJpW7Ma2 +-----END CERTIFICATE----- +)"; +} // namespace fs +} // namespace arrow diff --git a/cpp/src/arrow/filesystem/s3_test_util.cc b/cpp/src/arrow/filesystem/s3_test_util.cc index db0c60f2e80f2..554b14f26e713 100644 --- a/cpp/src/arrow/filesystem/s3_test_util.cc +++ b/cpp/src/arrow/filesystem/s3_test_util.cc @@ -19,6 +19,7 @@ # include #endif +#include "arrow/filesystem/s3_test_cert.h" #include "arrow/filesystem/s3_test_util.h" #include "arrow/filesystem/s3fs.h" #include "arrow/testing/process.h" @@ -69,7 +70,27 @@ std::string MinioTestServer::access_key() const { return impl_->access_key_; } std::string MinioTestServer::secret_key() const { return impl_->secret_key_; } -Status MinioTestServer::Start() { +Status MinioTestServer::GenerateCertificateFile() { + PlatformFilename public_crt_file, private_key_file; + ASSERT_OK_AND_ASSIGN( + public_crt_file, + PlatformFilename::FromString(impl_->temp_dir_->path().ToString() + "/public.crt")); + ASSERT_OK_AND_ASSIGN(FileDescriptor public_cert_fd, FileOpenWritable(public_crt_file)); + ASSERT_OK(FileWrite(public_cert_fd, reinterpret_cast(kMinioPublicCert), + strlen(kMinioPublicCert)); + ASSERT_OK(FileClose(public_cert_fd)); + + ASSERT_OK_AND_ASSIGN( + private_key_file, + PlatformFilename::FromString(impl_->temp_dir_->path().ToString() + "/private.key")); + ASSERT_OK_AND_ASSIGN(FileDescriptor private_key_fd, FileOpenWritable(private_key_file)); + ASSERT_OK(FileWrite(private_key_fd, reinterpret_cast(kMinioPrivateKey), + strlen(kMinioPrivateKey)); + ASSERT_OK(FileClose(private_key_fd)); + return Status::OK(); +} + +Status MinioTestServer::Start(bool enable_tls) { const char* connect_str = std::getenv(kEnvConnectString); const char* access_key = std::getenv(kEnvAccessKey); const char* secret_key = std::getenv(kEnvSecretKey); @@ -91,9 +112,18 @@ Status MinioTestServer::Start() { impl_->connect_string_ = GenerateConnectString(); ARROW_RETURN_NOT_OK(impl_->server_process_->SetExecutable(kMinioExecutableName)); // NOTE: --quiet makes startup faster by suppressing remote version check - impl_->server_process_->SetArgs({"server", "--quiet", "--compat", "--address", - impl_->connect_string_, - impl_->temp_dir_->path().ToString()}); + std::vector start_args = {"server", + "--quiet", + "--compat", + "--address", + impl_->connect_string_, + impl_->temp_dir_->path().ToString()}; + if (enable_tls) { + ARROW_RETURN_NOT_OK(GenerateCertificateFile()); + start_args.push_back("--certs-dir"); + start_args.push_back(impl_->temp_dir_->path().ToString()); + } + impl_->server_process_->SetArgs(start_args); ARROW_RETURN_NOT_OK(impl_->server_process_->Execute()); return Status::OK(); } diff --git a/cpp/src/arrow/filesystem/s3_test_util.h b/cpp/src/arrow/filesystem/s3_test_util.h index e270a6e1c469a..7348ba096149d 100644 --- a/cpp/src/arrow/filesystem/s3_test_util.h +++ b/cpp/src/arrow/filesystem/s3_test_util.h @@ -40,7 +40,7 @@ class MinioTestServer { MinioTestServer(); ~MinioTestServer(); - Status Start(); + Status Start(bool enable_tls = true); Status Stop(); @@ -51,6 +51,7 @@ class MinioTestServer { std::string secret_key() const; private: + Status GenerateCertificateFile(); struct Impl; std::unique_ptr impl_; }; diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index d7f1bf8c89b19..d7f96d96d3b86 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -206,7 +206,8 @@ class S3TestMixin : public AwsTestMixin { ARROW_ASSIGN_OR_RAISE(minio_, GetMinioEnv()->GetOneServer()); client_config_.reset(new Aws::Client::ClientConfiguration()); client_config_->endpointOverride = ToAwsString(minio_->connect_string()); - client_config_->scheme = Aws::Http::Scheme::HTTP; + client_config_->scheme = Aws::Http::Scheme::HTTPS; + client_config_->verifySSL = false; client_config_->retryStrategy = std::make_shared(kRetryInterval, kMaxRetryDuration); credentials_ = {ToAwsString(minio_->access_key()), ToAwsString(minio_->secret_key())}; @@ -531,7 +532,7 @@ class TestS3FS : public S3TestMixin { } Result> MakeNewFileSystem( - io::IOContext io_context = io::default_io_context(), bool use_https = false) { + io::IOContext io_context = io::default_io_context(), bool use_https = true) { options_.ConfigureAccessKey(minio_->access_key(), minio_->secret_key()); options_.scheme = use_https ? "https" : "http"; options_.endpoint_override = minio_->connect_string(); @@ -541,7 +542,7 @@ class TestS3FS : public S3TestMixin { return S3FileSystem::Make(options_, io_context); } - void MakeFileSystem(bool use_https = false) { + void MakeFileSystem(bool use_https = true) { ASSERT_OK_AND_ASSIGN(fs_, MakeNewFileSystem(io::default_io_context(), use_https)); } @@ -1295,9 +1296,7 @@ TEST_F(TestS3FS, SSECustomerKeyMatch) { // normal write/read with correct SSEC key std::shared_ptr stream; options_.sse_customer_key = "12345678123456781234567812345678"; - MakeFileSystem(true); // need to use https, otherwise get 'InvalidRequest Message: - // Requests specifying Server Side Encryption with Customer - // provided keys must be made over a secure connection.' + MakeFileSystem(); ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile_with_sse_c")); ASSERT_OK(stream->Write("some")); ASSERT_OK(stream->Close()); @@ -1310,7 +1309,7 @@ TEST_F(TestS3FS, SSECustomerKeyMatch) { TEST_F(TestS3FS, SSECustomerKeyMismatch) { std::shared_ptr stream; options_.sse_customer_key = "12345678123456781234567812345678"; - MakeFileSystem(true); + MakeFileSystem(); ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile_with_sse_c")); ASSERT_OK(stream->Write("some")); ASSERT_OK(stream->Close()); @@ -1443,7 +1442,7 @@ TEST_F(TestS3FS, FileSystemFromUri) { std::stringstream ss; ss << "s3://" << minio_->access_key() << ":" << minio_->secret_key() << "@bucket/somedir/subdir/subfile" - << "?scheme=http&endpoint_override=" << UriEscape(minio_->connect_string()); + << "?scheme=https&endpoint_override=" << UriEscape(minio_->connect_string()); std::string path; ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(ss.str(), &path)); @@ -1545,7 +1544,7 @@ class TestS3FSGeneric : public S3TestMixin, public GenericFileSystemTest { } options_.ConfigureAccessKey(minio_->access_key(), minio_->secret_key()); - options_.scheme = "http"; + options_.scheme = "https"; options_.endpoint_override = minio_->connect_string(); options_.retry_strategy = std::make_shared(); ASSERT_OK_AND_ASSIGN(s3fs_, S3FileSystem::Make(options_));