diff --git a/rust/Cargo.lock b/rust/Cargo.lock index d1f80abe16..59ff70724e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -22,6 +22,11 @@ name = "agent" version = "0.1.0" dependencies = [ "ngt", + "prost", + "proto", + "tokio", + "tokio-stream", + "tonic", ] [[package]] @@ -343,6 +348,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "http" version = "0.2.12" @@ -475,6 +486,16 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "matchit" version = "0.7.3" @@ -562,6 +583,16 @@ dependencies = [ "rand", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.32.2" @@ -583,6 +614,29 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" +[[package]] +name = "parking_lot" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.5", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -717,6 +771,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -742,6 +805,12 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "scratch" version = "1.0.7" @@ -768,6 +837,15 @@ dependencies = [ "syn", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.9" @@ -777,6 +855,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "smawk" version = "0.3.2" @@ -891,8 +975,12 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.48.0", ] @@ -906,6 +994,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -915,6 +1014,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -932,9 +1032,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", @@ -959,9 +1059,9 @@ dependencies = [ [[package]] name = "tonic-types" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b39bd850e4bf99146b3fd244019562cafd30338db068c5795c55b448eb02411" +checksum = "f4aa089471d8d4c60ec3aef047739713a4695f0b309d4cea0073bc55201064f4" dependencies = [ "prost", "prost-types", diff --git a/rust/bin/agent/Cargo.toml b/rust/bin/agent/Cargo.toml index 5fc426d196..fdfce09733 100644 --- a/rust/bin/agent/Cargo.toml +++ b/rust/bin/agent/Cargo.toml @@ -22,3 +22,8 @@ edition = "2021" [dependencies] ngt = { version = "0.1.0", path = "../../libs/ngt" } +prost = "0.12.4" +proto = { version = "0.1.0", path = "../../libs/proto" } +tokio = { version = "1.37.0", features = ["full"] } +tokio-stream = { version = "0.1.15", features = ["full"] } +tonic = "0.11.0" diff --git a/rust/bin/agent/src/handler.rs b/rust/bin/agent/src/handler.rs new file mode 100644 index 0000000000..7baa1b6237 --- /dev/null +++ b/rust/bin/agent/src/handler.rs @@ -0,0 +1,27 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +mod common; +mod index; +mod insert; +mod remove; +mod search; +mod update; +mod upsert; + +#[derive(Default, Debug)] +pub struct Agent { + +} diff --git a/rust/bin/agent/src/handler/common.rs b/rust/bin/agent/src/handler/common.rs new file mode 100644 index 0000000000..3e251cfc95 --- /dev/null +++ b/rust/bin/agent/src/handler/common.rs @@ -0,0 +1,21 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#[macro_export] +macro_rules! stream_type { + ($t:ty) => { + std::pin::Pin> + Send>> + }; +} diff --git a/rust/bin/agent/src/handler/index.rs b/rust/bin/agent/src/handler/index.rs new file mode 100644 index 0000000000..af0ac55d01 --- /dev/null +++ b/rust/bin/agent/src/handler/index.rs @@ -0,0 +1,60 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +use proto::{ + core::v1::agent_server, + payload::v1::{control, info, object, Empty}, +}; + +#[tonic::async_trait] +impl agent_server::Agent for super::Agent { + async fn create_index( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + async fn save_index( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Represent the creating and saving index RPC.\n"] + async fn create_and_save_index( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Represent the RPC to get the agent index information.\n"] + async fn index_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Represent the RPC to get the vector metadata. This RPC is mainly used for index correction process\n"] + async fn get_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/insert.rs b/rust/bin/agent/src/handler/insert.rs new file mode 100644 index 0000000000..e0073a0818 --- /dev/null +++ b/rust/bin/agent/src/handler/insert.rs @@ -0,0 +1,47 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +use proto::{ + payload::v1::{insert, object}, + vald::v1::insert_server, +}; +#[tonic::async_trait] +impl insert_server::Insert for super::Agent { + async fn insert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamInsert method."] + type StreamInsertStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to add new multiple vectors by bidirectional streaming.\n"] + async fn stream_insert( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to add new multiple vectors in a single request.\n"] + async fn multi_insert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/remove.rs b/rust/bin/agent/src/handler/remove.rs new file mode 100644 index 0000000000..803f33564d --- /dev/null +++ b/rust/bin/agent/src/handler/remove.rs @@ -0,0 +1,56 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +use proto::{ + payload::v1::{object, remove}, + vald::v1::remove_server, +}; + +#[tonic::async_trait] +impl remove_server::Remove for super::Agent { + async fn remove( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to remove an indexed vector based on timestamp.\n"] + async fn remove_by_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamRemove method."] + type StreamRemoveStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to remove multiple indexed vectors by bidirectional streaming.\n"] + async fn stream_remove( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to remove multiple indexed vectors in a single request.\n"] + async fn multi_remove( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/search.rs b/rust/bin/agent/src/handler/search.rs new file mode 100644 index 0000000000..d522d456ae --- /dev/null +++ b/rust/bin/agent/src/handler/search.rs @@ -0,0 +1,127 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +use proto::{payload::v1::search, vald::v1::search_server}; + +#[tonic::async_trait] +impl search_server::Search for super::Agent { + async fn search( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + todo!() + } + + #[doc = " A method to search indexed vectors by ID.\n"] + async fn search_by_id( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamSearch method."] + type StreamSearchStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to search indexed vectors by multiple vectors.\n"] + async fn stream_search( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamSearchByID method."] + type StreamSearchByIDStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to search indexed vectors by multiple IDs.\n"] + async fn stream_search_by_id( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to search indexed vectors by multiple vectors in a single request.\n"] + async fn multi_search( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to search indexed vectors by multiple IDs in a single request.\n"] + async fn multi_search_by_id( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to linear search indexed vectors by a raw vector.\n"] + async fn linear_search( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to linear search indexed vectors by ID.\n"] + async fn linear_search_by_id( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamLinearSearch method."] + type StreamLinearSearchStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to linear search indexed vectors by multiple vectors.\n"] + async fn stream_linear_search( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamLinearSearchByID method."] + type StreamLinearSearchByIDStream = crate::stream_type!(search::StreamResponse); + + #[doc = " A method to linear search indexed vectors by multiple IDs.\n"] + async fn stream_linear_search_by_id( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> + { + todo!() + } + + #[doc = " A method to linear search indexed vectors by multiple vectors in a single\n request.\n"] + async fn multi_linear_search( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to linear search indexed vectors by multiple IDs in a single\n request.\n"] + async fn multi_linear_search_by_id( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/update.rs b/rust/bin/agent/src/handler/update.rs new file mode 100644 index 0000000000..c738a3dfd5 --- /dev/null +++ b/rust/bin/agent/src/handler/update.rs @@ -0,0 +1,48 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +use proto::{ + payload::v1::{object, update}, + vald::v1::update_server, +}; + +#[tonic::async_trait] +impl update_server::Update for super::Agent { + async fn update( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamUpdate method."] + type StreamUpdateStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to update multiple indexed vectors by bidirectional streaming.\n"] + async fn stream_update( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to update multiple indexed vectors in a single request.\n"] + async fn multi_update( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/handler/upsert.rs b/rust/bin/agent/src/handler/upsert.rs new file mode 100644 index 0000000000..8191d736c4 --- /dev/null +++ b/rust/bin/agent/src/handler/upsert.rs @@ -0,0 +1,48 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +use proto::{ + payload::v1::{object, upsert}, + vald::v1::upsert_server, +}; + +#[tonic::async_trait] +impl upsert_server::Upsert for super::Agent { + async fn upsert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " Server streaming response type for the StreamUpsert method."] + type StreamUpsertStream = crate::stream_type!(object::StreamLocation); + + #[doc = " A method to insert/update multiple vectors by bidirectional streaming.\n"] + async fn stream_upsert( + &self, + request: tonic::Request>, + ) -> std::result::Result, tonic::Status> { + todo!() + } + + #[doc = " A method to insert/update multiple vectors in a single request.\n"] + async fn multi_upsert( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } +} diff --git a/rust/bin/agent/src/main.rs b/rust/bin/agent/src/main.rs index 7546cec810..5ef03c184d 100644 --- a/rust/bin/agent/src/main.rs +++ b/rust/bin/agent/src/main.rs @@ -13,6 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. // -fn main() { - println!("Hello, world!"); + +mod handler; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let addr = "[::1]:8081".parse()?; + let agent = handler::Agent::default(); + + tonic::transport::Server::builder() + .add_service(proto::core::v1::agent_server::AgentServer::new(agent)) + .serve(addr) + .await?; + + Ok(()) } diff --git a/rust/libs/proto/Cargo.toml b/rust/libs/proto/Cargo.toml index d185dbf655..72ea9d88f0 100644 --- a/rust/libs/proto/Cargo.toml +++ b/rust/libs/proto/Cargo.toml @@ -23,5 +23,5 @@ edition = "2021" [dependencies] futures-core = "0.3.30" prost = "0.12.3" -tonic = "0.10.2" -tonic-types = "0.10.2" +tonic = "0.11.0" +tonic-types = "0.11.0" diff --git a/rust/libs/proto/src/lib.rs b/rust/libs/proto/src/lib.rs index a25f31ae09..b9b1d0bac0 100644 --- a/rust/libs/proto/src/lib.rs +++ b/rust/libs/proto/src/lib.rs @@ -31,8 +31,8 @@ pub mod vald { } } -#[cfg(test)] -mod tests { - #[test] - fn it_works() {} +pub mod core { + pub mod v1 { + include!("core.v1.tonic.rs"); + } }