Skip to content

Commit

Permalink
add inner product
Browse files Browse the repository at this point in the history
  • Loading branch information
@masajiro
masajiro committed Apr 10, 2024
1 parent 441aa19 commit 6237d1d
Show file tree
Hide file tree
Showing 34 changed files with 3,222 additions and 1,156 deletions.
4 changes: 4 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@ if(${UNIX})
endif()
endif()

if(${NGT_BFLOAT_DISABLED})
message(STATUS "bfloat is disabled.")
endif()

add_subdirectory("${PROJECT_SOURCE_DIR}/lib")
add_subdirectory("${PROJECT_SOURCE_DIR}/bin")
add_subdirectory("${PROJECT_SOURCE_DIR}/samples")
Expand Down
2 changes: 1 addition & 1 deletion VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2.1.6
2.2.0
4 changes: 2 additions & 2 deletions bin/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ if(${UNIX})
include_directories("${PROJECT_SOURCE_DIR}/lib" "${PROJECT_BINARY_DIR}/lib/")
link_directories("${PROJECT_BINARY_DIR}/lib/NGT")
add_subdirectory("${PROJECT_SOURCE_DIR}/bin/ngt")
if(NOT ${NGT_SHARED_MEMORY_ALLOCATOR})
if(NOT DEFINED NGT_SHARED_MEMORY_ALLOCATOR OR (NOT ${NGT_SHARED_MEMORY_ALLOCATOR}))
add_subdirectory("${PROJECT_SOURCE_DIR}/bin/qbg")
endif()
endif()
endif()
4 changes: 4 additions & 0 deletions bin/ngt/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ Specify the distance function as follows.
- __j__: Jaccard distance. 1 byte unsigned integer should be specified for the data object type.
- __p__: Poincare distance
- __l__: Lorentz distance
- __i__: Inner product (Dot product)

**-n** *no\_of\_registration\_data*
Specify the number of data items to be registered. If not specified, all data in the specified file will be registered.
Expand Down Expand Up @@ -232,6 +233,9 @@ Specify the mode of the search parameter optimization.
**-I** *graph_type*
Specify the type of the specified index as input_index. For not ANNG, the index is converted to ANNG before graph reconstruction.
- __a__: ANNG
- __r__: RANNG. ANNG with reduced shortcut edges.
- __i__: IANNG. ANNG with reduced excess edges.
- __R__: RIANNG. ANNG with reduced shortcut or excess edges.
- __o__: The others

Examples of using ngt command
Expand Down
141 changes: 30 additions & 111 deletions lib/NGT/Capi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ NGTIndex ngt_create_graph_and_tree_in_memory(NGTProperty prop, NGTError error) {
return NULL;
#else
try{
NGT::Index *index = new NGT::GraphAndTreeIndex(*(static_cast<NGT::Property*>(prop)));
NGT::Index *index = new NGT::Index(*(static_cast<NGT::Property*>(prop)));
index->disableLog();
return static_cast<NGTIndex>(index);
}catch(std::exception &err){
Expand Down Expand Up @@ -249,148 +249,68 @@ bool ngt_set_property_object_type_integer(NGTProperty prop, NGTError error) {
return true;
}

bool ngt_set_property_distance_type_l1(NGTProperty prop, NGTError error) {
bool ngt_set_property_distance_type(NGTProperty prop, NGT::Index::Property::DistanceType type, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeL1;
(*static_cast<NGT::Property*>(prop)).distanceType = type;
return true;
}

bool ngt_set_property_distance_type_l2(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}
bool ngt_set_property_distance_type_l1(NGTProperty prop, NGTError error) {
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeL1, error);
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2;
return true;
bool ngt_set_property_distance_type_l2(NGTProperty prop, NGTError error) {
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeL2, error);
}

bool ngt_set_property_distance_type_angle(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeAngle;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeAngle, error);
}

bool ngt_set_property_distance_type_hamming(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeHamming;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeHamming, error);
}

bool ngt_set_property_distance_type_poincare(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypePoincare;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypePoincare, error);
}

bool ngt_set_property_distance_type_lorentz(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeLorentz;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeLorentz, error);
}

bool ngt_set_property_distance_type_jaccard(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeJaccard;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeJaccard, error);
}

bool ngt_set_property_distance_type_sparse_jaccard(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeSparseJaccard;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeSparseJaccard, error);
}

bool ngt_set_property_distance_type_normalized_l2(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeNormalizedL2;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeNormalizedL2, error);
}

bool ngt_set_property_distance_type_cosine(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeCosine;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeCosine, error);
}

bool ngt_set_property_distance_type_normalized_angle(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeNormalizedAngle;
return true;
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeNormalizedAngle, error);
}

bool ngt_set_property_distance_type_normalized_cosine(NGTProperty prop, NGTError error) {
if(prop == NULL){
std::stringstream ss;
ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
operate_error_string_(ss, error);
return false;
}
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeNormalizedCosine, error);
}

(*static_cast<NGT::Property*>(prop)).distanceType = NGT::Index::Property::DistanceType::DistanceTypeNormalizedCosine;
return true;
bool ngt_set_property_distance_type_inner_product(NGTProperty prop, NGTError error) {
return ngt_set_property_distance_type(prop, NGT::Index::Property::DistanceType::DistanceTypeInnerProduct, error);
}

NGTObjectDistances ngt_create_empty_results(NGTError error) {
Expand Down Expand Up @@ -557,7 +477,7 @@ bool ngt_search_index_with_query(NGTIndex index, NGTQuery query, NGTObjectDistan
}

NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

NGT::Object *ngtquery = NULL;

Expand Down Expand Up @@ -597,7 +517,7 @@ bool ngt_search_index_with_query_float(NGTIndex index, NGTQueryFloat query, NGTO
}

NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

NGT::Object *ngtquery = NULL;

Expand Down Expand Up @@ -630,7 +550,7 @@ bool ngt_search_index_with_query_uint8(NGTIndex index, NGTQueryUint8 query, NGTO
}

NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

NGT::Object *ngtquery = NULL;

Expand Down Expand Up @@ -663,7 +583,7 @@ bool ngt_search_index_with_query_float16(NGTIndex index, NGTQueryFloat16 query,
}

NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

NGT::Object *ngtquery = NULL;

Expand Down Expand Up @@ -768,7 +688,7 @@ bool ngt_linear_search_index_with_query(NGTIndex index, NGTQuery query, NGTObje
}

NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

NGT::Object *ngtquery = NULL;

Expand Down Expand Up @@ -1004,13 +924,12 @@ bool ngt_batch_append_index(NGTIndex index, float *obj, uint32_t data_count, NGT

bool ngt_batch_insert_index(NGTIndex index, float *obj, uint32_t data_count, uint32_t *ids, NGTError error) {
NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();

int32_t dim = pindex->getDimension();
bool status = true;
float *objptr = obj;
for (size_t idx = 0; idx < data_count; idx++, objptr += dim) {
try{
std::vector<double> vobj(objptr, objptr + dim);
std::vector<float> vobj(objptr, objptr + dim);
ids[idx] = pindex->insert(vobj);
}catch(std::exception &err) {
status = false;
Expand Down Expand Up @@ -1038,7 +957,7 @@ bool ngt_batch_append_index_as_uint8(NGTIndex index, uint8_t *obj, uint32_t data

bool ngt_batch_insert_index_as_uint8(NGTIndex index, uint8_t *obj, uint32_t data_count, uint32_t *ids, NGTError error) {
NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

bool status = true;
uint8_t *objptr = obj;
Expand Down Expand Up @@ -1073,7 +992,7 @@ bool ngt_batch_append_index_as_float16(NGTIndex index, NGTFloat16 *obj, uint32_t

bool ngt_batch_insert_index_as_float16(NGTIndex index, NGTFloat16 *obj, uint32_t data_count, uint32_t *ids, NGTError error) {
NGT::Index* pindex = static_cast<NGT::Index*>(index);
int32_t dim = pindex->getObjectSpace().getDimension();
int32_t dim = pindex->getDimension();

bool status = true;
NGT::float16 *objptr = static_cast<NGT::float16*>(obj);
Expand Down
8 changes: 5 additions & 3 deletions lib/NGT/Capi.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,17 @@ typedef struct {
} NGTQueryParameters;

typedef struct {
float *query;
float *query;
NGTQueryParameters params;
} NGTQueryFloat;

typedef struct {
uint8_t *query;
uint8_t *query;
NGTQueryParameters params;
} NGTQueryUint8;

typedef struct {
NGTFloat16 *query;
NGTFloat16 *query;
NGTQueryParameters params;
} NGTQueryFloat16;

Expand Down Expand Up @@ -140,6 +140,8 @@ bool ngt_set_property_distance_type_normalized_angle(NGTProperty, NGTError);

bool ngt_set_property_distance_type_normalized_cosine(NGTProperty, NGTError);

bool ngt_set_property_distance_type_inner_product(NGTProperty, NGTError);

NGTObjectDistances ngt_create_empty_results(NGTError);

bool ngt_search_index(NGTIndex, double*, int32_t, size_t, float, float, NGTObjectDistances, NGTError);
Expand Down
15 changes: 0 additions & 15 deletions lib/NGT/Clustering.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,20 +445,6 @@ namespace NGT {
clusters[entry.centroidID].members.push_back(entry);
soi++;
} else {
#if 0
double mind = DBL_MAX;
size_t mincidx = -1;
for (auto cit = clusters.begin(); cit != clusters.end(); ++cit) {
if ((*cit).members.size() >= clusterSize) {
continue;
}
double d = distanceL2(vectors[entry.vectorID], (*cit).centroid);
if (d < mind) {
mind = d;
mincidx = distance(clusters.begin(), cit);
}
}
#else
std::vector<float> ds(clusters.size());
#pragma omp parallel for
for (size_t idx = 0; idx < clusters.size(); idx++) {
Expand All @@ -476,7 +462,6 @@ namespace NGT {
mincidx = idx;
}
}
#endif
entry = Entry(entry.vectorID, mincidx, mind);
int pt = distance(sortedObjects.rbegin(), soi);
std::sort(sortedObjects.begin(), soi.base());
Expand Down
Loading

0 comments on commit 6237d1d

Please sign in to comment.