-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1690 from kuzudb/blob-type
Implement blob dataType
- Loading branch information
Showing
37 changed files
with
2,667 additions
and
2,352 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
#include "common/types/blob.h" | ||
|
||
#include "common/exception.h" | ||
#include "common/string_utils.h" | ||
|
||
namespace kuzu { | ||
namespace common { | ||
|
||
const int HexFormatConstants::HEX_MAP[256] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, | ||
-1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, | ||
-1}; | ||
|
||
static bool isRegularChar(char c) { | ||
return c >= 32 && c <= 126 && c != '\\' && c != '\'' && c != '"'; | ||
} | ||
|
||
uint64_t Blob::getBlobSize(const ku_string_t& blob) { | ||
uint64_t blobSize = 0; | ||
auto length = blob.len; | ||
auto blobStr = blob.getData(); | ||
for (auto i = 0; i < length; i++) { | ||
if (blobStr[i] == '\\') { | ||
validateHexCode(blobStr, length, i); | ||
blobSize++; | ||
i += HexFormatConstants::LENGTH - 1; | ||
} else if (blobStr[i] <= 127) { | ||
blobSize++; | ||
} else { | ||
throw ConversionException(StringUtils::string_format( | ||
"Invalid byte encountered in STRING -> BLOB conversion. All non-ascii characters " | ||
"must be escaped with hex codes (e.g. \\xAA)")); | ||
} | ||
} | ||
return blobSize; | ||
} | ||
|
||
void Blob::fromString(ku_string_t& str, uint8_t* resultBuffer) { | ||
auto resultPos = 0u; | ||
auto blobData = str.getData(); | ||
for (auto i = 0u; i < str.len; i++) { | ||
if (blobData[i] == '\\') { | ||
validateHexCode(blobData, str.len, i); | ||
auto firstByte = | ||
HexFormatConstants::HEX_MAP[blobData[i + HexFormatConstants::FIRST_BYTE_POS]]; | ||
auto secondByte = | ||
HexFormatConstants::HEX_MAP[blobData[i + HexFormatConstants::SECOND_BYTES_POS]]; | ||
resultBuffer[resultPos++] = | ||
(firstByte << HexFormatConstants::NUM_BYTES_TO_SHIFT_FOR_FIRST_BYTE) + secondByte; | ||
i += HexFormatConstants::LENGTH - 1; | ||
} else if (blobData[i] <= 127) { | ||
resultBuffer[resultPos++] = blobData[i]; | ||
} else { | ||
throw ConversionException( | ||
"Invalid byte encountered in STRING -> BLOB conversion. All non-ascii characters " | ||
"must be escaped with hex codes (e.g. \\xAA)"); | ||
} | ||
} | ||
} | ||
|
||
std::string Blob::toString(blob_t& blob) { | ||
std::string result; | ||
auto blobData = (uint8_t*)blob.value.getData(); | ||
for (auto i = 0u; i < blob.value.len; i++) { | ||
if (isRegularChar(blobData[i])) { | ||
// ascii characters are rendered as-is. | ||
result += blobData[i]; | ||
} else { | ||
auto firstByte = blobData[i] >> HexFormatConstants::NUM_BYTES_TO_SHIFT_FOR_FIRST_BYTE; | ||
auto secondByte = blobData[i] & HexFormatConstants::SECOND_BYTE_MASK; | ||
// non-ascii characters are rendered as hexadecimal (e.g. \x00). | ||
result += '\\'; | ||
result += 'x'; | ||
result += HexFormatConstants::HEX_TABLE[firstByte]; | ||
result += HexFormatConstants::HEX_TABLE[secondByte]; | ||
} | ||
} | ||
return result; | ||
} | ||
|
||
void Blob::validateHexCode(const uint8_t* blobStr, uint64_t length, uint64_t curPos) { | ||
if (curPos + HexFormatConstants::LENGTH > length) { | ||
throw ConversionException( | ||
"Invalid hex escape code encountered in string -> blob conversion: " | ||
"unterminated escape code at end of blob"); | ||
} | ||
if (memcmp(blobStr + curPos, HexFormatConstants::PREFIX, HexFormatConstants::PREFIX_LENGTH) != | ||
0 || | ||
HexFormatConstants::HEX_MAP[blobStr[curPos + HexFormatConstants::FIRST_BYTE_POS]] < 0 || | ||
HexFormatConstants::HEX_MAP[blobStr[curPos + HexFormatConstants::SECOND_BYTES_POS]] < 0) { | ||
throw ConversionException(StringUtils::string_format( | ||
"Invalid hex escape code encountered in string -> blob conversion: {}", | ||
std::string((char*)blobStr + curPos, HexFormatConstants::LENGTH))); | ||
} | ||
} | ||
|
||
} // namespace common | ||
} // namespace kuzu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
#include "function/blob/vector_blob_operations.h" | ||
|
||
#include "function/blob/operations/decode_operation.h" | ||
#include "function/blob/operations/encode_operation.h" | ||
#include "function/blob/operations/octet_length_operation.h" | ||
#include "function/string/vector_string_operations.h" | ||
|
||
namespace kuzu { | ||
namespace function { | ||
|
||
vector_operation_definitions OctetLengthVectorOperations::getDefinitions() { | ||
vector_operation_definitions definitions; | ||
definitions.push_back(make_unique<VectorOperationDefinition>(common::OCTET_LENGTH_FUNC_NAME, | ||
std::vector<common::LogicalTypeID>{common::LogicalTypeID::BLOB}, | ||
common::LogicalTypeID::INT64, | ||
UnaryExecFunction<common::blob_t, int64_t, operation::OctetLength>, nullptr, nullptr, | ||
nullptr, false /* isVarLength */)); | ||
return definitions; | ||
} | ||
|
||
vector_operation_definitions EncodeVectorOperations::getDefinitions() { | ||
vector_operation_definitions definitions; | ||
definitions.push_back(make_unique<VectorOperationDefinition>(common::ENCODE_FUNC_NAME, | ||
std::vector<common::LogicalTypeID>{common::LogicalTypeID::STRING}, | ||
common::LogicalTypeID::BLOB, | ||
VectorStringOperations::UnaryStringExecFunction<common::ku_string_t, common::blob_t, | ||
operation::Encode>, | ||
nullptr, false /* isVarLength */)); | ||
return definitions; | ||
} | ||
|
||
vector_operation_definitions DecodeVectorOperations::getDefinitions() { | ||
vector_operation_definitions definitions; | ||
definitions.push_back(make_unique<VectorOperationDefinition>(common::DECODE_FUNC_NAME, | ||
std::vector<common::LogicalTypeID>{common::LogicalTypeID::BLOB}, | ||
common::LogicalTypeID::STRING, | ||
VectorStringOperations::UnaryStringExecFunction<common::blob_t, common::ku_string_t, | ||
operation::Decode>, | ||
nullptr, false /* isVarLength */)); | ||
return definitions; | ||
} | ||
|
||
} // namespace function | ||
} // namespace kuzu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.