Skip to content

Commit

Permalink
Remove csv reader
Browse files Browse the repository at this point in the history
  • Loading branch information
mewim committed Mar 3, 2023
1 parent afed1c7 commit b1bd21f
Show file tree
Hide file tree
Showing 14 changed files with 169 additions and 530 deletions.
6 changes: 3 additions & 3 deletions dataset/copy-node-property-test/vPerson.csv
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
0,
1,"ozwhvnetnq"
2,"kuk,qg\\nrspmk"
3,"wmz,1234\lamo"
2,"kuk,qgnrspmk"
3,"wmz,1234lamo"
4,"tudoojdduf"
5,"qifidjufri"
6,"gqpnpbdmrb"
7,"dgzbiqjkaz"
8,"ebf,,uq\buqma"
8,"ebf,,uqbuqma"
9,"rwhnybogfy"
10,"enqpnymvdb"
11,"axgwwhhohf"
Expand Down
2 changes: 1 addition & 1 deletion src/common/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
add_subdirectory(arrow)
add_subdirectory(csv_reader)
add_subdirectory(copier_config)
add_subdirectory(data_chunk)
add_subdirectory(task_system)
add_subdirectory(types)
Expand Down
7 changes: 7 additions & 0 deletions src/common/copier_config/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
add_library(kuzu_common_copier_config
OBJECT
copier_config.cpp)

set(ALL_OBJECT_FILES
${ALL_OBJECT_FILES} $<TARGET_OBJECTS:kuzu_common_copier_config>
PARENT_SCOPE)
79 changes: 79 additions & 0 deletions src/common/copier_config/copier_config.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#include "common/copier_config/copier_config.h"

#include "common/constants.h"
#include "common/type_utils.h"
#include "common/utils.h"
#include "spdlog/spdlog.h"
#include "utf8proc_wrapper.h"

using namespace kuzu::utf8proc;

namespace kuzu {
namespace common {
CopyDescription::CopyDescription(const std::string& filePath, CSVReaderConfig csvReaderConfig)
: filePath{filePath}, csvReaderConfig{nullptr}, fileType{FileType::CSV} {
setFileType(filePath);
if (fileType == FileType::CSV) {
this->csvReaderConfig = std::make_unique<CSVReaderConfig>(csvReaderConfig);
}
}

CopyDescription::CopyDescription(const CopyDescription& copyDescription)
: filePath{copyDescription.filePath}, csvReaderConfig{nullptr}, fileType{
copyDescription.fileType} {
if (fileType == FileType::CSV) {
this->csvReaderConfig = std::make_unique<CSVReaderConfig>(*copyDescription.csvReaderConfig);
}
}

std::string CopyDescription::getFileTypeName(FileType fileType) {
switch (fileType) {
case FileType::CSV:
return "csv";

case FileType::ARROW:
return "arrow";

case FileType::PARQUET:
return "parquet";
}
}

std::string CopyDescription::getFileTypeSuffix(FileType fileType) {
return "." + getFileTypeName(fileType);
}

void CopyDescription::setFileType(std::string const& fileName) {
auto csvSuffix = getFileTypeSuffix(FileType::CSV);
auto arrowSuffix = getFileTypeSuffix(FileType::ARROW);
auto parquetSuffix = getFileTypeSuffix(FileType::PARQUET);

if (fileName.length() >= csvSuffix.length()) {
if (!fileName.compare(
fileName.length() - csvSuffix.length(), csvSuffix.length(), csvSuffix)) {
fileType = FileType::CSV;
return;
}
}

if (fileName.length() >= arrowSuffix.length()) {
if (!fileName.compare(
fileName.length() - arrowSuffix.length(), arrowSuffix.length(), arrowSuffix)) {
fileType = FileType::ARROW;
return;
}
}

if (fileName.length() >= parquetSuffix.length()) {
if (!fileName.compare(fileName.length() - parquetSuffix.length(), parquetSuffix.length(),
parquetSuffix)) {
fileType = FileType::PARQUET;
return;
}
}

throw CopyException("Unsupported file type: " + fileName);
}

} // namespace common
} // namespace kuzu
7 changes: 0 additions & 7 deletions src/common/csv_reader/CMakeLists.txt

This file was deleted.

Loading

0 comments on commit b1bd21f

Please sign in to comment.