Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Support int64 data type in CSVIter #11446

Merged
merged 1 commit into from
Jun 29, 2018
Merged

Support int64 data type in CSVIter #11446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 3rdparty/dmlc-core
Open in desktop
Submodule dmlc-core updated 14 files
+3 −3 CMakeLists.txt
+8 −0 include/dmlc/base.h
+28 −1 include/dmlc/endian.h
+43 −9 include/dmlc/io.h
+1 −1 include/dmlc/omp.h
+3 −3 include/dmlc/parameter.h
+68 −39 include/dmlc/serializer.h
+64 −18 src/data.cc
+6 −3 src/data/csv_parser.h
+118 −86 src/io/s3_filesys.cc
+87 −0 src/io/single_threaded_input_split.h
+21 −4 test/unittest/unittest_parser.cc
+26 −13 test/unittest/unittest_serializer.cc
+3 −3 tracker/dmlc_tracker/mpi.py
1 change: 1 addition & 0 deletions src/io/image_iter_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ struct PrefetcherParam : public dmlc::Parameter<PrefetcherParam> {
.add_enum("float32", mshadow::kFloat32)
.add_enum("float64", mshadow::kFloat64)
.add_enum("float16", mshadow::kFloat16)
.add_enum("int64", mshadow::kInt64)
.add_enum("int32", mshadow::kInt32)
.add_enum("uint8", mshadow::kUint8)
.set_default(dmlc::optional<int>())
Expand Down
16 changes: 11 additions & 5 deletions src/io/iter_csv.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,15 +174,21 @@ class CSVIter: public IIterator<DataInst> {
for (const auto& arg : kwargs) {
if (arg.first == "dtype") {
dtype_has_value = true;
if (arg.second == "int32" || arg.second == "float32") {
target_dtype = (arg.second == "int32") ? mshadow::kInt32 : mshadow::kFloat32;
if (arg.second == "int32") {
target_dtype = mshadow::kInt32;
} else if (arg.second == "int64") {
target_dtype = mshadow::kInt64;
} else if (arg.second == "float32") {
target_dtype = mshadow::kFloat32;
} else {
CHECK(false) << arg.second << " is not supported for CSVIter";
}
}
}
if (dtype_has_value && target_dtype == mshadow::kInt32) {
iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<int>()));
iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<int32_t>()));
} else if (dtype_has_value && target_dtype == mshadow::kInt64) {
iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<int64_t>()));
} else if (!dtype_has_value || target_dtype == mshadow::kFloat32) {
iterator_.reset(reinterpret_cast<CSVIterBase*>(new CSVIterTyped<float>()));
}
Expand Down Expand Up @@ -229,8 +235,8 @@ If ``data_csv = 'data/'`` is set, then all the files in this directory will be r
``reset()`` is expected to be called only after a complete pass of data.

By default, the CSVIter parses all entries in the data file as float32 data type,
if `dtype` argument is set to be 'int32' then CSVIter will parse all entries in the file
as int32 data type.
if `dtype` argument is set to be 'int32' or 'int64' then CSVIter will parse all entries in the file
as int32 or int64 data type accordingly.

Examples::

Expand Down