csv.hpp
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: Apache-2.0
4  */
5 
6 #pragma once
7 
8 #include <cudf/io/detail/utils.hpp>
9 #include <cudf/io/types.hpp>
11 #include <cudf/types.hpp>
12 #include <cudf/utilities/error.hpp>
14 
15 #include <memory>
16 #include <string>
17 #include <unordered_map>
18 #include <utility>
19 #include <variant>
20 #include <vector>
21 
27 namespace CUDF_EXPORT cudf {
28 namespace io {
29 
38 class csv_reader_options_builder;
39 
45  source_info _source;
46 
47  // Read settings
48 
49  // Specify the compression format of the source or infer from file extension
50  compression_type _compression = compression_type::AUTO;
51  // Bytes to skip from the source start
52  std::size_t _byte_range_offset = 0;
53  // Bytes to read; always reads complete rows
54  std::size_t _byte_range_size = 0;
55  // Names of all the columns; if empty then names are auto-generated
56  std::vector<std::string> _names;
57  // If there is no header or names, prepend this to the column ID as the name
58  std::string _prefix;
59  // Whether to rename duplicate column names
60  bool _mangle_dupe_cols = true;
61 
62  // Filter settings
63 
64  // Names of columns to read; empty is all columns
65  std::vector<std::string> _use_cols_names;
66  // Indexes of columns to read; empty is all columns
67  std::vector<int> _use_cols_indexes;
68  // Rows to read; -1 is all
69  size_type _nrows = -1;
70  // Rows to skip from the start
71  size_type _skiprows = 0;
72  // Rows to skip from the end
73  size_type _skipfooter = 0;
74  // Header row index
75  size_type _header = 0;
76 
77  // Parsing settings
78 
79  // Line terminator
80  char _lineterminator = '\n';
81  // Field delimiter
82  char _delimiter = ',';
83  // Numeric data thousands separator; cannot match delimiter
84  char _thousands = '\0';
85  // Decimal point character; cannot match delimiter
86  char _decimal = '.';
87  // Comment line start character
88  char _comment = '\0';
89  bool _windowslinetermination = false;
90  // Treat whitespace as field delimiter; overrides character delimiter
91  bool _delim_whitespace = false;
92  // Skip whitespace after the delimiter
93  bool _skipinitialspace = false;
94  // Ignore empty lines or parse line values as invalid
95  bool _skip_blank_lines = true;
96  // Treatment of quoting behavior
97  quote_style _quoting = quote_style::MINIMAL;
98  // Quoting character (if `quoting` is true)
99  char _quotechar = '"';
100  // Whether a quote inside a value is double-quoted
101  bool _doublequote = true;
102  // Whether to detect quotes surrounded by spaces e.g. ` "data" `. This flag has no effect when
103  // _doublequote is true
104  bool _detect_whitespace_around_quotes = false;
105  // Names of columns to read as datetime
106  std::vector<std::string> _parse_dates_names;
107  // Indexes of columns to read as datetime
108  std::vector<int> _parse_dates_indexes;
109  // Names of columns to parse as hexadecimal
110  std::vector<std::string> _parse_hex_names;
111  // Indexes of columns to parse as hexadecimal
112  std::vector<int> _parse_hex_indexes;
113 
114  // Conversion settings
115 
116  // Per-column types; disables type inference on those columns
117  std::variant<std::vector<data_type>, std::map<std::string, data_type>> _dtypes;
118  // Additional values to recognize as boolean true values
119  std::vector<std::string> _true_values{"True", "TRUE", "true"};
120  // Additional values to recognize as boolean false values
121  std::vector<std::string> _false_values{"False", "FALSE", "false"};
122  // Additional values to recognize as null values
123  std::vector<std::string> _na_values;
124  // Whether to keep the built-in default NA values
125  bool _keep_default_na = true;
126  // Whether to disable null filter; disabling can improve performance
127  bool _na_filter = true;
128  // Whether to parse dates as DD/MM versus MM/DD
129  bool _dayfirst = false;
130  // Cast timestamp columns to a specific type
131  data_type _timestamp_type{type_id::EMPTY};
132 
138  explicit csv_reader_options(source_info src) : _source{std::move(src)} {}
139 
141 
142  public:
148  csv_reader_options() = default;
149 
157 
163  [[nodiscard]] source_info const& get_source() const { return _source; }
164 
170  [[nodiscard]] compression_type get_compression() const { return _compression; }
171 
177  [[nodiscard]] std::size_t get_byte_range_offset() const { return _byte_range_offset; }
178 
184  [[nodiscard]] std::size_t get_byte_range_size() const { return _byte_range_size; }
185 
191  [[nodiscard]] std::size_t get_byte_range_size_with_padding() const
192  {
193  if (_byte_range_size == 0) {
194  return 0;
195  } else {
196  return _byte_range_size + get_byte_range_padding();
197  }
198  }
199 
205  [[nodiscard]] std::size_t get_byte_range_padding() const
206  {
207  auto const num_names = _names.size();
208  auto const num_dtypes = std::visit([](auto const& dtypes) { return dtypes.size(); }, _dtypes);
209  auto const num_columns = std::max(num_dtypes, num_names);
210 
211  auto const max_row_bytes = 16 * 1024; // 16KB
212  auto const column_bytes = 64;
213  auto const base_padding = 1024; // 1KB
214 
215  if (num_columns == 0) {
216  // Use flat size if the number of columns is not known
217  return max_row_bytes;
218  }
219 
220  // Expand the size based on the number of columns, if available
221  return base_padding + static_cast<size_t>(num_columns) * column_bytes;
222  }
223 
229  [[nodiscard]] std::vector<std::string> const& get_names() const { return _names; }
230 
236  [[nodiscard]] std::string get_prefix() const { return _prefix; }
237 
243  [[nodiscard]] bool is_enabled_mangle_dupe_cols() const { return _mangle_dupe_cols; }
244 
250  [[nodiscard]] std::vector<std::string> const& get_use_cols_names() const
251  {
252  return _use_cols_names;
253  }
254 
260  [[nodiscard]] std::vector<int> const& get_use_cols_indexes() const { return _use_cols_indexes; }
261 
267  [[nodiscard]] size_type get_nrows() const { return _nrows; }
268 
274  [[nodiscard]] size_type get_skiprows() const { return _skiprows; }
275 
281  [[nodiscard]] size_type get_skipfooter() const { return _skipfooter; }
282 
288  [[nodiscard]] size_type get_header() const { return _header; }
289 
295  [[nodiscard]] char get_lineterminator() const { return _lineterminator; }
296 
302  [[nodiscard]] char get_delimiter() const { return _delimiter; }
303 
309  [[nodiscard]] char get_thousands() const { return _thousands; }
310 
316  [[nodiscard]] char get_decimal() const { return _decimal; }
317 
323  [[nodiscard]] char get_comment() const { return _comment; }
324 
330  [[nodiscard]] bool is_enabled_windowslinetermination() const { return _windowslinetermination; }
331 
337  [[nodiscard]] bool is_enabled_delim_whitespace() const { return _delim_whitespace; }
338 
344  [[nodiscard]] bool is_enabled_skipinitialspace() const { return _skipinitialspace; }
345 
351  [[nodiscard]] bool is_enabled_skip_blank_lines() const { return _skip_blank_lines; }
352 
358  [[nodiscard]] quote_style get_quoting() const { return _quoting; }
359 
365  [[nodiscard]] char get_quotechar() const { return _quotechar; }
366 
372  [[nodiscard]] bool is_enabled_doublequote() const { return _doublequote; }
373 
381  {
382  return _detect_whitespace_around_quotes;
383  }
384 
390  [[nodiscard]] std::vector<std::string> const& get_parse_dates_names() const
391  {
392  return _parse_dates_names;
393  }
394 
400  [[nodiscard]] std::vector<int> const& get_parse_dates_indexes() const
401  {
402  return _parse_dates_indexes;
403  }
404 
410  [[nodiscard]] std::vector<std::string> const& get_parse_hex_names() const
411  {
412  return _parse_hex_names;
413  }
414 
420  [[nodiscard]] std::vector<int> const& get_parse_hex_indexes() const { return _parse_hex_indexes; }
421 
427  [[nodiscard]] std::variant<std::vector<data_type>, std::map<std::string, data_type>> const&
428  get_dtypes() const
429  {
430  return _dtypes;
431  }
432 
438  [[nodiscard]] std::vector<std::string> const& get_true_values() const { return _true_values; }
439 
445  [[nodiscard]] std::vector<std::string> const& get_false_values() const { return _false_values; }
446 
452  [[nodiscard]] std::vector<std::string> const& get_na_values() const { return _na_values; }
453 
459  [[nodiscard]] bool is_enabled_keep_default_na() const { return _keep_default_na; }
460 
466  [[nodiscard]] bool is_enabled_na_filter() const { return _na_filter; }
467 
473  [[nodiscard]] bool is_enabled_dayfirst() const { return _dayfirst; }
474 
480  [[nodiscard]] data_type get_timestamp_type() const { return _timestamp_type; }
481 
487  void set_source(source_info src) { _source = std::move(src); }
488 
494  void set_compression(compression_type comp) { _compression = comp; }
495 
501  void set_byte_range_offset(std::size_t offset)
502  {
503  if ((offset != 0) and ((_skiprows != 0) or (_skipfooter != 0) or (_nrows != -1))) {
504  CUDF_FAIL(
505  "When there is valid value in skiprows or skipfooter or nrows, offset can't have non-zero "
506  "value");
507  }
508  _byte_range_offset = offset;
509  }
510 
516  void set_byte_range_size(std::size_t size)
517  {
518  if ((size != 0) and ((_skiprows != 0) or (_skipfooter != 0) or (_nrows != -1))) {
519  CUDF_FAIL(
520  "If the value of any of skiprows, skipfooter or nrows is valid, range size cannot be "
521  "non-zero.");
522  }
523  _byte_range_size = size;
524  }
525 
531  void set_names(std::vector<std::string> col_names) { _names = std::move(col_names); }
532 
538  void set_prefix(std::string pfx) { _prefix = pfx; }
539 
545  void enable_mangle_dupe_cols(bool val) { _mangle_dupe_cols = val; }
546 
552  void set_use_cols_names(std::vector<std::string> col_names)
553  {
554  _use_cols_names = std::move(col_names);
555  }
556 
562  void set_use_cols_indexes(std::vector<int> col_indices)
563  {
564  _use_cols_indexes = std::move(col_indices);
565  }
566 
572  void set_nrows(size_type nrows)
573  {
574  CUDF_EXPECTS((nrows == 0) or (_skipfooter == 0), "Cannot use both `nrows` and `skipfooter`");
575  if ((nrows != -1) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
576  CUDF_FAIL(
577  "nrows can't be a non negative value if range offset and/or range size has been set");
578  }
579 
580  _nrows = nrows;
581  }
582 
588  void set_skiprows(size_type skiprows)
589  {
590  if ((skiprows != 0) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
591  CUDF_FAIL("skiprows must be zero if range offset or range size has been set",
592  std::invalid_argument);
593  }
594  _skiprows = skiprows;
595  }
596 
602  void set_skipfooter(size_type skipfooter)
603  {
604  CUDF_EXPECTS((skipfooter == 0) or (_nrows == -1),
605  "Cannot use both `nrows` and `skipfooter`",
606  std::invalid_argument);
607  if ((skipfooter != 0) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
608  CUDF_FAIL("skipfooter must be zero if range offset or range size has been set",
609  std::invalid_argument);
610  }
611 
612  _skipfooter = skipfooter;
613  }
614 
620  void set_header(size_type hdr) { _header = hdr; }
621 
627  void set_lineterminator(char term) { _lineterminator = term; }
628 
634  void set_delimiter(char delim) { _delimiter = delim; }
635 
641  void set_thousands(char val) { _thousands = val; }
642 
648  void set_decimal(char val) { _decimal = val; }
649 
655  void set_comment(char val) { _comment = val; }
656 
662  void enable_windowslinetermination(bool val) { _windowslinetermination = val; }
663 
669  void enable_delim_whitespace(bool val) { _delim_whitespace = val; }
670 
676  void enable_skipinitialspace(bool val) { _skipinitialspace = val; }
677 
683  void enable_skip_blank_lines(bool val) { _skip_blank_lines = val; }
684 
695  void set_quoting(quote_style quoting)
696  {
697  CUDF_EXPECTS(quoting == quote_style::MINIMAL || quoting == quote_style::NONE,
698  "Only MINIMAL and NONE are supported for quoting.");
699  _quoting = quoting;
700  }
701 
707  void set_quotechar(char ch) { _quotechar = ch; }
708 
714  void enable_doublequote(bool val) { _doublequote = val; }
715 
722  void enable_detect_whitespace_around_quotes(bool val) { _detect_whitespace_around_quotes = val; }
723 
729  void set_parse_dates(std::vector<std::string> col_names)
730  {
731  _parse_dates_names = std::move(col_names);
732  }
733 
739  void set_parse_dates(std::vector<int> col_indices)
740  {
741  _parse_dates_indexes = std::move(col_indices);
742  }
743 
749  void set_parse_hex(std::vector<std::string> col_names)
750  {
751  _parse_hex_names = std::move(col_names);
752  }
753 
759  void set_parse_hex(std::vector<int> col_indices) { _parse_hex_indexes = std::move(col_indices); }
760 
766  void set_dtypes(std::map<std::string, data_type> types) { _dtypes = std::move(types); }
767 
773  void set_dtypes(std::vector<data_type> types) { _dtypes = std::move(types); }
774 
780  void set_true_values(std::vector<std::string> vals)
781  {
782  _true_values.insert(_true_values.end(), vals.begin(), vals.end());
783  }
784 
790  void set_false_values(std::vector<std::string> vals)
791  {
792  _false_values.insert(_false_values.end(), vals.begin(), vals.end());
793  }
794 
800  void set_na_values(std::vector<std::string> vals)
801  {
802  CUDF_EXPECTS(vals.empty() or _na_filter, "Can't set na_values when na_filtering is disabled");
803 
804  _na_values = std::move(vals);
805  }
806 
812  void enable_keep_default_na(bool val) { _keep_default_na = val; }
813 
819  void enable_na_filter(bool val)
820  {
821  if (!val) { _na_values.clear(); }
822  _na_filter = val;
823  }
824 
830  void enable_dayfirst(bool val) { _dayfirst = val; }
831 
837  void set_timestamp_type(data_type type) { _timestamp_type = type; }
838 };
839 
845  csv_reader_options options;
846 
847  public:
854 
860  csv_reader_options_builder(source_info src) : options{std::move(src)} {}
861 
869  {
870  options._compression = comp;
871  return *this;
872  }
873 
881  {
882  options.set_byte_range_offset(offset);
883  return *this;
884  }
885 
893  {
894  options.set_byte_range_size(size);
895  return *this;
896  }
897 
904  csv_reader_options_builder& names(std::vector<std::string> col_names)
905  {
906  options._names = std::move(col_names);
907  return *this;
908  }
909 
917  {
918  options._prefix = std::move(pfx);
919  return *this;
920  }
921 
929  {
930  options._mangle_dupe_cols = val;
931  return *this;
932  }
933 
940  csv_reader_options_builder& use_cols_names(std::vector<std::string> col_names)
941  {
942  options._use_cols_names = std::move(col_names);
943  return *this;
944  }
945 
952  csv_reader_options_builder& use_cols_indexes(std::vector<int> col_indices)
953  {
954  options._use_cols_indexes = std::move(col_indices);
955  return *this;
956  }
957 
965  {
966  options.set_nrows(rows);
967  return *this;
968  }
969 
977  {
978  options.set_skiprows(skip);
979  return *this;
980  }
981 
989  {
990  options.set_skipfooter(skip);
991  return *this;
992  }
993 
1001  {
1002  options._header = hdr;
1003  return *this;
1004  }
1005 
1013  {
1014  options._lineterminator = term;
1015  return *this;
1016  }
1017 
1025  {
1026  options._delimiter = delim;
1027  return *this;
1028  }
1029 
1037  {
1038  options._thousands = val;
1039  return *this;
1040  }
1041 
1049  {
1050  options._decimal = val;
1051  return *this;
1052  }
1053 
1061  {
1062  options._comment = val;
1063  return *this;
1064  }
1065 
1073  {
1074  options._windowslinetermination = val;
1075  return *this;
1076  }
1077 
1085  {
1086  options._delim_whitespace = val;
1087  return *this;
1088  }
1089 
1097  {
1098  options._skipinitialspace = val;
1099  return *this;
1100  }
1101 
1109  {
1110  options._skip_blank_lines = val;
1111  return *this;
1112  }
1113 
1121  {
1122  options._quoting = style;
1123  return *this;
1124  }
1125 
1133  {
1134  options._quotechar = ch;
1135  return *this;
1136  }
1137 
1145  {
1146  options._doublequote = val;
1147  return *this;
1148  }
1149 
1158  {
1159  options._detect_whitespace_around_quotes = val;
1160  return *this;
1161  }
1162 
1169  csv_reader_options_builder& parse_dates(std::vector<std::string> col_names)
1170  {
1171  options._parse_dates_names = std::move(col_names);
1172  return *this;
1173  }
1174 
1181  csv_reader_options_builder& parse_dates(std::vector<int> col_indices)
1182  {
1183  options._parse_dates_indexes = std::move(col_indices);
1184  return *this;
1185  }
1186 
1193  csv_reader_options_builder& parse_hex(std::vector<std::string> col_names)
1194  {
1195  options._parse_hex_names = std::move(col_names);
1196  return *this;
1197  }
1198 
1205  csv_reader_options_builder& parse_hex(std::vector<int> col_indices)
1206  {
1207  options._parse_hex_indexes = std::move(col_indices);
1208  return *this;
1209  }
1210 
1217  csv_reader_options_builder& dtypes(std::map<std::string, data_type> types)
1218  {
1219  options._dtypes = std::move(types);
1220  return *this;
1221  }
1222 
1229  csv_reader_options_builder& dtypes(std::vector<data_type> types)
1230  {
1231  options._dtypes = std::move(types);
1232  return *this;
1233  }
1234 
1241  csv_reader_options_builder& true_values(std::vector<std::string> vals)
1242  {
1243  options._true_values.insert(options._true_values.end(), vals.begin(), vals.end());
1244  return *this;
1245  }
1246 
1253  csv_reader_options_builder& false_values(std::vector<std::string> vals)
1254  {
1255  options._false_values.insert(options._false_values.end(), vals.begin(), vals.end());
1256  return *this;
1257  }
1258 
1265  csv_reader_options_builder& na_values(std::vector<std::string> vals)
1266  {
1267  options.set_na_values(std::move(vals));
1268  return *this;
1269  }
1270 
1278  {
1279  options.enable_keep_default_na(val);
1280  return *this;
1281  }
1282 
1290  {
1291  options.enable_na_filter(val);
1292  return *this;
1293  }
1294 
1302  {
1303  options._dayfirst = val;
1304  return *this;
1305  }
1306 
1314  {
1315  options._timestamp_type = type;
1316  return *this;
1317  }
1318 
1322  operator csv_reader_options&&() { return std::move(options); }
1323 
1331  csv_reader_options&& build() { return std::move(options); }
1332 };
1333 
1352  csv_reader_options options,
1355  // end of group
1366 
1371  // Specify the sink to use for writer output
1372  sink_info _sink;
1373  // Set of columns to output
1374  table_view _table;
1375  // string to use for null entries
1376  std::string _na_rep = "";
1377  // Indicates whether to write headers to csv
1378  bool _include_header = true;
1379  // maximum number of rows to write in each chunk (limits memory use)
1380  size_type _rows_per_chunk = std::numeric_limits<size_type>::max();
1381  // character to use for separating lines (default "\n")
1382  std::string _line_terminator = "\n";
1383  // character to use for separating column values (default ",")
1384  char _inter_column_delimiter = ',';
1385  // string to use for values != 0 in INT8 types (default 'true')
1386  std::string _true_value = std::string{"true"};
1387  // string to use for values == 0 in INT8 types (default 'false')
1388  std::string _false_value = std::string{"false"};
1389  // Names of all columns; if empty, writer will generate column names
1390  std::vector<std::string> _names;
1391  // Quote style. Currently only MINIMAL and NONE are supported.
1392  quote_style _quoting = quote_style::MINIMAL;
1393 
1400  explicit csv_writer_options(sink_info sink, table_view const& table)
1401  : _sink(std::move(sink)), _table(table), _rows_per_chunk(table.num_rows())
1402  {
1403  }
1404 
1406 
1407  public:
1413  explicit csv_writer_options() = default;
1414 
1424 
1430  [[nodiscard]] sink_info const& get_sink() const { return _sink; }
1431 
1437  [[nodiscard]] table_view const& get_table() const { return _table; }
1438 
1444  [[nodiscard]] std::vector<std::string> const& get_names() const { return _names; }
1445 
1451  [[nodiscard]] std::string const& get_na_rep() const { return _na_rep; }
1452 
1458  [[nodiscard]] bool is_enabled_include_header() const { return _include_header; }
1459 
1465  [[nodiscard]] size_type get_rows_per_chunk() const { return _rows_per_chunk; }
1466 
1472  [[nodiscard]] std::string const& get_line_terminator() const { return _line_terminator; }
1473 
1479  [[nodiscard]] char get_inter_column_delimiter() const { return _inter_column_delimiter; }
1480 
1486  [[nodiscard]] std::string const& get_true_value() const { return _true_value; }
1487 
1493  [[nodiscard]] std::string const& get_false_value() const { return _false_value; }
1494 
1505  [[nodiscard]] quote_style get_quoting() const { return _quoting; }
1506 
1507  // Setter
1513  void set_names(std::vector<std::string> names) { _names = std::move(names); }
1514 
1520  void set_na_rep(std::string val) { _na_rep = std::move(val); }
1521 
1527  void enable_include_header(bool val) { _include_header = val; }
1528 
1534  void set_rows_per_chunk(size_type val) { _rows_per_chunk = val; }
1535 
1541  void set_line_terminator(std::string term) { _line_terminator = std::move(term); }
1542 
1548  void set_inter_column_delimiter(char delim) { _inter_column_delimiter = delim; }
1549 
1555  void set_true_value(std::string val) { _true_value = std::move(val); }
1556 
1562  void set_false_value(std::string val) { _false_value = std::move(val); }
1563 
1569  void set_table(table_view const& table) { _table = table; }
1570 
1581  void set_quoting(quote_style quoting)
1582  {
1583  CUDF_EXPECTS(quoting == quote_style::MINIMAL || quoting == quote_style::NONE,
1584  "Only MINIMAL and NONE are supported for quoting.");
1585  _quoting = quoting;
1586  }
1587 };
1588 
1593  csv_writer_options options;
1594 
1595  public:
1601  explicit csv_writer_options_builder() = default;
1602 
1610  : options{sink, table}
1611  {
1612  }
1613 
1620  csv_writer_options_builder& names(std::vector<std::string> names)
1621  {
1622  options._names = names;
1623  return *this;
1624  }
1625 
1633  {
1634  options._na_rep = val;
1635  return *this;
1636  };
1637 
1645  {
1646  options._include_header = val;
1647  return *this;
1648  }
1649 
1657  {
1658  options._rows_per_chunk = val;
1659  return *this;
1660  }
1661 
1669  {
1670  options._line_terminator = term;
1671  return *this;
1672  }
1673 
1681  {
1682  options._inter_column_delimiter = delim;
1683  return *this;
1684  }
1685 
1693  {
1694  options._true_value = val;
1695  return *this;
1696  }
1697 
1705  {
1706  options._false_value = val;
1707  return *this;
1708  }
1709 
1719  {
1720  options.set_quoting(quoting);
1721  return *this;
1722  }
1723 
1727  operator csv_writer_options&&() { return std::move(options); }
1728 
1736  csv_writer_options&& build() { return std::move(options); }
1737 };
1738 
1756 void write_csv(csv_writer_options const& options,
1758 
1760 struct is_supported_csv_write_type_fn {
1761  template <typename T>
1762  constexpr bool operator()() const
1763  {
1764  return cudf::io::detail::is_convertible_to_string_column<T>();
1765  }
1766 };
1768 
1775 constexpr bool is_supported_write_csv(data_type type)
1776 {
1777  return cudf::type_dispatcher(type, is_supported_csv_write_type_fn{});
1778 }
1779  // end of group
1781 } // namespace io
1782 } // namespace CUDF_EXPORT cudf
Indicator for the logical data type of an element in a column.
Definition: types.hpp:286
Builder to build options for read_csv().
Definition: csv.hpp:844
csv_reader_options_builder & dtypes(std::vector< data_type > types)
Sets per-column types.
Definition: csv.hpp:1229
csv_reader_options_builder & false_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean false values.
Definition: csv.hpp:1253
csv_reader_options_builder & use_cols_names(std::vector< std::string > col_names)
Sets names of the columns to be read.
Definition: csv.hpp:940
csv_reader_options_builder & doublequote(bool val)
Sets a quote inside a value is double-quoted.
Definition: csv.hpp:1144
csv_reader_options_builder & parse_hex(std::vector< std::string > col_names)
Sets names of columns to parse as hexadecimal.
Definition: csv.hpp:1193
csv_reader_options_builder & byte_range_offset(std::size_t offset)
Sets number of bytes to skip from source start.
Definition: csv.hpp:880
csv_reader_options_builder & delim_whitespace(bool val)
Sets whether to treat whitespace as field delimiter.
Definition: csv.hpp:1084
csv_reader_options_builder & skiprows(size_type skip)
Sets number of rows to skip from start.
Definition: csv.hpp:976
csv_reader_options_builder & skip_blank_lines(bool val)
Sets whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:1108
csv_reader_options && build()
move csv_reader_options member once it's built.
Definition: csv.hpp:1331
csv_reader_options_builder & dtypes(std::map< std::string, data_type > types)
Sets per-column types.
Definition: csv.hpp:1217
csv_reader_options_builder & quotechar(char ch)
Sets quoting character.
Definition: csv.hpp:1132
csv_reader_options_builder & na_values(std::vector< std::string > vals)
Sets additional values to recognize as null values.
Definition: csv.hpp:1265
csv_reader_options_builder & true_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean true values.
Definition: csv.hpp:1241
csv_reader_options_builder & decimal(char val)
Sets decimal point character.
Definition: csv.hpp:1048
csv_reader_options_builder & na_filter(bool val)
Sets whether to disable null filter.
Definition: csv.hpp:1289
csv_reader_options_builder & thousands(char val)
Sets numeric data thousands separator.
Definition: csv.hpp:1036
csv_reader_options_builder & parse_hex(std::vector< int > col_indices)
Sets indexes of columns to parse as hexadecimal.
Definition: csv.hpp:1205
csv_reader_options_builder & detect_whitespace_around_quotes(bool val)
Sets whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doubleq...
Definition: csv.hpp:1157
csv_reader_options_builder & windowslinetermination(bool val)
Sets whether to treat \r\n as line terminator.
Definition: csv.hpp:1072
csv_reader_options_builder & parse_dates(std::vector< int > col_indices)
Sets indexes of columns to read as datetime.
Definition: csv.hpp:1181
csv_reader_options_builder & nrows(size_type rows)
Sets number of rows to read.
Definition: csv.hpp:964
csv_reader_options_builder & names(std::vector< std::string > col_names)
Sets names of the column.
Definition: csv.hpp:904
csv_reader_options_builder & timestamp_type(data_type type)
Sets timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:1313
csv_reader_options_builder & mangle_dupe_cols(bool val)
Sets whether to rename duplicate column names.
Definition: csv.hpp:928
csv_reader_options_builder & skipfooter(size_type skip)
Sets number of rows to skip from end.
Definition: csv.hpp:988
csv_reader_options_builder()=default
Default constructor.
csv_reader_options_builder & byte_range_size(std::size_t size)
Sets number of bytes to read.
Definition: csv.hpp:892
csv_reader_options_builder & keep_default_na(bool val)
Sets whether to keep the built-in default NA values.
Definition: csv.hpp:1277
csv_reader_options_builder & quoting(quote_style style)
Sets quoting style.
Definition: csv.hpp:1120
csv_reader_options_builder & lineterminator(char term)
Sets line terminator.
Definition: csv.hpp:1012
csv_reader_options_builder & delimiter(char delim)
Sets field delimiter.
Definition: csv.hpp:1024
csv_reader_options_builder & use_cols_indexes(std::vector< int > col_indices)
Sets indexes of columns to read.
Definition: csv.hpp:952
csv_reader_options_builder & parse_dates(std::vector< std::string > col_names)
Sets names of columns to read as datetime.
Definition: csv.hpp:1169
csv_reader_options_builder(source_info src)
Constructor from source info.
Definition: csv.hpp:860
csv_reader_options_builder & comment(char val)
Sets comment line start character.
Definition: csv.hpp:1060
csv_reader_options_builder & compression(compression_type comp)
Sets compression format of the source.
Definition: csv.hpp:868
csv_reader_options_builder & header(size_type hdr)
Sets header row index.
Definition: csv.hpp:1000
csv_reader_options_builder & dayfirst(bool val)
Sets whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:1301
csv_reader_options_builder & prefix(std::string pfx)
Sets prefix to be used for column ID.
Definition: csv.hpp:916
csv_reader_options_builder & skipinitialspace(bool val)
Sets whether to skip whitespace after the delimiter.
Definition: csv.hpp:1096
Settings to use for read_csv().
Definition: csv.hpp:44
void enable_doublequote(bool val)
Sets a quote inside a value is double-quoted.
Definition: csv.hpp:714
void set_use_cols_indexes(std::vector< int > col_indices)
Sets indexes of columns to read.
Definition: csv.hpp:562
size_type get_skiprows() const
Returns number of rows to skip from start.
Definition: csv.hpp:274
bool is_enabled_delim_whitespace() const
Whether to treat whitespace as field delimiter.
Definition: csv.hpp:337
std::vector< int > const & get_parse_dates_indexes() const
Returns indexes of columns to read as datetime.
Definition: csv.hpp:400
void set_byte_range_offset(std::size_t offset)
Sets number of bytes to skip from source start.
Definition: csv.hpp:501
quote_style get_quoting() const
Returns quoting style.
Definition: csv.hpp:358
void set_parse_dates(std::vector< std::string > col_names)
Sets names of columns to read as datetime.
Definition: csv.hpp:729
void set_parse_dates(std::vector< int > col_indices)
Sets indexes of columns to read as datetime.
Definition: csv.hpp:739
bool is_enabled_doublequote() const
Whether a quote inside a value is double-quoted.
Definition: csv.hpp:372
char get_delimiter() const
Returns field delimiter.
Definition: csv.hpp:302
char get_lineterminator() const
Returns line terminator.
Definition: csv.hpp:295
csv_reader_options()=default
Default constructor.
void enable_detect_whitespace_around_quotes(bool val)
Sets whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doubleq...
Definition: csv.hpp:722
void set_false_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean false values.
Definition: csv.hpp:790
void set_decimal(char val)
Sets decimal point character.
Definition: csv.hpp:648
std::string get_prefix() const
Returns prefix to be used for column ID.
Definition: csv.hpp:236
bool is_enabled_detect_whitespace_around_quotes() const
Whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doublequote ...
Definition: csv.hpp:380
std::vector< std::string > const & get_na_values() const
Returns additional values to recognize as null values.
Definition: csv.hpp:452
char get_thousands() const
Returns numeric data thousands separator.
Definition: csv.hpp:309
bool is_enabled_mangle_dupe_cols() const
Whether to rename duplicate column names.
Definition: csv.hpp:243
void set_dtypes(std::map< std::string, data_type > types)
Sets per-column types.
Definition: csv.hpp:766
std::size_t get_byte_range_size_with_padding() const
Returns number of bytes to read with padding.
Definition: csv.hpp:191
std::vector< int > const & get_parse_hex_indexes() const
Returns indexes of columns to read as hexadecimal.
Definition: csv.hpp:420
std::vector< std::string > const & get_false_values() const
Returns additional values to recognize as boolean false values.
Definition: csv.hpp:445
void set_source(source_info src)
Sets source info.
Definition: csv.hpp:487
void enable_dayfirst(bool val)
Sets whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:830
void set_na_values(std::vector< std::string > vals)
Sets additional values to recognize as null values.
Definition: csv.hpp:800
void set_quoting(quote_style quoting)
Sets the expected quoting style used in the input CSV data.
Definition: csv.hpp:695
std::variant< std::vector< data_type >, std::map< std::string, data_type > > const & get_dtypes() const
Returns per-column types.
Definition: csv.hpp:428
void set_timestamp_type(data_type type)
Sets timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:837
std::size_t get_byte_range_offset() const
Returns number of bytes to skip from source start.
Definition: csv.hpp:177
data_type get_timestamp_type() const
Returns timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:480
bool is_enabled_na_filter() const
Whether to disable null filter.
Definition: csv.hpp:466
bool is_enabled_skip_blank_lines() const
Whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:351
char get_comment() const
Returns comment line start character.
Definition: csv.hpp:323
void set_lineterminator(char term)
Sets line terminator.
Definition: csv.hpp:627
void set_quotechar(char ch)
Sets quoting character.
Definition: csv.hpp:707
bool is_enabled_windowslinetermination() const
Whether to treat \r\n as line terminator.
Definition: csv.hpp:330
void enable_skip_blank_lines(bool val)
Sets whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:683
void enable_windowslinetermination(bool val)
Sets whether to treat \r\n as line terminator.
Definition: csv.hpp:662
void set_skiprows(size_type skiprows)
Sets number of rows to skip from start.
Definition: csv.hpp:588
void set_compression(compression_type comp)
Sets compression format of the source.
Definition: csv.hpp:494
void enable_delim_whitespace(bool val)
Sets whether to treat whitespace as field delimiter.
Definition: csv.hpp:669
std::vector< std::string > const & get_names() const
Returns names of the columns.
Definition: csv.hpp:229
void set_dtypes(std::vector< data_type > types)
Sets per-column types.
Definition: csv.hpp:773
void set_skipfooter(size_type skipfooter)
Sets number of rows to skip from end.
Definition: csv.hpp:602
bool is_enabled_dayfirst() const
Whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:473
std::size_t get_byte_range_padding() const
Returns number of bytes to pad when reading.
Definition: csv.hpp:205
void set_names(std::vector< std::string > col_names)
Sets names of the column.
Definition: csv.hpp:531
source_info const & get_source() const
Returns source info.
Definition: csv.hpp:163
void enable_keep_default_na(bool val)
Sets whether to keep the built-in default NA values.
Definition: csv.hpp:812
std::vector< std::string > const & get_parse_dates_names() const
Returns names of columns to read as datetime.
Definition: csv.hpp:390
void set_prefix(std::string pfx)
Sets prefix to be used for column ID.
Definition: csv.hpp:538
static csv_reader_options_builder builder(source_info src)
Creates a csv_reader_options_builder which will build csv_reader_options.
std::size_t get_byte_range_size() const
Returns number of bytes to read.
Definition: csv.hpp:184
std::vector< int > const & get_use_cols_indexes() const
Returns indexes of columns to read.
Definition: csv.hpp:260
std::vector< std::string > const & get_use_cols_names() const
Returns names of the columns to be read.
Definition: csv.hpp:250
void set_use_cols_names(std::vector< std::string > col_names)
Sets names of the columns to be read.
Definition: csv.hpp:552
compression_type get_compression() const
Returns compression format of the source.
Definition: csv.hpp:170
char get_quotechar() const
Returns quoting character.
Definition: csv.hpp:365
void set_true_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean true values.
Definition: csv.hpp:780
bool is_enabled_keep_default_na() const
Whether to keep the built-in default NA values.
Definition: csv.hpp:459
void set_header(size_type hdr)
Sets header row index.
Definition: csv.hpp:620
char get_decimal() const
Returns decimal point character.
Definition: csv.hpp:316
std::vector< std::string > const & get_true_values() const
Returns additional values to recognize as boolean true values.
Definition: csv.hpp:438
void set_parse_hex(std::vector< int > col_indices)
Sets indexes of columns to parse as hexadecimal.
Definition: csv.hpp:759
void set_thousands(char val)
Sets numeric data thousands separator.
Definition: csv.hpp:641
void enable_na_filter(bool val)
Sets whether to disable null filter.
Definition: csv.hpp:819
void set_byte_range_size(std::size_t size)
Sets number of bytes to read.
Definition: csv.hpp:516
void set_delimiter(char delim)
Sets field delimiter.
Definition: csv.hpp:634
void enable_mangle_dupe_cols(bool val)
Sets whether to rename duplicate column names.
Definition: csv.hpp:545
size_type get_nrows() const
Returns number of rows to read.
Definition: csv.hpp:267
std::vector< std::string > const & get_parse_hex_names() const
Returns names of columns to read as hexadecimal.
Definition: csv.hpp:410
void enable_skipinitialspace(bool val)
Sets whether to skip whitespace after the delimiter.
Definition: csv.hpp:676
size_type get_skipfooter() const
Returns number of rows to skip from end.
Definition: csv.hpp:281
void set_nrows(size_type nrows)
Sets number of rows to read.
Definition: csv.hpp:572
void set_comment(char val)
Sets comment line start character.
Definition: csv.hpp:655
bool is_enabled_skipinitialspace() const
Whether to skip whitespace after the delimiter.
Definition: csv.hpp:344
size_type get_header() const
Returns header row index.
Definition: csv.hpp:288
void set_parse_hex(std::vector< std::string > col_names)
Sets names of columns to parse as hexadecimal.
Definition: csv.hpp:749
Builder to build options for writer_csv()
Definition: csv.hpp:1592
csv_writer_options_builder()=default
Default constructor.
csv_writer_options && build()
move csv_writer_options member once it's built.
Definition: csv.hpp:1736
csv_writer_options_builder & quoting(quote_style quoting)
Sets the quote style for the writer.
Definition: csv.hpp:1718
csv_writer_options_builder & true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: csv.hpp:1692
csv_writer_options_builder & include_header(bool val)
Enables/Disables headers being written to csv.
Definition: csv.hpp:1644
csv_writer_options_builder & na_rep(std::string val)
Sets string to used for null entries.
Definition: csv.hpp:1632
csv_writer_options_builder & line_terminator(std::string term)
Sets character used for separating lines.
Definition: csv.hpp:1668
csv_writer_options_builder & false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: csv.hpp:1704
csv_writer_options_builder & names(std::vector< std::string > names)
Sets optional column names.
Definition: csv.hpp:1620
csv_writer_options_builder & inter_column_delimiter(char delim)
Sets character used for separating column values.
Definition: csv.hpp:1680
csv_writer_options_builder(sink_info const &sink, table_view const &table)
Constructor from sink and table.
Definition: csv.hpp:1609
csv_writer_options_builder & rows_per_chunk(int val)
Sets maximum number of rows to process for each file write.
Definition: csv.hpp:1656
Settings to use for write_csv().
Definition: csv.hpp:1370
void set_table(table_view const &table)
(Re)sets the table being written.
Definition: csv.hpp:1569
void set_rows_per_chunk(size_type val)
Sets maximum number of rows to process for each file write.
Definition: csv.hpp:1534
void set_quoting(quote_style quoting)
Sets the quote style for the writer.
Definition: csv.hpp:1581
void set_line_terminator(std::string term)
Sets character used for separating lines.
Definition: csv.hpp:1541
csv_writer_options()=default
Default constructor.
void set_inter_column_delimiter(char delim)
Sets character used for separating column values.
Definition: csv.hpp:1548
void set_true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: csv.hpp:1555
static csv_writer_options_builder builder(sink_info const &sink, table_view const &table)
Create builder to create csv_writer_options.
table_view const & get_table() const
Returns table that would be written to output.
Definition: csv.hpp:1437
void enable_include_header(bool val)
Enables/Disables headers being written to csv.
Definition: csv.hpp:1527
bool is_enabled_include_header() const
Whether to write headers to csv.
Definition: csv.hpp:1458
void set_na_rep(std::string val)
Sets string to used for null entries.
Definition: csv.hpp:1520
char get_inter_column_delimiter() const
Returns character used for separating column values.
Definition: csv.hpp:1479
std::string const & get_false_value() const
Returns string used for values == 0 in INT8 types.
Definition: csv.hpp:1493
std::string const & get_line_terminator() const
Returns character used for separating lines.
Definition: csv.hpp:1472
sink_info const & get_sink() const
Returns sink used for writer output.
Definition: csv.hpp:1430
std::vector< std::string > const & get_names() const
Returns names of the columns.
Definition: csv.hpp:1444
quote_style get_quoting() const
Returns the quote style for the writer.
Definition: csv.hpp:1505
std::string const & get_true_value() const
Returns string used for values != 0 in INT8 types.
Definition: csv.hpp:1486
void set_false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: csv.hpp:1562
size_type get_rows_per_chunk() const
Returns maximum number of rows to process for each file write.
Definition: csv.hpp:1465
std::string const & get_na_rep() const
Returns string to used for null entries.
Definition: csv.hpp:1451
void set_names(std::vector< std::string > names)
Sets optional associated column names.
Definition: csv.hpp:1513
A set of cudf::column_view's of the same size.
Definition: table_view.hpp:206
A set of cudf::column's of the same size.
Definition: table.hpp:31
size_type num_rows() const noexcept
Returns the number of rows.
Definition: table.hpp:103
Exception types and error-checking macros used throughout libcudf.
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
table_with_metadata read_csv(csv_reader_options options, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Reads a CSV dataset into a set of columns.
quote_style
Behavior when handling quotations in field data.
Definition: types.hpp:75
compression_type
Compression algorithms.
Definition: types.hpp:46
void write_csv(csv_writer_options const &options, rmm::cuda_stream_view stream=cudf::get_default_stream())
Writes a set of columns to CSV format.
constexpr bool is_supported_write_csv(data_type type)
Checks if a cudf::data_type is supported for CSV writing.
Definition: csv.hpp:1775
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::resource_ref< cuda::mr::device_accessible > device_async_resource_ref
CUDF_HOST_DEVICE constexpr decltype(auto) __forceinline__ type_dispatcher(cudf::data_type dtype, Functor f, Ts &&... args)
Invokes an operator() template with the type instantiation based on the specified cudf::data_type's i...
#define CUDF_EXPECTS(...)
Macro for checking (pre-)conditions that throws an exception when a condition is violated.
Definition: error.hpp:182
#define CUDF_FAIL(...)
Indicates that an erroneous code path has been taken.
Definition: error.hpp:223
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:84
Type definitions for the cuDF-IO API.
APIs for getting and setting the current device memory resource.
cuDF interfaces
Definition: host_udf.hpp:26
Destination information for write interfaces.
Definition: types.hpp:493
Source information for read interfaces.
Definition: types.hpp:306
Table with table metadata used by io readers to return the metadata by value.
Definition: types.hpp:271
Class definitions for (mutable)_table_view
Type declarations for libcudf.