csv.hpp
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: Apache-2.0
4  */
5 
6 #pragma once
7 
8 #include <cudf/io/detail/utils.hpp>
9 #include <cudf/io/types.hpp>
11 #include <cudf/types.hpp>
12 #include <cudf/utilities/error.hpp>
14 
15 #include <memory>
16 #include <string>
17 #include <unordered_map>
18 #include <utility>
19 #include <variant>
20 #include <vector>
21 
27 namespace CUDF_EXPORT cudf {
28 namespace io {
29 
38 class csv_reader_options_builder;
39 
45  source_info _source;
46 
47  // Read settings
48 
49  // Specify the compression format of the source or infer from file extension
50  compression_type _compression = compression_type::AUTO;
51  // Bytes to skip from the source start
52  std::size_t _byte_range_offset = 0;
53  // Bytes to read; always reads complete rows
54  std::size_t _byte_range_size = 0;
55  // Names of all the columns; if empty then names are auto-generated
56  std::vector<std::string> _names;
57  // If there is no header or names, prepend this to the column ID as the name
58  std::string _prefix;
59  // Whether to rename duplicate column names
60  bool _mangle_dupe_cols = true;
61 
62  // Filter settings
63 
64  // Names of columns to read; empty is all columns
65  std::vector<std::string> _use_cols_names;
66  // Indexes of columns to read; empty is all columns
67  std::vector<int> _use_cols_indexes;
68  // Rows to read; -1 is all
69  size_type _nrows = -1;
70  // Rows to skip from the start
71  size_type _skiprows = 0;
72  // Rows to skip from the end
73  size_type _skipfooter = 0;
74  // Header row index
75  size_type _header = 0;
76 
77  // Parsing settings
78 
79  // Line terminator
80  char _lineterminator = '\n';
81  // Field delimiter
82  char _delimiter = ',';
83  // Numeric data thousands separator; cannot match delimiter
84  char _thousands = '\0';
85  // Decimal point character; cannot match delimiter
86  char _decimal = '.';
87  // Comment line start character
88  char _comment = '\0';
89  bool _windowslinetermination = false;
90  // Treat whitespace as field delimiter; overrides character delimiter
91  bool _delim_whitespace = false;
92  // Skip whitespace after the delimiter
93  bool _skipinitialspace = false;
94  // Ignore empty lines or parse line values as invalid
95  bool _skip_blank_lines = true;
96  // Treatment of quoting behavior
97  quote_style _quoting = quote_style::MINIMAL;
98  // Quoting character (if `quoting` is true)
99  char _quotechar = '"';
100  // Whether a quote inside a value is double-quoted
101  bool _doublequote = true;
102  // Whether to detect quotes surrounded by spaces e.g. ` "data" `. This flag has no effect when
103  // _doublequote is true
104  bool _detect_whitespace_around_quotes = false;
105  // Names of columns to read as datetime
106  std::vector<std::string> _parse_dates_names;
107  // Indexes of columns to read as datetime
108  std::vector<int> _parse_dates_indexes;
109  // Names of columns to parse as hexadecimal
110  std::vector<std::string> _parse_hex_names;
111  // Indexes of columns to parse as hexadecimal
112  std::vector<int> _parse_hex_indexes;
113 
114  // Conversion settings
115 
116  // Per-column types; disables type inference on those columns
117  std::variant<std::vector<data_type>, std::map<std::string, data_type>> _dtypes;
118  // Additional values to recognize as boolean true values
119  std::vector<std::string> _true_values{"True", "TRUE", "true"};
120  // Additional values to recognize as boolean false values
121  std::vector<std::string> _false_values{"False", "FALSE", "false"};
122  // Additional values to recognize as null values
123  std::vector<std::string> _na_values;
124  // Whether to keep the built-in default NA values
125  bool _keep_default_na = true;
126  // Whether to disable null filter; disabling can improve performance
127  bool _na_filter = true;
128  // Whether to parse dates as DD/MM versus MM/DD
129  bool _dayfirst = false;
130  // Cast timestamp columns to a specific type
131  data_type _timestamp_type{type_id::EMPTY};
132 
138  explicit csv_reader_options(source_info src) : _source{std::move(src)} {}
139 
141 
142  public:
148  csv_reader_options() = default;
149 
157 
163  [[nodiscard]] source_info const& get_source() const { return _source; }
164 
170  [[nodiscard]] compression_type get_compression() const { return _compression; }
171 
177  [[nodiscard]] std::size_t get_byte_range_offset() const { return _byte_range_offset; }
178 
184  [[nodiscard]] std::size_t get_byte_range_size() const { return _byte_range_size; }
185 
191  [[nodiscard]] std::size_t get_byte_range_size_with_padding() const
192  {
193  if (_byte_range_size == 0) {
194  return 0;
195  } else {
196  return _byte_range_size + get_byte_range_padding();
197  }
198  }
199 
205  [[nodiscard]] std::size_t get_byte_range_padding() const
206  {
207  auto const num_names = _names.size();
208  auto const num_dtypes = std::visit([](auto const& dtypes) { return dtypes.size(); }, _dtypes);
209  auto const num_columns = std::max(num_dtypes, num_names);
210 
211  auto const max_row_bytes = 16 * 1024; // 16KB
212  auto const column_bytes = 64;
213  auto const base_padding = 1024; // 1KB
214 
215  if (num_columns == 0) {
216  // Use flat size if the number of columns is not known
217  return max_row_bytes;
218  }
219 
220  // Expand the size based on the number of columns, if available
221  return base_padding + static_cast<size_t>(num_columns) * column_bytes;
222  }
223 
229  [[nodiscard]] std::vector<std::string> const& get_names() const { return _names; }
230 
236  [[nodiscard]] std::string get_prefix() const { return _prefix; }
237 
243  [[nodiscard]] bool is_enabled_mangle_dupe_cols() const { return _mangle_dupe_cols; }
244 
250  [[nodiscard]] std::vector<std::string> const& get_use_cols_names() const
251  {
252  return _use_cols_names;
253  }
254 
260  [[nodiscard]] std::vector<int> const& get_use_cols_indexes() const { return _use_cols_indexes; }
261 
267  [[nodiscard]] size_type get_nrows() const { return _nrows; }
268 
274  [[nodiscard]] size_type get_skiprows() const { return _skiprows; }
275 
281  [[nodiscard]] size_type get_skipfooter() const { return _skipfooter; }
282 
288  [[nodiscard]] size_type get_header() const { return _header; }
289 
295  [[nodiscard]] char get_lineterminator() const { return _lineterminator; }
296 
302  [[nodiscard]] char get_delimiter() const { return _delimiter; }
303 
309  [[nodiscard]] char get_thousands() const { return _thousands; }
310 
316  [[nodiscard]] char get_decimal() const { return _decimal; }
317 
323  [[nodiscard]] char get_comment() const { return _comment; }
324 
330  [[nodiscard]] bool is_enabled_windowslinetermination() const { return _windowslinetermination; }
331 
337  [[nodiscard]] bool is_enabled_delim_whitespace() const { return _delim_whitespace; }
338 
344  [[nodiscard]] bool is_enabled_skipinitialspace() const { return _skipinitialspace; }
345 
351  [[nodiscard]] bool is_enabled_skip_blank_lines() const { return _skip_blank_lines; }
352 
358  [[nodiscard]] quote_style get_quoting() const { return _quoting; }
359 
365  [[nodiscard]] char get_quotechar() const { return _quotechar; }
366 
372  [[nodiscard]] bool is_enabled_doublequote() const { return _doublequote; }
373 
381  {
382  return _detect_whitespace_around_quotes;
383  }
384 
390  [[nodiscard]] std::vector<std::string> const& get_parse_dates_names() const
391  {
392  return _parse_dates_names;
393  }
394 
400  [[nodiscard]] std::vector<int> const& get_parse_dates_indexes() const
401  {
402  return _parse_dates_indexes;
403  }
404 
410  [[nodiscard]] std::vector<std::string> const& get_parse_hex_names() const
411  {
412  return _parse_hex_names;
413  }
414 
420  [[nodiscard]] std::vector<int> const& get_parse_hex_indexes() const { return _parse_hex_indexes; }
421 
427  [[nodiscard]] std::variant<std::vector<data_type>, std::map<std::string, data_type>> const&
428  get_dtypes() const
429  {
430  return _dtypes;
431  }
432 
438  [[nodiscard]] std::vector<std::string> const& get_true_values() const { return _true_values; }
439 
445  [[nodiscard]] std::vector<std::string> const& get_false_values() const { return _false_values; }
446 
452  [[nodiscard]] std::vector<std::string> const& get_na_values() const { return _na_values; }
453 
459  [[nodiscard]] bool is_enabled_keep_default_na() const { return _keep_default_na; }
460 
466  [[nodiscard]] bool is_enabled_na_filter() const { return _na_filter; }
467 
473  [[nodiscard]] bool is_enabled_dayfirst() const { return _dayfirst; }
474 
480  [[nodiscard]] data_type get_timestamp_type() const { return _timestamp_type; }
481 
487  void set_source(source_info src) { _source = std::move(src); }
488 
494  void set_compression(compression_type comp) { _compression = comp; }
495 
501  void set_byte_range_offset(std::size_t offset)
502  {
503  if ((offset != 0) and ((_skiprows != 0) or (_skipfooter != 0) or (_nrows != -1))) {
504  CUDF_FAIL(
505  "When there is valid value in skiprows or skipfooter or nrows, offset can't have non-zero "
506  "value");
507  }
508  _byte_range_offset = offset;
509  }
510 
516  void set_byte_range_size(std::size_t size)
517  {
518  if ((size != 0) and ((_skiprows != 0) or (_skipfooter != 0) or (_nrows != -1))) {
519  CUDF_FAIL(
520  "If the value of any of skiprows, skipfooter or nrows is valid, range size cannot be "
521  "non-zero.");
522  }
523  _byte_range_size = size;
524  }
525 
531  void set_names(std::vector<std::string> col_names) { _names = std::move(col_names); }
532 
538  void set_prefix(std::string pfx) { _prefix = std::move(pfx); }
539 
545  void enable_mangle_dupe_cols(bool val) { _mangle_dupe_cols = val; }
546 
552  void set_use_cols_names(std::vector<std::string> col_names)
553  {
554  _use_cols_names = std::move(col_names);
555  }
556 
562  void set_use_cols_indexes(std::vector<int> col_indices)
563  {
564  _use_cols_indexes = std::move(col_indices);
565  }
566 
572  void set_nrows(size_type nrows)
573  {
574  CUDF_EXPECTS((nrows == 0) or (_skipfooter == 0), "Cannot use both `nrows` and `skipfooter`");
575  if ((nrows != -1) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
576  CUDF_FAIL(
577  "nrows can't be a non negative value if range offset and/or range size has been set");
578  }
579 
580  _nrows = nrows;
581  }
582 
588  void set_skiprows(size_type skiprows)
589  {
590  if ((skiprows != 0) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
591  CUDF_FAIL("skiprows must be zero if range offset or range size has been set",
592  std::invalid_argument);
593  }
594  _skiprows = skiprows;
595  }
596 
602  void set_skipfooter(size_type skipfooter)
603  {
604  CUDF_EXPECTS((skipfooter == 0) or (_nrows == -1),
605  "Cannot use both `nrows` and `skipfooter`",
606  std::invalid_argument);
607  if ((skipfooter != 0) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
608  CUDF_FAIL("skipfooter must be zero if range offset or range size has been set",
609  std::invalid_argument);
610  }
611 
612  _skipfooter = skipfooter;
613  }
614 
620  void set_header(size_type hdr) { _header = hdr; }
621 
627  void set_lineterminator(char term) { _lineterminator = term; }
628 
634  void set_delimiter(char delim) { _delimiter = delim; }
635 
641  void set_thousands(char val) { _thousands = val; }
642 
648  void set_decimal(char val) { _decimal = val; }
649 
655  void set_comment(char val) { _comment = val; }
656 
662  void enable_windowslinetermination(bool val) { _windowslinetermination = val; }
663 
669  void enable_delim_whitespace(bool val) { _delim_whitespace = val; }
670 
676  void enable_skipinitialspace(bool val) { _skipinitialspace = val; }
677 
683  void enable_skip_blank_lines(bool val) { _skip_blank_lines = val; }
684 
693  void set_quoting(quote_style quoting)
694  {
695  CUDF_EXPECTS(quoting == quote_style::MINIMAL || quoting == quote_style::ALL ||
696  quoting == quote_style::NONNUMERIC || quoting == quote_style::NONE,
697  "Unsupported quoting style.");
698  _quoting = quoting;
699  }
700 
706  void set_quotechar(char ch) { _quotechar = ch; }
707 
713  void enable_doublequote(bool val) { _doublequote = val; }
714 
721  void enable_detect_whitespace_around_quotes(bool val) { _detect_whitespace_around_quotes = val; }
722 
728  void set_parse_dates(std::vector<std::string> col_names)
729  {
730  _parse_dates_names = std::move(col_names);
731  }
732 
738  void set_parse_dates(std::vector<int> col_indices)
739  {
740  _parse_dates_indexes = std::move(col_indices);
741  }
742 
748  void set_parse_hex(std::vector<std::string> col_names)
749  {
750  _parse_hex_names = std::move(col_names);
751  }
752 
758  void set_parse_hex(std::vector<int> col_indices) { _parse_hex_indexes = std::move(col_indices); }
759 
765  void set_dtypes(std::map<std::string, data_type> types) { _dtypes = std::move(types); }
766 
772  void set_dtypes(std::vector<data_type> types) { _dtypes = std::move(types); }
773 
779  void set_true_values(std::vector<std::string> vals)
780  {
781  _true_values.insert(_true_values.end(), vals.begin(), vals.end());
782  }
783 
789  void set_false_values(std::vector<std::string> vals)
790  {
791  _false_values.insert(_false_values.end(), vals.begin(), vals.end());
792  }
793 
799  void set_na_values(std::vector<std::string> vals)
800  {
801  CUDF_EXPECTS(vals.empty() or _na_filter, "Can't set na_values when na_filtering is disabled");
802 
803  _na_values = std::move(vals);
804  }
805 
811  void enable_keep_default_na(bool val) { _keep_default_na = val; }
812 
818  void enable_na_filter(bool val)
819  {
820  if (!val) { _na_values.clear(); }
821  _na_filter = val;
822  }
823 
829  void enable_dayfirst(bool val) { _dayfirst = val; }
830 
836  void set_timestamp_type(data_type type) { _timestamp_type = type; }
837 };
838 
844  csv_reader_options options;
845 
846  public:
853 
859  csv_reader_options_builder(source_info src) : options{std::move(src)} {}
860 
868  {
869  options.set_compression(comp);
870  return *this;
871  }
872 
880  {
881  options.set_byte_range_offset(offset);
882  return *this;
883  }
884 
892  {
893  options.set_byte_range_size(size);
894  return *this;
895  }
896 
903  csv_reader_options_builder& names(std::vector<std::string> col_names)
904  {
905  options.set_names(std::move(col_names));
906  return *this;
907  }
908 
916  {
917  options.set_prefix(std::move(pfx));
918  return *this;
919  }
920 
928  {
929  options.enable_mangle_dupe_cols(val);
930  return *this;
931  }
932 
939  csv_reader_options_builder& use_cols_names(std::vector<std::string> col_names)
940  {
941  options.set_use_cols_names(std::move(col_names));
942  return *this;
943  }
944 
951  csv_reader_options_builder& use_cols_indexes(std::vector<int> col_indices)
952  {
953  options.set_use_cols_indexes(std::move(col_indices));
954  return *this;
955  }
956 
964  {
965  options.set_nrows(rows);
966  return *this;
967  }
968 
976  {
977  options.set_skiprows(skip);
978  return *this;
979  }
980 
988  {
989  options.set_skipfooter(skip);
990  return *this;
991  }
992 
1000  {
1001  options.set_header(hdr);
1002  return *this;
1003  }
1004 
1012  {
1013  options.set_lineterminator(term);
1014  return *this;
1015  }
1016 
1024  {
1025  options.set_delimiter(delim);
1026  return *this;
1027  }
1028 
1036  {
1037  options.set_thousands(val);
1038  return *this;
1039  }
1040 
1048  {
1049  options.set_decimal(val);
1050  return *this;
1051  }
1052 
1060  {
1061  options.set_comment(val);
1062  return *this;
1063  }
1064 
1072  {
1073  options.enable_windowslinetermination(val);
1074  return *this;
1075  }
1076 
1084  {
1085  options.enable_delim_whitespace(val);
1086  return *this;
1087  }
1088 
1096  {
1097  options.enable_skipinitialspace(val);
1098  return *this;
1099  }
1100 
1108  {
1109  options.enable_skip_blank_lines(val);
1110  return *this;
1111  }
1112 
1120  {
1121  options.set_quoting(style);
1122  return *this;
1123  }
1124 
1132  {
1133  options.set_quotechar(ch);
1134  return *this;
1135  }
1136 
1144  {
1145  options.enable_doublequote(val);
1146  return *this;
1147  }
1148 
1157  {
1159  return *this;
1160  }
1161 
1168  csv_reader_options_builder& parse_dates(std::vector<std::string> col_names)
1169  {
1170  options.set_parse_dates(std::move(col_names));
1171  return *this;
1172  }
1173 
1180  csv_reader_options_builder& parse_dates(std::vector<int> col_indices)
1181  {
1182  options.set_parse_dates(std::move(col_indices));
1183  return *this;
1184  }
1185 
1192  csv_reader_options_builder& parse_hex(std::vector<std::string> col_names)
1193  {
1194  options.set_parse_hex(std::move(col_names));
1195  return *this;
1196  }
1197 
1204  csv_reader_options_builder& parse_hex(std::vector<int> col_indices)
1205  {
1206  options.set_parse_hex(std::move(col_indices));
1207  return *this;
1208  }
1209 
1216  csv_reader_options_builder& dtypes(std::map<std::string, data_type> types)
1217  {
1218  options.set_dtypes(std::move(types));
1219  return *this;
1220  }
1221 
1228  csv_reader_options_builder& dtypes(std::vector<data_type> types)
1229  {
1230  options.set_dtypes(std::move(types));
1231  return *this;
1232  }
1233 
1240  csv_reader_options_builder& true_values(std::vector<std::string> vals)
1241  {
1242  options.set_true_values(std::move(vals));
1243  return *this;
1244  }
1245 
1252  csv_reader_options_builder& false_values(std::vector<std::string> vals)
1253  {
1254  options.set_false_values(std::move(vals));
1255  return *this;
1256  }
1257 
1264  csv_reader_options_builder& na_values(std::vector<std::string> vals)
1265  {
1266  options.set_na_values(std::move(vals));
1267  return *this;
1268  }
1269 
1277  {
1278  options.enable_keep_default_na(val);
1279  return *this;
1280  }
1281 
1289  {
1290  options.enable_na_filter(val);
1291  return *this;
1292  }
1293 
1301  {
1302  options.enable_dayfirst(val);
1303  return *this;
1304  }
1305 
1313  {
1314  options.set_timestamp_type(type);
1315  return *this;
1316  }
1317 
1321  operator csv_reader_options&&() { return std::move(options); }
1322 
1330  csv_reader_options&& build() { return std::move(options); }
1331 };
1332 
1351  csv_reader_options options,
1354  // end of group
1365 
1370  // Specify the sink to use for writer output
1371  sink_info _sink;
1372  // Set of columns to output
1373  table_view _table;
1374  // string to use for null entries
1375  std::string _na_rep = "";
1376  // Indicates whether to write headers to csv
1377  bool _include_header = true;
1378  // maximum number of rows to write in each chunk (limits memory use)
1379  size_type _rows_per_chunk = std::numeric_limits<size_type>::max();
1380  // character to use for separating lines (default "\n")
1381  std::string _line_terminator = "\n";
1382  // character to use for separating column values (default ",")
1383  char _inter_column_delimiter = ',';
1384  // string to use for values != 0 in INT8 types (default 'true')
1385  std::string _true_value = std::string{"true"};
1386  // string to use for values == 0 in INT8 types (default 'false')
1387  std::string _false_value = std::string{"false"};
1388  // Names of all columns; if empty, writer will generate column names
1389  std::vector<std::string> _names;
1390  // Quote style. Currently only MINIMAL and NONE are supported.
1391  quote_style _quoting = quote_style::MINIMAL;
1392 
1399  explicit csv_writer_options(sink_info sink, table_view const& table)
1400  : _sink(std::move(sink)), _table(table), _rows_per_chunk(table.num_rows())
1401  {
1402  }
1403 
1405 
1406  public:
1412  explicit csv_writer_options() = default;
1413 
1423 
1429  [[nodiscard]] sink_info const& get_sink() const { return _sink; }
1430 
1436  [[nodiscard]] table_view const& get_table() const { return _table; }
1437 
1443  [[nodiscard]] std::vector<std::string> const& get_names() const { return _names; }
1444 
1450  [[nodiscard]] std::string const& get_na_rep() const { return _na_rep; }
1451 
1457  [[nodiscard]] bool is_enabled_include_header() const { return _include_header; }
1458 
1464  [[nodiscard]] size_type get_rows_per_chunk() const { return _rows_per_chunk; }
1465 
1471  [[nodiscard]] std::string const& get_line_terminator() const { return _line_terminator; }
1472 
1478  [[nodiscard]] char get_inter_column_delimiter() const { return _inter_column_delimiter; }
1479 
1485  [[nodiscard]] std::string const& get_true_value() const { return _true_value; }
1486 
1492  [[nodiscard]] std::string const& get_false_value() const { return _false_value; }
1493 
1504  [[nodiscard]] quote_style get_quoting() const { return _quoting; }
1505 
1506  // Setter
1512  void set_names(std::vector<std::string> names) { _names = std::move(names); }
1513 
1519  void set_na_rep(std::string val) { _na_rep = std::move(val); }
1520 
1526  void enable_include_header(bool val) { _include_header = val; }
1527 
1533  void set_rows_per_chunk(size_type val) { _rows_per_chunk = val; }
1534 
1540  void set_line_terminator(std::string term) { _line_terminator = std::move(term); }
1541 
1547  void set_inter_column_delimiter(char delim) { _inter_column_delimiter = delim; }
1548 
1554  void set_true_value(std::string val) { _true_value = std::move(val); }
1555 
1561  void set_false_value(std::string val) { _false_value = std::move(val); }
1562 
1568  void set_table(table_view const& table) { _table = table; }
1569 
1580  void set_quoting(quote_style quoting)
1581  {
1582  CUDF_EXPECTS(quoting == quote_style::MINIMAL || quoting == quote_style::NONE,
1583  "Only MINIMAL and NONE are supported for quoting.");
1584  _quoting = quoting;
1585  }
1586 };
1587 
1592  csv_writer_options options;
1593 
1594  public:
1600  explicit csv_writer_options_builder() = default;
1601 
1609  : options{sink, table}
1610  {
1611  }
1612 
1619  csv_writer_options_builder& names(std::vector<std::string> names)
1620  {
1621  options.set_names(std::move(names));
1622  return *this;
1623  }
1624 
1632  {
1633  options.set_na_rep(std::move(val));
1634  return *this;
1635  };
1636 
1644  {
1645  options.enable_include_header(val);
1646  return *this;
1647  }
1648 
1656  {
1657  options.set_rows_per_chunk(val);
1658  return *this;
1659  }
1660 
1668  {
1669  options.set_line_terminator(std::move(term));
1670  return *this;
1671  }
1672 
1680  {
1681  options.set_inter_column_delimiter(delim);
1682  return *this;
1683  }
1684 
1692  {
1693  options.set_true_value(std::move(val));
1694  return *this;
1695  }
1696 
1704  {
1705  options.set_false_value(std::move(val));
1706  return *this;
1707  }
1708 
1718  {
1719  options.set_quoting(quoting);
1720  return *this;
1721  }
1722 
1726  operator csv_writer_options&&() { return std::move(options); }
1727 
1735  csv_writer_options&& build() { return std::move(options); }
1736 };
1737 
1755 void write_csv(csv_writer_options const& options,
1757 
1759 struct is_supported_csv_write_type_fn {
1760  template <typename T>
1761  constexpr bool operator()() const
1762  {
1763  return cudf::io::detail::is_convertible_to_string_column<T>();
1764  }
1765 };
1767 
1774 constexpr bool is_supported_write_csv(data_type type)
1775 {
1776  return cudf::type_dispatcher(type, is_supported_csv_write_type_fn{});
1777 }
1778  // end of group
1780 } // namespace io
1781 } // namespace CUDF_EXPORT cudf
Indicator for the logical data type of an element in a column.
Definition: types.hpp:278
Builder to build options for read_csv().
Definition: csv.hpp:843
csv_reader_options_builder & dtypes(std::vector< data_type > types)
Sets per-column types.
Definition: csv.hpp:1228
csv_reader_options_builder & false_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean false values.
Definition: csv.hpp:1252
csv_reader_options_builder & use_cols_names(std::vector< std::string > col_names)
Sets names of the columns to be read.
Definition: csv.hpp:939
csv_reader_options_builder & doublequote(bool val)
Sets a quote inside a value is double-quoted.
Definition: csv.hpp:1143
csv_reader_options_builder & parse_hex(std::vector< std::string > col_names)
Sets names of columns to parse as hexadecimal.
Definition: csv.hpp:1192
csv_reader_options_builder & byte_range_offset(std::size_t offset)
Sets number of bytes to skip from source start.
Definition: csv.hpp:879
csv_reader_options_builder & delim_whitespace(bool val)
Sets whether to treat whitespace as field delimiter.
Definition: csv.hpp:1083
csv_reader_options_builder & skiprows(size_type skip)
Sets number of rows to skip from start.
Definition: csv.hpp:975
csv_reader_options_builder & skip_blank_lines(bool val)
Sets whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:1107
csv_reader_options && build()
move csv_reader_options member once it's built.
Definition: csv.hpp:1330
csv_reader_options_builder & dtypes(std::map< std::string, data_type > types)
Sets per-column types.
Definition: csv.hpp:1216
csv_reader_options_builder & quotechar(char ch)
Sets quoting character.
Definition: csv.hpp:1131
csv_reader_options_builder & na_values(std::vector< std::string > vals)
Sets additional values to recognize as null values.
Definition: csv.hpp:1264
csv_reader_options_builder & true_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean true values.
Definition: csv.hpp:1240
csv_reader_options_builder & decimal(char val)
Sets decimal point character.
Definition: csv.hpp:1047
csv_reader_options_builder & na_filter(bool val)
Sets whether to disable null filter.
Definition: csv.hpp:1288
csv_reader_options_builder & thousands(char val)
Sets numeric data thousands separator.
Definition: csv.hpp:1035
csv_reader_options_builder & parse_hex(std::vector< int > col_indices)
Sets indexes of columns to parse as hexadecimal.
Definition: csv.hpp:1204
csv_reader_options_builder & detect_whitespace_around_quotes(bool val)
Sets whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doubleq...
Definition: csv.hpp:1156
csv_reader_options_builder & windowslinetermination(bool val)
Sets whether to treat \r\n as line terminator.
Definition: csv.hpp:1071
csv_reader_options_builder & parse_dates(std::vector< int > col_indices)
Sets indexes of columns to read as datetime.
Definition: csv.hpp:1180
csv_reader_options_builder & nrows(size_type rows)
Sets number of rows to read.
Definition: csv.hpp:963
csv_reader_options_builder & names(std::vector< std::string > col_names)
Sets names of the column.
Definition: csv.hpp:903
csv_reader_options_builder & timestamp_type(data_type type)
Sets timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:1312
csv_reader_options_builder & mangle_dupe_cols(bool val)
Sets whether to rename duplicate column names.
Definition: csv.hpp:927
csv_reader_options_builder & skipfooter(size_type skip)
Sets number of rows to skip from end.
Definition: csv.hpp:987
csv_reader_options_builder()=default
Default constructor.
csv_reader_options_builder & byte_range_size(std::size_t size)
Sets number of bytes to read.
Definition: csv.hpp:891
csv_reader_options_builder & keep_default_na(bool val)
Sets whether to keep the built-in default NA values.
Definition: csv.hpp:1276
csv_reader_options_builder & quoting(quote_style style)
Sets quoting style.
Definition: csv.hpp:1119
csv_reader_options_builder & lineterminator(char term)
Sets line terminator.
Definition: csv.hpp:1011
csv_reader_options_builder & delimiter(char delim)
Sets field delimiter.
Definition: csv.hpp:1023
csv_reader_options_builder & use_cols_indexes(std::vector< int > col_indices)
Sets indexes of columns to read.
Definition: csv.hpp:951
csv_reader_options_builder & parse_dates(std::vector< std::string > col_names)
Sets names of columns to read as datetime.
Definition: csv.hpp:1168
csv_reader_options_builder(source_info src)
Constructor from source info.
Definition: csv.hpp:859
csv_reader_options_builder & comment(char val)
Sets comment line start character.
Definition: csv.hpp:1059
csv_reader_options_builder & compression(compression_type comp)
Sets compression format of the source.
Definition: csv.hpp:867
csv_reader_options_builder & header(size_type hdr)
Sets header row index.
Definition: csv.hpp:999
csv_reader_options_builder & dayfirst(bool val)
Sets whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:1300
csv_reader_options_builder & prefix(std::string pfx)
Sets prefix to be used for column ID.
Definition: csv.hpp:915
csv_reader_options_builder & skipinitialspace(bool val)
Sets whether to skip whitespace after the delimiter.
Definition: csv.hpp:1095
Settings to use for read_csv().
Definition: csv.hpp:44
void enable_doublequote(bool val)
Sets a quote inside a value is double-quoted.
Definition: csv.hpp:713
void set_use_cols_indexes(std::vector< int > col_indices)
Sets indexes of columns to read.
Definition: csv.hpp:562
size_type get_skiprows() const
Returns number of rows to skip from start.
Definition: csv.hpp:274
bool is_enabled_delim_whitespace() const
Whether to treat whitespace as field delimiter.
Definition: csv.hpp:337
std::vector< int > const & get_parse_dates_indexes() const
Returns indexes of columns to read as datetime.
Definition: csv.hpp:400
void set_byte_range_offset(std::size_t offset)
Sets number of bytes to skip from source start.
Definition: csv.hpp:501
quote_style get_quoting() const
Returns quoting style.
Definition: csv.hpp:358
void set_parse_dates(std::vector< std::string > col_names)
Sets names of columns to read as datetime.
Definition: csv.hpp:728
void set_parse_dates(std::vector< int > col_indices)
Sets indexes of columns to read as datetime.
Definition: csv.hpp:738
bool is_enabled_doublequote() const
Whether a quote inside a value is double-quoted.
Definition: csv.hpp:372
char get_delimiter() const
Returns field delimiter.
Definition: csv.hpp:302
char get_lineterminator() const
Returns line terminator.
Definition: csv.hpp:295
csv_reader_options()=default
Default constructor.
void enable_detect_whitespace_around_quotes(bool val)
Sets whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doubleq...
Definition: csv.hpp:721
void set_false_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean false values.
Definition: csv.hpp:789
void set_decimal(char val)
Sets decimal point character.
Definition: csv.hpp:648
std::string get_prefix() const
Returns prefix to be used for column ID.
Definition: csv.hpp:236
bool is_enabled_detect_whitespace_around_quotes() const
Whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doublequote ...
Definition: csv.hpp:380
std::vector< std::string > const & get_na_values() const
Returns additional values to recognize as null values.
Definition: csv.hpp:452
char get_thousands() const
Returns numeric data thousands separator.
Definition: csv.hpp:309
bool is_enabled_mangle_dupe_cols() const
Whether to rename duplicate column names.
Definition: csv.hpp:243
void set_dtypes(std::map< std::string, data_type > types)
Sets per-column types.
Definition: csv.hpp:765
std::size_t get_byte_range_size_with_padding() const
Returns number of bytes to read with padding.
Definition: csv.hpp:191
std::vector< int > const & get_parse_hex_indexes() const
Returns indexes of columns to read as hexadecimal.
Definition: csv.hpp:420
std::vector< std::string > const & get_false_values() const
Returns additional values to recognize as boolean false values.
Definition: csv.hpp:445
void set_source(source_info src)
Sets source info.
Definition: csv.hpp:487
void enable_dayfirst(bool val)
Sets whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:829
void set_na_values(std::vector< std::string > vals)
Sets additional values to recognize as null values.
Definition: csv.hpp:799
void set_quoting(quote_style quoting)
Sets the expected quoting style used in the input CSV data.
Definition: csv.hpp:693
std::variant< std::vector< data_type >, std::map< std::string, data_type > > const & get_dtypes() const
Returns per-column types.
Definition: csv.hpp:428
void set_timestamp_type(data_type type)
Sets timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:836
std::size_t get_byte_range_offset() const
Returns number of bytes to skip from source start.
Definition: csv.hpp:177
data_type get_timestamp_type() const
Returns timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:480
bool is_enabled_na_filter() const
Whether to disable null filter.
Definition: csv.hpp:466
bool is_enabled_skip_blank_lines() const
Whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:351
char get_comment() const
Returns comment line start character.
Definition: csv.hpp:323
void set_lineterminator(char term)
Sets line terminator.
Definition: csv.hpp:627
void set_quotechar(char ch)
Sets quoting character.
Definition: csv.hpp:706
bool is_enabled_windowslinetermination() const
Whether to treat \r\n as line terminator.
Definition: csv.hpp:330
void enable_skip_blank_lines(bool val)
Sets whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:683
void enable_windowslinetermination(bool val)
Sets whether to treat \r\n as line terminator.
Definition: csv.hpp:662
void set_skiprows(size_type skiprows)
Sets number of rows to skip from start.
Definition: csv.hpp:588
void set_compression(compression_type comp)
Sets compression format of the source.
Definition: csv.hpp:494
void enable_delim_whitespace(bool val)
Sets whether to treat whitespace as field delimiter.
Definition: csv.hpp:669
std::vector< std::string > const & get_names() const
Returns names of the columns.
Definition: csv.hpp:229
void set_dtypes(std::vector< data_type > types)
Sets per-column types.
Definition: csv.hpp:772
void set_skipfooter(size_type skipfooter)
Sets number of rows to skip from end.
Definition: csv.hpp:602
bool is_enabled_dayfirst() const
Whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:473
std::size_t get_byte_range_padding() const
Returns number of bytes to pad when reading.
Definition: csv.hpp:205
void set_names(std::vector< std::string > col_names)
Sets names of the column.
Definition: csv.hpp:531
source_info const & get_source() const
Returns source info.
Definition: csv.hpp:163
void enable_keep_default_na(bool val)
Sets whether to keep the built-in default NA values.
Definition: csv.hpp:811
std::vector< std::string > const & get_parse_dates_names() const
Returns names of columns to read as datetime.
Definition: csv.hpp:390
void set_prefix(std::string pfx)
Sets prefix to be used for column ID.
Definition: csv.hpp:538
static csv_reader_options_builder builder(source_info src)
Creates a csv_reader_options_builder which will build csv_reader_options.
std::size_t get_byte_range_size() const
Returns number of bytes to read.
Definition: csv.hpp:184
std::vector< int > const & get_use_cols_indexes() const
Returns indexes of columns to read.
Definition: csv.hpp:260
std::vector< std::string > const & get_use_cols_names() const
Returns names of the columns to be read.
Definition: csv.hpp:250
void set_use_cols_names(std::vector< std::string > col_names)
Sets names of the columns to be read.
Definition: csv.hpp:552
compression_type get_compression() const
Returns compression format of the source.
Definition: csv.hpp:170
char get_quotechar() const
Returns quoting character.
Definition: csv.hpp:365
void set_true_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean true values.
Definition: csv.hpp:779
bool is_enabled_keep_default_na() const
Whether to keep the built-in default NA values.
Definition: csv.hpp:459
void set_header(size_type hdr)
Sets header row index.
Definition: csv.hpp:620
char get_decimal() const
Returns decimal point character.
Definition: csv.hpp:316
std::vector< std::string > const & get_true_values() const
Returns additional values to recognize as boolean true values.
Definition: csv.hpp:438
void set_parse_hex(std::vector< int > col_indices)
Sets indexes of columns to parse as hexadecimal.
Definition: csv.hpp:758
void set_thousands(char val)
Sets numeric data thousands separator.
Definition: csv.hpp:641
void enable_na_filter(bool val)
Sets whether to disable null filter.
Definition: csv.hpp:818
void set_byte_range_size(std::size_t size)
Sets number of bytes to read.
Definition: csv.hpp:516
void set_delimiter(char delim)
Sets field delimiter.
Definition: csv.hpp:634
void enable_mangle_dupe_cols(bool val)
Sets whether to rename duplicate column names.
Definition: csv.hpp:545
size_type get_nrows() const
Returns number of rows to read.
Definition: csv.hpp:267
std::vector< std::string > const & get_parse_hex_names() const
Returns names of columns to read as hexadecimal.
Definition: csv.hpp:410
void enable_skipinitialspace(bool val)
Sets whether to skip whitespace after the delimiter.
Definition: csv.hpp:676
size_type get_skipfooter() const
Returns number of rows to skip from end.
Definition: csv.hpp:281
void set_nrows(size_type nrows)
Sets number of rows to read.
Definition: csv.hpp:572
void set_comment(char val)
Sets comment line start character.
Definition: csv.hpp:655
bool is_enabled_skipinitialspace() const
Whether to skip whitespace after the delimiter.
Definition: csv.hpp:344
size_type get_header() const
Returns header row index.
Definition: csv.hpp:288
void set_parse_hex(std::vector< std::string > col_names)
Sets names of columns to parse as hexadecimal.
Definition: csv.hpp:748
Builder to build options for writer_csv()
Definition: csv.hpp:1591
csv_writer_options_builder()=default
Default constructor.
csv_writer_options && build()
move csv_writer_options member once it's built.
Definition: csv.hpp:1735
csv_writer_options_builder & quoting(quote_style quoting)
Sets the quote style for the writer.
Definition: csv.hpp:1717
csv_writer_options_builder & true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: csv.hpp:1691
csv_writer_options_builder & include_header(bool val)
Enables/Disables headers being written to csv.
Definition: csv.hpp:1643
csv_writer_options_builder & na_rep(std::string val)
Sets string to used for null entries.
Definition: csv.hpp:1631
csv_writer_options_builder & line_terminator(std::string term)
Sets character used for separating lines.
Definition: csv.hpp:1667
csv_writer_options_builder & false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: csv.hpp:1703
csv_writer_options_builder & names(std::vector< std::string > names)
Sets optional column names.
Definition: csv.hpp:1619
csv_writer_options_builder & inter_column_delimiter(char delim)
Sets character used for separating column values.
Definition: csv.hpp:1679
csv_writer_options_builder(sink_info const &sink, table_view const &table)
Constructor from sink and table.
Definition: csv.hpp:1608
csv_writer_options_builder & rows_per_chunk(int val)
Sets maximum number of rows to process for each file write.
Definition: csv.hpp:1655
Settings to use for write_csv().
Definition: csv.hpp:1369
void set_table(table_view const &table)
(Re)sets the table being written.
Definition: csv.hpp:1568
void set_rows_per_chunk(size_type val)
Sets maximum number of rows to process for each file write.
Definition: csv.hpp:1533
void set_quoting(quote_style quoting)
Sets the quote style for the writer.
Definition: csv.hpp:1580
void set_line_terminator(std::string term)
Sets character used for separating lines.
Definition: csv.hpp:1540
csv_writer_options()=default
Default constructor.
void set_inter_column_delimiter(char delim)
Sets character used for separating column values.
Definition: csv.hpp:1547
void set_true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: csv.hpp:1554
static csv_writer_options_builder builder(sink_info const &sink, table_view const &table)
Create builder to create csv_writer_options.
table_view const & get_table() const
Returns table that would be written to output.
Definition: csv.hpp:1436
void enable_include_header(bool val)
Enables/Disables headers being written to csv.
Definition: csv.hpp:1526
bool is_enabled_include_header() const
Whether to write headers to csv.
Definition: csv.hpp:1457
void set_na_rep(std::string val)
Sets string to used for null entries.
Definition: csv.hpp:1519
char get_inter_column_delimiter() const
Returns character used for separating column values.
Definition: csv.hpp:1478
std::string const & get_false_value() const
Returns string used for values == 0 in INT8 types.
Definition: csv.hpp:1492
std::string const & get_line_terminator() const
Returns character used for separating lines.
Definition: csv.hpp:1471
sink_info const & get_sink() const
Returns sink used for writer output.
Definition: csv.hpp:1429
std::vector< std::string > const & get_names() const
Returns names of the columns.
Definition: csv.hpp:1443
quote_style get_quoting() const
Returns the quote style for the writer.
Definition: csv.hpp:1504
std::string const & get_true_value() const
Returns string used for values != 0 in INT8 types.
Definition: csv.hpp:1485
void set_false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: csv.hpp:1561
size_type get_rows_per_chunk() const
Returns maximum number of rows to process for each file write.
Definition: csv.hpp:1464
std::string const & get_na_rep() const
Returns string to used for null entries.
Definition: csv.hpp:1450
void set_names(std::vector< std::string > names)
Sets optional associated column names.
Definition: csv.hpp:1512
A set of cudf::column_view's of the same size.
Definition: table_view.hpp:206
A set of cudf::column's of the same size.
Definition: table.hpp:31
size_type num_rows() const noexcept
Returns the number of rows.
Definition: table.hpp:103
Exception types and error-checking macros used throughout libcudf.
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
table_with_metadata read_csv(csv_reader_options options, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Reads a CSV dataset into a set of columns.
quote_style
Behavior when handling quotations in field data.
Definition: types.hpp:75
compression_type
Compression algorithms.
Definition: types.hpp:46
void write_csv(csv_writer_options const &options, rmm::cuda_stream_view stream=cudf::get_default_stream())
Writes a set of columns to CSV format.
constexpr bool is_supported_write_csv(data_type type)
Checks if a cudf::data_type is supported for CSV writing.
Definition: csv.hpp:1774
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::resource_ref< cuda::mr::device_accessible > device_async_resource_ref
CUDF_HOST_DEVICE constexpr decltype(auto) __forceinline__ type_dispatcher(cudf::data_type dtype, Functor f, Ts &&... args)
Invokes an operator() template with the type instantiation based on the specified cudf::data_type's i...
#define CUDF_EXPECTS(...)
Macro for checking (pre-)conditions that throws an exception when a condition is violated.
Definition: error.hpp:182
#define CUDF_FAIL(...)
Indicates that an erroneous code path has been taken.
Definition: error.hpp:223
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:76
Type definitions for the cuDF-IO API.
APIs for getting and setting the current device memory resource.
cuDF interfaces
Definition: host_udf.hpp:26
Destination information for write interfaces.
Definition: types.hpp:493
Source information for read interfaces.
Definition: types.hpp:306
Table with table metadata used by io readers to return the metadata by value.
Definition: types.hpp:271
Class definitions for (mutable)_table_view
Type declarations for libcudf.