csv.hpp
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: Apache-2.0
4  */
5 
6 #pragma once
7 
8 #include <cudf/io/detail/utils.hpp>
9 #include <cudf/io/types.hpp>
11 #include <cudf/types.hpp>
12 #include <cudf/utilities/error.hpp>
14 
15 #include <memory>
16 #include <string>
17 #include <unordered_map>
18 #include <utility>
19 #include <variant>
20 #include <vector>
21 
27 namespace CUDF_EXPORT cudf {
28 namespace io {
29 
38 class csv_reader_options_builder;
39 
45  source_info _source;
46 
47  // Read settings
48 
49  // Specify the compression format of the source or infer from file extension
50  compression_type _compression = compression_type::AUTO;
51  // Bytes to skip from the source start
52  std::size_t _byte_range_offset = 0;
53  // Bytes to read; always reads complete rows
54  std::size_t _byte_range_size = 0;
55  // Names of all the columns; if empty then names are auto-generated
56  std::vector<std::string> _names;
57  // If there is no header or names, prepend this to the column ID as the name
58  std::string _prefix;
59  // Whether to rename duplicate column names
60  bool _mangle_dupe_cols = true;
61 
62  // Filter settings
63 
64  // Names of columns to read; empty is all columns
65  std::vector<std::string> _use_cols_names;
66  // Indexes of columns to read; empty is all columns
67  std::vector<int> _use_cols_indexes;
68  // Rows to read; -1 is all
69  size_type _nrows = -1;
70  // Rows to skip from the start
71  size_type _skiprows = 0;
72  // Rows to skip from the end
73  size_type _skipfooter = 0;
74  // Header row index
75  size_type _header = 0;
76 
77  // Parsing settings
78 
79  // Line terminator
80  char _lineterminator = '\n';
81  // Field delimiter
82  char _delimiter = ',';
83  // Numeric data thousands separator; cannot match delimiter
84  char _thousands = '\0';
85  // Decimal point character; cannot match delimiter
86  char _decimal = '.';
87  // Comment line start character
88  char _comment = '\0';
89  bool _windowslinetermination = false;
90  // Treat whitespace as field delimiter; overrides character delimiter
91  bool _delim_whitespace = false;
92  // Skip whitespace after the delimiter
93  bool _skipinitialspace = false;
94  // Ignore empty lines or parse line values as invalid
95  bool _skip_blank_lines = true;
96  // Treatment of quoting behavior
97  quote_style _quoting = quote_style::MINIMAL;
98  // Quoting character (if `quoting` is true)
99  char _quotechar = '"';
100  // Whether a quote inside a value is double-quoted
101  bool _doublequote = true;
102  // Whether to detect quotes surrounded by spaces e.g. ` "data" `. This flag has no effect when
103  // _doublequote is true
104  bool _detect_whitespace_around_quotes = false;
105  // Names of columns to read as datetime
106  std::vector<std::string> _parse_dates_names;
107  // Indexes of columns to read as datetime
108  std::vector<int> _parse_dates_indexes;
109  // Names of columns to parse as hexadecimal
110  std::vector<std::string> _parse_hex_names;
111  // Indexes of columns to parse as hexadecimal
112  std::vector<int> _parse_hex_indexes;
113 
114  // Conversion settings
115 
116  // Per-column types; disables type inference on those columns
117  std::variant<std::vector<data_type>, std::map<std::string, data_type>> _dtypes;
118  // Additional values to recognize as boolean true values
119  std::vector<std::string> _true_values{"True", "TRUE", "true"};
120  // Additional values to recognize as boolean false values
121  std::vector<std::string> _false_values{"False", "FALSE", "false"};
122  // Additional values to recognize as null values
123  std::vector<std::string> _na_values;
124  // Whether to keep the built-in default NA values
125  bool _keep_default_na = true;
126  // Whether to disable null filter; disabling can improve performance
127  bool _na_filter = true;
128  // Whether to parse dates as DD/MM versus MM/DD
129  bool _dayfirst = false;
130  // Cast timestamp columns to a specific type
131  data_type _timestamp_type{type_id::EMPTY};
132 
138  explicit csv_reader_options(source_info src) : _source{std::move(src)} {}
139 
141 
142  public:
148  csv_reader_options() = default;
149 
157 
163  [[nodiscard]] source_info const& get_source() const { return _source; }
164 
170  [[nodiscard]] compression_type get_compression() const { return _compression; }
171 
177  [[nodiscard]] std::size_t get_byte_range_offset() const { return _byte_range_offset; }
178 
184  [[nodiscard]] std::size_t get_byte_range_size() const { return _byte_range_size; }
185 
191  [[nodiscard]] std::size_t get_byte_range_size_with_padding() const
192  {
193  if (_byte_range_size == 0) {
194  return 0;
195  } else {
196  return _byte_range_size + get_byte_range_padding();
197  }
198  }
199 
205  [[nodiscard]] std::size_t get_byte_range_padding() const
206  {
207  auto const num_names = _names.size();
208  auto const num_dtypes = std::visit([](auto const& dtypes) { return dtypes.size(); }, _dtypes);
209  auto const num_columns = std::max(num_dtypes, num_names);
210 
211  auto const max_row_bytes = 16 * 1024; // 16KB
212  auto const column_bytes = 64;
213  auto const base_padding = 1024; // 1KB
214 
215  if (num_columns == 0) {
216  // Use flat size if the number of columns is not known
217  return max_row_bytes;
218  }
219 
220  // Expand the size based on the number of columns, if available
221  return base_padding + static_cast<size_t>(num_columns) * column_bytes;
222  }
223 
229  [[nodiscard]] std::vector<std::string> const& get_names() const { return _names; }
230 
236  [[nodiscard]] std::string get_prefix() const { return _prefix; }
237 
243  [[nodiscard]] bool is_enabled_mangle_dupe_cols() const { return _mangle_dupe_cols; }
244 
250  [[nodiscard]] std::vector<std::string> const& get_use_cols_names() const
251  {
252  return _use_cols_names;
253  }
254 
260  [[nodiscard]] std::vector<int> const& get_use_cols_indexes() const { return _use_cols_indexes; }
261 
267  [[nodiscard]] size_type get_nrows() const { return _nrows; }
268 
274  [[nodiscard]] size_type get_skiprows() const { return _skiprows; }
275 
281  [[nodiscard]] size_type get_skipfooter() const { return _skipfooter; }
282 
288  [[nodiscard]] size_type get_header() const { return _header; }
289 
295  [[nodiscard]] char get_lineterminator() const { return _lineterminator; }
296 
302  [[nodiscard]] char get_delimiter() const { return _delimiter; }
303 
309  [[nodiscard]] char get_thousands() const { return _thousands; }
310 
316  [[nodiscard]] char get_decimal() const { return _decimal; }
317 
323  [[nodiscard]] char get_comment() const { return _comment; }
324 
333  [[nodiscard]] [[deprecated("CRLF input is supported using the default `\\n` line terminator.")]]
335  {
336  return _windowslinetermination;
337  }
338 
344  [[nodiscard]] bool is_enabled_delim_whitespace() const { return _delim_whitespace; }
345 
351  [[nodiscard]] bool is_enabled_skipinitialspace() const { return _skipinitialspace; }
352 
358  [[nodiscard]] bool is_enabled_skip_blank_lines() const { return _skip_blank_lines; }
359 
365  [[nodiscard]] quote_style get_quoting() const { return _quoting; }
366 
372  [[nodiscard]] char get_quotechar() const { return _quotechar; }
373 
379  [[nodiscard]] bool is_enabled_doublequote() const { return _doublequote; }
380 
388  {
389  return _detect_whitespace_around_quotes;
390  }
391 
397  [[nodiscard]] std::vector<std::string> const& get_parse_dates_names() const
398  {
399  return _parse_dates_names;
400  }
401 
407  [[nodiscard]] std::vector<int> const& get_parse_dates_indexes() const
408  {
409  return _parse_dates_indexes;
410  }
411 
417  [[nodiscard]] std::vector<std::string> const& get_parse_hex_names() const
418  {
419  return _parse_hex_names;
420  }
421 
427  [[nodiscard]] std::vector<int> const& get_parse_hex_indexes() const { return _parse_hex_indexes; }
428 
434  [[nodiscard]] std::variant<std::vector<data_type>, std::map<std::string, data_type>> const&
435  get_dtypes() const
436  {
437  return _dtypes;
438  }
439 
445  [[nodiscard]] std::vector<std::string> const& get_true_values() const { return _true_values; }
446 
452  [[nodiscard]] std::vector<std::string> const& get_false_values() const { return _false_values; }
453 
459  [[nodiscard]] std::vector<std::string> const& get_na_values() const { return _na_values; }
460 
466  [[nodiscard]] bool is_enabled_keep_default_na() const { return _keep_default_na; }
467 
473  [[nodiscard]] bool is_enabled_na_filter() const { return _na_filter; }
474 
480  [[nodiscard]] bool is_enabled_dayfirst() const { return _dayfirst; }
481 
487  [[nodiscard]] data_type get_timestamp_type() const { return _timestamp_type; }
488 
494  void set_source(source_info src) { _source = std::move(src); }
495 
501  void set_compression(compression_type comp) { _compression = comp; }
502 
508  void set_byte_range_offset(std::size_t offset)
509  {
510  if ((offset != 0) and ((_skiprows != 0) or (_skipfooter != 0) or (_nrows != -1))) {
511  CUDF_FAIL(
512  "When there is valid value in skiprows or skipfooter or nrows, offset can't have non-zero "
513  "value");
514  }
515  _byte_range_offset = offset;
516  }
517 
523  void set_byte_range_size(std::size_t size)
524  {
525  if ((size != 0) and ((_skiprows != 0) or (_skipfooter != 0) or (_nrows != -1))) {
526  CUDF_FAIL(
527  "If the value of any of skiprows, skipfooter or nrows is valid, range size cannot be "
528  "non-zero.");
529  }
530  _byte_range_size = size;
531  }
532 
538  void set_names(std::vector<std::string> col_names) { _names = std::move(col_names); }
539 
545  void set_prefix(std::string pfx) { _prefix = std::move(pfx); }
546 
552  void enable_mangle_dupe_cols(bool val) { _mangle_dupe_cols = val; }
553 
559  void set_use_cols_names(std::vector<std::string> col_names)
560  {
561  _use_cols_names = std::move(col_names);
562  }
563 
569  void set_use_cols_indexes(std::vector<int> col_indices)
570  {
571  _use_cols_indexes = std::move(col_indices);
572  }
573 
579  void set_nrows(size_type nrows)
580  {
581  CUDF_EXPECTS((nrows == 0) or (_skipfooter == 0), "Cannot use both `nrows` and `skipfooter`");
582  if ((nrows != -1) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
583  CUDF_FAIL(
584  "nrows can't be a non negative value if range offset and/or range size has been set");
585  }
586 
587  _nrows = nrows;
588  }
589 
595  void set_skiprows(size_type skiprows)
596  {
597  if ((skiprows != 0) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
598  CUDF_FAIL("skiprows must be zero if range offset or range size has been set",
599  std::invalid_argument);
600  }
601  _skiprows = skiprows;
602  }
603 
609  void set_skipfooter(size_type skipfooter)
610  {
611  CUDF_EXPECTS((skipfooter == 0) or (_nrows == -1),
612  "Cannot use both `nrows` and `skipfooter`",
613  std::invalid_argument);
614  if ((skipfooter != 0) and ((_byte_range_offset != 0) or (_byte_range_size != 0))) {
615  CUDF_FAIL("skipfooter must be zero if range offset or range size has been set",
616  std::invalid_argument);
617  }
618 
619  _skipfooter = skipfooter;
620  }
621 
627  void set_header(size_type hdr) { _header = hdr; }
628 
634  void set_lineterminator(char term) { _lineterminator = term; }
635 
641  void set_delimiter(char delim) { _delimiter = delim; }
642 
648  void set_thousands(char val) { _thousands = val; }
649 
655  void set_decimal(char val) { _decimal = val; }
656 
662  void set_comment(char val) { _comment = val; }
663 
672  [[deprecated("CRLF input is supported using the default `\\n` line terminator.")]]
674  {
675  _windowslinetermination = val;
676  }
677 
683  void enable_delim_whitespace(bool val) { _delim_whitespace = val; }
684 
690  void enable_skipinitialspace(bool val) { _skipinitialspace = val; }
691 
697  void enable_skip_blank_lines(bool val) { _skip_blank_lines = val; }
698 
707  void set_quoting(quote_style quoting)
708  {
709  CUDF_EXPECTS(quoting == quote_style::MINIMAL || quoting == quote_style::ALL ||
710  quoting == quote_style::NONNUMERIC || quoting == quote_style::NONE,
711  "Unsupported quoting style.");
712  _quoting = quoting;
713  }
714 
720  void set_quotechar(char ch) { _quotechar = ch; }
721 
727  void enable_doublequote(bool val) { _doublequote = val; }
728 
735  void enable_detect_whitespace_around_quotes(bool val) { _detect_whitespace_around_quotes = val; }
736 
742  void set_parse_dates(std::vector<std::string> col_names)
743  {
744  _parse_dates_names = std::move(col_names);
745  }
746 
752  void set_parse_dates(std::vector<int> col_indices)
753  {
754  _parse_dates_indexes = std::move(col_indices);
755  }
756 
762  void set_parse_hex(std::vector<std::string> col_names)
763  {
764  _parse_hex_names = std::move(col_names);
765  }
766 
772  void set_parse_hex(std::vector<int> col_indices) { _parse_hex_indexes = std::move(col_indices); }
773 
779  void set_dtypes(std::map<std::string, data_type> types) { _dtypes = std::move(types); }
780 
786  void set_dtypes(std::vector<data_type> types) { _dtypes = std::move(types); }
787 
793  void set_true_values(std::vector<std::string> vals)
794  {
795  _true_values.insert(_true_values.end(), vals.begin(), vals.end());
796  }
797 
803  void set_false_values(std::vector<std::string> vals)
804  {
805  _false_values.insert(_false_values.end(), vals.begin(), vals.end());
806  }
807 
813  void set_na_values(std::vector<std::string> vals)
814  {
815  CUDF_EXPECTS(vals.empty() or _na_filter, "Can't set na_values when na_filtering is disabled");
816 
817  _na_values = std::move(vals);
818  }
819 
825  void enable_keep_default_na(bool val) { _keep_default_na = val; }
826 
832  void enable_na_filter(bool val)
833  {
834  if (!val) { _na_values.clear(); }
835  _na_filter = val;
836  }
837 
843  void enable_dayfirst(bool val) { _dayfirst = val; }
844 
850  void set_timestamp_type(data_type type) { _timestamp_type = type; }
851 };
852 
858  csv_reader_options options;
859 
860  public:
867 
873  csv_reader_options_builder(source_info src) : options{std::move(src)} {}
874 
882  {
883  options.set_compression(comp);
884  return *this;
885  }
886 
894  {
895  options.set_byte_range_offset(offset);
896  return *this;
897  }
898 
906  {
907  options.set_byte_range_size(size);
908  return *this;
909  }
910 
917  csv_reader_options_builder& names(std::vector<std::string> col_names)
918  {
919  options.set_names(std::move(col_names));
920  return *this;
921  }
922 
930  {
931  options.set_prefix(std::move(pfx));
932  return *this;
933  }
934 
942  {
943  options.enable_mangle_dupe_cols(val);
944  return *this;
945  }
946 
953  csv_reader_options_builder& use_cols_names(std::vector<std::string> col_names)
954  {
955  options.set_use_cols_names(std::move(col_names));
956  return *this;
957  }
958 
965  csv_reader_options_builder& use_cols_indexes(std::vector<int> col_indices)
966  {
967  options.set_use_cols_indexes(std::move(col_indices));
968  return *this;
969  }
970 
978  {
979  options.set_nrows(rows);
980  return *this;
981  }
982 
990  {
991  options.set_skiprows(skip);
992  return *this;
993  }
994 
1002  {
1003  options.set_skipfooter(skip);
1004  return *this;
1005  }
1006 
1014  {
1015  options.set_header(hdr);
1016  return *this;
1017  }
1018 
1026  {
1027  options.set_lineterminator(term);
1028  return *this;
1029  }
1030 
1038  {
1039  options.set_delimiter(delim);
1040  return *this;
1041  }
1042 
1050  {
1051  options.set_thousands(val);
1052  return *this;
1053  }
1054 
1062  {
1063  options.set_decimal(val);
1064  return *this;
1065  }
1066 
1074  {
1075  options.set_comment(val);
1076  return *this;
1077  }
1078 
1088  [[deprecated("CRLF input is supported using the default `\\n` line terminator.")]]
1090  {
1091  options._windowslinetermination = val;
1092  return *this;
1093  }
1094 
1102  {
1103  options.enable_delim_whitespace(val);
1104  return *this;
1105  }
1106 
1114  {
1115  options.enable_skipinitialspace(val);
1116  return *this;
1117  }
1118 
1126  {
1127  options.enable_skip_blank_lines(val);
1128  return *this;
1129  }
1130 
1138  {
1139  options.set_quoting(style);
1140  return *this;
1141  }
1142 
1150  {
1151  options.set_quotechar(ch);
1152  return *this;
1153  }
1154 
1162  {
1163  options.enable_doublequote(val);
1164  return *this;
1165  }
1166 
1175  {
1177  return *this;
1178  }
1179 
1186  csv_reader_options_builder& parse_dates(std::vector<std::string> col_names)
1187  {
1188  options.set_parse_dates(std::move(col_names));
1189  return *this;
1190  }
1191 
1198  csv_reader_options_builder& parse_dates(std::vector<int> col_indices)
1199  {
1200  options.set_parse_dates(std::move(col_indices));
1201  return *this;
1202  }
1203 
1210  csv_reader_options_builder& parse_hex(std::vector<std::string> col_names)
1211  {
1212  options.set_parse_hex(std::move(col_names));
1213  return *this;
1214  }
1215 
1222  csv_reader_options_builder& parse_hex(std::vector<int> col_indices)
1223  {
1224  options.set_parse_hex(std::move(col_indices));
1225  return *this;
1226  }
1227 
1234  csv_reader_options_builder& dtypes(std::map<std::string, data_type> types)
1235  {
1236  options.set_dtypes(std::move(types));
1237  return *this;
1238  }
1239 
1246  csv_reader_options_builder& dtypes(std::vector<data_type> types)
1247  {
1248  options.set_dtypes(std::move(types));
1249  return *this;
1250  }
1251 
1258  csv_reader_options_builder& true_values(std::vector<std::string> vals)
1259  {
1260  options.set_true_values(std::move(vals));
1261  return *this;
1262  }
1263 
1270  csv_reader_options_builder& false_values(std::vector<std::string> vals)
1271  {
1272  options.set_false_values(std::move(vals));
1273  return *this;
1274  }
1275 
1282  csv_reader_options_builder& na_values(std::vector<std::string> vals)
1283  {
1284  options.set_na_values(std::move(vals));
1285  return *this;
1286  }
1287 
1295  {
1296  options.enable_keep_default_na(val);
1297  return *this;
1298  }
1299 
1307  {
1308  options.enable_na_filter(val);
1309  return *this;
1310  }
1311 
1319  {
1320  options.enable_dayfirst(val);
1321  return *this;
1322  }
1323 
1331  {
1332  options.set_timestamp_type(type);
1333  return *this;
1334  }
1335 
1339  operator csv_reader_options&&() { return std::move(options); }
1340 
1348  csv_reader_options&& build() { return std::move(options); }
1349 };
1350 
1369  csv_reader_options options,
1370  cuda::stream_ref stream = cudf::get_default_stream(),
1372  // end of group
1383 
1387 size_t constexpr default_csv_compression_block_size = 1024 * 1024;
1388 
1393  // Specify the sink to use for writer output
1394  sink_info _sink;
1395  // Set of columns to output
1396  table_view _table;
1397  // string to use for null entries
1398  std::string _na_rep = "";
1399  // Indicates whether to write headers to csv
1400  bool _include_header = true;
1401  // maximum number of rows to write in each chunk (limits memory use)
1402  size_type _rows_per_chunk = std::numeric_limits<size_type>::max();
1403  // character to use for separating lines (default "\n")
1404  std::string _line_terminator = "\n";
1405  // character to use for separating column values (default ",")
1406  char _inter_column_delimiter = ',';
1407  // string to use for values != 0 in INT8 types (default 'true')
1408  std::string _true_value = std::string{"true"};
1409  // string to use for values == 0 in INT8 types (default 'false')
1410  std::string _false_value = std::string{"false"};
1411  // Names of all columns; if empty, writer will generate column names
1412  std::vector<std::string> _names;
1413  // Quote style. Currently only MINIMAL and NONE are supported.
1414  quote_style _quoting = quote_style::MINIMAL;
1415  // Compression type for output (default: NONE)
1416  compression_type _compression = compression_type::NONE;
1417  // Size of the blocks the output is compressed in, independent of `_rows_per_chunk`
1418  size_t _compression_block_size = default_csv_compression_block_size;
1419 
1426  explicit csv_writer_options(sink_info sink, table_view const& table)
1427  : _sink(std::move(sink)), _table(table), _rows_per_chunk(table.num_rows())
1428  {
1429  }
1430 
1432 
1433  public:
1439  explicit csv_writer_options() = default;
1440 
1450 
1456  [[nodiscard]] sink_info const& get_sink() const { return _sink; }
1457 
1463  [[nodiscard]] table_view const& get_table() const { return _table; }
1464 
1470  [[nodiscard]] std::vector<std::string> const& get_names() const { return _names; }
1471 
1477  [[nodiscard]] std::string const& get_na_rep() const { return _na_rep; }
1478 
1484  [[nodiscard]] bool is_enabled_include_header() const { return _include_header; }
1485 
1491  [[nodiscard]] size_type get_rows_per_chunk() const { return _rows_per_chunk; }
1492 
1498  [[nodiscard]] std::string const& get_line_terminator() const { return _line_terminator; }
1499 
1505  [[nodiscard]] char get_inter_column_delimiter() const { return _inter_column_delimiter; }
1506 
1512  [[nodiscard]] std::string const& get_true_value() const { return _true_value; }
1513 
1519  [[nodiscard]] std::string const& get_false_value() const { return _false_value; }
1520 
1531  [[nodiscard]] quote_style get_quoting() const { return _quoting; }
1532 
1538  [[nodiscard]] compression_type get_compression() const { return _compression; }
1539 
1545  [[nodiscard]] size_t get_compression_block_size() const { return _compression_block_size; }
1546 
1547  // Setter
1553  void set_names(std::vector<std::string> names) { _names = std::move(names); }
1554 
1560  void set_na_rep(std::string val) { _na_rep = std::move(val); }
1561 
1567  void enable_include_header(bool val) { _include_header = val; }
1568 
1574  void set_rows_per_chunk(size_type val) { _rows_per_chunk = val; }
1575 
1581  void set_line_terminator(std::string term) { _line_terminator = std::move(term); }
1582 
1588  void set_inter_column_delimiter(char delim) { _inter_column_delimiter = delim; }
1589 
1595  void set_true_value(std::string val) { _true_value = std::move(val); }
1596 
1602  void set_false_value(std::string val) { _false_value = std::move(val); }
1603 
1609  void set_table(table_view const& table) { _table = table; }
1610 
1621  void set_quoting(quote_style quoting)
1622  {
1623  CUDF_EXPECTS(quoting == quote_style::MINIMAL || quoting == quote_style::NONE,
1624  "Only MINIMAL and NONE are supported for quoting.");
1625  _quoting = quoting;
1626  }
1627 
1638  {
1639  CUDF_EXPECTS(comp == compression_type::NONE || comp == compression_type::ZSTD,
1640  "Only NONE and ZSTD compression are supported for CSV writer");
1641  _compression = comp;
1642  }
1643 
1653  void set_compression_block_size(size_t size)
1654  {
1655  CUDF_EXPECTS(size > 0, "Compression block size must be greater than zero");
1656  _compression_block_size = size;
1657  }
1658 };
1659 
1664  csv_writer_options options;
1665 
1666  public:
1672  explicit csv_writer_options_builder() = default;
1673 
1681  : options{sink, table}
1682  {
1683  }
1684 
1691  csv_writer_options_builder& names(std::vector<std::string> names)
1692  {
1693  options.set_names(std::move(names));
1694  return *this;
1695  }
1696 
1704  {
1705  options.set_na_rep(std::move(val));
1706  return *this;
1707  };
1708 
1716  {
1717  options.enable_include_header(val);
1718  return *this;
1719  }
1720 
1728  {
1729  options.set_rows_per_chunk(val);
1730  return *this;
1731  }
1732 
1740  {
1741  options.set_line_terminator(std::move(term));
1742  return *this;
1743  }
1744 
1752  {
1753  options.set_inter_column_delimiter(delim);
1754  return *this;
1755  }
1756 
1764  {
1765  options.set_true_value(std::move(val));
1766  return *this;
1767  }
1768 
1776  {
1777  options.set_false_value(std::move(val));
1778  return *this;
1779  }
1780 
1790  {
1791  options.set_quoting(quoting);
1792  return *this;
1793  }
1794 
1805  {
1806  options.set_compression(comp);
1807  return *this;
1808  }
1809 
1820  {
1821  options.set_compression_block_size(size);
1822  return *this;
1823  }
1824 
1828  operator csv_writer_options&&() { return std::move(options); }
1829 
1837  csv_writer_options&& build() { return std::move(options); }
1838 };
1839 
1857 void write_csv(csv_writer_options const& options,
1858  cuda::stream_ref stream = cudf::get_default_stream());
1859 
1861 struct is_supported_csv_write_type_fn {
1862  template <typename T>
1863  constexpr bool operator()() const
1864  {
1865  return cudf::io::detail::is_convertible_to_string_column<T>();
1866  }
1867 };
1869 
1876 constexpr bool is_supported_write_csv(data_type type)
1877 {
1878  return cudf::type_dispatcher(type, is_supported_csv_write_type_fn{});
1879 }
1880  // end of group
1882 } // namespace io
1883 } // namespace CUDF_EXPORT cudf
Indicator for the logical data type of an element in a column.
Definition: types.hpp:279
Builder to build options for read_csv().
Definition: csv.hpp:857
csv_reader_options_builder & dtypes(std::vector< data_type > types)
Sets per-column types.
Definition: csv.hpp:1246
csv_reader_options_builder & false_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean false values.
Definition: csv.hpp:1270
csv_reader_options_builder & use_cols_names(std::vector< std::string > col_names)
Sets names of the columns to be read.
Definition: csv.hpp:953
csv_reader_options_builder & doublequote(bool val)
Sets a quote inside a value is double-quoted.
Definition: csv.hpp:1161
csv_reader_options_builder & parse_hex(std::vector< std::string > col_names)
Sets names of columns to parse as hexadecimal.
Definition: csv.hpp:1210
csv_reader_options_builder & byte_range_offset(std::size_t offset)
Sets number of bytes to skip from source start.
Definition: csv.hpp:893
csv_reader_options_builder & delim_whitespace(bool val)
Sets whether to treat whitespace as field delimiter.
Definition: csv.hpp:1101
csv_reader_options_builder & skiprows(size_type skip)
Sets number of rows to skip from start.
Definition: csv.hpp:989
csv_reader_options_builder & skip_blank_lines(bool val)
Sets whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:1125
csv_reader_options && build()
move csv_reader_options member once it's built.
Definition: csv.hpp:1348
csv_reader_options_builder & dtypes(std::map< std::string, data_type > types)
Sets per-column types.
Definition: csv.hpp:1234
csv_reader_options_builder & quotechar(char ch)
Sets quoting character.
Definition: csv.hpp:1149
csv_reader_options_builder & na_values(std::vector< std::string > vals)
Sets additional values to recognize as null values.
Definition: csv.hpp:1282
csv_reader_options_builder & true_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean true values.
Definition: csv.hpp:1258
csv_reader_options_builder & decimal(char val)
Sets decimal point character.
Definition: csv.hpp:1061
csv_reader_options_builder & na_filter(bool val)
Sets whether to disable null filter.
Definition: csv.hpp:1306
csv_reader_options_builder & thousands(char val)
Sets numeric data thousands separator.
Definition: csv.hpp:1049
csv_reader_options_builder & parse_hex(std::vector< int > col_indices)
Sets indexes of columns to parse as hexadecimal.
Definition: csv.hpp:1222
csv_reader_options_builder & detect_whitespace_around_quotes(bool val)
Sets whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doubleq...
Definition: csv.hpp:1174
csv_reader_options_builder & windowslinetermination(bool val)
Sets whether to treat \r\n as line terminator.
Definition: csv.hpp:1089
csv_reader_options_builder & parse_dates(std::vector< int > col_indices)
Sets indexes of columns to read as datetime.
Definition: csv.hpp:1198
csv_reader_options_builder & nrows(size_type rows)
Sets number of rows to read.
Definition: csv.hpp:977
csv_reader_options_builder & names(std::vector< std::string > col_names)
Sets names of the column.
Definition: csv.hpp:917
csv_reader_options_builder & timestamp_type(data_type type)
Sets timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:1330
csv_reader_options_builder & mangle_dupe_cols(bool val)
Sets whether to rename duplicate column names.
Definition: csv.hpp:941
csv_reader_options_builder & skipfooter(size_type skip)
Sets number of rows to skip from end.
Definition: csv.hpp:1001
csv_reader_options_builder()=default
Default constructor.
csv_reader_options_builder & byte_range_size(std::size_t size)
Sets number of bytes to read.
Definition: csv.hpp:905
csv_reader_options_builder & keep_default_na(bool val)
Sets whether to keep the built-in default NA values.
Definition: csv.hpp:1294
csv_reader_options_builder & quoting(quote_style style)
Sets quoting style.
Definition: csv.hpp:1137
csv_reader_options_builder & lineterminator(char term)
Sets line terminator.
Definition: csv.hpp:1025
csv_reader_options_builder & delimiter(char delim)
Sets field delimiter.
Definition: csv.hpp:1037
csv_reader_options_builder & use_cols_indexes(std::vector< int > col_indices)
Sets indexes of columns to read.
Definition: csv.hpp:965
csv_reader_options_builder & parse_dates(std::vector< std::string > col_names)
Sets names of columns to read as datetime.
Definition: csv.hpp:1186
csv_reader_options_builder(source_info src)
Constructor from source info.
Definition: csv.hpp:873
csv_reader_options_builder & comment(char val)
Sets comment line start character.
Definition: csv.hpp:1073
csv_reader_options_builder & compression(compression_type comp)
Sets compression format of the source.
Definition: csv.hpp:881
csv_reader_options_builder & header(size_type hdr)
Sets header row index.
Definition: csv.hpp:1013
csv_reader_options_builder & dayfirst(bool val)
Sets whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:1318
csv_reader_options_builder & prefix(std::string pfx)
Sets prefix to be used for column ID.
Definition: csv.hpp:929
csv_reader_options_builder & skipinitialspace(bool val)
Sets whether to skip whitespace after the delimiter.
Definition: csv.hpp:1113
Settings to use for read_csv().
Definition: csv.hpp:44
void enable_doublequote(bool val)
Sets a quote inside a value is double-quoted.
Definition: csv.hpp:727
void set_use_cols_indexes(std::vector< int > col_indices)
Sets indexes of columns to read.
Definition: csv.hpp:569
size_type get_skiprows() const
Returns number of rows to skip from start.
Definition: csv.hpp:274
bool is_enabled_delim_whitespace() const
Whether to treat whitespace as field delimiter.
Definition: csv.hpp:344
std::vector< int > const & get_parse_dates_indexes() const
Returns indexes of columns to read as datetime.
Definition: csv.hpp:407
void set_byte_range_offset(std::size_t offset)
Sets number of bytes to skip from source start.
Definition: csv.hpp:508
quote_style get_quoting() const
Returns quoting style.
Definition: csv.hpp:365
void set_parse_dates(std::vector< std::string > col_names)
Sets names of columns to read as datetime.
Definition: csv.hpp:742
void set_parse_dates(std::vector< int > col_indices)
Sets indexes of columns to read as datetime.
Definition: csv.hpp:752
bool is_enabled_doublequote() const
Whether a quote inside a value is double-quoted.
Definition: csv.hpp:379
char get_delimiter() const
Returns field delimiter.
Definition: csv.hpp:302
char get_lineterminator() const
Returns line terminator.
Definition: csv.hpp:295
csv_reader_options()=default
Default constructor.
void enable_detect_whitespace_around_quotes(bool val)
Sets whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doubleq...
Definition: csv.hpp:735
void set_false_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean false values.
Definition: csv.hpp:803
void set_decimal(char val)
Sets decimal point character.
Definition: csv.hpp:655
std::string get_prefix() const
Returns prefix to be used for column ID.
Definition: csv.hpp:236
bool is_enabled_detect_whitespace_around_quotes() const
Whether to detect quotes surrounded by spaces e.g. "data". This flag has no effect when _doublequote ...
Definition: csv.hpp:387
std::vector< std::string > const & get_na_values() const
Returns additional values to recognize as null values.
Definition: csv.hpp:459
char get_thousands() const
Returns numeric data thousands separator.
Definition: csv.hpp:309
bool is_enabled_mangle_dupe_cols() const
Whether to rename duplicate column names.
Definition: csv.hpp:243
void set_dtypes(std::map< std::string, data_type > types)
Sets per-column types.
Definition: csv.hpp:779
std::size_t get_byte_range_size_with_padding() const
Returns number of bytes to read with padding.
Definition: csv.hpp:191
std::vector< int > const & get_parse_hex_indexes() const
Returns indexes of columns to read as hexadecimal.
Definition: csv.hpp:427
std::vector< std::string > const & get_false_values() const
Returns additional values to recognize as boolean false values.
Definition: csv.hpp:452
void set_source(source_info src)
Sets source info.
Definition: csv.hpp:494
void enable_dayfirst(bool val)
Sets whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:843
void set_na_values(std::vector< std::string > vals)
Sets additional values to recognize as null values.
Definition: csv.hpp:813
void set_quoting(quote_style quoting)
Sets the expected quoting style used in the input CSV data.
Definition: csv.hpp:707
std::variant< std::vector< data_type >, std::map< std::string, data_type > > const & get_dtypes() const
Returns per-column types.
Definition: csv.hpp:435
void set_timestamp_type(data_type type)
Sets timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:850
std::size_t get_byte_range_offset() const
Returns number of bytes to skip from source start.
Definition: csv.hpp:177
data_type get_timestamp_type() const
Returns timestamp_type to which all timestamp columns will be cast.
Definition: csv.hpp:487
bool is_enabled_na_filter() const
Whether to disable null filter.
Definition: csv.hpp:473
bool is_enabled_skip_blank_lines() const
Whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:358
char get_comment() const
Returns comment line start character.
Definition: csv.hpp:323
void set_lineterminator(char term)
Sets line terminator.
Definition: csv.hpp:634
void set_quotechar(char ch)
Sets quoting character.
Definition: csv.hpp:720
bool is_enabled_windowslinetermination() const
Whether to treat \r\n as line terminator.
Definition: csv.hpp:334
void enable_skip_blank_lines(bool val)
Sets whether to ignore empty lines or parse line values as invalid.
Definition: csv.hpp:697
void enable_windowslinetermination(bool val)
Sets whether to treat \r\n as line terminator.
Definition: csv.hpp:673
void set_skiprows(size_type skiprows)
Sets number of rows to skip from start.
Definition: csv.hpp:595
void set_compression(compression_type comp)
Sets compression format of the source.
Definition: csv.hpp:501
void enable_delim_whitespace(bool val)
Sets whether to treat whitespace as field delimiter.
Definition: csv.hpp:683
std::vector< std::string > const & get_names() const
Returns names of the columns.
Definition: csv.hpp:229
void set_dtypes(std::vector< data_type > types)
Sets per-column types.
Definition: csv.hpp:786
void set_skipfooter(size_type skipfooter)
Sets number of rows to skip from end.
Definition: csv.hpp:609
bool is_enabled_dayfirst() const
Whether to parse dates as DD/MM versus MM/DD.
Definition: csv.hpp:480
std::size_t get_byte_range_padding() const
Returns number of bytes to pad when reading.
Definition: csv.hpp:205
void set_names(std::vector< std::string > col_names)
Sets names of the column.
Definition: csv.hpp:538
source_info const & get_source() const
Returns source info.
Definition: csv.hpp:163
void enable_keep_default_na(bool val)
Sets whether to keep the built-in default NA values.
Definition: csv.hpp:825
std::vector< std::string > const & get_parse_dates_names() const
Returns names of columns to read as datetime.
Definition: csv.hpp:397
void set_prefix(std::string pfx)
Sets prefix to be used for column ID.
Definition: csv.hpp:545
static csv_reader_options_builder builder(source_info src)
Creates a csv_reader_options_builder which will build csv_reader_options.
std::size_t get_byte_range_size() const
Returns number of bytes to read.
Definition: csv.hpp:184
std::vector< int > const & get_use_cols_indexes() const
Returns indexes of columns to read.
Definition: csv.hpp:260
std::vector< std::string > const & get_use_cols_names() const
Returns names of the columns to be read.
Definition: csv.hpp:250
void set_use_cols_names(std::vector< std::string > col_names)
Sets names of the columns to be read.
Definition: csv.hpp:559
compression_type get_compression() const
Returns compression format of the source.
Definition: csv.hpp:170
char get_quotechar() const
Returns quoting character.
Definition: csv.hpp:372
void set_true_values(std::vector< std::string > vals)
Sets additional values to recognize as boolean true values.
Definition: csv.hpp:793
bool is_enabled_keep_default_na() const
Whether to keep the built-in default NA values.
Definition: csv.hpp:466
void set_header(size_type hdr)
Sets header row index.
Definition: csv.hpp:627
char get_decimal() const
Returns decimal point character.
Definition: csv.hpp:316
std::vector< std::string > const & get_true_values() const
Returns additional values to recognize as boolean true values.
Definition: csv.hpp:445
void set_parse_hex(std::vector< int > col_indices)
Sets indexes of columns to parse as hexadecimal.
Definition: csv.hpp:772
void set_thousands(char val)
Sets numeric data thousands separator.
Definition: csv.hpp:648
void enable_na_filter(bool val)
Sets whether to disable null filter.
Definition: csv.hpp:832
void set_byte_range_size(std::size_t size)
Sets number of bytes to read.
Definition: csv.hpp:523
void set_delimiter(char delim)
Sets field delimiter.
Definition: csv.hpp:641
void enable_mangle_dupe_cols(bool val)
Sets whether to rename duplicate column names.
Definition: csv.hpp:552
size_type get_nrows() const
Returns number of rows to read.
Definition: csv.hpp:267
std::vector< std::string > const & get_parse_hex_names() const
Returns names of columns to read as hexadecimal.
Definition: csv.hpp:417
void enable_skipinitialspace(bool val)
Sets whether to skip whitespace after the delimiter.
Definition: csv.hpp:690
size_type get_skipfooter() const
Returns number of rows to skip from end.
Definition: csv.hpp:281
void set_nrows(size_type nrows)
Sets number of rows to read.
Definition: csv.hpp:579
void set_comment(char val)
Sets comment line start character.
Definition: csv.hpp:662
bool is_enabled_skipinitialspace() const
Whether to skip whitespace after the delimiter.
Definition: csv.hpp:351
size_type get_header() const
Returns header row index.
Definition: csv.hpp:288
void set_parse_hex(std::vector< std::string > col_names)
Sets names of columns to parse as hexadecimal.
Definition: csv.hpp:762
Builder to build options for writer_csv()
Definition: csv.hpp:1663
csv_writer_options_builder & compression_block_size(size_t size)
Sets the size of the blocks that the output is compressed in.
Definition: csv.hpp:1819
csv_writer_options_builder()=default
Default constructor.
csv_writer_options && build()
move csv_writer_options member once it's built.
Definition: csv.hpp:1837
csv_writer_options_builder & quoting(quote_style quoting)
Sets the quote style for the writer.
Definition: csv.hpp:1789
csv_writer_options_builder & compression(compression_type comp)
Sets the compression type for the output.
Definition: csv.hpp:1804
csv_writer_options_builder & true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: csv.hpp:1763
csv_writer_options_builder & include_header(bool val)
Enables/Disables headers being written to csv.
Definition: csv.hpp:1715
csv_writer_options_builder & na_rep(std::string val)
Sets string to used for null entries.
Definition: csv.hpp:1703
csv_writer_options_builder & line_terminator(std::string term)
Sets character used for separating lines.
Definition: csv.hpp:1739
csv_writer_options_builder & false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: csv.hpp:1775
csv_writer_options_builder & names(std::vector< std::string > names)
Sets optional column names.
Definition: csv.hpp:1691
csv_writer_options_builder & inter_column_delimiter(char delim)
Sets character used for separating column values.
Definition: csv.hpp:1751
csv_writer_options_builder(sink_info const &sink, table_view const &table)
Constructor from sink and table.
Definition: csv.hpp:1680
csv_writer_options_builder & rows_per_chunk(int val)
Sets maximum number of rows to process for each file write.
Definition: csv.hpp:1727
Settings to use for write_csv().
Definition: csv.hpp:1392
void set_table(table_view const &table)
(Re)sets the table being written.
Definition: csv.hpp:1609
void set_rows_per_chunk(size_type val)
Sets maximum number of rows to process for each file write.
Definition: csv.hpp:1574
void set_quoting(quote_style quoting)
Sets the quote style for the writer.
Definition: csv.hpp:1621
void set_line_terminator(std::string term)
Sets character used for separating lines.
Definition: csv.hpp:1581
csv_writer_options()=default
Default constructor.
void set_compression(compression_type comp)
Sets the compression type for the output.
Definition: csv.hpp:1637
compression_type get_compression() const
Returns the compression type for the output.
Definition: csv.hpp:1538
void set_compression_block_size(size_t size)
Sets the size of the blocks that the output is compressed in.
Definition: csv.hpp:1653
void set_inter_column_delimiter(char delim)
Sets character used for separating column values.
Definition: csv.hpp:1588
void set_true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: csv.hpp:1595
static csv_writer_options_builder builder(sink_info const &sink, table_view const &table)
Create builder to create csv_writer_options.
table_view const & get_table() const
Returns table that would be written to output.
Definition: csv.hpp:1463
void enable_include_header(bool val)
Enables/Disables headers being written to csv.
Definition: csv.hpp:1567
bool is_enabled_include_header() const
Whether to write headers to csv.
Definition: csv.hpp:1484
void set_na_rep(std::string val)
Sets string to used for null entries.
Definition: csv.hpp:1560
char get_inter_column_delimiter() const
Returns character used for separating column values.
Definition: csv.hpp:1505
std::string const & get_false_value() const
Returns string used for values == 0 in INT8 types.
Definition: csv.hpp:1519
std::string const & get_line_terminator() const
Returns character used for separating lines.
Definition: csv.hpp:1498
sink_info const & get_sink() const
Returns sink used for writer output.
Definition: csv.hpp:1456
std::vector< std::string > const & get_names() const
Returns names of the columns.
Definition: csv.hpp:1470
quote_style get_quoting() const
Returns the quote style for the writer.
Definition: csv.hpp:1531
size_t get_compression_block_size() const
Returns the size of the blocks that the output is compressed in.
Definition: csv.hpp:1545
std::string const & get_true_value() const
Returns string used for values != 0 in INT8 types.
Definition: csv.hpp:1512
void set_false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: csv.hpp:1602
size_type get_rows_per_chunk() const
Returns maximum number of rows to process for each file write.
Definition: csv.hpp:1491
std::string const & get_na_rep() const
Returns string to used for null entries.
Definition: csv.hpp:1477
void set_names(std::vector< std::string > names)
Sets optional associated column names.
Definition: csv.hpp:1553
A set of cudf::column_view's of the same size.
Definition: table_view.hpp:206
A set of cudf::column's of the same size.
Definition: table.hpp:31
size_type num_rows() const noexcept
Returns the number of rows.
Definition: table.hpp:103
Exception types and error-checking macros used throughout libcudf.
cuda::stream_ref const get_default_stream()
Get the current default stream.
table_with_metadata read_csv(csv_reader_options options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Reads a CSV dataset into a set of columns.
quote_style
Behavior when handling quotations in field data.
Definition: types.hpp:75
compression_type
Compression algorithms.
Definition: types.hpp:46
void write_csv(csv_writer_options const &options, cuda::stream_ref stream=cudf::get_default_stream())
Writes a set of columns to CSV format.
constexpr size_t default_csv_compression_block_size
Default size of the blocks that compressed CSV output is split into.
Definition: csv.hpp:1387
constexpr bool is_supported_write_csv(data_type type)
Checks if a cudf::data_type is supported for CSV writing.
Definition: csv.hpp:1876
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::resource_ref< cuda::mr::device_accessible > device_async_resource_ref
CUDF_HOST_DEVICE constexpr decltype(auto) __forceinline__ type_dispatcher(cudf::data_type dtype, Functor f, Ts &&... args)
Invokes an operator() template with the type instantiation based on the specified cudf::data_type's i...
#define CUDF_EXPECTS(...)
Macro for checking (pre-)conditions that throws an exception when a condition is violated.
Definition: error.hpp:182
#define CUDF_FAIL(...)
Indicates that an erroneous code path has been taken.
Definition: error.hpp:223
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:76
Type definitions for the cuDF-IO API.
APIs for getting and setting the current device memory resource.
cuDF interfaces
Definition: host_udf.hpp:27
Destination information for write interfaces.
Definition: types.hpp:493
Source information for read interfaces.
Definition: types.hpp:306
Table with table metadata used by io readers to return the metadata by value.
Definition: types.hpp:271
Class definitions for (mutable)_table_view
Type declarations for libcudf.