io/types.hpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019-2023, NVIDIA CORPORATION.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
22 #pragma once
23 
24 #include <cudf/table/table.hpp>
25 #include <cudf/types.hpp>
26 #include <cudf/utilities/span.hpp>
27 
28 #include <map>
29 #include <memory>
30 #include <optional>
31 #include <string>
32 #include <unordered_map>
33 #include <vector>
34 
35 namespace cudf {
37 namespace io {
38 class data_sink;
39 class datasource;
40 } // namespace io
41 } // namespace cudf
42 
44 namespace cudf {
46 namespace io {
50 enum class compression_type {
51  NONE,
52  AUTO,
53  SNAPPY,
54  GZIP,
55  BZIP2,
56  BROTLI,
57  ZIP,
58  XZ,
59  ZLIB,
60  LZ4,
61  LZO,
62  ZSTD
63 };
64 
68 enum class io_type {
69  FILEPATH,
70  HOST_BUFFER,
72  VOID,
74 };
75 
79 enum class quote_style {
80  MINIMAL,
81  ALL,
82  NONNUMERIC,
83  NONE
84 };
85 
94 };
95 
100  public:
105 
115  size_t num_failed_bytes,
116  size_t num_skipped_bytes,
117  size_t num_compressed_output_bytes)
118  : _num_compressed_bytes(num_compressed_bytes),
119  _num_failed_bytes(num_failed_bytes),
120  _num_skipped_bytes(num_skipped_bytes),
121  _num_compressed_output_bytes(num_compressed_output_bytes)
122  {
123  }
124 
132  {
133  _num_compressed_bytes += other._num_compressed_bytes;
134  _num_failed_bytes += other._num_failed_bytes;
135  _num_skipped_bytes += other._num_skipped_bytes;
136  _num_compressed_output_bytes += other._num_compressed_output_bytes;
137  return *this;
138  }
139 
148  [[nodiscard]] auto num_compressed_bytes() const noexcept { return _num_compressed_bytes; }
149 
155  [[nodiscard]] auto num_failed_bytes() const noexcept { return _num_failed_bytes; }
156 
162  [[nodiscard]] auto num_skipped_bytes() const noexcept { return _num_skipped_bytes; }
163 
169  [[nodiscard]] auto num_total_input_bytes() const noexcept
170  {
172  }
173 
182  [[nodiscard]] auto compression_ratio() const noexcept
183  {
184  return static_cast<double>(num_compressed_bytes()) / _num_compressed_output_bytes;
185  }
186 
187  private:
188  std::size_t _num_compressed_bytes = 0;
189  std::size_t _num_failed_bytes = 0;
190  std::size_t _num_skipped_bytes = 0;
191  std::size_t _num_compressed_output_bytes = 0;
192 };
193 
200  ALWAYS
201 };
202 
210  std::string name;
211  std::optional<bool> is_nullable;
212  std::vector<column_name_info> children;
213 
220  column_name_info(std::string const& _name, std::optional<bool> _is_nullable = std::nullopt)
221  : name(_name), is_nullable(_is_nullable)
222  {
223  }
224 
225  column_name_info() = default;
226 };
227 
232  std::vector<column_name_info>
234  std::map<std::string, std::string> user_data;
236  std::vector<std::unordered_map<std::string, std::string>>
238 };
239 
244  std::unique_ptr<table> tbl;
246 };
247 
255 struct host_buffer {
256  // TODO: to be replaced by `host_span`
257  char const* data = nullptr;
258  size_t size = 0;
259  host_buffer() = default;
266  host_buffer(char const* data, size_t size) : data(data), size(size) {}
267 };
268 
276 template <typename T>
277 constexpr inline auto is_byte_like_type()
278 {
279  using non_cv_T = std::remove_cv_t<T>;
280  return std::is_same_v<non_cv_T, int8_t> || std::is_same_v<non_cv_T, char> ||
281  std::is_same_v<non_cv_T, uint8_t> || std::is_same_v<non_cv_T, unsigned char> ||
282  std::is_same_v<non_cv_T, std::byte>;
283 }
284 
288 struct source_info {
289  source_info() = default;
290 
296  explicit source_info(std::vector<std::string> const& file_paths) : _filepaths(file_paths) {}
297 
303  explicit source_info(std::string const& file_path) : _filepaths({file_path}) {}
304 
312  explicit source_info(std::vector<host_buffer> const& host_buffers) : _type(io_type::HOST_BUFFER)
313  {
314  _host_buffers.reserve(host_buffers.size());
316  host_buffers.end(),
317  std::back_inserter(_host_buffers),
318  [](auto const hb) {
319  return cudf::host_span<std::byte const>{
320  reinterpret_cast<std::byte const*>(hb.data), hb.size};
321  });
322  }
323 
332  explicit source_info(char const* host_data, size_t size)
333  : _type(io_type::HOST_BUFFER),
334  _host_buffers(
335  {cudf::host_span<std::byte const>(reinterpret_cast<std::byte const*>(host_data), size)})
336  {
337  }
338 
344  template <typename T, CUDF_ENABLE_IF(is_byte_like_type<std::remove_cv_t<T>>())>
345  explicit source_info(cudf::host_span<cudf::host_span<T>> const host_buffers)
346  : _type(io_type::HOST_BUFFER)
347  {
348  if constexpr (not std::is_same_v<std::remove_cv_t<T>, std::byte>) {
349  _host_buffers.reserve(host_buffers.size());
350  std::transform(host_buffers.begin(),
351  host_buffers.end(),
352  std::back_inserter(_host_buffers),
353  [](auto const s) {
354  return cudf::host_span<std::byte const>{
355  reinterpret_cast<std::byte const*>(s.data()), s.size()};
356  });
357  } else {
358  _host_buffers.assign(host_buffers.begin(), host_buffers.end());
359  }
360  }
361 
367  template <typename T, CUDF_ENABLE_IF(is_byte_like_type<std::remove_cv_t<T>>())>
368  explicit source_info(cudf::host_span<T> host_data)
369  : _type(io_type::HOST_BUFFER),
370  _host_buffers{cudf::host_span<std::byte const>(
371  reinterpret_cast<std::byte const*>(host_data.data()), host_data.size())}
372  {
373  }
374 
381  : _type(io_type::DEVICE_BUFFER), _device_buffers(device_buffers.begin(), device_buffers.end())
382  {
383  }
384 
391  : _type(io_type::DEVICE_BUFFER), _device_buffers({{d_buffer}})
392  {
393  }
394 
400  explicit source_info(std::vector<cudf::io::datasource*> const& sources)
401  : _type(io_type::USER_IMPLEMENTED), _user_sources(sources)
402  {
403  }
404 
411  : _type(io_type::USER_IMPLEMENTED), _user_sources({source})
412  {
413  }
414 
420  [[nodiscard]] auto type() const { return _type; }
426  [[nodiscard]] auto const& filepaths() const { return _filepaths; }
432  [[nodiscard]] auto const& host_buffers() const { return _host_buffers; }
438  [[nodiscard]] auto const& device_buffers() const { return _device_buffers; }
444  [[nodiscard]] auto const& user_sources() const { return _user_sources; }
445 
446  private:
447  io_type _type = io_type::FILEPATH;
448  std::vector<std::string> _filepaths;
449  std::vector<cudf::host_span<std::byte const>> _host_buffers;
450  std::vector<cudf::device_span<std::byte const>> _device_buffers;
451  std::vector<cudf::io::datasource*> _user_sources;
452 };
453 
457 struct sink_info {
458  sink_info() = default;
464  sink_info(size_t num_sinks) : _num_sinks(num_sinks) {}
465 
471  explicit sink_info(std::vector<std::string> const& file_paths)
472  : _type(io_type::FILEPATH), _num_sinks(file_paths.size()), _filepaths(file_paths)
473  {
474  }
475 
481  explicit sink_info(std::string const& file_path)
482  : _type(io_type::FILEPATH), _filepaths({file_path})
483  {
484  }
485 
491  explicit sink_info(std::vector<std::vector<char>*> const& buffers)
492  : _type(io_type::HOST_BUFFER), _num_sinks(buffers.size()), _buffers(buffers)
493  {
494  }
500  explicit sink_info(std::vector<char>* buffer) : _type(io_type::HOST_BUFFER), _buffers({buffer}) {}
501 
507  explicit sink_info(std::vector<cudf::io::data_sink*> const& user_sinks)
508  : _type(io_type::USER_IMPLEMENTED), _num_sinks(user_sinks.size()), _user_sinks(user_sinks)
509  {
510  }
511 
517  explicit sink_info(class cudf::io::data_sink* user_sink)
518  : _type(io_type::USER_IMPLEMENTED), _user_sinks({user_sink})
519  {
520  }
521 
527  [[nodiscard]] auto type() const { return _type; }
533  [[nodiscard]] auto num_sinks() const { return _num_sinks; }
539  [[nodiscard]] auto const& filepaths() const { return _filepaths; }
545  [[nodiscard]] auto const& buffers() const { return _buffers; }
551  [[nodiscard]] auto const& user_sinks() const { return _user_sinks; }
552 
553  private:
554  io_type _type = io_type::VOID;
555  size_t _num_sinks = 1;
556  std::vector<std::string> _filepaths;
557  std::vector<std::vector<char>*> _buffers;
558  std::vector<cudf::io::data_sink*> _user_sinks;
559 };
560 
561 class table_input_metadata;
562 
567  friend table_input_metadata;
568  std::string _name = "";
569  std::optional<bool> _nullable;
570  bool _list_column_is_map = false;
571  bool _use_int96_timestamp = false;
572  bool _output_as_binary = false;
573  std::optional<uint8_t> _decimal_precision;
574  std::optional<int32_t> _parquet_field_id;
575  std::vector<column_in_metadata> children;
576 
577  public:
578  column_in_metadata() = default;
584  column_in_metadata(std::string_view name) : _name{name} {}
592  {
593  children.push_back(child);
594  return *this;
595  }
596 
603  column_in_metadata& set_name(std::string const& name) noexcept
604  {
605  _name = name;
606  return *this;
607  }
608 
616  {
617  _nullable = nullable;
618  return *this;
619  }
620 
629  {
630  _list_column_is_map = true;
631  return *this;
632  }
633 
643  {
644  _use_int96_timestamp = req;
645  return *this;
646  }
647 
655  column_in_metadata& set_decimal_precision(uint8_t precision) noexcept
656  {
657  _decimal_precision = precision;
658  return *this;
659  }
660 
667  column_in_metadata& set_parquet_field_id(int32_t field_id) noexcept
668  {
669  _parquet_field_id = field_id;
670  return *this;
671  }
672 
682  {
683  _output_as_binary = binary;
684  return *this;
685  }
686 
693  column_in_metadata& child(size_type i) noexcept { return children[i]; }
694 
701  [[nodiscard]] column_in_metadata const& child(size_type i) const noexcept { return children[i]; }
702 
708  [[nodiscard]] std::string get_name() const noexcept { return _name; }
709 
715  [[nodiscard]] bool is_nullability_defined() const noexcept { return _nullable.has_value(); }
716 
724  [[nodiscard]] bool nullable() const { return _nullable.value(); }
725 
731  [[nodiscard]] bool is_map() const noexcept { return _list_column_is_map; }
732 
739  [[nodiscard]] bool is_enabled_int96_timestamps() const noexcept { return _use_int96_timestamp; }
740 
746  [[nodiscard]] bool is_decimal_precision_set() const noexcept
747  {
748  return _decimal_precision.has_value();
749  }
750 
758  [[nodiscard]] uint8_t get_decimal_precision() const { return _decimal_precision.value(); }
759 
765  [[nodiscard]] bool is_parquet_field_id_set() const noexcept
766  {
767  return _parquet_field_id.has_value();
768  }
769 
777  [[nodiscard]] int32_t get_parquet_field_id() const { return _parquet_field_id.value(); }
778 
784  [[nodiscard]] size_type num_children() const noexcept { return children.size(); }
785 
791  [[nodiscard]] bool is_enabled_output_as_binary() const noexcept { return _output_as_binary; }
792 };
793 
798  public:
799  table_input_metadata() = default; // Required by cython
800 
809 
818  explicit table_input_metadata(table_metadata const& metadata);
819 
820  std::vector<column_in_metadata> column_metadata;
821 };
822 
832 
833  partition_info() = default;
840  partition_info(size_type start_row, size_type num_rows) : start_row(start_row), num_rows(num_rows)
841  {
842  }
843 };
844 
850  // Whether to read binary data as a string column
851  bool _convert_binary_to_strings{true};
852 
853  std::vector<reader_column_schema> children;
854 
855  public:
856  reader_column_schema() = default;
857 
863  reader_column_schema(size_type number_of_children) { children.resize(number_of_children); }
864 
871  {
872  children.assign(child_span.begin(), child_span.end());
873  }
874 
882  {
883  children.push_back(child);
884  return *this;
885  }
886 
893  [[nodiscard]] reader_column_schema& child(size_type i) { return children[i]; }
894 
901  [[nodiscard]] reader_column_schema const& child(size_type i) const { return children[i]; }
902 
912  {
913  _convert_binary_to_strings = convert_to_string;
914  return *this;
915  }
916 
922  [[nodiscard]] bool is_enabled_convert_binary_to_strings() const
923  {
924  return _convert_binary_to_strings;
925  }
926 
932  [[nodiscard]] size_t get_num_children() const { return children.size(); }
933 };
934 
935 } // namespace io
936 } // namespace cudf
constexpr iterator end() const noexcept
Returns an iterator to the element following the last element of the span.
Definition: span.hpp:123
constexpr iterator begin() const noexcept
Returns an iterator to the first element of the span.
Definition: span.hpp:115
Metadata for a column.
Definition: io/types.hpp:566
column_in_metadata & set_name(std::string const &name) noexcept
Set the name of this column.
Definition: io/types.hpp:603
column_in_metadata & add_child(column_in_metadata const &child)
Add the children metadata of this column.
Definition: io/types.hpp:591
bool is_enabled_output_as_binary() const noexcept
Get whether to encode this column as binary or string data.
Definition: io/types.hpp:791
column_in_metadata & set_parquet_field_id(int32_t field_id) noexcept
Set the parquet field id of this column.
Definition: io/types.hpp:667
column_in_metadata & set_int96_timestamps(bool req) noexcept
Specifies whether this timestamp column should be encoded using the deprecated int96 physical type....
Definition: io/types.hpp:642
column_in_metadata & set_decimal_precision(uint8_t precision) noexcept
Set the decimal precision of this column. Only valid if this column is a decimal (fixed-point) type.
Definition: io/types.hpp:655
bool is_enabled_int96_timestamps() const noexcept
Get whether to encode this timestamp column using deprecated int96 physical type.
Definition: io/types.hpp:739
bool is_parquet_field_id_set() const noexcept
Get whether parquet field id has been set for this column.
Definition: io/types.hpp:765
bool is_decimal_precision_set() const noexcept
Get whether precision has been set for this decimal column.
Definition: io/types.hpp:746
bool nullable() const
Gets the explicitly set nullability for this column.
Definition: io/types.hpp:724
size_type num_children() const noexcept
Get the number of children of this column.
Definition: io/types.hpp:784
bool is_map() const noexcept
If this is the metadata of a list column, returns whether it is to be encoded as a map.
Definition: io/types.hpp:731
uint8_t get_decimal_precision() const
Get the decimal precision that was set for this column.
Definition: io/types.hpp:758
column_in_metadata & set_output_as_binary(bool binary) noexcept
Specifies whether this column should be written as binary or string data Only valid for the following...
Definition: io/types.hpp:681
column_in_metadata & child(size_type i) noexcept
Get reference to a child of this column.
Definition: io/types.hpp:693
int32_t get_parquet_field_id() const
Get the parquet field id that was set for this column.
Definition: io/types.hpp:777
std::string get_name() const noexcept
Get the name of this column.
Definition: io/types.hpp:708
column_in_metadata & set_list_column_as_map() noexcept
Specify that this list column should be encoded as a map in the written file.
Definition: io/types.hpp:628
column_in_metadata(std::string_view name)
Construct a new column in metadata object.
Definition: io/types.hpp:584
column_in_metadata & set_nullability(bool nullable) noexcept
Set the nullability of this column.
Definition: io/types.hpp:615
bool is_nullability_defined() const noexcept
Get whether nullability has been explicitly set for this column.
Definition: io/types.hpp:715
column_in_metadata const & child(size_type i) const noexcept
Get const reference to a child of this column.
Definition: io/types.hpp:701
Interface class for storing the output data from the writers.
Definition: data_sink.hpp:43
Interface class for providing input data to the readers.
Definition: datasource.hpp:41
schema element for reader
Definition: io/types.hpp:849
reader_column_schema const & child(size_type i) const
Get const reference to a child of this column.
Definition: io/types.hpp:901
bool is_enabled_convert_binary_to_strings() const
Get whether to encode this column as binary or string data.
Definition: io/types.hpp:922
reader_column_schema(host_span< reader_column_schema > const &child_span)
Construct a new reader column schema object with a span defining the children.
Definition: io/types.hpp:870
reader_column_schema & set_convert_binary_to_strings(bool convert_to_string)
Specifies whether this column should be written as binary or string data Only valid for the following...
Definition: io/types.hpp:911
size_t get_num_children() const
Get the number of child objects.
Definition: io/types.hpp:932
reader_column_schema & add_child(reader_column_schema const &child)
Add the children metadata of this column.
Definition: io/types.hpp:881
reader_column_schema & child(size_type i)
Get reference to a child of this column.
Definition: io/types.hpp:893
reader_column_schema(size_type number_of_children)
Construct a new reader column schema object.
Definition: io/types.hpp:863
Metadata for a table.
Definition: io/types.hpp:797
table_input_metadata(table_view const &table)
Construct a new table_input_metadata from a table_view.
table_input_metadata(table_metadata const &metadata)
Construct a new table_input_metadata from a table_metadata object.
std::vector< column_in_metadata > column_metadata
List of column metadata.
Definition: io/types.hpp:820
Statistics about compression performed by a writer.
Definition: io/types.hpp:99
auto compression_ratio() const noexcept
Returns the compression ratio for the successfully compressed blocks.
Definition: io/types.hpp:182
auto num_total_input_bytes() const noexcept
Returns the total size of compression inputs.
Definition: io/types.hpp:169
writer_compression_statistics & operator+=(writer_compression_statistics const &other) noexcept
Adds the values from another writer_compression_statistics object.
Definition: io/types.hpp:131
auto num_failed_bytes() const noexcept
Returns the number of bytes in blocks that failed to compress.
Definition: io/types.hpp:155
writer_compression_statistics()=default
Default constructor.
auto num_skipped_bytes() const noexcept
Returns the number of bytes in blocks that were skipped during compression.
Definition: io/types.hpp:162
writer_compression_statistics(size_t num_compressed_bytes, size_t num_failed_bytes, size_t num_skipped_bytes, size_t num_compressed_output_bytes)
Constructor with initial values.
Definition: io/types.hpp:114
auto num_compressed_bytes() const noexcept
Returns the number of bytes in blocks that were successfully compressed.
Definition: io/types.hpp:148
A set of cudf::column_view's of the same size.
Definition: table_view.hpp:187
A set of cudf::column's of the same size.
Definition: table.hpp:40
std::unique_ptr< column > transform(column_view const &input, std::string const &unary_udf, data_type output_type, bool is_ptx, rmm::mr::device_memory_resource *mr=rmm::mr::get_current_device_resource())
Creates a new column by applying a unary function against every element of an input column.
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:80
io_type
Data source or destination types.
Definition: io/types.hpp:68
@ HOST_BUFFER
Input/output is a buffer in host memory.
@ USER_IMPLEMENTED
Input/output is handled by a custom user class.
@ VOID
Input/output is nothing. No work is done. Useful for benchmarking.
@ FILEPATH
Input/output is a file path.
@ DEVICE_BUFFER
Input/output is a buffer in device memory.
constexpr auto is_byte_like_type()
Returns true if the type is byte-like, meaning it is reasonable to pass as a pointer to bytes.
Definition: io/types.hpp:277
compression_type
Compression algorithms.
Definition: io/types.hpp:50
@ BROTLI
BROTLI format, using LZ77 + Huffman + 2nd order context modeling.
@ SNAPPY
Snappy format, using byte-oriented LZ77.
@ XZ
XZ format, using LZMA(2) algorithm.
@ ZIP
ZIP format, using DEFLATE algorithm.
@ LZO
Lempel–Ziv–Oberhumer format.
@ BZIP2
BZIP2 format, using Burrows-Wheeler transform.
@ ZSTD
Zstandard format.
@ ZLIB
ZLIB format, using DEFLATE algorithm.
@ LZ4
LZ4 format, using LZ77.
@ AUTO
Automatically detect or select compression format.
@ GZIP
GZIP format, using DEFLATE algorithm.
quote_style
Behavior when handling quotations in field data.
Definition: io/types.hpp:79
@ MINIMAL
Quote only fields which contain special characters.
@ ALL
Quote all fields.
@ NONE
Never quote fields; disable quotation parsing.
@ NONNUMERIC
Quote all non-numeric fields.
statistics_freq
Column statistics granularity type for parquet/orc writers.
Definition: io/types.hpp:89
@ STATISTICS_COLUMN
Full column and offset indices. Implies STATISTICS_ROWGROUP.
Definition: io/types.hpp:93
@ STATISTICS_ROWGROUP
Per-Rowgroup column statistics.
Definition: io/types.hpp:91
@ STATISTICS_NONE
No column statistics.
Definition: io/types.hpp:90
@ STATISTICS_PAGE
Per-page column statistics.
Definition: io/types.hpp:92
dictionary_policy
Control use of dictionary encoding for parquet writer.
Definition: io/types.hpp:197
@ ALWAYS
Use dictionary reqardless of impact on compression.
Definition: io/types.hpp:200
@ ADAPTIVE
Use dictionary when it will not impact compression.
Definition: io/types.hpp:199
@ NEVER
Never use dictionary encoding.
Definition: io/types.hpp:198
cuDF interfaces
Definition: aggregation.hpp:34
bool nullable(table_view const &view)
Returns True if any of the columns in the table is nullable. (not entire hierarchy)
Definition: table_view.hpp:305
Device version of C++20 std::span with reduced feature set.
Definition: span.hpp:277
C++20 std::span with reduced feature set.
Definition: span.hpp:210
Detailed name (and optionally nullability) information for output columns.
Definition: io/types.hpp:209
std::optional< bool > is_nullable
Column nullability.
Definition: io/types.hpp:211
std::vector< column_name_info > children
Child column names.
Definition: io/types.hpp:212
column_name_info(std::string const &_name, std::optional< bool > _is_nullable=std::nullopt)
Construct a column name info with a name, optional nullabilty, and no children.
Definition: io/types.hpp:220
std::string name
Column name.
Definition: io/types.hpp:210
Non-owning view of a host memory buffer.
Definition: io/types.hpp:255
char const * data
Pointer to the buffer.
Definition: io/types.hpp:257
size_t size
Size of the buffer.
Definition: io/types.hpp:258
host_buffer(char const *data, size_t size)
Construct a new host buffer object.
Definition: io/types.hpp:266
Information used while writing partitioned datasets.
Definition: io/types.hpp:829
partition_info(size_type start_row, size_type num_rows)
Construct a new partition_info.
Definition: io/types.hpp:840
size_type start_row
The start row of the partition.
Definition: io/types.hpp:830
size_type num_rows
The number of rows in the partition.
Definition: io/types.hpp:831
Destination information for write interfaces.
Definition: io/types.hpp:457
auto const & buffers() const
Get the host buffers of the input.
Definition: io/types.hpp:545
sink_info(std::vector< std::vector< char > * > const &buffers)
Construct a new sink info object for multiple host buffers.
Definition: io/types.hpp:491
sink_info(std::string const &file_path)
Construct a new sink info object for a single file.
Definition: io/types.hpp:481
auto const & filepaths() const
Get the filepaths of the input.
Definition: io/types.hpp:539
sink_info(class cudf::io::data_sink *user_sink)
Construct a new sink info object for a single user-implemented sink.
Definition: io/types.hpp:517
sink_info(std::vector< cudf::io::data_sink * > const &user_sinks)
Construct a new sink info object for multiple user-implemented sinks.
Definition: io/types.hpp:507
auto num_sinks() const
Get the number of sinks.
Definition: io/types.hpp:533
auto const & user_sinks() const
Get the user sinks of the input.
Definition: io/types.hpp:551
sink_info(std::vector< std::string > const &file_paths)
Construct a new sink info object for multiple files.
Definition: io/types.hpp:471
sink_info(size_t num_sinks)
Construct a new sink info object.
Definition: io/types.hpp:464
auto type() const
Get the type of the input.
Definition: io/types.hpp:527
sink_info(std::vector< char > *buffer)
Construct a new sink info object for a single host buffer.
Definition: io/types.hpp:500
Source information for read interfaces.
Definition: io/types.hpp:288
auto const & device_buffers() const
Get the device buffers of the input.
Definition: io/types.hpp:438
source_info(char const *host_data, size_t size)
Construct a new source info object for a single buffer.
Definition: io/types.hpp:332
auto const & filepaths() const
Get the filepaths of the input.
Definition: io/types.hpp:426
source_info(cudf::host_span< T > host_data)
Construct a new source info object for a single buffer.
Definition: io/types.hpp:368
source_info(std::string const &file_path)
Construct a new source info object for a single file.
Definition: io/types.hpp:303
source_info(cudf::host_span< cudf::host_span< T >> const host_buffers)
Construct a new source info object for multiple buffers in host memory.
Definition: io/types.hpp:345
source_info(cudf::device_span< std::byte const > d_buffer)
Construct a new source info object from a device buffer.
Definition: io/types.hpp:390
source_info(cudf::io::datasource *source)
Construct a new source info object for a single user-implemented source.
Definition: io/types.hpp:410
source_info(std::vector< cudf::io::datasource * > const &sources)
Construct a new source info object for multiple user-implemented sources.
Definition: io/types.hpp:400
source_info(cudf::host_span< cudf::device_span< std::byte const >> device_buffers)
Construct a new source info object for multiple buffers in device memory.
Definition: io/types.hpp:380
auto const & host_buffers() const
Get the host buffers of the input.
Definition: io/types.hpp:432
auto type() const
Get the type of the input.
Definition: io/types.hpp:420
source_info(std::vector< std::string > const &file_paths)
Construct a new source info object for multiple files.
Definition: io/types.hpp:296
auto const & user_sources() const
Get the user sources of the input.
Definition: io/types.hpp:444
source_info(std::vector< host_buffer > const &host_buffers)
Construct a new source info object for multiple buffers in host memory.
Definition: io/types.hpp:312
Table metadata returned by IO readers.
Definition: io/types.hpp:231
std::vector< std::unordered_map< std::string, std::string > > per_file_user_data
Per file format-dependent metadata as key-values pairs.
Definition: io/types.hpp:237
std::vector< column_name_info > schema_info
Detailed name information for the entire output hierarchy.
Definition: io/types.hpp:233
std::map< std::string, std::string > user_data
Definition: io/types.hpp:234
Table with table metadata used by io readers to return the metadata by value.
Definition: io/types.hpp:243
std::unique_ptr< table > tbl
Table.
Definition: io/types.hpp:244
table_metadata metadata
Table metadata.
Definition: io/types.hpp:245
Class definition for cudf::table.
Type declarations for libcudf.