Readers

Classes
class	cudf::io::avro_reader_options
	Settings to use for read_avro(). More...
class	cudf::io::avro_reader_options_builder
	Builder to build options for read_avro(). More...
class	cudf::io::csv_reader_options
	Settings to use for read_csv(). More...
class	cudf::io::csv_reader_options_builder
	Builder to build options for read_csv(). More...
class	cudf::io::experimental::cudftable_reader_options
	Settings for read_cudftable(). More...
class	cudf::io::experimental::cudftable_reader_options_builder
	Class to build cudftable_reader_options. More...
struct	cudf::io::parquet::experimental::deletion_vector_info
	Struct used to specify information about deletion vectors and the index column to the experimental parquet reader. More...
class	cudf::io::parquet::experimental::chunked_parquet_reader
	The chunked parquet reader class to read a Parquet source iteratively in a series of tables, chunk by chunk. Each chunk is prepended with a row index column built using the specified row group offsets and row counts. The resultant table chunk is filtered using the supplied serialized roaring64 bitmap deletion vector and returned. More...
class	cudf::io::parquet::experimental::hybrid_scan_metadata
	Shareable, pre-parsed Parquet file metadata for the Hybrid Scan reader. More...
class	cudf::io::parquet::experimental::hybrid_scan_reader
	The experimental parquet reader class to optimally read parquet files subject to highly selective filters, called a Hybrid Scan operation. More...
class	cudf::io::parquet::experimental::hybrid_scan_multifile
	Multi-file variant of the experimental Hybrid Scan Parquet reader. More...
struct	cudf::io::schema_element
	Allows specifying the target types for nested JSON data via json_reader_options' set_dtypes method. More...
class	cudf::io::json_reader_options
	Input arguments to the read_json interface. More...
class	cudf::io::json_reader_options_builder
	Builds settings to use for read_json(). More...
struct	cudf::io::json_reader_diagnostics
	Optional diagnostics produced by read_json_with_diagnostics. More...
struct	cudf::io::json_reader_result
	Result of read_json_with_diagnostics: the parsed table together with reader-specific diagnostics. More...
struct	cudf::io::json_reader_row_diagnostics
	Optional row-level diagnostics produced by read_json_with_row_diagnostics. More...
struct	cudf::io::json_reader_result_with_row_diagnostics
	Result of read_json_with_row_diagnostics: the parsed table together with reader-specific row-level diagnostics. More...
class	cudf::io::orc_reader_options
	Settings to use for read_orc(). More...
class	cudf::io::orc_reader_options_builder
	Builds settings to use for read_orc(). More...
class	cudf::io::chunked_orc_reader
	The chunked orc reader class to read an ORC file iteratively into a series of tables, chunk by chunk. More...
class	cudf::io::parquet_reader_options
	Settings for read_parquet(). More...
class	cudf::io::parquet_reader_options_builder
	Builds parquet_reader_options to use for read_parquet(). More...
class	cudf::io::chunked_parquet_reader
	The chunked parquet reader class to read Parquet file iteratively in to a series of tables, chunk by chunk. More...
class	cudf::io::text::byte_range_info
	stores offset and size used to indicate a byte range More...
class	cudf::io::text::device_data_chunk
	A contract guaranteeing stream-ordered memory access to the underlying device data. More...
class	cudf::io::text::data_chunk_reader
	a reader capable of producing views over device memory. More...
class	cudf::io::text::data_chunk_source
	a data source capable of creating a reader which can produce views of the data source in device memory. More...
struct	cudf::io::text::parse_options
	Parsing options for multibyte_split. More...

Enumerations
enum class	cudf::io::parquet::experimental::use_data_page_mask : bool { YES = true , NO = false }
	Whether to compute and use a page mask using the row mask to skip decompression and decoding of the masked pages. More...
enum class	cudf::io::json_recovery_mode_t { cudf::io::FAIL , cudf::io::RECOVER_WITH_NULL }
	Control the error recovery behavior of the json parser. More...

Functions
table_with_metadata	cudf::io::read_avro (avro_reader_options const &options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads an Avro dataset into a set of columns. More...
table_with_metadata	cudf::io::read_csv (csv_reader_options options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a CSV dataset into a set of columns. More...
packed_table	cudf::io::experimental::read_cudftable (cudftable_reader_options const &options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Read a table in CudfTable binary format. More...
table_with_metadata	cudf::io::parquet::experimental::read_parquet (parquet_reader_options const &options, deletion_vector_info const &deletion_vector_info, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a table from parquet source, prepends an index column to it, deserializes the specified 64-bit roaring bitmap deletion vectors and applies them to the read table. More...
size_t	cudf::io::parquet::experimental::compute_num_deleted_rows (deletion_vector_info const &deletion_vector_info, cudf::size_type max_chunk_rows=std::numeric_limits< size_type >::max(), cuda::stream_ref stream=cudf::get_default_stream())
	Computes the number of rows deleted by the serialized 64-bit roaring bitmap deletion vectors. More...
std::unique_ptr< column >	cudf::io::parquet::experimental::get_variant_field (column_view const &variant_column, std::string_view path, std::optional< mutable_column_view > status=std::nullopt, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Extract the raw VARIANT-encoded bytes of a nested field by JSONPath-like path. More...
std::unique_ptr< column >	cudf::io::parquet::experimental::cast_variant (column_view const &values, data_type desired_type, std::optional< mutable_column_view > status=std::nullopt, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Decode a VARIANT value column's blobs into a typed cuDF column. More...
std::unique_ptr< column >	cudf::io::parquet::experimental::extract_variant_field (column_view const &variant_column, std::string_view path, data_type desired_type, std::optional< mutable_column_view > status=std::nullopt, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Convenience wrapper: extract a nested object value by path and decode into a typed column. More...
std::unique_ptr< column >	cudf::io::parquet::experimental::get_variant_type_id (column_view const &values, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Return the logical type of each VARIANT value blob in a list<uint8> column. More...
table_with_metadata	cudf::io::read_json (json_reader_options options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a JSON dataset into a set of columns. More...
json_reader_result	cudf::io::read_json_with_diagnostics (json_reader_options options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a JSON dataset into a set of columns, additionally reporting reader diagnostics that do not belong on the standard table_metadata. More...
json_reader_result_with_row_diagnostics	cudf::io::read_json_with_row_diagnostics (json_reader_options options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a JSON dataset into a set of columns, additionally reporting reader diagnostics with row-level schema mismatch details. More...
bool	cudf::io::is_supported_read_orc (compression_type compression)
	Check if the compression type is supported for reading ORC files. More...
bool	cudf::io::is_supported_write_orc (compression_type compression)
	Check if the compression type is supported for writing ORC files. More...
table_with_metadata	cudf::io::read_orc (orc_reader_options const &options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads an ORC dataset into a set of columns. More...
raw_orc_statistics	cudf::io::read_raw_orc_statistics (source_info const &src_info, cuda::stream_ref stream=cudf::get_default_stream())
	Reads file-level and stripe-level statistics of ORC dataset. More...
parsed_orc_statistics	cudf::io::read_parsed_orc_statistics (source_info const &src_info, cuda::stream_ref stream=cudf::get_default_stream())
	Reads file-level and stripe-level statistics of ORC dataset. More...
orc_metadata	cudf::io::read_orc_metadata (source_info const &src_info, cuda::stream_ref stream=cudf::get_default_stream())
	Reads metadata of ORC dataset. More...
bool	cudf::io::is_supported_read_parquet (compression_type compression)
	Check if the compression type is supported for reading Parquet files. More...
bool	cudf::io::is_supported_write_parquet (compression_type compression)
	Check if the compression type is supported for writing Parquet files. More...
table_with_metadata	cudf::io::read_parquet (parquet_reader_options const &options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a Parquet dataset into a set of columns. More...
table_with_metadata	cudf::io::read_parquet (std::vector< std::unique_ptr< cudf::io::datasource >> &&sources, std::vector< parquet::FileMetaData > &&parquet_metadatas, parquet_reader_options const &options, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Reads a Parquet dataset into a set of columns using pre-existing Parquet datasources and file metadatas. More...
parquet_metadata	cudf::io::read_parquet_metadata (source_info const &src_info)
	Reads metadata of parquet dataset. More...
std::vector< parquet::FileMetaData >	cudf::io::read_parquet_footers (std::span< std::unique_ptr< cudf::io::datasource > const > sources)
	Constructs FileMetaData objects from parquet dataset. More...
std::unique_ptr< table >	cudf::io::read_parquet_column_chunk_bounds (std::span< parquet::FileMetaData const > parquet_metadatas, std::span< std::string const > column_names, cuda::stream_ref stream=cudf::get_default_stream(), cudf::memory_resources mr=cudf::get_current_device_resource_ref())
	Decode parquet column-chunk min/max statistics for selected leaf columns. More...
std::vector< byte_range_info >	cudf::io::text::create_byte_range_infos_consecutive (int64_t total_bytes, int64_t range_count)
	Create a collection of consecutive ranges between [0, total_bytes). More...
byte_range_info	cudf::io::text::create_byte_range_info_max ()
	Create a byte_range_info which represents as much of a file as possible. Specifically, [0, numeric_limits<int64_t>:\:max()). More...
std::unique_ptr< cudf::column >	cudf::io::text::multibyte_split (data_chunk_source const &source, std::string_view delimiter, parse_options options={}, cuda::stream_ref stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
	Splits the source text into a strings column using a multiple byte delimiter. More...

Variables
constexpr size_t	cudf::io::default_stripe_size_bytes = 64 * 1024 * 1024
	64MB default orc stripe size
constexpr size_type	cudf::io::default_stripe_size_rows = 1000000
	1M rows default orc stripe rows
constexpr size_type	cudf::io::default_row_index_stride = 10000
	10K rows default orc row index stride
constexpr size_t	cudf::io::default_row_group_size_bytes
	Infinite bytes per row group. More...
constexpr size_type	cudf::io::default_row_group_size_rows = 1'000'000
	1 million rows per row group
constexpr size_t	cudf::io::default_max_page_size_bytes = 512 * 1024
	512KB per page
constexpr size_type	cudf::io::default_max_page_size_rows = 20000
	20k rows per page
constexpr int32_t	cudf::io::default_column_index_truncate_length = 64
	truncate to 64 bytes
constexpr size_t	cudf::io::default_max_dictionary_size = 1024 * 1024
	1MB dictionary size
constexpr size_type	cudf::io::default_max_page_fragment_size = 5000
	5000 rows per page fragment

Detailed Description

Enumeration Type Documentation

json_recovery_mode_t

enum cudf::io::json_recovery_mode_t

strong

#include <cudf/io/json.hpp>

Control the error recovery behavior of the json parser.

Enumerator
FAIL	Does not recover from an error when encountering an invalid format.
RECOVER_WITH_NULL	Recovers from an error, replacing invalid records with null.

Definition at line 61 of file json.hpp.

use_data_page_mask

enum cudf::io::parquet::experimental::use_data_page_mask : bool

strong

#include <cudf/io/experimental/hybrid_scan.hpp>

Whether to compute and use a page mask using the row mask to skip decompression and decoding of the masked pages.

Enumerator
YES	Compute and use a data page mask.
NO	Do not compute or use a data page mask.

Definition at line 56 of file hybrid_scan.hpp.

Function Documentation

cast_variant()

std::unique_ptr<column> cudf::io::parquet::experimental::cast_variant	(	column_view const &	values,
		data_type	desired_type,
		std::optional< mutable_column_view >	status = std::nullopt,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/experimental/variant.hpp>

Decode a VARIANT value column's blobs into a typed cuDF column.

A null value is produced when the input row is null or the encoded type does not match desired_type.

For a decimal desired_type, every encoded width is accepted and each value is rescaled from its own encoded scale to desired_type.scale(), truncating toward zero; a value that no longer fits produces a null row with variant_operation_status::OVERFLOW.

Parameters

values	list<uint8> column of VARIANT-encoded value bytes
desired_type	Target cuDF type (STRING, INT8/INT16/INT32/INT64, FLOAT32/FLOAT64, BOOL8, or DECIMAL32/DECIMAL64/DECIMAL128)
status	Optional in-out parameter, variant_operation_status values, one per row. Must be non-nullable, UINT8, and have the same row count as values. On input, its existing values are treated as status from a prior get_variant_field call: rows already marked non-success are propagated directly to the output without decoding. It is then overwritten in place with the final per-row status. Callers with no prior status to propagate must initialize every row to variant_operation_status::SUCCESS before calling
stream	CUDA stream
mr	Device memory resource

Returns

Typed column decoded from the VARIANT value blobs

Exceptions

std::invalid_argument

if values is not a list<uint8> column; if desired_type is not one of the supported types (STRING, INT8/INT16/INT32/INT64, FLOAT32/FLOAT64, BOOL8, or DECIMAL32/DECIMAL64/DECIMAL128); or if status is provided but is nullable, not UINT8, or has a different row count than values

compute_num_deleted_rows()

size_t cudf::io::parquet::experimental::compute_num_deleted_rows	(	deletion_vector_info const &	deletion_vector_info,
		cudf::size_type	max_chunk_rows = std::numeric_limits< size_type >::max(),
		cuda::stream_ref	stream = cudf::get_default_stream()
	)

#include <cudf/io/experimental/deletion_vectors.hpp>

Computes the number of rows deleted by the serialized 64-bit roaring bitmap deletion vectors.

Parameters

deletion_vector_info	Information about the deletion vectors and the index column
max_chunk_rows	Maximum number of rows to process at a time
stream	CUDA stream used for device memory operations and kernel launches

Returns

Number of rows deleted by the specified 64-bit roaring bitmap deletion vectors

create_byte_range_info_max()

byte_range_info cudf::io::text::create_byte_range_info_max

(

)

#include <cudf/io/text/byte_range_info.hpp>

Create a byte_range_info which represents as much of a file as possible. Specifically, [0, numeric_limits<int64_t>:\:max()).

Returns

Byte range info of size [0, numeric_limits<int64_t>:\:max())

create_byte_range_infos_consecutive()

std::vector<byte_range_info> cudf::io::text::create_byte_range_infos_consecutive	(	int64_t	total_bytes,
		int64_t	range_count
	)

#include <cudf/io/text/byte_range_info.hpp>

Create a collection of consecutive ranges between [0, total_bytes).

Each range wil be the same size except if total_bytes is not evenly divisible by range_count, in which case the last range size will be the remainder.

Parameters

total_bytes	total number of bytes in all ranges
range_count	total number of ranges in which to divide bytes

Returns

Vector of range objects

extract_variant_field()

std::unique_ptr<column> cudf::io::parquet::experimental::extract_variant_field	(	column_view const &	variant_column,
		std::string_view	path,
		data_type	desired_type,
		std::optional< mutable_column_view >	status = std::nullopt,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/experimental/variant.hpp>

Convenience wrapper: extract a nested object value by path and decode into a typed column.

Semantically equivalent to extracting the field with get_variant_field and then decoding the extracted list<uint8> values with cast_variant.

Parameters

variant_column	Struct column (VARIANT materialization)
path	JSONPath-like path string (see get_variant_field for syntax)
desired_type	Target type: STRING, INT8/INT16/INT32/INT64, FLOAT32/FLOAT64, BOOL8, or DECIMAL32/DECIMAL64/DECIMAL128 (see cast_variant for decimal rescaling)
status	Optional. When provided, filled with variant_operation_status values, one per row. Must be non-nullable, UINT8, and have the same row count as variant_column
stream	CUDA stream
mr	Device memory resource

Returns

Column of desired_type

Exceptions

std::invalid_argument

on empty path or malformed syntax; or if status is provided but is nullable, not UINT8, or has a different row count than variant_column

get_variant_field()

std::unique_ptr<column> cudf::io::parquet::experimental::get_variant_field	(	column_view const &	variant_column,
		std::string_view	path,
		std::optional< mutable_column_view >	status = std::nullopt,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/experimental/variant.hpp>

Extract the raw VARIANT-encoded bytes of a nested field by JSONPath-like path.

Path grammar: path := "$"? first_step step* first := name | "." name | "[" index "]" step := "." name | "[" index "]" name := any sequence of bytes other than '.' or '[' index := non-negative base-10 integer (leading zeros are allowed, e.g. "[01]" == "[1]")

Examples: "x" -> top-level field "x" (leading $ optional) "$.foo" -> top-level field "foo" "$.foo.bar" -> object descent foo -> bar "$[0]" -> first element of a top-level array "$.a[0].b" -> object key "a" -> first array element -> object key "b"

Parameters

variant_column	Struct column (VARIANT materialization) with list<uint8> children (metadata, value), plus optional shredded siblings
path	JSONPath-like path string identifying the target field
status	Optional. When provided, filled with variant_operation_status values, one per row. Must be non-nullable, UINT8, and have the same row count as variant_column
stream	CUDA stream
mr	Device memory resource

Returns

list<uint8> column with the extracted value's encoded bytes. A row is null when the input row is null, a name is absent, an index is out of bounds, or a step does not match the current value.

Exceptions

std::invalid_argument

on empty path or malformed syntax ([*] wildcards, negative indices, out-of-range indices, and quoted names inside [...] are not supported); or if status is provided but is nullable, not UINT8, or has a different row count than variant_column

get_variant_type_id()

std::unique_ptr<column> cudf::io::parquet::experimental::get_variant_type_id	(	column_view const &	values,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/experimental/variant.hpp>

Return the logical type of each VARIANT value blob in a list<uint8> column.

Classifies only the value_metadata header byte; does not validate the remaining payload. A recognized header returns its logical type even when the payload is truncated. A null output row is produced when the input row is null, the blob is empty, or the header carries an unrecognized type. An encoded Variant null (NULLVAL) produces a valid NULL_VALUE row.

Parameters

values	list<uint8> column of VARIANT-encoded value bytes
stream	CUDA stream
mr	Device memory resource

Returns

UINT8 column of variant_logical_type values cast to uint8_t

Exceptions

std::invalid_argument

if values is not a list<uint8> column

is_supported_read_orc()

bool cudf::io::is_supported_read_orc

(

compression_type

compression

)

#include <cudf/io/orc.hpp>

Check if the compression type is supported for reading ORC files.

Note

This is a runtime check. Some compression types may not be supported because of the current system configuration.

Parameters

compression

Compression type

Returns

Boolean indicating if the compression type is supported

is_supported_read_parquet()

bool cudf::io::is_supported_read_parquet

(

compression_type

compression

)

#include <cudf/io/parquet.hpp>

Check if the compression type is supported for reading Parquet files.

Note

This is a runtime check. Some compression types may not be supported because of the current system configuration.

Parameters

compression

Compression type

Returns

Boolean indicating if the compression type is supported

is_supported_write_orc()

bool cudf::io::is_supported_write_orc

(

compression_type

compression

)

#include <cudf/io/orc.hpp>

Check if the compression type is supported for writing ORC files.

Note

This is a runtime check. Some compression types may not be supported because of the current system configuration.

Parameters

compression

Compression type

Returns

Boolean indicating if the compression type is supported

is_supported_write_parquet()

bool cudf::io::is_supported_write_parquet

(

compression_type

compression

)

#include <cudf/io/parquet.hpp>

Check if the compression type is supported for writing Parquet files.

Note

This is a runtime check. Some compression types may not be supported because of the current system configuration.

Parameters

compression

Compression type

Returns

Boolean indicating if the compression type is supported

multibyte_split()

std::unique_ptr<cudf::column> cudf::io::text::multibyte_split	(	data_chunk_source const &	source,
		std::string_view	delimiter,
		parse_options	options = {},
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/text/multibyte_split.hpp>

Splits the source text into a strings column using a multiple byte delimiter.

Providing a byte range allows multibyte_split to read a file partially, only returning the offsets of delimiters which begin within the range. If thinking in terms of "records", where each delimiter dictates the end of a record, all records which begin within the byte range provided will be returned, including any record which may begin in the range but end outside of the range. Records which begin outside of the range will ignored, even if those records end inside the range.

Examples:
 source:     "abc..def..ghi..jkl.."
 delimiter:  ".."
 
 byte_range: nullopt
 return:     ["abc..", "def..", "ghi..", jkl..", ""]
 
 byte_range: [0, 2)
 return:     ["abc.."]
 
 byte_range: [2, 9)
 return:     ["def..", "ghi.."]
 
 byte_range: [11, 2)
 return:     []
 
 byte_range: [13, 7)
 return:     ["jkl..", ""]

Parameters

source	The source string
delimiter	UTF-8 encoded string for which to find offsets in the source
options	the parsing options to use (including byte range)
stream	CUDA stream used for device memory operations and kernel launches
mr	Memory resource to use for the device memory allocation

Returns

The strings found by splitting the source by the delimiter within the relevant byte range.

read_avro()

table_with_metadata cudf::io::read_avro	(	avro_reader_options const &	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/avro.hpp>

Reads an Avro dataset into a set of columns.

The following code snippet demonstrates how to read a dataset from a file:

auto source  = cudf::io::source_info("dataset.avro");
auto options = cudf::io::avro_reader_options::builder(source);
auto result  = cudf::io::read_avro(options);

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned table_with_metadata

Returns

The set of columns along with metadata

read_csv()

table_with_metadata cudf::io::read_csv	(	csv_reader_options	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/csv.hpp>

Reads a CSV dataset into a set of columns.

The following code snippet demonstrates how to read a dataset from a file:

auto source  = cudf::io::source_info("dataset.csv");
auto options = cudf::io::csv_reader_options::builder(source);
auto result  = cudf::io::read_csv(options);

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned table_with_metadata

Returns

The set of columns along with metadata

read_cudftable()

packed_table cudf::io::experimental::read_cudftable	(	cudftable_reader_options const &	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/experimental/cudftable.hpp>

Read a table in CudfTable binary format.

This function reads the header from the datasource, validates the format, and uses cudf::unpack to deserialize the table.

Returns a packed_table containing a table_view and the underlying packed_columns data. After reading the data from the source, the unpacking operation creates views without copying - the table_view points directly into the contiguous memory buffers owned by packed_columns.

It is the caller's responsibility to ensure the table_view does not outlive the packed_columns data.

Parameters

options	Options specifying the source to read from
stream	CUDA stream used for device memory operations and kernel launches
mr	An optional memory resource to use for all device allocations

Returns

A packed_table containing the deserialized table view and its backing data

read_json()

table_with_metadata cudf::io::read_json	(	json_reader_options	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/json.hpp>

Reads a JSON dataset into a set of columns.

The following code snippet demonstrates how to read a dataset from a file:

auto source  = cudf::io::source_info("dataset.json");
auto options = cudf::io::read_json_options::builder(source);
auto result  = cudf::io::read_json(options);

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned table_with_metadata.

Returns

The set of columns along with metadata

read_json_with_diagnostics()

json_reader_result cudf::io::read_json_with_diagnostics	(	json_reader_options	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/json.hpp>

Reads a JSON dataset into a set of columns, additionally reporting reader diagnostics that do not belong on the standard table_metadata.

Behaves identically to read_json for column construction. The additional diagnostics field carries reader-specific observations such as top_level_columns_with_schema_mismatch.

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned result.

Returns

The parsed table, its metadata, and reader diagnostics

read_json_with_row_diagnostics()

json_reader_result_with_row_diagnostics cudf::io::read_json_with_row_diagnostics	(	json_reader_options	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/json.hpp>

Reads a JSON dataset into a set of columns, additionally reporting reader diagnostics with row-level schema mismatch details.

Behaves identically to read_json for column construction. The additional diagnostics field carries reader-specific observations such as top_level_columns_with_schema_mismatch_rows.

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned result.

Returns

The parsed table, its metadata, and reader diagnostics with row-level details

read_orc()

table_with_metadata cudf::io::read_orc	(	orc_reader_options const &	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/orc.hpp>

Reads an ORC dataset into a set of columns.

The following code snippet demonstrates how to read a dataset from a file:

auto source  = cudf::io::source_info("dataset.orc");
auto options = cudf::io::orc_reader_options::builder(source);
auto result  = cudf::io::read_orc(options);

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned table_with_metadata.

Returns

The set of columns

read_orc_metadata()

orc_metadata cudf::io::read_orc_metadata	(	source_info const &	src_info,
		cuda::stream_ref	stream = cudf::get_default_stream()
	)

#include <cudf/io/orc_metadata.hpp>

Reads metadata of ORC dataset.

Parameters

src_info	Dataset source
stream	CUDA stream used for device memory operations and kernel launches

Returns

orc_metadata with ORC schema, number of rows and number of stripes.

read_parquet() [1/3]

table_with_metadata cudf::io::read_parquet	(	parquet_reader_options const &	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/parquet.hpp>

Reads a Parquet dataset into a set of columns.

The following code snippet demonstrates how to read a dataset from a file:

auto source  = cudf::io::source_info("dataset.parquet");
auto options = cudf::io::parquet_reader_options::builder(source);
auto result  = cudf::io::read_parquet(options);

Row-group selection and output ordering are described in parquet_reader_options::set_row_groups().

Parameters

options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned table_with_metadata

Returns

The set of columns along with metadata

read_parquet() [2/3]

table_with_metadata cudf::io::parquet::experimental::read_parquet	(	parquet_reader_options const &	options,
		deletion_vector_info const &	deletion_vector_info,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/experimental/deletion_vectors.hpp>

Reads a table from parquet source, prepends an index column to it, deserializes the specified 64-bit roaring bitmap deletion vectors and applies them to the read table.

Reads a table from a parquet source, builds a row index column to the table using the specified row group offsets and row counts and prepends it to the table, deserializes the specified 64-bit roaring bitmap deletion vectors and applies them to the read table using the specified deletion vector row counts. If the row group offsets and row counts are empty, the index column is simply a sequence of UINT64 from 0 to the total number of rows in the table. If the serialized roaring64 bitmap span is empty, the read table (prepended with the index column) is returned as is.

Parameters

options	Parquet reader options
deletion_vector_info	Information about the deletion vectors and the index column
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the returned table

Returns

Read table with a prepended index column filtered using the deletion vector, along with its metadata

read_parquet() [3/3]

table_with_metadata cudf::io::read_parquet	(	std::vector< std::unique_ptr< cudf::io::datasource >> &&	sources,
		std::vector< parquet::FileMetaData > &&	parquet_metadatas,
		parquet_reader_options const &	options,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		rmm::device_async_resource_ref	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/parquet.hpp>

Reads a Parquet dataset into a set of columns using pre-existing Parquet datasources and file metadatas.

The following code snippet demonstrates how to read a dataset from a file:

auto sources = cudf::io::make_datasources(cudf::io::source_info("dataset.parquet"));
auto metadatas = cudf::io::read_parquet_footers(sources);
auto options = cudf::io::parquet_reader_options::builder();
auto result  = cudf::io::read_parquet(std::move(sources), std::move(metadatas), options);

Row-group selection and output ordering are described in parquet_reader_options::set_row_groups().

Parameters

sources	Input datasource objects to read the dataset from
parquet_metadatas	Pre-materialized Parquet file metadata(s). Read from sources if empty
options	Settings for controlling reading behavior
stream	CUDA stream used for device memory operations and kernel launches
mr	Device memory resource used to allocate device memory of the table in the returned table_with_metadata

Returns

The set of columns along with metadata

read_parquet_column_chunk_bounds()

std::unique_ptr<table> cudf::io::read_parquet_column_chunk_bounds	(	std::span< parquet::FileMetaData const >	parquet_metadatas,
		std::span< std::string const >	column_names,
		cuda::stream_ref	stream = cudf::get_default_stream(),
		cudf::memory_resources	mr = cudf::get_current_device_resource_ref()
	)

#include <cudf/io/parquet_metadata.hpp>

Decode parquet column-chunk min/max statistics for selected leaf columns.

Missing min/max statistics are represented as nulls in the corresponding output column. Parquet min/max exactness flags are not interpreted by this function. The requested column names are resolved against each file's schema. The returned table contains one row per source row group. Column 0 is the source file index, column 1 is the file-local row-group index, and subsequent columns are min/max pairs in the order of column_names.

Parameters

parquet_metadatas	Parquet file metadata, one per source
column_names	Dotted leaf-column paths to decode statistics for
stream	CUDA stream used for device memory operations
mr	Memory resources to use for device memory allocation

Returns

Table of row-group identifiers and decoded min/max bounds. For requested column column_names[i], the min column is at 2 + 2 * i and the max column is at 3 + 2 * i.

Exceptions

std::invalid_argument	If a requested leaf-column path is missing or ambiguous.
std::invalid_argument	If a requested column has unsupported or compound statistics dtype.
std::invalid_argument	If a requested column has mismatching statistics dtype across sources.

read_parquet_footers()

std::vector<parquet::FileMetaData> cudf::io::read_parquet_footers

(

std::span< std::unique_ptr< cudf::io::datasource > const >

sources

)

#include <cudf/io/parquet_metadata.hpp>

Constructs FileMetaData objects from parquet dataset.

Parameters

sources

Input datasource objects to read the dataset from

Returns

List of FileMetaData objects, one per parquet source

read_parquet_metadata()

parquet_metadata cudf::io::read_parquet_metadata

(

source_info const &

src_info

)

#include <cudf/io/parquet_metadata.hpp>

Reads metadata of parquet dataset.

Parameters

src_info

Dataset source information

Returns

parquet_metadata with parquet schema, number of rows, number of row groups and key-value metadata

read_parsed_orc_statistics()

parsed_orc_statistics cudf::io::read_parsed_orc_statistics	(	source_info const &	src_info,
		cuda::stream_ref	stream = cudf::get_default_stream()
	)

#include <cudf/io/orc_metadata.hpp>

Reads file-level and stripe-level statistics of ORC dataset.

Parameters

src_info	Dataset source
stream	CUDA stream used for device memory operations and kernel launches

Returns

Column names and decoded ORC statistics

read_raw_orc_statistics()

raw_orc_statistics cudf::io::read_raw_orc_statistics	(	source_info const &	src_info,
		cuda::stream_ref	stream = cudf::get_default_stream()
	)

#include <cudf/io/orc_metadata.hpp>

Reads file-level and stripe-level statistics of ORC dataset.

The following code snippet demonstrates how to read statistics of a dataset from a file:

auto result = cudf::read_raw_orc_statistics(cudf::source_info("dataset.orc"));

Parameters

src_info	Dataset source
stream	CUDA stream used for device memory operations and kernel launches

Returns

Column names and encoded ORC statistics

Variable Documentation

default_row_group_size_bytes

constexpr size_t cudf::io::default_row_group_size_bytes

constexpr

#include <cudf/io/parquet.hpp>

Initial value:

=
  std::numeric_limits<size_t>::max()

Infinite bytes per row group.

Definition at line 34 of file parquet.hpp.