parquet_io_utils.hpp
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: Apache-2.0
4  */
5 
6 #pragma once
7 
8 #include <cudf/io/datasource.hpp>
9 #include <cudf/io/text/byte_range_info.hpp>
10 #include <cudf/utilities/memory_resource.hpp>
11 
12 #include <rmm/device_buffer.hpp>
13 #include <rmm/resource_ref.hpp>
14 
15 #include <cuda/stream>
16 
17 #include <cstddef>
18 #include <cstdint>
19 #include <functional>
20 #include <future>
21 #include <span>
22 #include <tuple>
23 #include <utility>
24 #include <vector>
25 
31 namespace CUDF_EXPORT cudf {
32 namespace io::parquet {
33 
40 using cudf::io::text::byte_range_info;
41 
45 enum class io_submission_policy : int8_t {
46  SERIALIZE,
47  INTERLEAVE
48 };
49 
67 [[nodiscard]] std::size_t metadata_size_hint();
68 
77 [[nodiscard]] std::unique_ptr<cudf::io::datasource::buffer> fetch_footer_to_host(
78  cudf::io::datasource& datasource);
79 
91 [[nodiscard]] std::vector<std::unique_ptr<cudf::io::datasource::buffer>> fetch_footers_to_host(
92  cudf::host_span<std::reference_wrapper<cudf::io::datasource> const> datasources);
93 
103 [[nodiscard]] std::unique_ptr<cudf::io::datasource::buffer> fetch_page_index_to_host(
104  cudf::io::datasource& datasource, byte_range_info const page_index_bytes);
105 
119 [[nodiscard]] std::vector<std::unique_ptr<cudf::io::datasource::buffer>> fetch_page_indexes_to_host(
120  cudf::host_span<std::reference_wrapper<cudf::io::datasource> const> datasources,
121  cudf::host_span<byte_range_info const> page_index_bytes_per_source);
122 
137 std::tuple<std::vector<rmm::device_buffer>,
138  std::vector<cudf::device_span<uint8_t const>>,
139  std::future<void>>
140 fetch_byte_ranges_to_device_async(
141  cudf::io::datasource& datasource,
142  std::span<byte_range_info const> byte_ranges,
143  io_submission_policy policy,
144  cuda::stream_ref stream = cudf::get_default_stream(),
145  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
146 
161 std::tuple<std::vector<rmm::device_buffer>,
162  std::vector<std::vector<cudf::device_span<uint8_t const>>>,
163  std::future<void>>
164 fetch_byte_ranges_to_device_async(
165  cudf::host_span<std::reference_wrapper<cudf::io::datasource> const> datasources,
166  cudf::host_span<std::vector<byte_range_info> const> byte_ranges_per_source,
167  io_submission_policy policy,
168  cuda::stream_ref stream = cudf::get_default_stream(),
169  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
170 
186 std::pair<std::vector<rmm::device_buffer>, std::vector<cudf::device_span<uint8_t const>>>
187 fetch_bloom_filters_to_device(cudf::io::datasource& datasource,
188  cudf::host_span<byte_range_info const> bloom_filter_byte_ranges,
189  io_submission_policy policy,
190  cuda::stream_ref stream = cudf::get_default_stream(),
191  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
192 
208 std::pair<std::vector<rmm::device_buffer>,
209  std::vector<std::vector<cudf::device_span<uint8_t const>>>>
210 fetch_bloom_filters_to_device(
211  cudf::host_span<std::reference_wrapper<cudf::io::datasource> const> datasources,
212  cudf::host_span<std::vector<byte_range_info> const> bloom_filter_byte_ranges_per_source,
213  io_submission_policy policy,
214  cuda::stream_ref stream = cudf::get_default_stream(),
215  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
216 
235 [[deprecated("Use the overload that takes io_submission_policy.")]]
236 std::tuple<std::vector<rmm::device_buffer>,
237  std::vector<cudf::device_span<uint8_t const>>,
238  std::future<void>>
239 fetch_byte_ranges_to_device_async(
240  cudf::io::datasource& datasource,
241  std::span<byte_range_info const> byte_ranges,
242  cuda::stream_ref stream = cudf::get_default_stream(),
243  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
244 
263 [[deprecated("Use the overload that takes io_submission_policy.")]]
264 std::tuple<std::vector<rmm::device_buffer>,
265  std::vector<std::vector<cudf::device_span<uint8_t const>>>,
266  std::future<void>>
267 fetch_byte_ranges_to_device_async(
268  cudf::host_span<std::reference_wrapper<cudf::io::datasource> const> datasources,
269  cudf::host_span<std::vector<byte_range_info> const> byte_ranges_per_source,
270  cuda::stream_ref stream = cudf::get_default_stream(),
271  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
272 
292 [[deprecated("Use the overload that takes io_submission_policy.")]]
293 std::pair<std::vector<rmm::device_buffer>, std::vector<cudf::device_span<uint8_t const>>>
294 fetch_bloom_filters_to_device(cudf::io::datasource& datasource,
295  cudf::host_span<byte_range_info const> bloom_filter_byte_ranges,
296  cuda::stream_ref stream = cudf::get_default_stream(),
297  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
298 
318 [[deprecated("Use the overload that takes io_submission_policy.")]]
319 std::pair<std::vector<rmm::device_buffer>,
320  std::vector<std::vector<cudf::device_span<uint8_t const>>>>
321 fetch_bloom_filters_to_device(
322  cudf::host_span<std::reference_wrapper<cudf::io::datasource> const> datasources,
323  cudf::host_span<std::vector<byte_range_info> const> bloom_filter_byte_ranges_per_source,
324  cuda::stream_ref stream = cudf::get_default_stream(),
325  cudf::memory_resources mr = cudf::get_current_device_resource_ref());
326  // end of group
328 } // namespace io::parquet
329 } // namespace CUDF_EXPORT cudf
byte_range_info.hpp
Class and helper functions for specifying a byte range within a data source.
cudf::io::datasource
Interface class for providing input data to the readers.
Definition: datasource.hpp:37
cudf::io::text::byte_range_info
stores offset and size used to indicate a byte range
Definition: byte_range_info.hpp:29
cudf::memory_resources
Non-owning references to the memory resources used by a cuDF operation.
Definition: memory_resource.hpp:47
datasource.hpp
Interface classes for providing input data to the readers from files, host memory,...
device_buffer.hpp
cudf::get_default_stream
cuda::stream_ref const get_default_stream()
Get the current default stream.
cudf::io::parquet::fetch_page_index_to_host
std::unique_ptr< cudf::io::datasource::buffer > fetch_page_index_to_host(cudf::io::datasource &datasource, byte_range_info const page_index_bytes)
Fetches a host buffer of Parquet page index from the input data source.
cudf::io::parquet::metadata_size_hint
std::size_t metadata_size_hint()
Returns the Parquet reader's footer speculative read size in bytes.
cudf::io::parquet::fetch_footer_to_host
std::unique_ptr< cudf::io::datasource::buffer > fetch_footer_to_host(cudf::io::datasource &datasource)
Fetches a host buffer of Parquet footer bytes from the input data source.
cudf::io::parquet::fetch_bloom_filters_to_device
std::pair< std::vector< rmm::device_buffer >, std::vector< std::vector< cudf::device_span< uint8_t const > > > > fetch_bloom_filters_to_device(cudf::host_span< std::reference_wrapper< cudf::io::datasource > const > datasources, cudf::host_span< std::vector< byte_range_info > const > bloom_filter_byte_ranges_per_source, cuda::stream_ref stream=cudf::get_default_stream(), cudf::memory_resources mr=cudf::get_current_device_resource_ref())
Fetches Parquet bloom filter bitsets from multiple datasources into device buffers.
cudf::io::parquet::io_submission_policy
io_submission_policy
Controls whether I/O submissions are serialized across callers.
Definition: parquet_io_utils.hpp:45
cudf::io::parquet::fetch_page_indexes_to_host
std::vector< std::unique_ptr< cudf::io::datasource::buffer > > fetch_page_indexes_to_host(cudf::host_span< std::reference_wrapper< cudf::io::datasource > const > datasources, cudf::host_span< byte_range_info const > page_index_bytes_per_source)
Fetches host buffers of Parquet page index bytes from multiple input data sources.
cudf::io::parquet::fetch_byte_ranges_to_device_async
std::tuple< std::vector< rmm::device_buffer >, std::vector< std::vector< cudf::device_span< uint8_t const > > >, std::future< void > > fetch_byte_ranges_to_device_async(cudf::host_span< std::reference_wrapper< cudf::io::datasource > const > datasources, cudf::host_span< std::vector< byte_range_info > const > byte_ranges_per_source, cuda::stream_ref stream=cudf::get_default_stream(), cudf::memory_resources mr=cudf::get_current_device_resource_ref())
Fetches lists of byte ranges from multiple datasources into device buffers.
cudf::io::parquet::fetch_footers_to_host
std::vector< std::unique_ptr< cudf::io::datasource::buffer > > fetch_footers_to_host(cudf::host_span< std::reference_wrapper< cudf::io::datasource > const > datasources)
Fetches host buffers of Parquet footer bytes from multiple input data sources.
cudf::io::parquet::io_submission_policy::SERIALIZE
@ SERIALIZE
Serialize submissions across callers that use this policy.
cudf::io::parquet::io_submission_policy::INTERLEAVE
@ INTERLEAVE
Allow submissions from different callers to interleave.
cudf::get_current_device_resource_ref
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
Definition: memory_resource.hpp:34
memory_resource.hpp
APIs for getting and setting the current device memory resource.
cudf
cuDF interfaces
Definition: host_udf.hpp:27
resource_ref.hpp
cudf::host_span
Host span, a non-owning view over a contiguous sequence of host-accessible elements.
Definition: span.hpp:65
Generated by doxygen 1.9.1