hybrid_scan_multifile.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 */
 
#pragma once
 
#include <cudf/io/experimental/hybrid_scan.hpp>
#include <cudf/io/parquet.hpp>
#include <cudf/io/parquet_schema.hpp>
#include <cudf/io/text/byte_range_info.hpp>
#include <cudf/io/types.hpp>
#include <cudf/types.hpp>
#include <cudf/utilities/export.hpp>
 
#include <rmm/resource_ref.hpp>
 
#include <cuda/stream>
 
#include <memory>
#include <span>
#include <utility>
#include <vector>
 
namespace cudf::io::parquet::experimental::detail {
class hybrid_scan_reader_impl;
}  // namespace cudf::io::parquet::experimental::detail
 
using cudf::io::text::byte_range_info;
 
namespace CUDF_EXPORT cudf {
namespace io::parquet::experimental {
class hybrid_scan_multifile {
 public:
  explicit hybrid_scan_multifile(cudf::host_span<cudf::host_span<uint8_t const> const> footer_bytes,
                                 parquet_reader_options const& options);
 
  explicit hybrid_scan_multifile(cudf::host_span<FileMetaData const> parquet_metadata,
                                 parquet_reader_options const& options);
 
  explicit hybrid_scan_multifile(std::vector<FileMetaData>&& parquet_metadata,
                                 parquet_reader_options const& options);
 
  ~hybrid_scan_multifile();
 
  [[nodiscard]] std::vector<FileMetaData> parquet_metadatas() const;
 
  [[nodiscard]] std::vector<byte_range_info> page_index_byte_ranges() const;
 
  void setup_page_indexes(
    cudf::host_span<cudf::host_span<uint8_t const> const> page_index_bytes) const;
 
  [[nodiscard]] std::vector<std::vector<size_type>> all_row_groups(
    parquet_reader_options const& options) const;
 
  [[nodiscard]] size_type total_rows_in_row_groups(
    cudf::host_span<std::vector<size_type> const> row_group_indices) const;
 
  void reset_column_selection() const;
 
  [[nodiscard]] std::vector<std::vector<size_type>> filter_row_groups_with_byte_range(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<std::vector<size_type>> filter_row_groups_with_stats(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<size_type>>
  bloom_filters_byte_ranges(std::span<std::vector<size_type> const> row_group_indices,
                            parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<std::vector<size_type>> filter_row_groups_with_bloom_filters(
    std::span<cudf::device_span<uint8_t const> const> bloom_filter_data,
    std::span<std::vector<size_type> const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<size_type>>
  dictionary_pages_byte_ranges(cudf::host_span<std::vector<size_type> const> row_group_indices,
                               parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<std::vector<size_type>> filter_row_groups_with_dictionary_pages(
    cudf::host_span<cudf::device_span<uint8_t const> const> dictionary_page_data,
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream) const;
 
  [[nodiscard]] std::unique_ptr<cudf::column> build_all_true_row_mask(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::unique_ptr<cudf::column> build_row_mask_with_page_index_stats(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<size_type>>
  filter_column_chunks_byte_ranges(cudf::host_span<std::vector<size_type> const> row_group_indices,
                                   parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_filter_columns(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::host_span<cudf::device_span<uint8_t const> const> column_chunk_data,
    cudf::mutable_column_view& row_mask,
    use_data_page_mask mask_data_pages,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<size_type>>
  payload_column_chunks_byte_ranges(cudf::host_span<std::vector<size_type> const> row_group_indices,
                                    parquet_reader_options const& options) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<size_type>>
  payload_pages_byte_ranges(cudf::host_span<std::vector<size_type> const> row_group_indices,
                            cudf::column_view const& row_mask,
                            parquet_reader_options const& options,
                            cuda::stream_ref stream) const;
 
  [[nodiscard]] table_with_metadata materialize_payload_columns(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::host_span<cudf::device_span<uint8_t const> const> column_chunk_data,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<size_type>>
  all_column_chunks_byte_ranges(cudf::host_span<std::vector<size_type> const> row_group_indices,
                                parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_all_columns(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::host_span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  void setup_chunking_for_filter_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    cudf::host_span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] table_with_metadata materialize_filter_columns_chunk(
    cudf::mutable_column_view& row_mask) const;
 
  void setup_chunking_for_payload_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    cudf::host_span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  void setup_chunking_for_payload_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::column_view const& row_mask,
    cudf::host_span<cudf::device_span<uint8_t const> const> page_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] table_with_metadata materialize_payload_columns_chunk(
    cudf::column_view const& row_mask) const;
 
  void setup_chunking_for_all_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    cudf::host_span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] table_with_metadata materialize_all_columns_chunk() const;
 
  [[nodiscard]] std::vector<std::vector<std::vector<size_type>>> construct_row_group_passes(
    cudf::host_span<std::vector<size_type> const> row_group_indices,
    std::size_t pass_read_limit) const;
 
  [[nodiscard]] bool has_next_table_chunk() const;
 
 private:
  std::unique_ptr<detail::hybrid_scan_reader_impl> _impl;
};
  // end of group
 
}  // namespace io::parquet::experimental
}  // namespace CUDF_EXPORT cudf