hybrid_scan.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 */
 
#pragma once
 
#include <cudf/io/parquet.hpp>
#include <cudf/io/parquet_schema.hpp>
#include <cudf/io/text/byte_range_info.hpp>
#include <cudf/io/types.hpp>
#include <cudf/types.hpp>
#include <cudf/utilities/export.hpp>
 
#include <rmm/resource_ref.hpp>
 
#include <cuda/stream>
 
#include <memory>
#include <span>
#include <utility>
#include <vector>
 
namespace cudf::io::parquet::experimental::detail {
class hybrid_scan_reader_impl;
 
class aggregate_reader_metadata;
}  // namespace cudf::io::parquet::experimental::detail
 
using cudf::io::text::byte_range_info;
 
namespace CUDF_EXPORT cudf {
namespace io::parquet::experimental {
enum class use_data_page_mask : bool {
  YES = true,  
  NO  = false  
};
 
class hybrid_scan_metadata {
 public:
  hybrid_scan_metadata(cudf::host_span<uint8_t const> footer_bytes,
                       parquet_reader_options const& options);
 
  hybrid_scan_metadata(FileMetaData const& parquet_metadata, parquet_reader_options const& options);
 
  ~hybrid_scan_metadata();
 
  hybrid_scan_metadata(hybrid_scan_metadata const&) = default;  
  hybrid_scan_metadata(hybrid_scan_metadata&&)      = default;  
 
  hybrid_scan_metadata& operator=(hybrid_scan_metadata const&) = default;
 
  hybrid_scan_metadata& operator=(hybrid_scan_metadata&&) = default;
 
 private:
  std::shared_ptr<detail::aggregate_reader_metadata> _metadata;
  friend class hybrid_scan_reader;
};
 
class hybrid_scan_reader {
 public:
  explicit hybrid_scan_reader(cudf::host_span<uint8_t const> footer_bytes,
                              parquet_reader_options const& options);
 
  explicit hybrid_scan_reader(FileMetaData const& parquet_metadata,
                              parquet_reader_options const& options);
 
  explicit hybrid_scan_reader(hybrid_scan_metadata metadata);
 
  ~hybrid_scan_reader();
 
  [[nodiscard]] FileMetaData parquet_metadata() const;
 
  [[nodiscard]] byte_range_info page_index_byte_range() const;
 
  void setup_page_index(cudf::host_span<uint8_t const> page_index_bytes) const;
 
  [[nodiscard]] std::vector<size_type> all_row_groups(parquet_reader_options const& options) const;
 
  [[nodiscard]] std::size_t total_rows_in_row_groups(
    std::span<size_type const> row_group_indices) const;
 
  void reset_column_selection() const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_byte_range(
    std::span<size_type const> row_group_indices, parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_stats(
    std::span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream) const;
 
  [[nodiscard]] std::vector<byte_range_info> bloom_filters_byte_ranges(
    std::span<size_type const> row_group_indices, parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_bloom_filters(
    std::span<cudf::device_span<uint8_t const> const> bloom_filter_data,
    std::span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream) const;
 
  [[nodiscard]] std::vector<byte_range_info> dictionary_pages_byte_ranges(
    std::span<size_type const> row_group_indices, parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_dictionary_pages(
    std::span<cudf::device_span<uint8_t const> const> dictionary_page_data,
    std::span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream) const;
 
  [[nodiscard]] std::unique_ptr<cudf::column> build_all_true_row_mask(
    std::span<size_type const> row_group_indices,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::unique_ptr<cudf::column> build_row_mask_with_page_index_stats(
    std::span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::vector<byte_range_info> filter_column_chunks_byte_ranges(
    std::span<size_type const> row_group_indices, parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_filter_columns(
    std::span<size_type const> row_group_indices,
    std::span<cudf::device_span<uint8_t const> const> column_chunk_data,
    cudf::mutable_column_view& row_mask,
    use_data_page_mask mask_data_pages,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::vector<byte_range_info> payload_column_chunks_byte_ranges(
    std::span<size_type const> row_group_indices, parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_payload_columns(
    std::span<size_type const> row_group_indices,
    std::span<cudf::device_span<uint8_t const> const> column_chunk_data,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::vector<byte_range_info> all_column_chunks_byte_ranges(
    std::span<size_type const> row_group_indices, parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_all_columns(
    std::span<size_type const> row_group_indices,
    std::span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
  void setup_chunking_for_filter_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    std::span<size_type const> row_group_indices,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    std::span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] table_with_metadata materialize_filter_columns_chunk(
    cudf::mutable_column_view& row_mask) const;
 
  void setup_chunking_for_payload_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    std::span<size_type const> row_group_indices,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    std::span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] table_with_metadata materialize_payload_columns_chunk(
    cudf::column_view const& row_mask) const;
 
  void setup_chunking_for_all_columns(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    std::span<size_type const> row_group_indices,
    std::span<cudf::device_span<uint8_t const> const> column_chunk_data,
    parquet_reader_options const& options,
    cuda::stream_ref stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] table_with_metadata materialize_all_columns_chunk() const;
 
  [[nodiscard]] std::vector<std::vector<cudf::size_type>> construct_row_group_passes(
    std::span<cudf::size_type const> row_group_indices, std::size_t pass_read_limit) const;
 
  [[nodiscard]] bool has_next_table_chunk() const;
 
 private:
  std::unique_ptr<detail::hybrid_scan_reader_impl> _impl;
};
  // end of group
 
}  // namespace io::parquet::experimental
}  // namespace CUDF_EXPORT cudf