hybrid_scan.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION.
 * SPDX-License-Identifier: Apache-2.0
 */
 
#pragma once
 
#include <cudf/io/parquet.hpp>
#include <cudf/io/parquet_schema.hpp>
#include <cudf/io/text/byte_range_info.hpp>
#include <cudf/io/types.hpp>
#include <cudf/types.hpp>
#include <cudf/utilities/export.hpp>
 
#include <thrust/host_vector.h>
 
#include <memory>
#include <utility>
#include <vector>
 
namespace CUDF_EXPORT cudf {
namespace io::parquet::experimental::detail {
class hybrid_scan_reader_impl;
}  // namespace io::parquet::experimental::detail
}  // namespace CUDF_EXPORT cudf
 
using cudf::io::text::byte_range_info;
 
namespace CUDF_EXPORT cudf {
namespace io::parquet::experimental {
enum class use_data_page_mask : bool {
  YES = true,  
  NO  = false  
};
 
class hybrid_scan_reader {
 public:
  explicit hybrid_scan_reader(cudf::host_span<uint8_t const> footer_bytes,
                              parquet_reader_options const& options);
 
  explicit hybrid_scan_reader(FileMetaData const& parquet_metadata,
                              parquet_reader_options const& options);
 
  ~hybrid_scan_reader();
 
  [[nodiscard]] FileMetaData parquet_metadata() const;
 
  [[nodiscard]] byte_range_info page_index_byte_range() const;
 
  void setup_page_index(cudf::host_span<uint8_t const> page_index_bytes) const;
 
  [[nodiscard]] std::vector<size_type> all_row_groups(parquet_reader_options const& options) const;
 
  [[nodiscard]] size_type total_rows_in_row_groups(
    cudf::host_span<size_type const> row_group_indices) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_byte_range(
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_stats(
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] std::pair<std::vector<byte_range_info>, std::vector<byte_range_info>>
  secondary_filters_byte_ranges(cudf::host_span<size_type const> row_group_indices,
                                parquet_reader_options const& options) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_dictionary_pages(
    cudf::host_span<rmm::device_buffer> dictionary_page_data,
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] std::vector<size_type> filter_row_groups_with_bloom_filters(
    cudf::host_span<rmm::device_buffer> bloom_filter_data,
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] std::unique_ptr<cudf::column> build_row_mask_with_page_index_stats(
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream,
    rmm::device_async_resource_ref mr) const;
 
  [[nodiscard]] std::vector<byte_range_info> filter_column_chunks_byte_ranges(
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_filter_columns(
    cudf::host_span<size_type const> row_group_indices,
    std::vector<rmm::device_buffer>&& column_chunk_buffers,
    cudf::mutable_column_view& row_mask,
    use_data_page_mask mask_data_pages,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] std::vector<byte_range_info> payload_column_chunks_byte_ranges(
    cudf::host_span<size_type const> row_group_indices,
    parquet_reader_options const& options) const;
 
  [[nodiscard]] table_with_metadata materialize_payload_columns(
    cudf::host_span<size_type const> row_group_indices,
    std::vector<rmm::device_buffer>&& column_chunk_buffers,
    cudf::column_view const& row_mask,
    use_data_page_mask mask_data_pages,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream) const;
 
  void setup_chunking_for_filter_columns(std::size_t chunk_read_limit,
                                         std::size_t pass_read_limit,
                                         cudf::host_span<size_type const> row_group_indices,
                                         cudf::column_view const& row_mask,
                                         use_data_page_mask mask_data_pages,
                                         std::vector<rmm::device_buffer>&& column_chunk_buffers,
                                         parquet_reader_options const& options,
                                         rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] table_with_metadata materialize_filter_columns_chunk(
    cudf::mutable_column_view& row_mask, rmm::cuda_stream_view stream) const;
 
  void setup_chunking_for_payload_columns(std::size_t chunk_read_limit,
                                          std::size_t pass_read_limit,
                                          cudf::host_span<size_type const> row_group_indices,
                                          cudf::column_view const& row_mask,
                                          use_data_page_mask mask_data_pages,
                                          std::vector<rmm::device_buffer>&& column_chunk_buffers,
                                          parquet_reader_options const& options,
                                          rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] table_with_metadata materialize_payload_columns_chunk(
    cudf::column_view const& row_mask, rmm::cuda_stream_view stream) const;
 
  [[nodiscard]] bool has_next_table_chunk() const;
 
 private:
  std::unique_ptr<detail::hybrid_scan_reader_impl> _impl;
};
  // end of group
 
}  // namespace io::parquet::experimental
}  // namespace CUDF_EXPORT cudf