deletion_vectors.hpp

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 */
 
#pragma once
 
#include <cudf/io/parquet.hpp>
#include <cudf/io/types.hpp>
#include <cudf/types.hpp>
#include <cudf/utilities/export.hpp>
#include <cudf/utilities/roaring_bitmap.hpp>
 
#include <queue>
 
namespace CUDF_EXPORT cudf {
namespace io::parquet::experimental {
 
struct deletion_vector_info {
  // Following vectors specify the data spans of input deletion vectors and the number of rows
  // spanned by each deletion vector in order. Deletion vectors are applied in order of their
  // appearance in the vectors. These vectors if empty will result in no table filtration.
 
  std::vector<cudf::host_span<cuda::std::byte const>> serialized_roaring_bitmaps;
  std::vector<size_type> deletion_vector_row_counts;
 
  // Following vectors customize the row index column prepended to the read table from the Parquet
  // source(s). These vectors if empty will result in an index column that is a sequence from 0 to
  // the total number of rows in the table.
 
  std::vector<size_t> row_group_offsets;
  std::vector<size_type> row_group_num_rows;
};
 
class chunked_parquet_reader {
 public:
  chunked_parquet_reader(
    std::size_t chunk_read_limit,
    parquet_reader_options const& options,
    deletion_vector_info const& deletion_vector_info,
    rmm::cuda_stream_view stream      = cudf::get_default_stream(),
    rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
  chunked_parquet_reader(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    parquet_reader_options const& options,
    deletion_vector_info const& deletion_vector_info,
    rmm::cuda_stream_view stream      = cudf::get_default_stream(),
    rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
  ~chunked_parquet_reader();
 
  [[nodiscard]] bool has_next() const;
 
  [[nodiscard]] table_with_metadata read_chunk();
 
 private:
  std::unique_ptr<cudf::io::chunked_parquet_reader> _reader;
  std::queue<size_t> _row_group_row_offsets;
  std::queue<size_type> _row_group_row_counts;
  std::queue<cudf::roaring_bitmap> _deletion_vectors;
  std::queue<size_type> _deletion_vector_row_counts;
  size_t _start_row;
  bool _is_unspecified_row_group_data;
  rmm::cuda_stream_view _stream;
  rmm::device_async_resource_ref _mr;
  rmm::device_async_resource_ref _table_mr;
};
 
table_with_metadata read_parquet(
  parquet_reader_options const& options,
  deletion_vector_info const& deletion_vector_info,
  rmm::cuda_stream_view stream      = cudf::get_default_stream(),
  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
[[nodiscard]] size_t compute_num_deleted_rows(
  deletion_vector_info const& deletion_vector_info,
  cudf::size_type max_chunk_rows = std::numeric_limits<size_type>::max(),
  rmm::cuda_stream_view stream   = cudf::get_default_stream());
  // end of group
 
}  // namespace io::parquet::experimental
}  // namespace CUDF_EXPORT cudf