parquet.hpp

Go to the documentation of this file.

/*
 * Copyright (c) 2020-2025, NVIDIA CORPORATION.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#include <cudf/ast/expressions.hpp>
#include <cudf/io/detail/parquet.hpp>
#include <cudf/io/types.hpp>
#include <cudf/table/table_view.hpp>
#include <cudf/types.hpp>
#include <cudf/utilities/export.hpp>
#include <cudf/utilities/memory_resource.hpp>
 
#include <iostream>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>
 
namespace CUDF_EXPORT cudf {
namespace io {
constexpr size_t default_row_group_size_bytes =
  std::numeric_limits<size_t>::max();                          
constexpr size_type default_row_group_size_rows = 1'000'000;   
constexpr size_t default_max_page_size_bytes    = 512 * 1024;  
constexpr size_type default_max_page_size_rows  = 20000;       
constexpr int32_t default_column_index_truncate_length = 64;   
constexpr size_t default_max_dictionary_size           = 1024 * 1024;  
constexpr size_type default_max_page_fragment_size     = 5000;  
 
[[nodiscard]] bool is_supported_read_parquet(compression_type compression);
 
[[nodiscard]] bool is_supported_write_parquet(compression_type compression);
 
class parquet_reader_options_builder;
 
class parquet_reader_options {
  source_info _source;
 
  // Path in schema of column to read; `nullopt` is all
  std::optional<std::vector<std::string>> _columns;
 
  // List of individual row groups to read (ignored if empty)
  std::vector<std::vector<size_type>> _row_groups;
  // Number of rows to skip from the start; Parquet stores the number of rows as int64_t
  int64_t _skip_rows = 0;
  // Number of rows to read; `nullopt` is all
  std::optional<size_type> _num_rows;
 
  // Read row groups that start at or after this byte offset into the source
  size_t _skip_bytes = 0;
  // Read row groups that start before _num_bytes bytes after _skip_bytes into the source
  std::optional<size_t> _num_bytes;
 
  // Predicate filter as AST to filter output rows.
  std::optional<std::reference_wrapper<ast::expression const>> _filter;
 
  // Whether to store string data as categorical type
  bool _convert_strings_to_categories = false;
  // Whether to use PANDAS metadata to load columns
  bool _use_pandas_metadata = true;
  // Whether to read and use ARROW schema
  bool _use_arrow_schema = true;
  // Whether to allow reading matching select columns from mismatched Parquet files.
  bool _allow_mismatched_pq_schemas = false;
  // Cast timestamp columns to a specific type
  data_type _timestamp_type{type_id::EMPTY};
  // Whether to use JIT compilation for filtering
  bool _use_jit_filter = false;
 
  std::optional<std::vector<reader_column_schema>> _reader_column_schema;
 
  explicit parquet_reader_options(source_info src) : _source{std::move(src)} {}
 
  friend parquet_reader_options_builder;
 
 public:
  explicit parquet_reader_options() = default;
 
  static parquet_reader_options_builder builder(source_info src = source_info{});
 
  [[nodiscard]] source_info const& get_source() const { return _source; }
 
  [[nodiscard]] bool is_enabled_convert_strings_to_categories() const
  {
    return _convert_strings_to_categories;
  }
 
  [[nodiscard]] bool is_enabled_use_pandas_metadata() const { return _use_pandas_metadata; }
 
  [[nodiscard]] bool is_enabled_use_arrow_schema() const { return _use_arrow_schema; }
 
  [[nodiscard]] bool is_enabled_allow_mismatched_pq_schemas() const
  {
    return _allow_mismatched_pq_schemas;
  }
 
  [[nodiscard]] std::optional<std::vector<reader_column_schema>> get_column_schema() const
  {
    return _reader_column_schema;
  }
 
  [[nodiscard]] int64_t get_skip_rows() const { return _skip_rows; }
 
  [[nodiscard]] std::optional<size_type> const& get_num_rows() const { return _num_rows; }
 
  [[nodiscard]] size_t get_skip_bytes() const { return _skip_bytes; }
 
  [[nodiscard]] std::optional<size_t> const& get_num_bytes() const { return _num_bytes; }
 
  [[nodiscard]] auto const& get_columns() const { return _columns; }
 
  [[nodiscard]] auto const& get_row_groups() const { return _row_groups; }
 
  [[nodiscard]] auto const& get_filter() const { return _filter; }
 
  [[nodiscard]] data_type get_timestamp_type() const { return _timestamp_type; }
 
  [[nodiscard]] bool is_enabled_use_jit_filter() const { return _use_jit_filter; }
 
  void set_columns(std::vector<std::string> col_names) { _columns = std::move(col_names); }
 
  void set_row_groups(std::vector<std::vector<size_type>> row_groups);
 
  void set_filter(ast::expression const& filter) { _filter = filter; }
 
  void enable_convert_strings_to_categories(bool val) { _convert_strings_to_categories = val; }
 
  void enable_use_pandas_metadata(bool val) { _use_pandas_metadata = val; }
 
  void enable_use_arrow_schema(bool val) { _use_arrow_schema = val; }
 
  void enable_allow_mismatched_pq_schemas(bool val) { _allow_mismatched_pq_schemas = val; }
 
  void set_column_schema(std::vector<reader_column_schema> val)
  {
    _reader_column_schema = std::move(val);
  }
 
  void set_skip_rows(int64_t val);
 
  void set_num_rows(size_type val);
 
  void set_skip_bytes(size_t val);
 
  void set_num_bytes(size_t val);
 
  void set_timestamp_type(data_type type) { _timestamp_type = type; }
};
 
class parquet_reader_options_builder {
  parquet_reader_options options;
 
 public:
  parquet_reader_options_builder() = default;
 
  explicit parquet_reader_options_builder(source_info src) : options{std::move(src)} {}
 
  parquet_reader_options_builder& columns(std::vector<std::string> col_names)
  {
    options._columns = std::move(col_names);
    return *this;
  }
 
  parquet_reader_options_builder& row_groups(std::vector<std::vector<size_type>> row_groups)
  {
    options.set_row_groups(std::move(row_groups));
    return *this;
  }
 
  parquet_reader_options_builder& filter(ast::expression const& filter)
  {
    options.set_filter(filter);
    return *this;
  }
 
  parquet_reader_options_builder& convert_strings_to_categories(bool val)
  {
    options._convert_strings_to_categories = val;
    return *this;
  }
 
  parquet_reader_options_builder& use_pandas_metadata(bool val)
  {
    options._use_pandas_metadata = val;
    return *this;
  }
 
  parquet_reader_options_builder& use_arrow_schema(bool val)
  {
    options._use_arrow_schema = val;
    return *this;
  }
 
  parquet_reader_options_builder& allow_mismatched_pq_schemas(bool val)
  {
    options._allow_mismatched_pq_schemas = val;
    return *this;
  }
 
  parquet_reader_options_builder& set_column_schema(std::vector<reader_column_schema> val)
  {
    options._reader_column_schema = std::move(val);
    return *this;
  }
 
  parquet_reader_options_builder& skip_rows(int64_t val)
  {
    options.set_skip_rows(val);
    return *this;
  }
 
  parquet_reader_options_builder& num_rows(size_type val)
  {
    options.set_num_rows(val);
    return *this;
  }
 
  parquet_reader_options_builder& skip_bytes(size_t val)
  {
    options.set_skip_bytes(val);
    return *this;
  }
 
  parquet_reader_options_builder& num_bytes(size_t val)
  {
    options.set_num_bytes(val);
    return *this;
  }
 
  parquet_reader_options_builder& timestamp_type(data_type type)
  {
    options._timestamp_type = type;
    return *this;
  }
 
  parquet_reader_options_builder& use_jit_filter(bool use_jit_filter)
  {
    options._use_jit_filter = use_jit_filter;
    return *this;
  }
 
  operator parquet_reader_options&&() { return std::move(options); }
 
  parquet_reader_options&& build() { return std::move(options); }
};
 
table_with_metadata read_parquet(
  parquet_reader_options const& options,
  rmm::cuda_stream_view stream      = cudf::get_default_stream(),
  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
class chunked_parquet_reader {
 public:
  chunked_parquet_reader();
 
  chunked_parquet_reader(
    std::size_t chunk_read_limit,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream      = cudf::get_default_stream(),
    rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
  chunked_parquet_reader(
    std::size_t chunk_read_limit,
    std::size_t pass_read_limit,
    parquet_reader_options const& options,
    rmm::cuda_stream_view stream      = cudf::get_default_stream(),
    rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
  ~chunked_parquet_reader();
 
  [[nodiscard]] bool has_next() const;
 
  [[nodiscard]] table_with_metadata read_chunk() const;
 
 private:
  std::unique_ptr<cudf::io::parquet::detail::chunked_reader> reader;
};
  // end of group
struct sorting_column {
  int column_idx{};           
  bool is_descending{false};  
  bool is_nulls_first{true};  
};
 
class parquet_writer_options_base {
  // Specify the sink to use for writer output
  sink_info _sink;
  // Specify the compression format to use
  compression_type _compression = compression_type::SNAPPY;
  // Specify the level of statistics in the output file
  statistics_freq _stats_level = statistics_freq::STATISTICS_ROWGROUP;
  // Optional associated metadata
  std::optional<table_input_metadata> _metadata;
  // Optional footer key_value_metadata
  std::vector<std::map<std::string, std::string>> _user_data;
  // Parquet writer can write INT96 or TIMESTAMP_MICROS. Defaults to TIMESTAMP_MICROS.
  // If true then overrides any per-column setting in _metadata.
  bool _write_timestamps_as_int96 = false;
  // Parquet writer can write timestamps as UTC
  // Defaults to true because libcudf timestamps are implicitly UTC
  bool _write_timestamps_as_UTC = true;
  // Whether to write ARROW schema
  bool _write_arrow_schema = false;
  // Maximum size of each row group (unless smaller than a single page)
  size_t _row_group_size_bytes = default_row_group_size_bytes;
  // Maximum number of rows in row group (unless smaller than a single page)
  size_type _row_group_size_rows = default_row_group_size_rows;
  // Maximum size of each page (uncompressed)
  size_t _max_page_size_bytes = default_max_page_size_bytes;
  // Maximum number of rows in a page
  size_type _max_page_size_rows = default_max_page_size_rows;
  // Maximum size of min or max values in column index
  int32_t _column_index_truncate_length = default_column_index_truncate_length;
  // When to use dictionary encoding for data
  dictionary_policy _dictionary_policy = dictionary_policy::ADAPTIVE;
  // Maximum size of column chunk dictionary (in bytes)
  size_t _max_dictionary_size = default_max_dictionary_size;
  // Maximum number of rows in a page fragment
  std::optional<size_type> _max_page_fragment_size;
  // Optional compression statistics
  std::shared_ptr<writer_compression_statistics> _compression_stats;
  // write V2 page headers?
  bool _v2_page_headers = false;
  // Which columns in _table are used for sorting
  std::optional<std::vector<sorting_column>> _sorting_columns;
 
 protected:
  explicit parquet_writer_options_base(sink_info sink) : _sink(std::move(sink)) {}
 
 public:
  parquet_writer_options_base() = default;
 
  [[nodiscard]] sink_info const& get_sink() const { return _sink; }
 
  [[nodiscard]] compression_type get_compression() const { return _compression; }
 
  [[nodiscard]] statistics_freq get_stats_level() const { return _stats_level; }
 
  [[nodiscard]] auto const& get_metadata() const { return _metadata; }
 
  [[nodiscard]] std::vector<std::map<std::string, std::string>> const& get_key_value_metadata()
    const
  {
    return _user_data;
  }
 
  [[nodiscard]] bool is_enabled_int96_timestamps() const { return _write_timestamps_as_int96; }
 
  [[nodiscard]] auto is_enabled_utc_timestamps() const { return _write_timestamps_as_UTC; }
 
  [[nodiscard]] auto is_enabled_write_arrow_schema() const { return _write_arrow_schema; }
 
  [[nodiscard]] auto get_row_group_size_bytes() const { return _row_group_size_bytes; }
 
  [[nodiscard]] auto get_row_group_size_rows() const { return _row_group_size_rows; }
 
  [[nodiscard]] auto get_max_page_size_bytes() const
  {
    return std::min(_max_page_size_bytes, get_row_group_size_bytes());
  }
 
  [[nodiscard]] auto get_max_page_size_rows() const
  {
    return std::min(_max_page_size_rows, get_row_group_size_rows());
  }
 
  [[nodiscard]] auto get_column_index_truncate_length() const
  {
    return _column_index_truncate_length;
  }
 
  [[nodiscard]] dictionary_policy get_dictionary_policy() const { return _dictionary_policy; }
 
  [[nodiscard]] auto get_max_dictionary_size() const { return _max_dictionary_size; }
 
  [[nodiscard]] auto get_max_page_fragment_size() const { return _max_page_fragment_size; }
 
  [[nodiscard]] std::shared_ptr<writer_compression_statistics> get_compression_statistics() const
  {
    return _compression_stats;
  }
 
  [[nodiscard]] auto is_enabled_write_v2_headers() const { return _v2_page_headers; }
 
  [[nodiscard]] auto const& get_sorting_columns() const { return _sorting_columns; }
 
  void set_metadata(table_input_metadata metadata);
 
  void set_key_value_metadata(std::vector<std::map<std::string, std::string>> metadata);
 
  void set_stats_level(statistics_freq sf);
  void set_compression(compression_type compression);
 
  void enable_int96_timestamps(bool req);
 
  void enable_utc_timestamps(bool val);
 
  void enable_write_arrow_schema(bool val);
 
  void set_row_group_size_bytes(size_t size_bytes);
 
  void set_row_group_size_rows(size_type size_rows);
 
  void set_max_page_size_bytes(size_t size_bytes);
 
  void set_max_page_size_rows(size_type size_rows);
 
  void set_column_index_truncate_length(int32_t size_bytes);
 
  void set_dictionary_policy(dictionary_policy policy);
 
  void set_max_dictionary_size(size_t size_bytes);
 
  void set_max_page_fragment_size(size_type size_rows);
 
  void set_compression_statistics(std::shared_ptr<writer_compression_statistics> comp_stats);
 
  void enable_write_v2_headers(bool val);
 
  void set_sorting_columns(std::vector<sorting_column> sorting_columns);
};
 
template <class BuilderT, class OptionsT>
class parquet_writer_options_builder_base {
  OptionsT _options;
 
 protected:
  inline OptionsT& get_options() { return _options; }
 
  explicit parquet_writer_options_builder_base(OptionsT options);
 
 public:
  explicit parquet_writer_options_builder_base() = default;
 
  BuilderT& metadata(table_input_metadata metadata);
 
  BuilderT& key_value_metadata(std::vector<std::map<std::string, std::string>> metadata);
 
  BuilderT& stats_level(statistics_freq sf);
 
  BuilderT& compression(compression_type compression);
 
  BuilderT& row_group_size_bytes(size_t val);
 
  BuilderT& row_group_size_rows(size_type val);
 
  BuilderT& max_page_size_bytes(size_t val);
 
  BuilderT& max_page_size_rows(size_type val);
 
  BuilderT& column_index_truncate_length(int32_t val);
 
  BuilderT& dictionary_policy(enum dictionary_policy val);
 
  BuilderT& max_dictionary_size(size_t val);
 
  BuilderT& max_page_fragment_size(size_type val);
 
  BuilderT& compression_statistics(
    std::shared_ptr<writer_compression_statistics> const& comp_stats);
 
  BuilderT& int96_timestamps(bool enabled);
 
  BuilderT& utc_timestamps(bool enabled);
 
  BuilderT& write_arrow_schema(bool enabled);
 
  BuilderT& write_v2_headers(bool enabled);
 
  BuilderT& sorting_columns(std::vector<sorting_column> sorting_columns);
 
  operator OptionsT&&();
 
  OptionsT&& build();
};
 
class parquet_writer_options_builder;
 
class parquet_writer_options : public parquet_writer_options_base {
  // Sets of columns to output
  table_view _table;
  // Partitions described as {start_row, num_rows} pairs
  std::vector<partition_info> _partitions;
  // Column chunks file paths to be set in the raw output metadata. One per output file
  std::vector<std::string> _column_chunks_file_paths;
 
  friend parquet_writer_options_builder;
 
  explicit parquet_writer_options(sink_info const& sink, table_view table);
 
 public:
  parquet_writer_options() = default;
 
  static parquet_writer_options_builder builder(sink_info const& sink, table_view const& table);
 
  static parquet_writer_options_builder builder();
 
  [[nodiscard]] table_view get_table() const { return _table; }
 
  [[nodiscard]] std::vector<partition_info> const& get_partitions() const { return _partitions; }
 
  [[nodiscard]] std::vector<std::string> const& get_column_chunks_file_paths() const
  {
    return _column_chunks_file_paths;
  }
 
  void set_partitions(std::vector<partition_info> partitions);
 
  void set_column_chunks_file_paths(std::vector<std::string> file_paths);
};
 
class parquet_writer_options_builder
  : public parquet_writer_options_builder_base<parquet_writer_options_builder,
                                               parquet_writer_options> {
 public:
  explicit parquet_writer_options_builder() = default;
 
  explicit parquet_writer_options_builder(sink_info const& sink, table_view const& table);
 
  parquet_writer_options_builder& partitions(std::vector<partition_info> partitions);
 
  parquet_writer_options_builder& column_chunks_file_paths(std::vector<std::string> file_paths);
};
 
std::unique_ptr<std::vector<uint8_t>> write_parquet(
  parquet_writer_options const& options, rmm::cuda_stream_view stream = cudf::get_default_stream());
 
std::unique_ptr<std::vector<uint8_t>> merge_row_group_metadata(
  std::vector<std::unique_ptr<std::vector<uint8_t>>> const& metadata_list);
 
class chunked_parquet_writer_options_builder;
 
class chunked_parquet_writer_options : public parquet_writer_options_base {
  explicit chunked_parquet_writer_options(sink_info sink);
 
  friend chunked_parquet_writer_options_builder;
 
 public:
  chunked_parquet_writer_options() = default;
 
  static chunked_parquet_writer_options_builder builder(sink_info const& sink);
};
 
class chunked_parquet_writer_options_builder
  : public parquet_writer_options_builder_base<chunked_parquet_writer_options_builder,
                                               chunked_parquet_writer_options> {
 public:
  chunked_parquet_writer_options_builder() = default;
 
  chunked_parquet_writer_options_builder(sink_info const& sink);
};
 
class chunked_parquet_writer {
 public:
  chunked_parquet_writer();
 
  chunked_parquet_writer(chunked_parquet_writer_options const& options,
                         rmm::cuda_stream_view stream = cudf::get_default_stream());
  ~chunked_parquet_writer();
 
  chunked_parquet_writer& write(table_view const& table,
                                std::vector<partition_info> const& partitions = {});
 
  std::unique_ptr<std::vector<uint8_t>> close(
    std::vector<std::string> const& column_chunks_file_paths = {});
 
  std::unique_ptr<parquet::detail::writer> writer;
};
 
using parquet_chunked_writer [[deprecated("Use chunked_parquet_writer instead")]] =
  chunked_parquet_writer;
  // end of group
 
}  // namespace io
}  // namespace CUDF_EXPORT cudf