parquet_schema.hpp

Go to the documentation of this file.

/*
 * Copyright (c) 2025, NVIDIA CORPORATION.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#include <cudf/types.hpp>
 
#include <cuda/std/optional>
 
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
 
namespace CUDF_EXPORT cudf {
namespace io::parquet {
enum class Type : int8_t {
  UNDEFINED            = -1,  // Undefined for non-leaf nodes
  BOOLEAN              = 0,
  INT32                = 1,
  INT64                = 2,
  INT96                = 3,  // Deprecated
  FLOAT                = 4,
  DOUBLE               = 5,
  BYTE_ARRAY           = 6,
  FIXED_LEN_BYTE_ARRAY = 7,
};
 
enum class ConvertedType : int8_t {
  UNKNOWN = -1,  // No type information present
  UTF8    = 0,   // a BYTE_ARRAY may contain UTF8 encoded chars
  MAP     = 1,   // a map is converted as an optional field containing a repeated key/value pair
  MAP_KEY_VALUE = 2,  // a key/value pair is converted into a group of two fields
  LIST =
    3,  // a list is converted into an optional field containing a repeated field for its values
  ENUM    = 4,      // an enum is converted into a binary field
  DECIMAL = 5,      // A decimal value. 10^(-scale) encoded as 2's complement big endian
                    // (precision=number of digits, scale=location of decimal point)
  DATE        = 6,  // A Date, stored as days since Unix epoch, encoded as the INT32 physical type.
  TIME_MILLIS = 7,  // A time. The total number of milliseconds since midnight.The value is stored
                    // as an INT32 physical type.
  TIME_MICROS = 8,  // A time. The total number of microseconds since midnight.  The value is stored
                    // as an INT64 physical type.
  TIMESTAMP_MILLIS = 9,   // A date/time combination, recorded as milliseconds since the Unix epoch
                          // using physical type of INT64.
  TIMESTAMP_MICROS = 10,  // A date/time combination, microseconds since the Unix epoch as INT64
  UINT_8           = 11,  // An unsigned integer 8-bit value as INT32
  UINT_16          = 12,  // An unsigned integer 16-bit value as INT32
  UINT_32          = 13,  // An unsigned integer 32-bit value as INT32
  UINT_64          = 14,  // An unsigned integer 64-bit value as INT64
  INT_8            = 15,  // A signed integer 8-bit value as INT32
  INT_16           = 16,  // A signed integer 16-bit value as INT32
  INT_32           = 17,  // A signed integer 32-bit value as INT32
  INT_64           = 18,  // A signed integer 8-bit value as INT64
  JSON             = 19,  // A JSON document embedded within a single UTF8 column.
  BSON             = 20,  // A BSON document embedded within a single BINARY column.
  INTERVAL = 21,  // This type annotates a time interval stored as a FIXED_LEN_BYTE_ARRAY of length
                  // 12 for 3 integers {months,days,milliseconds}
  NA = 25,        // No Type information, For eg, all-nulls.
};
 
enum class Encoding : uint8_t {
  PLAIN                   = 0,
  GROUP_VAR_INT           = 1,  // Deprecated, never used
  PLAIN_DICTIONARY        = 2,
  RLE                     = 3,
  BIT_PACKED              = 4,  // Deprecated by parquet-format in 2013, superseded by RLE
  DELTA_BINARY_PACKED     = 5,
  DELTA_LENGTH_BYTE_ARRAY = 6,
  DELTA_BYTE_ARRAY        = 7,
  RLE_DICTIONARY          = 8,
  BYTE_STREAM_SPLIT       = 9,
  NUM_ENCODINGS           = 10,
};
 
enum class Compression : uint8_t {
  UNCOMPRESSED = 0,
  SNAPPY       = 1,
  GZIP         = 2,
  LZO          = 3,
  BROTLI       = 4,  // Added in 2.3.2
  LZ4          = 5,  // deprecated; based on LZ4, but with an additional undocumented framing scheme
  ZSTD         = 6,  // Added in 2.3.2
  LZ4_RAW      = 7,  // "standard" LZ4 block format
};
 
enum class FieldRepetitionType : int8_t {
  UNSPECIFIED = -1,
  REQUIRED    = 0,  // This field is required (can not be null) and each record has exactly 1 value.
  OPTIONAL    = 1,  // The field is optional (can be null) and each record has 0 or 1 values.
  REPEATED    = 2,  // The field is repeated and can contain 0 or more values
};
 
enum class PageType : uint8_t {
  DATA_PAGE       = 0,
  INDEX_PAGE      = 1,
  DICTIONARY_PAGE = 2,
  DATA_PAGE_V2    = 3,
};
 
enum class BoundaryOrder : uint8_t {
  UNORDERED  = 0,
  ASCENDING  = 1,
  DESCENDING = 2,
};
 
enum class FieldType : uint8_t {
  BOOLEAN_TRUE  = 1,
  BOOLEAN_FALSE = 2,
  I8            = 3,
  I16           = 4,
  I32           = 5,
  I64           = 6,
  DOUBLE        = 7,
  BINARY        = 8,
  LIST          = 9,
  SET           = 10,
  MAP           = 11,
  STRUCT        = 12,
  UUID          = 13,
};
 
struct file_header_s {
  uint32_t magic;
};
 
struct file_ender_s {
  uint32_t footer_len;
  uint32_t magic;
};
 
struct DecimalType {
  int32_t scale = 0;
  int32_t precision = 0;
};
 
struct TimeUnit {
  enum Type : uint8_t { UNDEFINED, MILLIS, MICROS, NANOS };
  Type type;
};
 
struct TimeType {
  bool isAdjustedToUTC = true;
  TimeUnit unit = {TimeUnit::Type::MILLIS};
};
 
struct TimestampType {
  bool isAdjustedToUTC = true;
  TimeUnit unit = {TimeUnit::Type::MILLIS};
};
 
struct IntType {
  int8_t bitWidth = 0;
  bool isSigned = false;
};
 
struct LogicalType {
  enum Type : uint8_t {
    UNDEFINED,
    STRING,
    MAP,
    LIST,
    ENUM,
    DECIMAL,
    DATE,
    TIME,
    TIMESTAMP,
    // 9 is reserved
    INTEGER = 10,
    UNKNOWN,
    JSON,
    BSON
  };
 
  Type type;
  cuda::std::optional<DecimalType> decimal_type;
  cuda::std::optional<TimeType> time_type;
  cuda::std::optional<TimestampType> timestamp_type;
  cuda::std::optional<IntType> int_type;
 
  LogicalType(Type tp = Type::UNDEFINED) : type(tp) {}
 
  LogicalType(DecimalType&& dt) : type(DECIMAL), decimal_type(dt) {}
 
  LogicalType(TimeType&& tt) : type(TIME), time_type(tt) {}
 
  LogicalType(TimestampType&& tst) : type(TIMESTAMP), timestamp_type(tst) {}
 
  LogicalType(IntType&& it) : type(INTEGER), int_type(it) {}
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr bool is_time_millis() const
  {
    return type == TIME and time_type->unit.type == TimeUnit::MILLIS;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr bool is_time_micros() const
  {
    return type == TIME and time_type->unit.type == TimeUnit::MICROS;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr bool is_time_nanos() const
  {
    return type == TIME and time_type->unit.type == TimeUnit::NANOS;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr bool is_timestamp_millis() const
  {
    return type == TIMESTAMP and timestamp_type->unit.type == TimeUnit::MILLIS;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr bool is_timestamp_micros() const
  {
    return type == TIMESTAMP and timestamp_type->unit.type == TimeUnit::MICROS;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr bool is_timestamp_nanos() const
  {
    return type == TIMESTAMP and timestamp_type->unit.type == TimeUnit::NANOS;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr int8_t bit_width() const
  {
    return type == INTEGER ? int_type->bitWidth : -1;
  }
 
  [[nodiscard]] constexpr bool is_signed() const { return type == INTEGER and int_type->isSigned; }
 
  [[nodiscard]] constexpr int32_t scale() const
  {
    return type == DECIMAL ? decimal_type->scale : -1;
  }
 
  [[nodiscard]] CUDF_HOST_DEVICE constexpr int32_t precision() const
  {
    return type == DECIMAL ? decimal_type->precision : -1;
  }
};
 
struct ColumnOrder {
  enum Type : uint8_t { UNDEFINED, TYPE_ORDER };
  Type type;
};
 
struct SchemaElement {
  Type type = Type::UNDEFINED;
  int32_t type_length = 0;
  FieldRepetitionType repetition_type = FieldRepetitionType::REQUIRED;
  std::string name = "";
  int32_t num_children = 0;
  std::optional<ConvertedType> converted_type;
  int32_t decimal_scale = 0;
  int32_t decimal_precision = 0;
  std::optional<int32_t> field_id;
  std::optional<LogicalType> logical_type;
 
  bool output_as_byte_array = false;
 
  std::optional<type_id> arrow_type;
 
  // The following fields are filled in later during schema initialization
 
  int max_definition_level = 0;
  int max_repetition_level = 0;
  size_type parent_idx = 0;
  std::vector<size_type> children_idx;
 
  bool operator==(SchemaElement const& other) const
  {
    return type == other.type && converted_type == other.converted_type &&
           type_length == other.type_length && name == other.name &&
           num_children == other.num_children && decimal_scale == other.decimal_scale &&
           decimal_precision == other.decimal_precision && field_id == other.field_id;
  }
 
  // the parquet format is a little squishy when it comes to interpreting
  // repeated fields. sometimes repeated fields act as "stubs" in the schema
  // that don't represent a true nesting level.
  //
  // this is the case with plain lists:
  //
  // optional group my_list (LIST) {
  //   repeated group element {        <-- not part of the output hierarchy
  //     required binary str (UTF8);
  //   };
  // }
  //
  // However, for backwards compatibility reasons, there are a few special cases, namely
  // List<Struct<>> (which also corresponds to how the map type is specified), where
  // this does not hold true
  //
  // optional group my_list (LIST) {
  //   repeated group element {        <-- part of the hierarchy because it represents a struct
  //     required binary str (UTF8);
  //     required int32 num;
  //  };
  // }
 
  [[nodiscard]] bool is_stub() const
  {
    return repetition_type == FieldRepetitionType::REPEATED && num_children == 1;
  }
 
  [[nodiscard]] bool is_one_level_list(SchemaElement const& parent) const
  {
    return repetition_type == FieldRepetitionType::REPEATED and num_children == 0 and
           not parent.is_list();
  }
 
  [[nodiscard]] bool is_list() const { return converted_type == ConvertedType::LIST; }
 
  [[nodiscard]] bool is_struct() const
  {
    return type == Type::UNDEFINED &&
           // this assumption might be a little weak.
           ((repetition_type != FieldRepetitionType::REPEATED) ||
            (repetition_type == FieldRepetitionType::REPEATED && num_children > 1));
  }
};
 
struct Statistics {
  std::optional<std::vector<uint8_t>> max;
  std::optional<std::vector<uint8_t>> min;
  std::optional<int64_t> null_count;
  std::optional<int64_t> distinct_count;
  std::optional<std::vector<uint8_t>> max_value;
  std::optional<std::vector<uint8_t>> min_value;
  std::optional<bool> is_max_value_exact;
  std::optional<bool> is_min_value_exact;
};
 
struct SizeStatistics {
  std::optional<int64_t> unencoded_byte_array_data_bytes;
  std::optional<std::vector<int64_t>> repetition_level_histogram;
 
  std::optional<std::vector<int64_t>> definition_level_histogram;
};
 
struct PageLocation {
  int64_t offset;
  int32_t compressed_page_size;
  int64_t first_row_index;
};
 
struct OffsetIndex {
  std::vector<PageLocation> page_locations;
  std::optional<std::vector<int64_t>> unencoded_byte_array_data_bytes;
};
 
struct ColumnIndex {
  std::vector<bool> null_pages;
  std::vector<std::vector<uint8_t>> min_values;
  std::vector<std::vector<uint8_t>> max_values;
  BoundaryOrder boundary_order = BoundaryOrder::UNORDERED;
  std::optional<std::vector<int64_t>> null_counts;
  std::optional<std::vector<int64_t>> repetition_level_histogram;
  std::optional<std::vector<int64_t>> definition_level_histogram;
};
 
struct PageEncodingStats {
  PageType page_type;
  Encoding encoding;
  int32_t count;
};
 
struct SortingColumn {
  int32_t column_idx;
  bool descending;
  bool nulls_first;
};
 
struct ColumnChunkMetaData {
  Type type = Type::BOOLEAN;
  std::vector<Encoding> encodings;
  std::vector<std::string> path_in_schema;
  Compression codec = Compression::UNCOMPRESSED;
  int64_t num_values = 0;
  int64_t total_uncompressed_size = 0;
  int64_t total_compressed_size = 0;
  int64_t data_page_offset = 0;
  int64_t index_page_offset = 0;
  int64_t dictionary_page_offset = 0;
  Statistics statistics;
  std::optional<std::vector<PageEncodingStats>> encoding_stats;
  std::optional<int64_t> bloom_filter_offset;
  std::optional<int32_t> bloom_filter_length;
  std::optional<SizeStatistics> size_statistics;
};
 
struct BloomFilterAlgorithm {
  enum Algorithm : uint8_t { UNDEFINED, SPLIT_BLOCK };
  Algorithm algorithm{Algorithm::SPLIT_BLOCK};
};
 
struct BloomFilterHash {
  enum Hash : uint8_t { UNDEFINED, XXHASH };
  Hash hash{Hash::XXHASH};
};
 
struct BloomFilterCompression {
  enum Compression : uint8_t { UNDEFINED, UNCOMPRESSED };
  Compression compression{Compression::UNCOMPRESSED};
};
 
struct BloomFilterHeader {
  int32_t num_bytes;
  BloomFilterAlgorithm algorithm;
  BloomFilterHash hash;
  BloomFilterCompression compression;
};
 
struct ColumnChunk {
  std::string file_path = "";
  int64_t file_offset = 0;
  ColumnChunkMetaData meta_data;
  int64_t offset_index_offset = 0;
  int32_t offset_index_length = 0;
  int64_t column_index_offset = 0;
  int32_t column_index_length = 0;
 
  // Following fields are derived from other fields
 
  int schema_idx = -1;
 
  // The indexes don't really live here, but it's a convenient place to hang them.
 
  std::optional<OffsetIndex> offset_index;
  std::optional<ColumnIndex> column_index;
};
 
struct RowGroup {
  std::vector<ColumnChunk> columns;
  int64_t total_byte_size = 0;
  int64_t num_rows = 0;
  std::optional<std::vector<SortingColumn>> sorting_columns;
  std::optional<int64_t> file_offset;
  std::optional<int64_t> total_compressed_size;
  std::optional<int16_t> ordinal;
};
 
struct KeyValue {
  std::string key;
  std::string value;
};
 
struct FileMetaData {
  int32_t version = 0;
  std::vector<SchemaElement> schema;
  int64_t num_rows = 0;
  std::vector<RowGroup> row_groups;
  std::vector<KeyValue> key_value_metadata;
  std::string created_by = "";
  std::optional<std::vector<ColumnOrder>> column_orders;
};
 
struct DataPageHeader {
  int32_t num_values = 0;
  Encoding encoding = Encoding::PLAIN;
  Encoding definition_level_encoding = Encoding::PLAIN;
  Encoding repetition_level_encoding = Encoding::PLAIN;
};
 
struct DataPageHeaderV2 {
  int32_t num_values = 0;
  int32_t num_nulls = 0;
  int32_t num_rows = 0;
  Encoding encoding = Encoding::PLAIN;
  int32_t definition_levels_byte_length = 0;
  int32_t repetition_levels_byte_length = 0;
  bool is_compressed = true;
};
 
struct DictionaryPageHeader {
  int32_t num_values = 0;
  Encoding encoding = Encoding::PLAIN;
};
 
struct PageHeader {
  PageType type = PageType::DATA_PAGE;
  int32_t uncompressed_page_size = 0;
  int32_t compressed_page_size = 0;
 
  // Headers for page specific data. One only will be set.
 
  DataPageHeader data_page_header;
  DictionaryPageHeader dictionary_page_header;
  DataPageHeaderV2 data_page_header_v2;
};
  // end of group
}  // namespace io::parquet
}  // namespace CUDF_EXPORT cudf