Chunk with multiple messages. This class contains two buffers for concatenated metadata and data. More...

#include <chunk.hpp>

Public Member Functions
	Chunk (Chunk &&other) noexcept=default
	move constructor More...

Chunk &	operator= (Chunk &&other) noexcept=default
	move assignment operator More...

	Chunk (Chunk const &)=delete

Chunk &	operator= (Chunk const &)=delete

constexpr ChunkID	chunk_id () const
	The ID of the chunk. More...

constexpr size_t	n_messages () const
	The number of messages in the chunk. More...

constexpr PartID	part_id (size_t i) const
	Partition ID of the i-th message. More...

constexpr size_t	expected_num_chunks (size_t i) const
	The expected number of chunks of the i-th message. More...

constexpr bool	is_control_message (size_t i) const
	Whether the i-th message is a control message. More...

Chunk	get_data (ChunkID new_chunk_id, size_t i, BufferResource *br)
	Get the data of the i-th message, as a new chunk. More...

constexpr uint32_t	metadata_size (size_t i) const
	Get the size of the metadata of the i-th message. More...

constexpr size_t	data_size (size_t i) const
	Get the size of the packed data of the i-th message. More...

void	set_data_buffer (std::unique_ptr< Buffer > data)
	Set the data buffer. More...

bool	is_data_buffer_set () const
	Whether the data buffer is set. More...

bool	is_metadata_buffer_set () const
	Whether the metadata buffer is set. More...

MemoryType	data_memory_type () const
	Get the memory type of the data buffer. More...

constexpr size_t	concat_data_size () const
	Get the size of the concatenated data. More...

constexpr size_t	concat_metadata_size () const
	Get the size of the concatenated metadata. More...

std::unique_ptr< std::vector< uint8_t > >	release_metadata_buffer ()
	Release the ownership of the metadata buffer. More...

std::unique_ptr< Buffer >	release_data_buffer ()
	Release the ownership of the data buffer. More...

bool	is_ready () const
	Whether the chunk is ready for consumption. More...

std::string	str () const
	Returns a description of this chunk. More...

std::unique_ptr< std::vector< uint8_t > >	serialize () const
	Returns a metadata message that represents this chunk. More...

Static Public Member Functions
static constexpr size_t	metadata_message_header_size (size_t n_messages)
	The size of the metadata message header. More...

static Chunk	from_packed_data (ChunkID chunk_id, PartID part_id, PackedData &&packed_data)
	Create a single-message chunk from a packed data. More...

static Chunk	from_finished_partition (ChunkID chunk_id, PartID part_id, size_t expected_num_chunks)
	Create a single-message chunk for a finished partition (control message). More...

static Chunk	deserialize (std::vector< uint8_t > const &msg, bool validate=true)
	Create a chunk by deserializing a metadata message. More...

static bool	validate_format (std::vector< uint8_t > const &serialized_buf)
	Validate if a deserialized buffer follows the Chunk format. More...

static Chunk	concat (std::vector< Chunk > &&chunks, ChunkID chunk_id, BufferResource *br, std::optional< MemoryType > preferred_mem_type=std::nullopt)
	Concatenate multiple chunks into a single chunk. More...

Friends
Chunk	make_dummy_chunk (ChunkID, PartID)

Detailed Description

Chunk with multiple messages. This class contains two buffers for concatenated metadata and data.

When the Chunk is serialized, the format is as follows:

chunk_id: uint64_t, ID of the chunk
n_elements: size_t, Number of messages in the chunk
[partition_ids]: vector<PartID>, Partition IDs of the messages, size = n_elements
[expected_num_chunks]: vector<size_t>, Expected number of chunks of the messages, size = n_elements
[meta_offsets]: vector<uint32_t>, Prefix sums (excluding 0) of the metadata sizes of the messages, size = n_elements
[data_offsets]: vector<uint64_t>, Prefix sums (excluding 0) of the data sizes of the messages, size = n_elements
[concat_metadata]: vector<uint8_t>, Concatenated metadata of the messages, size = meta_offsets[n_elements - 1]

For a chunk with N messages with M bytes of concat metadata the size of serialized buffer is sizeof(ChunkID) + sizeof(size_t) + N * (sizeof(PartID) + sizeof(size_t) + sizeof(uint32_t) + sizeof(uint64_t)) + M = 16 + N * 24 + M bytes.

For a chunk with a single control message (ie. M = 0), the size of the serialized buffer is 40 bytes.

For a chunk with a single message with M bytes of metadata, the size of the serialized buffer is 40 + M bytes.

Definition at line 58 of file chunk.hpp.

Constructor & Destructor Documentation

◆ Chunk()

rapidsmpf::shuffler::detail::Chunk::Chunk ( Chunk && other )

defaultnoexcept

move constructor

Parameters

other The chunk to move.

Member Function Documentation

◆ chunk_id()

constexpr ChunkID rapidsmpf::shuffler::detail::Chunk::chunk_id ( ) const

inlineconstexpr

The ID of the chunk.

Returns: The ID of the chunk.

Definition at line 100 of file chunk.hpp.

◆ concat()

static Chunk rapidsmpf::shuffler::detail::Chunk::concat	(	std::vector< Chunk > &&	chunks,
		ChunkID	chunk_id,
		BufferResource *	br,
		std::optional< MemoryType >	preferred_mem_type = `std::nullopt`
	)

static

Concatenate multiple chunks into a single chunk.

Using a new stream from br->stream_pool() for the concatenation.

Parameters

chunks	Vector of chunks to concatenate.
chunk_id	The ID for the resulting concatenated chunk.
br	The buffer resource to use for memory allocation.
preferred_mem_type	The preferred memory type to use for the concatenated data buffer.

Returns: Chunk The concatenated chunk.

Exceptions

std::logic_error if the input vector is empty or the concatenated chunk contains duplicate partition IDs.

Note: The chunks vector should contain unique partition IDs.

◆ concat_data_size()

constexpr size_t rapidsmpf::shuffler::detail::Chunk::concat_data_size ( ) const

inlineconstexpr

Get the size of the concatenated data.

Returns: The size of the concatenated data.

Definition at line 232 of file chunk.hpp.

◆ concat_metadata_size()

constexpr size_t rapidsmpf::shuffler::detail::Chunk::concat_metadata_size ( ) const

inlineconstexpr

Get the size of the concatenated metadata.

Returns: The size of the concatenated metadata.

Definition at line 241 of file chunk.hpp.

◆ data_memory_type()

MemoryType rapidsmpf::shuffler::detail::Chunk::data_memory_type ( ) const

inline

Get the memory type of the data buffer.

Returns: The memory type of the data buffer.

Definition at line 222 of file chunk.hpp.

◆ data_size()

constexpr size_t rapidsmpf::shuffler::detail::Chunk::data_size ( size_t i ) const

inlineconstexpr

Get the size of the packed data of the i-th message.

Parameters

i	The index of the message.

Returns: The size of the packed data of the message. Zero when the message is a control message, otherwise the size of PackedData::data of the message.

Definition at line 184 of file chunk.hpp.

◆ deserialize()

static Chunk rapidsmpf::shuffler::detail::Chunk::deserialize	(	std::vector< uint8_t > const &	msg,
		bool	validate = `true`
	)

static

Create a chunk by deserializing a metadata message.

Parameters

msg	The metadata message received from another rank.
validate	Whether to validate the metadata buffer.

Returns: The chunk.

Exceptions

std::runtime_error if the metadata buffer does not follow the expected format and validate is true.

◆ expected_num_chunks()

constexpr size_t rapidsmpf::shuffler::detail::Chunk::expected_num_chunks ( size_t i ) const

inlineconstexpr

The expected number of chunks of the i-th message.

Parameters

i	The index of the message.

Returns: The expected number of chunks for the message. Non-zero when the message is a control message, otherwise zero (data message).

Definition at line 130 of file chunk.hpp.

◆ from_finished_partition()

static Chunk rapidsmpf::shuffler::detail::Chunk::from_finished_partition	(	ChunkID	chunk_id,
		PartID	part_id,
		size_t	expected_num_chunks
	)

static

Create a single-message chunk for a finished partition (control message).

Parameters

chunk_id	The ID of the chunk.
part_id	The ID of the partition.
expected_num_chunks	The expected number of chunks.

Returns: The chunk.

◆ from_packed_data()

static Chunk rapidsmpf::shuffler::detail::Chunk::from_packed_data	(	ChunkID	chunk_id,
		PartID	part_id,
		PackedData &&	packed_data
	)

static

Create a single-message chunk from a packed data.

Parameters

chunk_id	The ID of the chunk.
part_id	The ID of the partition.
packed_data	The packed data.

Returns: The chunk.

◆ get_data()

Chunk rapidsmpf::shuffler::detail::Chunk::get_data	(	ChunkID	new_chunk_id,
		size_t	i,
		BufferResource *	br
	)

Get the data of the i-th message, as a new chunk.

Parameters

new_chunk_id	The ID of the new chunk.
i	The index of the message.
br	The buffer resource to use for copying the data.

Returns: A new chunk containing the data of the i-th message.

Note

This will create a copy of the packed data using a new stream from br->stream_pool(). If there is only one message and the message is a data message, the buffers will be moved to the new chunk. Otherwise a new chunk will be created by copying data. If the i'th message is,

control message, the metadata and data buffers will be nullptr, else if
data message, both metadata and data buffers will be non-null (for a metadata-only message, the data buffer will be an empty HOST buffer)

Exceptions

std::out_of_range if the index is out of bounds.

◆ is_control_message()

constexpr bool rapidsmpf::shuffler::detail::Chunk::is_control_message ( size_t i ) const

inlineconstexpr

Whether the i-th message is a control message.

Parameters

i	The index of the message.

Returns: True if the message is a control message, false otherwise.

Definition at line 140 of file chunk.hpp.

◆ is_data_buffer_set()

bool rapidsmpf::shuffler::detail::Chunk::is_data_buffer_set ( ) const

inline

Whether the data buffer is set.

Returns: True if the data buffer is set, false otherwise.

Definition at line 204 of file chunk.hpp.

◆ is_metadata_buffer_set()

bool rapidsmpf::shuffler::detail::Chunk::is_metadata_buffer_set ( ) const

inline

Whether the metadata buffer is set.

Returns: True if the metadata buffer is set, false otherwise.

Definition at line 213 of file chunk.hpp.

◆ is_ready()

bool rapidsmpf::shuffler::detail::Chunk::is_ready ( ) const

inline

Whether the chunk is ready for consumption.

Returns: True if the chunk is ready, false otherwise.

Note: chunk is ready if it has no data or if the data is ready. data_ buffer could be set later, so we need to check if it is non-null.

Definition at line 316 of file chunk.hpp.

◆ metadata_message_header_size()

static constexpr size_t rapidsmpf::shuffler::detail::Chunk::metadata_message_header_size ( size_t n_messages )

inlinestaticconstexpr

The size of the metadata message header.

Parameters

n_messages The number of messages in the chunk.

Returns: The size of the metadata message header.

Definition at line 88 of file chunk.hpp.

◆ metadata_size()

constexpr uint32_t rapidsmpf::shuffler::detail::Chunk::metadata_size ( size_t i ) const

inlineconstexpr

Get the size of the metadata of the i-th message.

Parameters

i	The index of the message.

Returns: The size of the metadata of the message. Zero when the message is a control message, otherwise the size of PackedData::metadata.

Definition at line 172 of file chunk.hpp.

◆ n_messages()

constexpr size_t rapidsmpf::shuffler::detail::Chunk::n_messages ( ) const

inlineconstexpr

The number of messages in the chunk.

Returns: The number of messages in the chunk.

Definition at line 109 of file chunk.hpp.

◆ operator=()

Chunk& rapidsmpf::shuffler::detail::Chunk::operator= ( Chunk && other )

defaultnoexcept

move assignment operator

Parameters

other The chunk to move.

Returns: this chunk.

◆ part_id()

constexpr PartID rapidsmpf::shuffler::detail::Chunk::part_id ( size_t i ) const

inlineconstexpr

Partition ID of the i-th message.

Parameters

i	The index of the message.

Returns: The ID of the partition.

Definition at line 119 of file chunk.hpp.

◆ release_data_buffer()

std::unique_ptr<Buffer> rapidsmpf::shuffler::detail::Chunk::release_data_buffer ( )

inline

Release the ownership of the data buffer.

Returns: The data buffer.

Definition at line 259 of file chunk.hpp.

◆ release_metadata_buffer()

std::unique_ptr<std::vector<uint8_t> > rapidsmpf::shuffler::detail::Chunk::release_metadata_buffer ( )

inline

Release the ownership of the metadata buffer.

Returns: The metadata buffer.

Definition at line 250 of file chunk.hpp.

◆ serialize()

std::unique_ptr<std::vector<uint8_t> > rapidsmpf::shuffler::detail::Chunk::serialize ( ) const

Returns a metadata message that represents this chunk.

Returns: The metadata message as a serialized byte vector.

◆ set_data_buffer()

void rapidsmpf::shuffler::detail::Chunk::set_data_buffer ( std::unique_ptr< Buffer > data )

inline

Set the data buffer.

Parameters

data	The data buffer.

Definition at line 194 of file chunk.hpp.

◆ str()

std::string rapidsmpf::shuffler::detail::Chunk::str ( ) const

Returns a description of this chunk.

Returns: The description.

◆ validate_format()

static bool rapidsmpf::shuffler::detail::Chunk::validate_format ( std::vector< uint8_t > const & serialized_buf )

static

Validate if a deserialized buffer follows the Chunk format.

Parameters

serialized_buf The deserialized buffer to validate.

Returns: True if the deserialized buffer follows the Chunk format, false otherwise.

The documentation for this class was generated from the following file:

/__w/rapidsmpf/rapidsmpf/cpp/include/rapidsmpf/shuffler/chunk.hpp

Public Member Functions

Static Public Member Functions

Friends

Detailed Description

Constructor & Destructor Documentation

◆ Chunk()

Member Function Documentation

◆ chunk_id()

◆ concat()

◆ concat_data_size()

◆ concat_metadata_size()

◆ data_memory_type()

◆ data_size()

◆ deserialize()

◆ expected_num_chunks()

◆ from_finished_partition()

◆ from_packed_data()

◆ get_data()

◆ is_control_message()

◆ is_data_buffer_set()

◆ is_metadata_buffer_set()

◆ is_ready()

◆ metadata_message_header_size()

◆ metadata_size()

◆ n_messages()

◆ operator=()

◆ part_id()

◆ release_data_buffer()

◆ release_metadata_buffer()

◆ serialize()

◆ set_data_buffer()

◆ str()

◆ validate_format()