A partition chunk representing either a data message or a control message. More...

#include <chunk.hpp>

Public Member Functions
	Chunk (Chunk &&other) noexcept=default
	move constructor More...

Chunk &	operator= (Chunk &&other) noexcept=default
	move assignment operator More...

	Chunk (Chunk const &)=delete

Chunk &	operator= (Chunk const &)=delete

constexpr ChunkID	chunk_id () const
	The ID of the chunk. More...

constexpr PartID	part_id () const
	Partition ID of the message. More...

constexpr std::size_t	expected_num_chunks () const
	The expected number of chunks for the message. More...

constexpr bool	is_control_message () const
	Whether the message is a control message. More...

constexpr std::uint32_t	metadata_size () const
	Get the size of the metadata of the message. More...

constexpr std::size_t	data_size () const
	Get the size of the packed data of the message. More...

void	set_data_buffer (std::unique_ptr< Buffer > data)
	Set the data buffer. More...

bool	is_data_buffer_set () const
	Whether the data buffer is set. More...

bool	is_metadata_buffer_set () const
	Whether the metadata buffer is set. More...

MemoryType	data_memory_type () const
	Get the memory type of the data buffer. More...

std::unique_ptr< std::vector< std::uint8_t > >	release_metadata_buffer ()
	Release the ownership of the metadata buffer. More...

std::unique_ptr< Buffer >	release_data_buffer ()
	Release the ownership of the data buffer. More...

bool	is_ready () const
	Whether the chunk is ready for consumption. More...

std::string	str () const
	Returns a description of this chunk. More...

std::unique_ptr< std::vector< std::uint8_t > >	serialize () const
	Returns a metadata message that represents this chunk. More...

Static Public Member Functions
static constexpr std::size_t	metadata_message_header_size ()
	The size of the metadata message header. More...

static Chunk	from_packed_data (ChunkID chunk_id, PartID part_id, PackedData &&packed_data)
	Create a chunk from a packed data. More...

static Chunk	from_finished_partition (ChunkID chunk_id, PartID part_id, std::size_t expected_num_chunks)
	Create a chunk for a finished partition (control message). More...

static Chunk	deserialize (std::vector< std::uint8_t > const &msg, BufferResource *br, bool validate=true)
	Create a chunk by deserializing a metadata message. More...

static bool	validate_format (std::vector< std::uint8_t > const &serialized_buf)
	Validate if a deserialized buffer follows the Chunk format. More...

Friends
Chunk	make_dummy_chunk (ChunkID, PartID)

Detailed Description

A partition chunk representing either a data message or a control message.

A Chunk represents a single message for a partition in the shuffler. There are two types.

Data Message. Contains actual partition data (metadata and optionally GPU data buffer).

Used to transfer partition data between ranks.
expected_num_chunks is 0.
is_control_message() returns false.
Contains metadata buffer and optionally a data buffer.

Control Message. Signals partition completion without carrying data.

Used to indicate that all data chunks for a partition have been sent.
expected_num_chunks > 0 (indicates total number of data chunks expected).
is_control_message() returns true.
No metadata or data buffers (metadata_size = 0, data_size = 0).

When serialized, the format is:

chunk_id: std::uint64_t, ID of the chunk.
partition_id: PartID, Partition ID of the message.
expected_num_chunks: std::size_t, Expected number of chunks (0 for data, >0 for control).
metadata_size: std::uint32_t, Size of the metadata in bytes.
data_size: std::uint64_t, Size of the data in bytes.
metadata: vector<std::uint8_t>, Metadata buffer

Definition at line 58 of file chunk.hpp.

Constructor & Destructor Documentation

◆ Chunk()

rapidsmpf::shuffler::detail::Chunk::Chunk ( Chunk && other )

defaultnoexcept

move constructor

Parameters

other The chunk to move.

Member Function Documentation

◆ chunk_id()

constexpr ChunkID rapidsmpf::shuffler::detail::Chunk::chunk_id ( ) const

inlineconstexpr

The ID of the chunk.

Returns: The ID of the chunk.

Definition at line 97 of file chunk.hpp.

◆ data_memory_type()

MemoryType rapidsmpf::shuffler::detail::Chunk::data_memory_type ( ) const

inline

Get the memory type of the data buffer.

Returns: The memory type of the data buffer.

Definition at line 183 of file chunk.hpp.

◆ data_size()

constexpr std::size_t rapidsmpf::shuffler::detail::Chunk::data_size ( ) const

inlineconstexpr

Get the size of the packed data of the message.

Returns: The size of the packed data of the message. Zero when the message is a control message, otherwise the size of PackedData::data of the message.

Definition at line 146 of file chunk.hpp.

◆ deserialize()

static Chunk rapidsmpf::shuffler::detail::Chunk::deserialize	(	std::vector< std::uint8_t > const &	msg,
		BufferResource *	br,
		bool	validate = `true`
	)

static

Create a chunk by deserializing a metadata message.

Parameters

msg	The metadata message received from another rank.
br	Buffer resource for allocating a the data buffer of the deserialized message.
validate	Whether to validate the metadata buffer.

Returns: The chunk.

Exceptions

std::logic_error	if the chunk is not a control message and no buffer resource is provided.
std::runtime_error	if the metadata buffer does not follow the expected format and `validate` is true.

◆ expected_num_chunks()

constexpr std::size_t rapidsmpf::shuffler::detail::Chunk::expected_num_chunks ( ) const

inlineconstexpr

The expected number of chunks for the message.

Returns: The expected number of chunks for the message. Non-zero when the message is a control message, otherwise zero (data message).

Definition at line 116 of file chunk.hpp.

◆ from_finished_partition()

static Chunk rapidsmpf::shuffler::detail::Chunk::from_finished_partition	(	ChunkID	chunk_id,
		PartID	part_id,
		std::size_t	expected_num_chunks
	)

static

Create a chunk for a finished partition (control message).

Parameters

chunk_id	The ID of the chunk.
part_id	The ID of the partition.
expected_num_chunks	The expected number of chunks.

Returns: The chunk.

◆ from_packed_data()

static Chunk rapidsmpf::shuffler::detail::Chunk::from_packed_data	(	ChunkID	chunk_id,
		PartID	part_id,
		PackedData &&	packed_data
	)

static

Create a chunk from a packed data.

Parameters

chunk_id	The ID of the chunk.
part_id	The ID of the partition.
packed_data	The packed data.

Returns: The chunk.

◆ is_control_message()

constexpr bool rapidsmpf::shuffler::detail::Chunk::is_control_message ( ) const

inlineconstexpr

Whether the message is a control message.

Returns: True if the message is a control message, false otherwise.

Definition at line 125 of file chunk.hpp.

◆ is_data_buffer_set()

bool rapidsmpf::shuffler::detail::Chunk::is_data_buffer_set ( ) const

inline

Whether the data buffer is set.

Returns: True if the data buffer is set, false otherwise.

Definition at line 165 of file chunk.hpp.

◆ is_metadata_buffer_set()

bool rapidsmpf::shuffler::detail::Chunk::is_metadata_buffer_set ( ) const

inline

Whether the metadata buffer is set.

Returns: True if the metadata buffer is set, false otherwise.

Definition at line 174 of file chunk.hpp.

◆ is_ready()

bool rapidsmpf::shuffler::detail::Chunk::is_ready ( ) const

inline

Whether the chunk is ready for consumption.

Returns: True if the chunk is ready, false otherwise.

Note: chunk is ready if it has no data or if the data is ready. data_ buffer could be set later, so we need to check if it is non-null.

Definition at line 263 of file chunk.hpp.

◆ metadata_message_header_size()

static constexpr std::size_t rapidsmpf::shuffler::detail::Chunk::metadata_message_header_size ( )

inlinestaticconstexpr

The size of the metadata message header.

Returns: The size of the metadata message header.

Definition at line 87 of file chunk.hpp.

◆ metadata_size()

constexpr std::uint32_t rapidsmpf::shuffler::detail::Chunk::metadata_size ( ) const

inlineconstexpr

Get the size of the metadata of the message.

Returns: The size of the metadata of the message. Zero when the message is a control message, otherwise the size of PackedData::metadata.

Definition at line 136 of file chunk.hpp.

◆ operator=()

Chunk& rapidsmpf::shuffler::detail::Chunk::operator= ( Chunk && other )

defaultnoexcept

move assignment operator

Parameters

other The chunk to move.

Returns: this chunk.

◆ part_id()

constexpr PartID rapidsmpf::shuffler::detail::Chunk::part_id ( ) const

inlineconstexpr

Partition ID of the message.

Returns: The ID of the partition.

Definition at line 106 of file chunk.hpp.

◆ release_data_buffer()

std::unique_ptr<Buffer> rapidsmpf::shuffler::detail::Chunk::release_data_buffer ( )

inline

Release the ownership of the data buffer.

Returns: The data buffer.

Definition at line 202 of file chunk.hpp.

◆ release_metadata_buffer()

std::unique_ptr<std::vector<std::uint8_t> > rapidsmpf::shuffler::detail::Chunk::release_metadata_buffer ( )

inline

Release the ownership of the metadata buffer.

Returns: The metadata buffer.

Definition at line 193 of file chunk.hpp.

◆ serialize()

std::unique_ptr<std::vector<std::uint8_t> > rapidsmpf::shuffler::detail::Chunk::serialize ( ) const

Returns a metadata message that represents this chunk.

Returns: The metadata message as a serialized byte vector.

◆ set_data_buffer()

void rapidsmpf::shuffler::detail::Chunk::set_data_buffer ( std::unique_ptr< Buffer > data )

inline

Set the data buffer.

Parameters

data	The data buffer.

Definition at line 155 of file chunk.hpp.

◆ str()

std::string rapidsmpf::shuffler::detail::Chunk::str ( ) const

Returns a description of this chunk.

Returns: The description.

◆ validate_format()

static bool rapidsmpf::shuffler::detail::Chunk::validate_format ( std::vector< std::uint8_t > const & serialized_buf )

static

Validate if a deserialized buffer follows the Chunk format.

Parameters

serialized_buf The deserialized buffer to validate.

Returns: True if the deserialized buffer follows the Chunk format, false otherwise.

The documentation for this class was generated from the following file:

/__w/rapidsmpf/rapidsmpf/cpp/include/rapidsmpf/shuffler/chunk.hpp

Public Member Functions

Static Public Member Functions

Friends

Detailed Description

Constructor & Destructor Documentation

◆ Chunk()

Member Function Documentation

◆ chunk_id()

◆ data_memory_type()

◆ data_size()

◆ deserialize()

◆ expected_num_chunks()

◆ from_finished_partition()

◆ from_packed_data()

◆ is_control_message()

◆ is_data_buffer_set()

◆ is_metadata_buffer_set()

◆ is_ready()

◆ metadata_message_header_size()

◆ metadata_size()

◆ operator=()

◆ part_id()

◆ release_data_buffer()

◆ release_metadata_buffer()

◆ serialize()

◆ set_data_buffer()

◆ str()

◆ validate_format()