The table of merge pairs for the BPE encoder. More...
#include <byte_pair_encoding.hpp>
Public Member Functions | |
bpe_merge_pairs (std::unique_ptr< cudf::column > &&input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref()) | |
Construct a new bpe merge pairs object. More... | |
bpe_merge_pairs (cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref()) | |
Construct a new bpe merge pairs object. More... | |
The table of merge pairs for the BPE encoder.
To create an instance, call nvtext::load_merge_pairs
Definition at line 39 of file byte_pair_encoding.hpp.
nvtext::bpe_merge_pairs::bpe_merge_pairs | ( | std::unique_ptr< cudf::column > && | input, |
rmm::cuda_stream_view | stream = cudf::get_default_stream() , |
||
rmm::device_async_resource_ref | mr = cudf::get_current_device_resource_ref() |
||
) |
Construct a new bpe merge pairs object.
input | The input file containing the BPE merge pairs |
stream | CUDA stream used for device memory operations and kernel launches. |
mr | Device memory resource used to allocate the device memory |
nvtext::bpe_merge_pairs::bpe_merge_pairs | ( | cudf::strings_column_view const & | input, |
rmm::cuda_stream_view | stream = cudf::get_default_stream() , |
||
rmm::device_async_resource_ref | mr = cudf::get_current_device_resource_ref() |
||
) |
Construct a new bpe merge pairs object.
input | The input column of strings |
stream | CUDA stream used for device memory operations and kernel launches. |
mr | Device memory resource used to allocate the device memory |