std::unique_ptr< bpe_merge_pairs > load_merge_pairs(cudf::strings_column_view const &merge_pairs, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Create a nvtext::bpe_merge_pairs from a strings column.
std::unique_ptr< cudf::column > byte_pair_encoding(cudf::strings_column_view const &input, bpe_merge_pairs const &merges_pairs, cudf::string_scalar const &separator=cudf::string_scalar(" "), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Byte pair encode the input strings.
The table of merge pairs for the BPE encoder.
bpe_merge_pairs(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Construct a new bpe merge pairs object.
bpe_merge_pairs(std::unique_ptr< cudf::column > &&input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Construct a new bpe merge pairs object.