21 #include <cudf/utilities/export.hpp>
24 namespace CUDF_EXPORT
nvtext {
256 struct tokenize_vocabulary_impl;
257 tokenize_vocabulary_impl* _impl{};
A non-owning, immutable view of device data as a column of elements, some of which may be null as ind...
An owning class to represent a string in device memory.
Given a column-view of strings type, an instance of this class provides a wrapper on this compound co...
Class definition for cudf::column.
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::async_resource_ref< cuda::mr::device_accessible > device_async_resource_ref
std::unique_ptr< cudf::column > count_tokens(cudf::strings_column_view const &input, cudf::strings_column_view const &delimiters, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns the number of tokens in each string of a strings column by using multiple strings delimiters ...
std::unique_ptr< tokenize_vocabulary > load_vocabulary(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Create a tokenize_vocabulary object from a strings column.
std::unique_ptr< cudf::column > tokenize(cudf::strings_column_view const &input, cudf::strings_column_view const &delimiters, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns a single column of strings by tokenizing the input strings column using multiple strings as d...
std::unique_ptr< cudf::column > detokenize(cudf::strings_column_view const &input, cudf::column_view const &row_indices, cudf::string_scalar const &separator=cudf::string_scalar(" "), rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Creates a strings column from a strings column of tokens and an associated column of row ids.
std::unique_ptr< cudf::column > character_tokenize(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns a single column of strings by converting each character to a string.
std::unique_ptr< cudf::column > tokenize_with_vocabulary(cudf::strings_column_view const &input, tokenize_vocabulary const &vocabulary, cudf::string_scalar const &delimiter, cudf::size_type default_id=-1, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns the token ids for the input string by looking up each delimited token in the given vocabulary...
int32_t size_type
Row index type for columns and tables.
Class definitions for cudf::scalar.
Class definition for cudf::strings_column_view.
Vocabulary object to be used with nvtext::tokenize_with_vocabulary.
tokenize_vocabulary(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Vocabulary object constructor.