tokenize.hpp
Go to the documentation of this file.
1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2020-2024, NVIDIA CORPORATION.
3  * SPDX-License-Identifier: Apache-2.0
4  */
5 #pragma once
6 
7 #include <cudf/column/column.hpp>
8 #include <cudf/scalar/scalar.hpp>
9 #include <cudf/strings/strings_column_view.hpp>
10 #include <cudf/utilities/export.hpp>
11 #include <cudf/utilities/memory_resource.hpp>
12 
13 namespace CUDF_EXPORT nvtext {
50 std::unique_ptr<cudf::column> tokenize(
51  cudf::strings_column_view const& input,
52  cudf::string_scalar const& delimiter = cudf::string_scalar{""},
53  rmm::cuda_stream_view stream = cudf::get_default_stream(),
54  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
55 
86 std::unique_ptr<cudf::column> tokenize(
87  cudf::strings_column_view const& input,
88  cudf::strings_column_view const& delimiters,
89  rmm::cuda_stream_view stream = cudf::get_default_stream(),
90  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
91 
117 std::unique_ptr<cudf::column> count_tokens(
118  cudf::strings_column_view const& input,
119  cudf::string_scalar const& delimiter = cudf::string_scalar{""},
120  rmm::cuda_stream_view stream = cudf::get_default_stream(),
121  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
122 
149 std::unique_ptr<cudf::column> count_tokens(
150  cudf::strings_column_view const& input,
151  cudf::strings_column_view const& delimiters,
152  rmm::cuda_stream_view stream = cudf::get_default_stream(),
153  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
154 
176 std::unique_ptr<cudf::column> character_tokenize(
177  cudf::strings_column_view const& input,
178  rmm::cuda_stream_view stream = cudf::get_default_stream(),
179  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
180 
215 std::unique_ptr<cudf::column> detokenize(
216  cudf::strings_column_view const& input,
217  cudf::column_view const& row_indices,
218  cudf::string_scalar const& separator = cudf::string_scalar(" "),
219  rmm::cuda_stream_view stream = cudf::get_default_stream(),
220  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
221 
227 struct tokenize_vocabulary {
240  tokenize_vocabulary(cudf::strings_column_view const& input,
241  rmm::cuda_stream_view stream = cudf::get_default_stream(),
242  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
243  ~tokenize_vocabulary();
244 
245  struct tokenize_vocabulary_impl;
246  tokenize_vocabulary_impl* _impl{};
247 };
248 
262 std::unique_ptr<tokenize_vocabulary> load_vocabulary(
263  cudf::strings_column_view const& input,
264  rmm::cuda_stream_view stream = cudf::get_default_stream(),
265  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
266 
292 std::unique_ptr<cudf::column> tokenize_with_vocabulary(
293  cudf::strings_column_view const& input,
294  tokenize_vocabulary const& vocabulary,
295  cudf::string_scalar const& delimiter,
296  cudf::size_type default_id = -1,
297  rmm::cuda_stream_view stream = cudf::get_default_stream(),
298  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
299  // end of tokenize group
301 } // namespace CUDF_EXPORT nvtext
cudf::column_view
A non-owning, immutable view of device data as a column of elements, some of which may be null as ind...
Definition: column_view.hpp:318
cudf::string_scalar
An owning class to represent a string in device memory.
Definition: scalar.hpp:410
cudf::strings_column_view
Given a column-view of strings type, an instance of this class provides a wrapper on this compound co...
Definition: strings_column_view.hpp:27
rmm::cuda_stream_view
column.hpp
Class definition for cudf::column.
cudf::get_default_stream
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
cudf::get_current_device_resource_ref
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
Definition: memory_resource.hpp:40
rmm::device_async_resource_ref
detail::cccl_async_resource_ref< cuda::mr::resource_ref< cuda::mr::device_accessible > > device_async_resource_ref
nvtext::count_tokens
std::unique_ptr< cudf::column > count_tokens(cudf::strings_column_view const &input, cudf::strings_column_view const &delimiters, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns the number of tokens in each string of a strings column by using multiple strings delimiters ...
nvtext::load_vocabulary
std::unique_ptr< tokenize_vocabulary > load_vocabulary(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Create a tokenize_vocabulary object from a strings column.
nvtext::tokenize
std::unique_ptr< cudf::column > tokenize(cudf::strings_column_view const &input, cudf::strings_column_view const &delimiters, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns a single column of strings by tokenizing the input strings column using multiple strings as d...
nvtext::detokenize
std::unique_ptr< cudf::column > detokenize(cudf::strings_column_view const &input, cudf::column_view const &row_indices, cudf::string_scalar const &separator=cudf::string_scalar(" "), rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Creates a strings column from a strings column of tokens and an associated column of row ids.
nvtext::character_tokenize
std::unique_ptr< cudf::column > character_tokenize(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns a single column of strings by converting each character to a string.
nvtext::tokenize_with_vocabulary
std::unique_ptr< cudf::column > tokenize_with_vocabulary(cudf::strings_column_view const &input, tokenize_vocabulary const &vocabulary, cudf::string_scalar const &delimiter, cudf::size_type default_id=-1, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns the token ids for the input string by looking up each delimited token in the given vocabulary...
cudf::size_type
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:84
memory_resource.hpp
nvtext
NVText APIs.
Definition: byte_pair_encoding.hpp:15
scalar.hpp
Class definitions for cudf::scalar.
strings_column_view.hpp
Class definition for cudf::strings_column_view.
nvtext::tokenize_vocabulary
Vocabulary object to be used with nvtext::tokenize_with_vocabulary.
Definition: tokenize.hpp:227
nvtext::tokenize_vocabulary::tokenize_vocabulary
tokenize_vocabulary(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Vocabulary object constructor.
Generated by doxygen 1.9.1