tokenize.hpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020-2024, NVIDIA CORPORATION.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #pragma once
17 
18 #include <cudf/column/column.hpp>
19 #include <cudf/scalar/scalar.hpp>
21 #include <cudf/utilities/export.hpp>
23 
24 namespace CUDF_EXPORT nvtext {
61 std::unique_ptr<cudf::column> tokenize(
62  cudf::strings_column_view const& input,
63  cudf::string_scalar const& delimiter = cudf::string_scalar{""},
66 
97 std::unique_ptr<cudf::column> tokenize(
98  cudf::strings_column_view const& input,
99  cudf::strings_column_view const& delimiters,
102 
128 std::unique_ptr<cudf::column> count_tokens(
129  cudf::strings_column_view const& input,
130  cudf::string_scalar const& delimiter = cudf::string_scalar{""},
133 
160 std::unique_ptr<cudf::column> count_tokens(
161  cudf::strings_column_view const& input,
162  cudf::strings_column_view const& delimiters,
165 
187 std::unique_ptr<cudf::column> character_tokenize(
188  cudf::strings_column_view const& input,
191 
226 std::unique_ptr<cudf::column> detokenize(
227  cudf::strings_column_view const& input,
228  cudf::column_view const& row_indices,
229  cudf::string_scalar const& separator = cudf::string_scalar(" "),
232 
255 
256  struct tokenize_vocabulary_impl;
257  tokenize_vocabulary_impl* _impl{};
258 };
259 
273 std::unique_ptr<tokenize_vocabulary> load_vocabulary(
274  cudf::strings_column_view const& input,
277 
303 std::unique_ptr<cudf::column> tokenize_with_vocabulary(
304  cudf::strings_column_view const& input,
305  tokenize_vocabulary const& vocabulary,
306  cudf::string_scalar const& delimiter,
307  cudf::size_type default_id = -1,
310  // end of tokenize group
312 } // namespace CUDF_EXPORT nvtext
A non-owning, immutable view of device data as a column of elements, some of which may be null as ind...
An owning class to represent a string in device memory.
Definition: scalar.hpp:429
Given a column-view of strings type, an instance of this class provides a wrapper on this compound co...
Class definition for cudf::column.
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::async_resource_ref< cuda::mr::device_accessible > device_async_resource_ref
std::unique_ptr< cudf::column > count_tokens(cudf::strings_column_view const &input, cudf::strings_column_view const &delimiters, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns the number of tokens in each string of a strings column by using multiple strings delimiters ...
std::unique_ptr< tokenize_vocabulary > load_vocabulary(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Create a tokenize_vocabulary object from a strings column.
std::unique_ptr< cudf::column > tokenize(cudf::strings_column_view const &input, cudf::strings_column_view const &delimiters, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns a single column of strings by tokenizing the input strings column using multiple strings as d...
std::unique_ptr< cudf::column > detokenize(cudf::strings_column_view const &input, cudf::column_view const &row_indices, cudf::string_scalar const &separator=cudf::string_scalar(" "), rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Creates a strings column from a strings column of tokens and an associated column of row ids.
std::unique_ptr< cudf::column > character_tokenize(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns a single column of strings by converting each character to a string.
std::unique_ptr< cudf::column > tokenize_with_vocabulary(cudf::strings_column_view const &input, tokenize_vocabulary const &vocabulary, cudf::string_scalar const &delimiter, cudf::size_type default_id=-1, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Returns the token ids for the input string by looking up each delimited token in the given vocabulary...
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:95
NVText APIs.
Class definitions for cudf::scalar.
Class definition for cudf::strings_column_view.
Vocabulary object to be used with nvtext::tokenize_with_vocabulary.
Definition: tokenize.hpp:238
tokenize_vocabulary(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Vocabulary object constructor.