subword_tokenize.hpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020-2024, NVIDIA CORPORATION.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #pragma once
17 
18 #include <cudf/column/column.hpp>
21 #include <cudf/utilities/export.hpp>
23 
24 namespace CUDF_EXPORT nvtext {
25 
36  uint16_t first_token_id{};
37  uint16_t separator_token_id{};
38  uint16_t unknown_token_id{};
39  uint32_t outer_hash_a{};
40  uint32_t outer_hash_b{};
41  uint16_t num_bins{};
42  std::unique_ptr<cudf::column> table;
44  std::unique_ptr<cudf::column> bin_coefficients;
46  std::unique_ptr<cudf::column> bin_offsets;
48  std::unique_ptr<cudf::column>
50  std::unique_ptr<cudf::column>
52 };
53 
69 std::unique_ptr<hashed_vocabulary> load_vocabulary_file(
70  std::string const& filename_hashed_vocabulary,
73 
81  uint32_t nrows_tensor{};
85  uint32_t sequence_length{};
92  std::unique_ptr<cudf::column> tensor_token_ids;
98  std::unique_ptr<cudf::column> tensor_attention_mask;
105  std::unique_ptr<cudf::column> tensor_metadata;
106 };
107 
157  cudf::strings_column_view const& strings,
158  hashed_vocabulary const& vocabulary_table,
159  uint32_t max_sequence_length,
160  uint32_t stride,
161  bool do_lower_case,
162  bool do_truncate,
165  // end of group
167 } // namespace CUDF_EXPORT nvtext
Given a column-view of strings type, an instance of this class provides a wrapper on this compound co...
Class definition for cudf::column.
column view class definitions
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::async_resource_ref< cuda::mr::device_accessible > device_async_resource_ref
tokenizer_result subword_tokenize(cudf::strings_column_view const &strings, hashed_vocabulary const &vocabulary_table, uint32_t max_sequence_length, uint32_t stride, bool do_lower_case, bool do_truncate, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Creates a tokenizer that cleans the text, splits it into tokens and returns token-ids from an input v...
std::unique_ptr< hashed_vocabulary > load_vocabulary_file(std::string const &filename_hashed_vocabulary, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Load the hashed vocabulary file into device memory.
NVText APIs.
Class definition for cudf::strings_column_view.
The vocabulary data for use with the subword_tokenize function.
std::unique_ptr< cudf::column > bin_coefficients
std::unique_ptr< cudf::column > aux_cp_table
uint64 column, The auxiliary code point table to use for normalization
std::unique_ptr< cudf::column > bin_offsets
std::unique_ptr< cudf::column > table
std::unique_ptr< cudf::column > cp_metadata
uint32 column, The code point metadata table to use for normalization
Result object for the subword_tokenize functions.
std::unique_ptr< cudf::column > tensor_token_ids
A vector of token-ids for each row.
std::unique_ptr< cudf::column > tensor_metadata
The metadata for each tensor row.
std::unique_ptr< cudf::column > tensor_attention_mask
This mask identifies which tensor-token-ids are valid.