byte_pair_encoding.hpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2022-2024, NVIDIA CORPORATION.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #pragma once
17 
18 #include <cudf/column/column.hpp>
20 #include <cudf/scalar/scalar.hpp>
23 #include <cudf/utilities/export.hpp>
25 
26 namespace CUDF_EXPORT nvtext {
27 
40  struct bpe_merge_pairs_impl;
41 
49  bpe_merge_pairs(std::unique_ptr<cudf::column>&& input,
52 
63 
64  ~bpe_merge_pairs();
66 
67  private:
68  friend bpe_merge_pairs_impl const* get_bpe_merge_pairs_impl(bpe_merge_pairs const&);
69  bpe_merge_pairs_impl* impl{};
70 };
71 
97 std::unique_ptr<bpe_merge_pairs> load_merge_pairs(
98  cudf::strings_column_view const& merge_pairs,
101 
128 std::unique_ptr<cudf::column> byte_pair_encoding(
129  cudf::strings_column_view const& input,
130  bpe_merge_pairs const& merges_pairs,
131  cudf::string_scalar const& separator = cudf::string_scalar(" "),
133  // end of group
135 } // namespace CUDF_EXPORT nvtext
An owning class to represent a string in device memory.
Definition: scalar.hpp:429
Given a column-view of strings type, an instance of this class provides a wrapper on this compound co...
Class definition for cudf::column.
column view class definitions
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
rmm::device_async_resource_ref get_current_device_resource_ref()
Get the current device memory resource reference.
cuda::mr::async_resource_ref< cuda::mr::device_accessible > device_async_resource_ref
std::unique_ptr< bpe_merge_pairs > load_merge_pairs(cudf::strings_column_view const &merge_pairs, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Create a nvtext::bpe_merge_pairs from a strings column.
std::unique_ptr< cudf::column > byte_pair_encoding(cudf::strings_column_view const &input, bpe_merge_pairs const &merges_pairs, cudf::string_scalar const &separator=cudf::string_scalar(" "), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Byte pair encode the input strings.
NVText APIs.
Class definitions for cudf::scalar.
Class definition for cudf::strings_column_view.
The table of merge pairs for the BPE encoder.
bpe_merge_pairs(cudf::strings_column_view const &input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Construct a new bpe merge pairs object.
bpe_merge_pairs(std::unique_ptr< cudf::column > &&input, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::device_async_resource_ref mr=cudf::get_current_device_resource_ref())
Construct a new bpe merge pairs object.