18 #include "kafka_callback.hpp"
22 #include <librdkafka/rdkafkacpp.h>
62 python_callable_type python_callable,
63 kafka_oauth_callback_wrapper_type callable_wrapper);
86 python_callable_type python_callable,
87 kafka_oauth_callback_wrapper_type callable_wrapper,
88 std::string
const& topic_name,
93 std::string
const& delimiter);
103 std::unique_ptr<cudf::io::datasource::buffer>
host_read(
size_t offset,
size_t size)
override;
133 void commit_offset(std::string
const& topic,
int partition, int64_t offset);
177 std::map<std::string, std::vector<int32_t>>
list_topics(std::string specific_topic);
198 std::unique_ptr<RdKafka::Conf> kafka_conf;
199 std::unique_ptr<RdKafka::KafkaConsumer> consumer;
201 std::map<std::string, std::string> configs;
202 python_callable_type python_callable_;
203 kafka_oauth_callback_wrapper_type callable_wrapper_;
205 std::string topic_name;
207 int64_t start_offset;
210 int default_timeout = 10000;
211 std::string delimiter;
216 RdKafka::ErrorCode update_consumer_topic_partition_assignment(std::string
const& topic,
225 void consume_to_buffer();
Interface class for providing input data to the readers.
libcudf datasource for Apache Kafka
void unsubscribe()
Stop all active consumption and remove consumer subscriptions to topic/partition instances.
std::unique_ptr< cudf::io::datasource::buffer > host_read(size_t offset, size_t size) override
Returns a buffer with a subset of data from Kafka Topic.
std::map< std::string, std::string > current_configs()
Retrieve the current Kafka client configurations.
std::map< std::string, std::vector< int32_t > > list_topics(std::string specific_topic)
Query the Kafka broker for the list of Topic partitions for a Topic. If no topic is specified then th...
kafka_consumer(std::map< std::string, std::string > configs, python_callable_type python_callable, kafka_oauth_callback_wrapper_type callable_wrapper)
Creates an instance of the Kafka consumer object that is in a semi-ready state.
size_t host_read(size_t offset, size_t size, uint8_t *dst) override
Reads a selected range into a preallocated buffer.
void commit_offset(std::string const &topic, int partition, int64_t offset)
Commits an offset to a specified Kafka Topic/Partition instance.
kafka_consumer(std::map< std::string, std::string > configs, python_callable_type python_callable, kafka_oauth_callback_wrapper_type callable_wrapper, std::string const &topic_name, int partition, int64_t start_offset, int64_t end_offset, int batch_timeout, std::string const &delimiter)
Instantiate a Kafka consumer object. Documentation for librdkafka configurations can be found at http...
void close(int timeout)
Close the underlying socket connection to Kafka and clean up system resources.
size_t size() const override
Returns the size of the data in Kafka buffer.
int64_t get_committed_offset(std::string const &topic, int partition)
Get the latest offset that was successfully committed to the Kafka broker.
std::map< std::string, int64_t > get_watermark_offset(std::string const &topic, int partition, int timeout, bool cached)
Retrieve the watermark offset values for a topic/partition.