rdkafka.h

Go to the documentation of this file.

/*
 * librdkafka - Apache Kafka C library
 *
 * Copyright (c) 2012-2013 Magnus Edenhill
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met: 
 * 
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer. 
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution. 
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/* @cond NO_DOC */
#pragma once

#include <stdio.h>
#include <inttypes.h>
#include <sys/types.h>

#ifdef __cplusplus
extern "C" {
#if 0
} /* Restore indent */
#endif
#endif

#ifdef _MSC_VER
#include <basetsd.h>
#ifndef WIN32_MEAN_AND_LEAN
#define WIN32_MEAN_AND_LEAN
#endif
#include <Winsock2.h>  /* for sockaddr, .. */
typedef SSIZE_T ssize_t;
#define RD_UNUSED
#define RD_INLINE __inline
#define RD_DEPRECATED
#undef RD_EXPORT
#ifdef LIBRDKAFKA_STATICLIB
#define RD_EXPORT
#else
#ifdef LIBRDKAFKA_EXPORTS
#define RD_EXPORT __declspec(dllexport)
#else
#define RD_EXPORT __declspec(dllimport)
#endif
#ifndef LIBRDKAFKA_TYPECHECKS
#define LIBRDKAFKA_TYPECHECKS 0
#endif
#endif

#else
#include <sys/socket.h> /* for sockaddr, .. */

#define RD_UNUSED __attribute__((unused))
#define RD_INLINE inline
#define RD_EXPORT
#define RD_DEPRECATED __attribute__((deprecated))

#ifndef LIBRDKAFKA_TYPECHECKS
#define LIBRDKAFKA_TYPECHECKS 1
#endif
#endif


#if LIBRDKAFKA_TYPECHECKS
#define _LRK_TYPECHECK(RET,TYPE,ARG)                    \
        ({ if (0) { TYPE __t RD_UNUSED = (ARG); } RET; })

#define _LRK_TYPECHECK2(RET,TYPE,ARG,TYPE2,ARG2)        \
        ({                                              \
                if (0) {                                \
                        TYPE __t RD_UNUSED = (ARG);     \
                        TYPE2 __t2 RD_UNUSED = (ARG2);  \
                }                                       \
                RET; })
#else
#define _LRK_TYPECHECK(RET,TYPE,ARG)  (RET)
#define _LRK_TYPECHECK2(RET,TYPE,ARG,TYPE2,ARG2) (RET)
#endif

/* @endcond */


#define RD_KAFKA_VERSION  0x000905ff

RD_EXPORT
int rd_kafka_version(void);

RD_EXPORT
const char *rd_kafka_version_str (void);

typedef enum rd_kafka_type_t {
    RD_KAFKA_PRODUCER, 
    RD_KAFKA_CONSUMER  
} rd_kafka_type_t;


typedef enum rd_kafka_timestamp_type_t {
    RD_KAFKA_TIMESTAMP_NOT_AVAILABLE,   
    RD_KAFKA_TIMESTAMP_CREATE_TIME,     
    RD_KAFKA_TIMESTAMP_LOG_APPEND_TIME  
} rd_kafka_timestamp_type_t;



RD_EXPORT
const char *rd_kafka_get_debug_contexts(void);

#define RD_KAFKA_DEBUG_CONTEXTS \
    "all,generic,broker,topic,metadata,queue,msg,protocol,cgrp,security,fetch,feature"


/* @cond NO_DOC */
/* Private types to provide ABI compatibility */
typedef struct rd_kafka_s rd_kafka_t;
typedef struct rd_kafka_topic_s rd_kafka_topic_t;
typedef struct rd_kafka_conf_s rd_kafka_conf_t;
typedef struct rd_kafka_topic_conf_s rd_kafka_topic_conf_t;
typedef struct rd_kafka_queue_s rd_kafka_queue_t;
/* @endcond */


typedef enum {
    /* Internal errors to rdkafka: */
    RD_KAFKA_RESP_ERR__BEGIN = -200,
    RD_KAFKA_RESP_ERR__BAD_MSG = -199,
    RD_KAFKA_RESP_ERR__BAD_COMPRESSION = -198,
    RD_KAFKA_RESP_ERR__DESTROY = -197,
    RD_KAFKA_RESP_ERR__FAIL = -196,
    RD_KAFKA_RESP_ERR__TRANSPORT = -195,
    RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE = -194,
    RD_KAFKA_RESP_ERR__RESOLVE = -193,
    RD_KAFKA_RESP_ERR__MSG_TIMED_OUT = -192,
    RD_KAFKA_RESP_ERR__PARTITION_EOF = -191,
    RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION = -190,
    RD_KAFKA_RESP_ERR__FS = -189,
    RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC = -188,
    RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN = -187,
    RD_KAFKA_RESP_ERR__INVALID_ARG = -186,
    RD_KAFKA_RESP_ERR__TIMED_OUT = -185,
    RD_KAFKA_RESP_ERR__QUEUE_FULL = -184,
        RD_KAFKA_RESP_ERR__ISR_INSUFF = -183,
        RD_KAFKA_RESP_ERR__NODE_UPDATE = -182,
    RD_KAFKA_RESP_ERR__SSL = -181,
        RD_KAFKA_RESP_ERR__WAIT_COORD = -180,
        RD_KAFKA_RESP_ERR__UNKNOWN_GROUP = -179,
        RD_KAFKA_RESP_ERR__IN_PROGRESS = -178,
        RD_KAFKA_RESP_ERR__PREV_IN_PROGRESS = -177,
        RD_KAFKA_RESP_ERR__EXISTING_SUBSCRIPTION = -176,
        RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175,
        RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174,
        RD_KAFKA_RESP_ERR__CONFLICT = -173,
        RD_KAFKA_RESP_ERR__STATE = -172,
        RD_KAFKA_RESP_ERR__UNKNOWN_PROTOCOL = -171,
        RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED = -170,
    RD_KAFKA_RESP_ERR__AUTHENTICATION = -169,
    RD_KAFKA_RESP_ERR__NO_OFFSET = -168,
    RD_KAFKA_RESP_ERR__OUTDATED = -167,
    RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE = -166,
        RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE = -165,
        RD_KAFKA_RESP_ERR__WAIT_CACHE = -164,

    RD_KAFKA_RESP_ERR__END = -100,

    /* Kafka broker errors: */
    RD_KAFKA_RESP_ERR_UNKNOWN = -1,
    RD_KAFKA_RESP_ERR_NO_ERROR = 0,
    RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE = 1,
    RD_KAFKA_RESP_ERR_INVALID_MSG = 2,
    RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART = 3,
    RD_KAFKA_RESP_ERR_INVALID_MSG_SIZE = 4,
    RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE = 5,
    RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION = 6,
    RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT = 7,
    RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE = 8,
    RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE = 9,
    RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE = 10,
    RD_KAFKA_RESP_ERR_STALE_CTRL_EPOCH = 11,
    RD_KAFKA_RESP_ERR_OFFSET_METADATA_TOO_LARGE = 12,
    RD_KAFKA_RESP_ERR_NETWORK_EXCEPTION = 13,
        RD_KAFKA_RESP_ERR_GROUP_LOAD_IN_PROGRESS = 14,
        RD_KAFKA_RESP_ERR_GROUP_COORDINATOR_NOT_AVAILABLE = 15,
        RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP = 16,
        RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION = 17,
        RD_KAFKA_RESP_ERR_RECORD_LIST_TOO_LARGE = 18,
        RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS = 19,
        RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS_AFTER_APPEND = 20,
        RD_KAFKA_RESP_ERR_INVALID_REQUIRED_ACKS = 21,
        RD_KAFKA_RESP_ERR_ILLEGAL_GENERATION = 22,
        RD_KAFKA_RESP_ERR_INCONSISTENT_GROUP_PROTOCOL = 23,
    RD_KAFKA_RESP_ERR_INVALID_GROUP_ID = 24,
        RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID = 25,
        RD_KAFKA_RESP_ERR_INVALID_SESSION_TIMEOUT = 26,
    RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS = 27,
        RD_KAFKA_RESP_ERR_INVALID_COMMIT_OFFSET_SIZE = 28,
        RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED = 29,
    RD_KAFKA_RESP_ERR_GROUP_AUTHORIZATION_FAILED = 30,
    RD_KAFKA_RESP_ERR_CLUSTER_AUTHORIZATION_FAILED = 31,
    RD_KAFKA_RESP_ERR_INVALID_TIMESTAMP = 32,
    RD_KAFKA_RESP_ERR_UNSUPPORTED_SASL_MECHANISM = 33,
    RD_KAFKA_RESP_ERR_ILLEGAL_SASL_STATE = 34,
    RD_KAFKA_RESP_ERR_UNSUPPORTED_VERSION = 35,
    RD_KAFKA_RESP_ERR_TOPIC_ALREADY_EXISTS = 36,
    RD_KAFKA_RESP_ERR_INVALID_PARTITIONS = 37,
    RD_KAFKA_RESP_ERR_INVALID_REPLICATION_FACTOR = 38,
    RD_KAFKA_RESP_ERR_INVALID_REPLICA_ASSIGNMENT = 39,
    RD_KAFKA_RESP_ERR_INVALID_CONFIG = 40,
    RD_KAFKA_RESP_ERR_NOT_CONTROLLER = 41,
    RD_KAFKA_RESP_ERR_INVALID_REQUEST = 42,
    RD_KAFKA_RESP_ERR_UNSUPPORTED_FOR_MESSAGE_FORMAT = 43,

    RD_KAFKA_RESP_ERR_END_ALL,
} rd_kafka_resp_err_t;


struct rd_kafka_err_desc {
    rd_kafka_resp_err_t code;
    const char *name;      
    const char *desc;      
};


RD_EXPORT
void rd_kafka_get_err_descs (const struct rd_kafka_err_desc **errdescs,
                 size_t *cntp);




RD_EXPORT
const char *rd_kafka_err2str (rd_kafka_resp_err_t err);



RD_EXPORT
const char *rd_kafka_err2name (rd_kafka_resp_err_t err);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_last_error (void);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_errno2err(int errnox);


RD_EXPORT
int rd_kafka_errno (void);



typedef struct rd_kafka_topic_partition_s {
        char        *topic;             
        int32_t      partition;         
    int64_t      offset;            
        void        *metadata;          
        size_t       metadata_size;     
        void        *opaque;            
        rd_kafka_resp_err_t err;        
        void       *_private;           
} rd_kafka_topic_partition_t;


RD_EXPORT
void rd_kafka_topic_partition_destroy (rd_kafka_topic_partition_t *rktpar);


typedef struct rd_kafka_topic_partition_list_s {
        int cnt;               
        int size;              
        rd_kafka_topic_partition_t *elems; 
} rd_kafka_topic_partition_list_t;


RD_EXPORT
rd_kafka_topic_partition_list_t *rd_kafka_topic_partition_list_new (int size);


RD_EXPORT
void
rd_kafka_topic_partition_list_destroy (rd_kafka_topic_partition_list_t *rkparlist);

RD_EXPORT
rd_kafka_topic_partition_t *
rd_kafka_topic_partition_list_add (rd_kafka_topic_partition_list_t *rktparlist,
                                   const char *topic, int32_t partition);


RD_EXPORT
void
rd_kafka_topic_partition_list_add_range (rd_kafka_topic_partition_list_t
                                         *rktparlist,
                                         const char *topic,
                                         int32_t start, int32_t stop);



RD_EXPORT
int
rd_kafka_topic_partition_list_del (rd_kafka_topic_partition_list_t *rktparlist,
                   const char *topic, int32_t partition);


RD_EXPORT
int
rd_kafka_topic_partition_list_del_by_idx (
    rd_kafka_topic_partition_list_t *rktparlist,
    int idx);


RD_EXPORT
rd_kafka_topic_partition_list_t *
rd_kafka_topic_partition_list_copy (const rd_kafka_topic_partition_list_t *src);




RD_EXPORT
rd_kafka_resp_err_t rd_kafka_topic_partition_list_set_offset (
    rd_kafka_topic_partition_list_t *rktparlist,
    const char *topic, int32_t partition, int64_t offset);



RD_EXPORT
rd_kafka_topic_partition_t *
rd_kafka_topic_partition_list_find (rd_kafka_topic_partition_list_t *rktparlist,
                    const char *topic, int32_t partition);


RD_EXPORT void
rd_kafka_topic_partition_list_sort (rd_kafka_topic_partition_list_t *rktparlist,
                                    int (*cmp) (const void *a, const void *b,
                                                void *opaque),
                                    void *opaque);


typedef enum rd_kafka_vtype_t {
        RD_KAFKA_VTYPE_END,       
        RD_KAFKA_VTYPE_TOPIC,     
        RD_KAFKA_VTYPE_RKT,       
        RD_KAFKA_VTYPE_PARTITION, 
        RD_KAFKA_VTYPE_VALUE,     
        RD_KAFKA_VTYPE_KEY,       
        RD_KAFKA_VTYPE_OPAQUE,    
        RD_KAFKA_VTYPE_MSGFLAGS,  
        RD_KAFKA_VTYPE_TIMESTAMP, 
} rd_kafka_vtype_t;


#define RD_KAFKA_V_END RD_KAFKA_VTYPE_END

#define RD_KAFKA_V_TOPIC(topic)                                         \
        _LRK_TYPECHECK(RD_KAFKA_VTYPE_TOPIC, const char *, topic),      \
        (const char *)topic

#define RD_KAFKA_V_RKT(rkt)                                             \
        _LRK_TYPECHECK(RD_KAFKA_VTYPE_RKT, rd_kafka_topic_t *, rkt),    \
        (rd_kafka_topic_t *)rkt

#define RD_KAFKA_V_PARTITION(partition)                                 \
        _LRK_TYPECHECK(RD_KAFKA_VTYPE_PARTITION, int32_t, partition),   \
        (int32_t)partition

#define RD_KAFKA_V_VALUE(VALUE,LEN)                                     \
        _LRK_TYPECHECK2(RD_KAFKA_VTYPE_VALUE, void *, VALUE, size_t, LEN), \
        (void *)VALUE, (size_t)LEN

#define RD_KAFKA_V_KEY(KEY,LEN)                                         \
        _LRK_TYPECHECK2(RD_KAFKA_VTYPE_KEY, const void *, KEY, size_t, LEN), \
        (void *)KEY, (size_t)LEN

#define RD_KAFKA_V_OPAQUE(opaque)                                 \
        _LRK_TYPECHECK(RD_KAFKA_VTYPE_OPAQUE, void *, opaque),    \
        (void *)opaque

#define RD_KAFKA_V_MSGFLAGS(msgflags)                                 \
        _LRK_TYPECHECK(RD_KAFKA_VTYPE_MSGFLAGS, int, msgflags),       \
        (int)msgflags

#define RD_KAFKA_V_TIMESTAMP(timestamp)                                 \
        _LRK_TYPECHECK(RD_KAFKA_VTYPE_TIMESTAMP, int64_t, timestamp),   \
        (int64_t)timestamp

// FIXME: This doesn't show up in docs for some reason
// "Compound rd_kafka_message_t is not documented."

typedef struct rd_kafka_message_s {
    rd_kafka_resp_err_t err;   
    rd_kafka_topic_t *rkt;     
    int32_t partition;         
    void   *payload;           
    size_t  len;               
    void   *key;               
    size_t  key_len;           
    int64_t offset;            
    void  *_private;           
} rd_kafka_message_t;


RD_EXPORT
void rd_kafka_message_destroy(rd_kafka_message_t *rkmessage);




static RD_INLINE const char *
RD_UNUSED 
rd_kafka_message_errstr(const rd_kafka_message_t *rkmessage) {
    if (!rkmessage->err)
        return NULL;

    if (rkmessage->payload)
        return (const char *)rkmessage->payload;

    return rd_kafka_err2str(rkmessage->err);
}



RD_EXPORT
int64_t rd_kafka_message_timestamp (const rd_kafka_message_t *rkmessage,
                    rd_kafka_timestamp_type_t *tstype);



typedef enum {
    RD_KAFKA_CONF_UNKNOWN = -2, 
    RD_KAFKA_CONF_INVALID = -1, 
    RD_KAFKA_CONF_OK = 0        
} rd_kafka_conf_res_t;


RD_EXPORT
rd_kafka_conf_t *rd_kafka_conf_new(void);


RD_EXPORT
void rd_kafka_conf_destroy(rd_kafka_conf_t *conf);


RD_EXPORT
rd_kafka_conf_t *rd_kafka_conf_dup(const rd_kafka_conf_t *conf);


RD_EXPORT
rd_kafka_conf_res_t rd_kafka_conf_set(rd_kafka_conf_t *conf,
                       const char *name,
                       const char *value,
                       char *errstr, size_t errstr_size);


RD_EXPORT
void rd_kafka_conf_set_events(rd_kafka_conf_t *conf, int events);


RD_EXPORT
void rd_kafka_conf_set_dr_cb(rd_kafka_conf_t *conf,
                  void (*dr_cb) (rd_kafka_t *rk,
                         void *payload, size_t len,
                         rd_kafka_resp_err_t err,
                         void *opaque, void *msg_opaque));

RD_EXPORT
void rd_kafka_conf_set_dr_msg_cb(rd_kafka_conf_t *conf,
                                  void (*dr_msg_cb) (rd_kafka_t *rk,
                                                     const rd_kafka_message_t *
                                                     rkmessage,
                                                     void *opaque));


RD_EXPORT
void rd_kafka_conf_set_consume_cb (rd_kafka_conf_t *conf,
                                   void (*consume_cb) (rd_kafka_message_t *
                                                       rkmessage,
                                                       void *opaque));

RD_EXPORT
void rd_kafka_conf_set_rebalance_cb (
        rd_kafka_conf_t *conf,
        void (*rebalance_cb) (rd_kafka_t *rk,
                              rd_kafka_resp_err_t err,
                              rd_kafka_topic_partition_list_t *partitions,
                              void *opaque));



RD_EXPORT
void rd_kafka_conf_set_offset_commit_cb (
        rd_kafka_conf_t *conf,
        void (*offset_commit_cb) (rd_kafka_t *rk,
                                  rd_kafka_resp_err_t err,
                                  rd_kafka_topic_partition_list_t *offsets,
                                  void *opaque));


RD_EXPORT
void rd_kafka_conf_set_error_cb(rd_kafka_conf_t *conf,
                 void  (*error_cb) (rd_kafka_t *rk, int err,
                            const char *reason,
                            void *opaque));

RD_EXPORT
void rd_kafka_conf_set_throttle_cb (rd_kafka_conf_t *conf,
                    void (*throttle_cb) (
                        rd_kafka_t *rk,
                        const char *broker_name,
                        int32_t broker_id,
                        int throttle_time_ms,
                        void *opaque));


RD_EXPORT
void rd_kafka_conf_set_log_cb(rd_kafka_conf_t *conf,
              void (*log_cb) (const rd_kafka_t *rk, int level,
                                          const char *fac, const char *buf));


RD_EXPORT
void rd_kafka_conf_set_stats_cb(rd_kafka_conf_t *conf,
                 int (*stats_cb) (rd_kafka_t *rk,
                          char *json,
                          size_t json_len,
                          void *opaque));



RD_EXPORT
void rd_kafka_conf_set_socket_cb(rd_kafka_conf_t *conf,
                                  int (*socket_cb) (int domain, int type,
                                                    int protocol,
                                                    void *opaque));



RD_EXPORT void
rd_kafka_conf_set_connect_cb (rd_kafka_conf_t *conf,
                              int (*connect_cb) (int sockfd,
                                                 const struct sockaddr *addr,
                                                 int addrlen,
                                                 const char *id,
                                                 void *opaque));

RD_EXPORT void
rd_kafka_conf_set_closesocket_cb (rd_kafka_conf_t *conf,
                                  int (*closesocket_cb) (int sockfd,
                                                         void *opaque));



#ifndef _MSC_VER

RD_EXPORT
void rd_kafka_conf_set_open_cb (rd_kafka_conf_t *conf,
                                int (*open_cb) (const char *pathname,
                                                int flags, mode_t mode,
                                                void *opaque));
#endif

RD_EXPORT
void rd_kafka_conf_set_opaque(rd_kafka_conf_t *conf, void *opaque);

RD_EXPORT
void *rd_kafka_opaque(const rd_kafka_t *rk);



RD_EXPORT
void rd_kafka_conf_set_default_topic_conf (rd_kafka_conf_t *conf,
                                           rd_kafka_topic_conf_t *tconf);



RD_EXPORT
rd_kafka_conf_res_t rd_kafka_conf_get (const rd_kafka_conf_t *conf,
                                       const char *name,
                                       char *dest, size_t *dest_size);


RD_EXPORT
rd_kafka_conf_res_t rd_kafka_topic_conf_get (const rd_kafka_topic_conf_t *conf,
                                             const char *name,
                                             char *dest, size_t *dest_size);


RD_EXPORT
const char **rd_kafka_conf_dump(rd_kafka_conf_t *conf, size_t *cntp);


RD_EXPORT
const char **rd_kafka_topic_conf_dump(rd_kafka_topic_conf_t *conf,
                       size_t *cntp);

RD_EXPORT
void rd_kafka_conf_dump_free(const char **arr, size_t cnt);

RD_EXPORT
void rd_kafka_conf_properties_show(FILE *fp);

RD_EXPORT
rd_kafka_topic_conf_t *rd_kafka_topic_conf_new(void);


RD_EXPORT
rd_kafka_topic_conf_t *rd_kafka_topic_conf_dup(const rd_kafka_topic_conf_t
                        *conf);


RD_EXPORT
void rd_kafka_topic_conf_destroy(rd_kafka_topic_conf_t *topic_conf);


RD_EXPORT
rd_kafka_conf_res_t rd_kafka_topic_conf_set(rd_kafka_topic_conf_t *conf,
                         const char *name,
                         const char *value,
                         char *errstr, size_t errstr_size);

RD_EXPORT
void rd_kafka_topic_conf_set_opaque(rd_kafka_topic_conf_t *conf, void *opaque);


RD_EXPORT
void
rd_kafka_topic_conf_set_partitioner_cb (rd_kafka_topic_conf_t *topic_conf,
                    int32_t (*partitioner) (
                        const rd_kafka_topic_t *rkt,
                        const void *keydata,
                        size_t keylen,
                        int32_t partition_cnt,
                        void *rkt_opaque,
                        void *msg_opaque));

RD_EXPORT
int rd_kafka_topic_partition_available(const rd_kafka_topic_t *rkt,
                    int32_t partition);


/*******************************************************************
 *                                 *
 * Partitioners provided by rdkafka                                *
 *                                 *
 *******************************************************************/

RD_EXPORT
int32_t rd_kafka_msg_partitioner_random(const rd_kafka_topic_t *rkt,
                     const void *key, size_t keylen,
                     int32_t partition_cnt,
                     void *opaque, void *msg_opaque);

RD_EXPORT
int32_t rd_kafka_msg_partitioner_consistent (const rd_kafka_topic_t *rkt,
                     const void *key, size_t keylen,
                     int32_t partition_cnt,
                     void *opaque, void *msg_opaque);

RD_EXPORT
int32_t rd_kafka_msg_partitioner_consistent_random (const rd_kafka_topic_t *rkt,
           const void *key, size_t keylen,
           int32_t partition_cnt,
           void *opaque, void *msg_opaque);


RD_EXPORT
rd_kafka_t *rd_kafka_new(rd_kafka_type_t type, rd_kafka_conf_t *conf,
              char *errstr, size_t errstr_size);


RD_EXPORT
void        rd_kafka_destroy(rd_kafka_t *rk);



RD_EXPORT
const char *rd_kafka_name(const rd_kafka_t *rk);


RD_EXPORT
char *rd_kafka_memberid (const rd_kafka_t *rk);


RD_EXPORT
rd_kafka_topic_t *rd_kafka_topic_new(rd_kafka_t *rk, const char *topic,
                      rd_kafka_topic_conf_t *conf);



RD_EXPORT
void rd_kafka_topic_destroy(rd_kafka_topic_t *rkt);


RD_EXPORT
const char *rd_kafka_topic_name(const rd_kafka_topic_t *rkt);


RD_EXPORT
void *rd_kafka_topic_opaque (const rd_kafka_topic_t *rkt);


#define RD_KAFKA_PARTITION_UA  ((int32_t)-1)


RD_EXPORT
int rd_kafka_poll(rd_kafka_t *rk, int timeout_ms);


RD_EXPORT
void rd_kafka_yield (rd_kafka_t *rk);




RD_EXPORT rd_kafka_resp_err_t
rd_kafka_pause_partitions (rd_kafka_t *rk,
               rd_kafka_topic_partition_list_t *partitions);



RD_EXPORT rd_kafka_resp_err_t
rd_kafka_resume_partitions (rd_kafka_t *rk,
                rd_kafka_topic_partition_list_t *partitions);




RD_EXPORT rd_kafka_resp_err_t
rd_kafka_query_watermark_offsets (rd_kafka_t *rk,
              const char *topic, int32_t partition,
              int64_t *low, int64_t *high, int timeout_ms);


RD_EXPORT rd_kafka_resp_err_t
rd_kafka_get_watermark_offsets (rd_kafka_t *rk,
                const char *topic, int32_t partition,
                int64_t *low, int64_t *high);



RD_EXPORT rd_kafka_resp_err_t
rd_kafka_offsets_for_times (rd_kafka_t *rk,
                            rd_kafka_topic_partition_list_t *offsets,
                            int timeout_ms);


RD_EXPORT
void rd_kafka_mem_free (rd_kafka_t *rk, void *ptr);


RD_EXPORT
rd_kafka_queue_t *rd_kafka_queue_new(rd_kafka_t *rk);

RD_EXPORT
void rd_kafka_queue_destroy(rd_kafka_queue_t *rkqu);


RD_EXPORT
rd_kafka_queue_t *rd_kafka_queue_get_main (rd_kafka_t *rk);


RD_EXPORT
rd_kafka_queue_t *rd_kafka_queue_get_consumer (rd_kafka_t *rk);

RD_EXPORT
rd_kafka_queue_t *rd_kafka_queue_get_partition (rd_kafka_t *rk,
                                                const char *topic,
                                                int32_t partition);

RD_EXPORT
void rd_kafka_queue_forward (rd_kafka_queue_t *src, rd_kafka_queue_t *dst);

RD_EXPORT
rd_kafka_resp_err_t rd_kafka_set_log_queue (rd_kafka_t *rk,
                                            rd_kafka_queue_t *rkqu);


RD_EXPORT
size_t rd_kafka_queue_length (rd_kafka_queue_t *rkqu);


RD_EXPORT
void rd_kafka_queue_io_event_enable (rd_kafka_queue_t *rkqu, int fd,
                     const void *payload, size_t size);

#define RD_KAFKA_OFFSET_BEGINNING -2  
#define RD_KAFKA_OFFSET_END       -1  
#define RD_KAFKA_OFFSET_STORED -1000  
#define RD_KAFKA_OFFSET_INVALID -1001 
#define RD_KAFKA_OFFSET_TAIL_BASE -2000 /* internal: do not use */

#define RD_KAFKA_OFFSET_TAIL(CNT)  (RD_KAFKA_OFFSET_TAIL_BASE - (CNT))

RD_EXPORT
int rd_kafka_consume_start(rd_kafka_topic_t *rkt, int32_t partition,
                int64_t offset);

RD_EXPORT
int rd_kafka_consume_start_queue(rd_kafka_topic_t *rkt, int32_t partition,
                  int64_t offset, rd_kafka_queue_t *rkqu);

RD_EXPORT
int rd_kafka_consume_stop(rd_kafka_topic_t *rkt, int32_t partition);



RD_EXPORT
rd_kafka_resp_err_t rd_kafka_seek (rd_kafka_topic_t *rkt,
                                   int32_t partition,
                                   int64_t offset,
                                   int timeout_ms);


RD_EXPORT
rd_kafka_message_t *rd_kafka_consume(rd_kafka_topic_t *rkt, int32_t partition,
                      int timeout_ms);



RD_EXPORT
ssize_t rd_kafka_consume_batch(rd_kafka_topic_t *rkt, int32_t partition,
                int timeout_ms,
                rd_kafka_message_t **rkmessages,
                size_t rkmessages_size);



RD_EXPORT
int rd_kafka_consume_callback(rd_kafka_topic_t *rkt, int32_t partition,
                   int timeout_ms,
                   void (*consume_cb) (rd_kafka_message_t
                           *rkmessage,
                           void *opaque),
                   void *opaque);


RD_EXPORT
rd_kafka_message_t *rd_kafka_consume_queue(rd_kafka_queue_t *rkqu,
                        int timeout_ms);

RD_EXPORT
ssize_t rd_kafka_consume_batch_queue(rd_kafka_queue_t *rkqu,
                      int timeout_ms,
                      rd_kafka_message_t **rkmessages,
                      size_t rkmessages_size);

RD_EXPORT
int rd_kafka_consume_callback_queue(rd_kafka_queue_t *rkqu,
                     int timeout_ms,
                     void (*consume_cb) (rd_kafka_message_t
                             *rkmessage,
                             void *opaque),
                     void *opaque);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_offset_store(rd_kafka_topic_t *rkt,
                       int32_t partition, int64_t offset);
RD_EXPORT rd_kafka_resp_err_t
rd_kafka_subscribe (rd_kafka_t *rk,
                    const rd_kafka_topic_partition_list_t *topics);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_unsubscribe (rd_kafka_t *rk);


RD_EXPORT rd_kafka_resp_err_t
rd_kafka_subscription (rd_kafka_t *rk,
                       rd_kafka_topic_partition_list_t **topics);



RD_EXPORT
rd_kafka_message_t *rd_kafka_consumer_poll (rd_kafka_t *rk, int timeout_ms);

RD_EXPORT
rd_kafka_resp_err_t rd_kafka_consumer_close (rd_kafka_t *rk);



RD_EXPORT rd_kafka_resp_err_t
rd_kafka_assign (rd_kafka_t *rk,
                 const rd_kafka_topic_partition_list_t *partitions);

RD_EXPORT rd_kafka_resp_err_t
rd_kafka_assignment (rd_kafka_t *rk,
                     rd_kafka_topic_partition_list_t **partitions);




RD_EXPORT rd_kafka_resp_err_t
rd_kafka_commit (rd_kafka_t *rk, const rd_kafka_topic_partition_list_t *offsets,
                 int async);


RD_EXPORT rd_kafka_resp_err_t
rd_kafka_commit_message (rd_kafka_t *rk, const rd_kafka_message_t *rkmessage,
                         int async);


RD_EXPORT rd_kafka_resp_err_t
rd_kafka_commit_queue (rd_kafka_t *rk,
               const rd_kafka_topic_partition_list_t *offsets,
               rd_kafka_queue_t *rkqu,
               void (*cb) (rd_kafka_t *rk,
                   rd_kafka_resp_err_t err,
                   rd_kafka_topic_partition_list_t *offsets,
                   void *opaque),
               void *opaque);


RD_EXPORT rd_kafka_resp_err_t
rd_kafka_committed (rd_kafka_t *rk,
            rd_kafka_topic_partition_list_t *partitions,
            int timeout_ms);



RD_EXPORT rd_kafka_resp_err_t
rd_kafka_position (rd_kafka_t *rk,
           rd_kafka_topic_partition_list_t *partitions);


#define RD_KAFKA_MSG_F_FREE  0x1 
#define RD_KAFKA_MSG_F_COPY  0x2 
#define RD_KAFKA_MSG_F_BLOCK 0x4 
RD_EXPORT
int rd_kafka_produce(rd_kafka_topic_t *rkt, int32_t partition,
              int msgflags,
              void *payload, size_t len,
              const void *key, size_t keylen,
              void *msg_opaque);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_producev (rd_kafka_t *rk, ...);


RD_EXPORT
int rd_kafka_produce_batch(rd_kafka_topic_t *rkt, int32_t partition,
                            int msgflags,
                            rd_kafka_message_t *rkmessages, int message_cnt);




RD_EXPORT
rd_kafka_resp_err_t rd_kafka_flush (rd_kafka_t *rk, int timeout_ms);


typedef struct rd_kafka_metadata_broker {
        int32_t     id;             
        char       *host;           
        int         port;           
} rd_kafka_metadata_broker_t;

typedef struct rd_kafka_metadata_partition {
        int32_t     id;             
        rd_kafka_resp_err_t err;    
        int32_t     leader;         
        int         replica_cnt;    
        int32_t    *replicas;       
        int         isr_cnt;        
        int32_t    *isrs;           
} rd_kafka_metadata_partition_t;

typedef struct rd_kafka_metadata_topic {
        char       *topic;          
        int         partition_cnt;  
        struct rd_kafka_metadata_partition *partitions; 
        rd_kafka_resp_err_t err;    
} rd_kafka_metadata_topic_t;


typedef struct rd_kafka_metadata {
        int         broker_cnt;     
        struct rd_kafka_metadata_broker *brokers;  
        int         topic_cnt;      
        struct rd_kafka_metadata_topic *topics;    
        int32_t     orig_broker_id;   
        char       *orig_broker_name; 
} rd_kafka_metadata_t;


RD_EXPORT
rd_kafka_resp_err_t
rd_kafka_metadata (rd_kafka_t *rk, int all_topics,
                   rd_kafka_topic_t *only_rkt,
                   const struct rd_kafka_metadata **metadatap,
                   int timeout_ms);

RD_EXPORT
void rd_kafka_metadata_destroy(const struct rd_kafka_metadata *metadata);


struct rd_kafka_group_member_info {
        char *member_id;            
        char *client_id;            
        char *client_host;          
        void *member_metadata;      
        int   member_metadata_size; 
        void *member_assignment;    
        int   member_assignment_size; 
};

struct rd_kafka_group_info {
        struct rd_kafka_metadata_broker broker; 
        char *group;                            
        rd_kafka_resp_err_t err;                
        char *state;                            
        char *protocol_type;                    
        char *protocol;                         
        struct rd_kafka_group_member_info *members; 
        int   member_cnt;                       
};

struct rd_kafka_group_list {
        struct rd_kafka_group_info *groups;   
        int    group_cnt;                     
};


RD_EXPORT
rd_kafka_resp_err_t
rd_kafka_list_groups (rd_kafka_t *rk, const char *group,
                      const struct rd_kafka_group_list **grplistp,
                      int timeout_ms);

RD_EXPORT
void rd_kafka_group_list_destroy (const struct rd_kafka_group_list *grplist);


RD_EXPORT
int rd_kafka_brokers_add(rd_kafka_t *rk, const char *brokerlist);




RD_EXPORT RD_DEPRECATED
void rd_kafka_set_logger(rd_kafka_t *rk,
              void (*func) (const rd_kafka_t *rk, int level,
                    const char *fac, const char *buf));


RD_EXPORT
void rd_kafka_set_log_level(rd_kafka_t *rk, int level);


RD_EXPORT
void rd_kafka_log_print(const rd_kafka_t *rk, int level,
             const char *fac, const char *buf);


RD_EXPORT
void rd_kafka_log_syslog(const rd_kafka_t *rk, int level,
              const char *fac, const char *buf);


RD_EXPORT
int         rd_kafka_outq_len(rd_kafka_t *rk);



RD_EXPORT
void rd_kafka_dump(FILE *fp, rd_kafka_t *rk);



RD_EXPORT
int rd_kafka_thread_cnt(void);


RD_EXPORT
int rd_kafka_wait_destroyed(int timeout_ms);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_poll_set_consumer (rd_kafka_t *rk);


typedef int rd_kafka_event_type_t;
#define RD_KAFKA_EVENT_NONE          0x0
#define RD_KAFKA_EVENT_DR            0x1  
#define RD_KAFKA_EVENT_FETCH         0x2  
#define RD_KAFKA_EVENT_LOG           0x4  
#define RD_KAFKA_EVENT_ERROR         0x8  
#define RD_KAFKA_EVENT_REBALANCE     0x10 
#define RD_KAFKA_EVENT_OFFSET_COMMIT 0x20 
typedef struct rd_kafka_op_s rd_kafka_event_t;


RD_EXPORT
rd_kafka_event_type_t rd_kafka_event_type (const rd_kafka_event_t *rkev);

RD_EXPORT
const char *rd_kafka_event_name (const rd_kafka_event_t *rkev);


RD_EXPORT
void rd_kafka_event_destroy (rd_kafka_event_t *rkev);


RD_EXPORT
const rd_kafka_message_t *rd_kafka_event_message_next (rd_kafka_event_t *rkev);


RD_EXPORT
size_t rd_kafka_event_message_array (rd_kafka_event_t *rkev,
                     const rd_kafka_message_t **rkmessages,
                     size_t size);


RD_EXPORT
size_t rd_kafka_event_message_count (rd_kafka_event_t *rkev);


RD_EXPORT
rd_kafka_resp_err_t rd_kafka_event_error (rd_kafka_event_t *rkev);


RD_EXPORT
const char *rd_kafka_event_error_string (rd_kafka_event_t *rkev);



RD_EXPORT
void *rd_kafka_event_opaque (rd_kafka_event_t *rkev);


RD_EXPORT
int rd_kafka_event_log (rd_kafka_event_t *rkev,
            const char **fac, const char **str, int *level);


RD_EXPORT rd_kafka_topic_partition_list_t *
rd_kafka_event_topic_partition_list (rd_kafka_event_t *rkev);


RD_EXPORT rd_kafka_topic_partition_t *
rd_kafka_event_topic_partition (rd_kafka_event_t *rkev);


RD_EXPORT
rd_kafka_event_t *rd_kafka_queue_poll (rd_kafka_queue_t *rkqu, int timeout_ms);

RD_EXPORT
int rd_kafka_queue_poll_callback (rd_kafka_queue_t *rkqu, int timeout_ms);


#ifdef __cplusplus
}
#endif