totemknet.c
Go to the documentation of this file.
/*
* Copyright (c) 2016-2022 Red Hat, Inc.
*
* All rights reserved.
*
* Author: Christine Caulfield (ccaulfie@redhat.com)
* This software licensed under BSD license, the text of which follows:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* - Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* - Neither the name of the MontaVista Software, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <config.h>
#include <assert.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netdb.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <sys/param.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <pthread.h>
#include <sched.h>
#include <time.h>
#include <sys/time.h>
#include <sys/poll.h>
#include <sys/uio.h>
#include <limits.h>
#include <qb/qbdefs.h>
#include <qb/qbloop.h>
#ifdef HAVE_LIBNOZZLE
#include <libgen.h>
#include <libnozzle.h>
#endif
#include <corosync/sq.h>
#include <corosync/swab.h>
#include <corosync/logsys.h>
#include <corosync/icmap.h>
#include <corosync/totem/totemip.h>
#include "totemknet.h"
#include "main.h"
#include "util.h"
#include <libknet.h>
#include <corosync/totem/totemstats.h>
#ifndef MSG_NOSIGNAL
#define MSG_NOSIGNAL 0
#endif
#ifdef HAVE_LIBNOZZLE
static int setup_nozzle(void *knet_context);
#endif
/* Should match that used by cfg */
#define CFG_INTERFACE_STATUS_MAX_LEN 512
struct totemknet_instance {
struct crypto_instance *crypto_inst;
struct knet_handle_crypto_cfg last_good_crypto_cfg;
qb_loop_t *poll_handle;
knet_handle_t knet_handle;
int link_mode;
void *context;
int (*totemknet_deliver_fn) (
void *context,
const void *msg,
unsigned int msg_len,
const struct sockaddr_storage *system_from);
int (*totemknet_iface_change_fn) (
void *context,
const struct totem_ip_address *iface_address,
unsigned int link_no);
void (*totemknet_mtu_changed) (
void *context,
int net_mtu);
void (*totemknet_target_set_completed) (void *context);
/*
* Function and data used to log messages
*/
int totemknet_log_level_security;
int totemknet_log_level_error;
int totemknet_log_level_warning;
int totemknet_log_level_notice;
int totemknet_log_level_debug;
int totemknet_subsys_id;
int knet_subsys_id;
void (*totemknet_log_printf) (
int level,
int subsys,
const char *function,
const char *file,
int line,
const char *format,
...)__attribute__((format(printf, 6, 7)));
void *knet_context;
char iov_buffer[KNET_MAX_PACKET_SIZE + 1];
char *link_status[INTERFACE_MAX];
struct totem_ip_address my_ids[INTERFACE_MAX];
uint16_t ip_port[INTERFACE_MAX];
int our_nodeid;
int loopback_link;
struct totem_config *totem_config;
struct totem_ip_address token_target;
qb_loop_timer_handle timer_netif_check_timeout;
qb_loop_timer_handle timer_merge_detect_timeout;
int send_merge_detect_message;
unsigned int merge_detect_messages_sent_before_timeout;
int logpipes[2];
int knet_fd;
pthread_mutex_t log_mutex;
#ifdef HAVE_LIBNOZZLE
char *nozzle_name;
char *nozzle_ipaddr;
char *nozzle_prefix;
char *nozzle_macaddr;
nozzle_t nozzle_handle;
#endif
};
/* Awkward. But needed to get stats from knet */
struct totemknet_instance *global_instance;
struct work_item {
const void *msg;
unsigned int msg_len;
struct totemknet_instance *instance;
};
int totemknet_member_list_rebind_ip (
void *knet_context);
static int totemknet_configure_compression (
struct totemknet_instance *instance,
struct totem_config *totem_config);
static void totemknet_start_merge_detect_timeout(
void *knet_context);
static void totemknet_stop_merge_detect_timeout(
void *knet_context);
static void log_flush_messages (
void *knet_context);
static void totemknet_instance_initialize (struct totemknet_instance *instance)
{
int res;
memset (instance, 0, sizeof (struct totemknet_instance));
res = pthread_mutex_init(&instance->log_mutex, NULL);
/*
* There is not too much else what can be done.
*/
assert(res == 0);
}
#define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
do { \
(void)pthread_mutex_lock(&instance->log_mutex); \
instance->totemknet_log_printf ( \
level, subsys, function, file, line, \
(const char *)format, ##args); \
(void)pthread_mutex_unlock(&instance->log_mutex); \
} while (0);
#define knet_log_printf(level, format, args...) \
do { \
knet_log_printf_lock ( \
level, instance->totemknet_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
(const char *)format, ##args); \
} while (0);
#define libknet_log_printf(level, format, args...) \
do { \
knet_log_printf_lock ( \
level, instance->knet_subsys_id, \
__FUNCTION__, "libknet.h", __LINE__, \
(const char *)format, ##args); \
} while (0);
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
do { \
char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
instance->totemknet_log_printf ( \
level, instance->totemknet_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
fmt ": %s (%d)", ##args, _error_ptr, err_num); \
} while(0)
#ifdef HAVE_LIBNOZZLE
static inline uint8_t *ether_get_dst_mac(const unsigned char *packet, ssize_t packet_len)
{
/*
* Packet must be at least as long as one MAC address
*/
if (packet_len < 6) {
return NULL;
}
/*
* To ensure portability, this implementation parses raw packet data directly
* according to the Ethernet II standard.
*
* The system-provided struct ether_header is not used because its definition
* and the location of its header file vary between platforms like Linux, BSD,
* and Illumos. In the Ethernet II standard, the destination MAC address is
* always the first 6 bytes of the frame.
*/
return (uint8_t *)packet;
}
static inline int is_ether_addr_multicast(const uint8_t *addr)
{
return (addr[0] & 0x01);
}
static inline int is_ether_addr_zero(const uint8_t *addr)
{
return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
}
static int ether_host_filter_fn(void *private_data,
const unsigned char *outdata,
ssize_t outdata_len,
uint8_t tx_rx,
knet_node_id_t this_host_id,
knet_node_id_t src_host_id,
int8_t *channel,
knet_node_id_t *dst_host_ids,
size_t *dst_host_ids_entries)
{
uint8_t *dst_mac = ether_get_dst_mac(outdata, outdata_len);
uint16_t dst_host_id;
if (dst_mac == NULL) {
return -1;
}
if (is_ether_addr_zero(dst_mac))
return -1;
if (is_ether_addr_multicast(dst_mac)) {
return 1;
}
memmove(&dst_host_id, &dst_mac[4], 2);
dst_host_ids[0] = ntohs(dst_host_id);
*dst_host_ids_entries = 1;
return 0;
}
#endif
static int dst_host_filter_callback_fn(void *private_data,
const unsigned char *outdata,
ssize_t outdata_len,
uint8_t tx_rx,
knet_node_id_t this_host_id,
knet_node_id_t src_host_id,
int8_t *channel,
knet_node_id_t *dst_host_ids,
size_t *dst_host_ids_entries)
{
struct totem_message_header *header = (struct totem_message_header *)outdata;
int res;
#ifdef HAVE_LIBNOZZLE
if (*channel != 0) {
return ether_host_filter_fn(private_data,
outdata, outdata_len,
tx_rx,
this_host_id, src_host_id,
channel,
dst_host_ids,
dst_host_ids_entries);
}
#endif
if (header->target_nodeid) {
dst_host_ids[0] = header->target_nodeid;
*dst_host_ids_entries = 1;
res = 0; /* unicast message */
}
else {
*dst_host_ids_entries = 0;
res = 1; /* multicast message */
}
return res;
}
static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
{
struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
knet_handle_remove_datafd(instance->knet_handle, datafd);
}
}
static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
{
struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
// TODO: what? if anything.
knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
}
static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
{
struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
/* We don't need to tell corosync the actual knet MTU */
// instance->totemknet_mtu_changed(instance->context, data_mtu);
}
int totemknet_crypto_set (
void *knet_context,
const char *cipher_type,
const char *hash_type)
{
return (0);
}
static inline void ucast_sendmsg (
struct totemknet_instance *instance,
struct totem_ip_address *system_to,
const void *msg,
unsigned int msg_len)
{
int res = 0;
struct totem_message_header *header = (struct totem_message_header *)msg;
struct msghdr msg_ucast;
struct iovec iovec;
header->target_nodeid = system_to->nodeid;
iovec.iov_base = (void *)msg;
iovec.iov_len = msg_len;
/*
* Build unicast message
*/
memset(&msg_ucast, 0, sizeof(msg_ucast));
msg_ucast.msg_iov = (void *)&iovec;
msg_ucast.msg_iovlen = 1;
/*
* Transmit unicast message
* An error here is recovered by totemsrp
*/
res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
if (res < 0) {
KNET_LOGSYS_PERROR (errno, instance->totemknet_log_level_debug,
"sendmsg(ucast) failed (non-critical)");
}
}
static inline void mcast_sendmsg (
struct totemknet_instance *instance,
const void *msg,
unsigned int msg_len,
int only_active)
{
int res;
struct totem_message_header *header = (struct totem_message_header *)msg;
struct msghdr msg_mcast;
struct iovec iovec;
iovec.iov_base = (void *)msg;
iovec.iov_len = msg_len;
header->target_nodeid = 0;
/*
* Build multicast message
*/
memset(&msg_mcast, 0, sizeof(msg_mcast));
msg_mcast.msg_iov = (void *)&iovec;
msg_mcast.msg_iovlen = 1;
// log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
if (res < msg_len) {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
}
if (!only_active || instance->send_merge_detect_message) {
/*
* Current message was sent to all nodes
*/
instance->merge_detect_messages_sent_before_timeout++;
instance->send_merge_detect_message = 0;
}
}
static int node_compare(const void *aptr, const void *bptr)
{
uint16_t a,b;
a = *(uint16_t *)aptr;
b = *(uint16_t *)bptr;
return a > b;
}
#ifndef OWN_INDEX_NONE
#define OWN_INDEX_NONE -1
#endif
int totemknet_nodestatus_get (
void *knet_context,
unsigned int nodeid,
struct totem_node_status *node_status)
{
int i;
int res = 0;
struct knet_link_status link_status;
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
struct knet_host_status knet_host_status;
uint8_t link_list[KNET_MAX_LINK];
size_t num_links;
if (!instance->knet_handle) {
return CS_ERR_NOT_EXIST; /* Not using knet */
}
if (!node_status) {
return CS_ERR_INVALID_PARAM;
}
res = knet_host_get_status(instance->knet_handle,
nodeid,
&knet_host_status);
if (res) {
knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_host_status(%d) failed: %d", nodeid, res);
return (-1);
}
node_status->nodeid = nodeid;
node_status->reachable = knet_host_status.reachable;
node_status->remote = knet_host_status.remote;
node_status->external = knet_host_status.external;
#ifdef HAVE_KNET_ONWIRE_VER
res = knet_handle_get_onwire_ver(instance->knet_handle,
nodeid,
&node_status->onwire_min,
&node_status->onwire_max,
&node_status->onwire_ver);
if (res) {
knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_onwire_ver(%d) failed: %d", nodeid, res);
return (-1);
}
#endif
/* Get link info */
res = knet_link_get_link_list(instance->knet_handle,
nodeid, link_list, &num_links);
if (res) {
knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_list(%d) failed: %d", nodeid, res);
return (-1);
}
/* node_status[] has been zeroed for us in totempg.c */
for (i=0; i < num_links; i++) {
if (!instance->totem_config->interfaces[link_list[i]].configured) {
continue;
}
res = knet_link_get_status(instance->knet_handle,
nodeid,
link_list[i],
&link_status,
sizeof(link_status));
if (res == 0) {
node_status->link_status[link_list[i]].enabled = link_status.enabled;
node_status->link_status[link_list[i]].connected = link_status.connected;
node_status->link_status[link_list[i]].dynconnected = link_status.dynconnected;
node_status->link_status[link_list[i]].mtu = link_status.mtu;
memcpy(node_status->link_status[link_list[i]].src_ipaddr, link_status.src_ipaddr, KNET_MAX_HOST_LEN);
memcpy(node_status->link_status[link_list[i]].dst_ipaddr, link_status.dst_ipaddr, KNET_MAX_HOST_LEN);
} else {
knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_status(%d, %d) failed: %d", nodeid, link_list[i], res);
}
}
return res;
}
int totemknet_ifaces_get (void *knet_context,
char ***status,
unsigned int *iface_count)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
struct knet_link_status link_status;
knet_node_id_t host_list[KNET_MAX_HOST];
uint8_t link_list[KNET_MAX_LINK];
size_t num_hosts;
size_t num_links;
size_t link_idx;
int i,j;
char *ptr;
int res = 0;
/*
* Don't do the whole 'link_info' bit if the caller just wants
* a count of interfaces.
*/
if (status) {
int own_idx = OWN_INDEX_NONE;
res = knet_host_get_host_list(instance->knet_handle,
host_list, &num_hosts);
if (res) {
return (-1);
}
qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
for (j=0; j<num_hosts; j++) {
if (host_list[j] == instance->our_nodeid) {
own_idx = j;
break;
}
}
for (i=0; i<INTERFACE_MAX; i++) {
memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
if (own_idx != OWN_INDEX_NONE) {
instance->link_status[i][own_idx] = 'n';
}
instance->link_status[i][num_hosts] = '\0';
}
/* This is all a bit "inside-out" because "status" is a set of strings per link
* and knet orders things by host
*/
for (j=0; j<num_hosts; j++) {
if (own_idx != OWN_INDEX_NONE && j == own_idx) {
continue ;
}
res = knet_link_get_link_list(instance->knet_handle,
host_list[j], link_list, &num_links);
if (res) {
return (-1);
}
link_idx = 0;
for (i=0; i < num_links; i++) {
/*
* Skip over links that are unconfigured to corosync. This is basically
* link0 if corosync isn't using it for comms, as we will still
* have it set up for loopback.
*/
if (!instance->totem_config->interfaces[link_list[i]].configured) {
continue;
}
ptr = instance->link_status[link_idx++];
res = knet_link_get_status(instance->knet_handle,
host_list[j],
link_list[i],
&link_status,
sizeof(link_status));
if (res == 0) {
ptr[j] = '0' + (link_status.enabled |
link_status.connected<<1 |
link_status.dynconnected<<2);
}
else {
knet_log_printf (LOGSYS_LEVEL_ERROR,
"totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
ptr[j] = '?';
}
}
}
*status = instance->link_status;
}
*iface_count = INTERFACE_MAX;
return (res);
}
int totemknet_finalize (
void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
int i,j;
static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
uint8_t links[KNET_MAX_LINK];
size_t num_nodes;
size_t num_links;
knet_log_printf(LOG_DEBUG, "totemknet: finalize");
qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
/*
* Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
*/
res = knet_handle_setfwd(instance->knet_handle, 0);
if (res) {
knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
}
res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
if (res) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
/* Crash out anyway */
goto finalise_error;
}
/* Tidily shut down all nodes & links. */
for (i=0; i<num_nodes; i++) {
res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
if (res) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
goto finalise_error;
}
for (j=0; j<num_links; j++) {
res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
if (res) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
}
res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
if (res) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
}
}
res = knet_host_remove(instance->knet_handle, nodes[i]);
if (res) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
}
}
finalise_error:
res = knet_handle_free(instance->knet_handle);
if (res) {
knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
}
totemknet_stop_merge_detect_timeout(instance);
log_flush_messages(instance);
/*
* Error is deliberately ignored
*/
(void)pthread_mutex_destroy(&instance->log_mutex);
return (res);
}
static int log_deliver_fn (
int fd,
int revents,
void *data)
{
struct totemknet_instance *instance = (struct totemknet_instance *)data;
char buffer[sizeof(struct knet_log_msg)*4];
char *bufptr = buffer;
int done = 0;
int len;
len = read(fd, buffer, sizeof(buffer));
while (done < len) {
struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
switch (msg->msglevel) {
case KNET_LOG_ERR:
libknet_log_printf (LOGSYS_LEVEL_ERROR, "%s: %s",
knet_log_get_subsystem_name(msg->subsystem),
msg->msg);
break;
case KNET_LOG_WARN:
libknet_log_printf (LOGSYS_LEVEL_WARNING, "%s: %s",
knet_log_get_subsystem_name(msg->subsystem),
msg->msg);
break;
case KNET_LOG_INFO:
libknet_log_printf (LOGSYS_LEVEL_INFO, "%s: %s",
knet_log_get_subsystem_name(msg->subsystem),
msg->msg);
break;
case KNET_LOG_DEBUG:
libknet_log_printf (LOGSYS_LEVEL_DEBUG, "%s: %s",
knet_log_get_subsystem_name(msg->subsystem),
msg->msg);
break;
#ifdef KNET_LOG_TRACE
case KNET_LOG_TRACE:
libknet_log_printf (LOGSYS_LEVEL_TRACE, "%s: %s",
knet_log_get_subsystem_name(msg->subsystem),
msg->msg);
break;
#endif
}
bufptr += sizeof(struct knet_log_msg);
done += sizeof(struct knet_log_msg);
}
return 0;
}
static int data_deliver_fn (
int fd,
int revents,
void *data)
{
struct totemknet_instance *instance = (struct totemknet_instance *)data;
struct msghdr msg_hdr;
struct iovec iov_recv;
struct sockaddr_storage system_from;
ssize_t msg_len;
char *data_ptr = instance->iov_buffer;
iov_recv.iov_base = instance->iov_buffer;
iov_recv.iov_len = KNET_MAX_PACKET_SIZE + 1;
memset(&msg_hdr, 0, sizeof(msg_hdr));
msg_hdr.msg_name = &system_from;
msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
msg_hdr.msg_iov = &iov_recv;
msg_hdr.msg_iovlen = 1;
msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
if (msg_len <= 0) {
return (0);
}
if (msg_len >= KNET_MAX_PACKET_SIZE + 1) {
/*
* It this happens it is real bug, because knet always sends packet with maximum size
* of KNET_MAX_PACKET_SIZE.
* If received packet is MAX_PACKET_SIZE + 1 it means packet was truncated
* (iov_buffer size and iov_len are intentionally set to KNET_MAX_PACKET_SIZE + 1).
*/
knet_log_printf(instance->totemknet_log_level_error,
"Received truncated packet. Please report this bug. Dropping packet.");
return (0);
}
/*
* If it's from the knet fd then it will have the optional knet header on it
*/
#ifdef KNET_DATAFD_FLAG_RX_RETURN_INFO
if (fd == instance->knet_fd) {
struct knet_datafd_header *datafd_header = (struct knet_datafd_header *)data_ptr;
/* knet_log_printf (LOGSYS_LEVEL_DEBUG, "Packet from knet_fd nodeid: %d\n", datafd_header->src_nodeid); */
/* Advance past the ACTUAL header size, not the size we think it might be */
data_ptr += datafd_header->size;
msg_len -= datafd_header->size;
}
#endif
/*
* Handle incoming message
*/
instance->totemknet_deliver_fn (
instance->context,
data_ptr,
msg_len,
&system_from);
return (0);
}
static void timer_function_netif_check_timeout (
void *data)
{
struct totemknet_instance *instance = (struct totemknet_instance *)data;
int i;
int res = 0;
for (i=0; i < INTERFACE_MAX; i++) {
if (!instance->totem_config->interfaces[i].configured) {
continue;
}
res = instance->totemknet_iface_change_fn (instance->context,
&instance->my_ids[i],
i);
}
if (res != 0) {
/* This is only called at startup, so we can quit here.
Refresh takes a different path */
corosync_exit_error(COROSYNC_DONE_MAINCONFIGREAD);
}
}
static void knet_set_access_list_config(struct totemknet_instance *instance)
{
#ifdef HAVE_KNET_ACCESS_LIST
uint32_t value;
cs_error_t err;
value = instance->totem_config->block_unlisted_ips;
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
err = knet_handle_enable_access_lists(instance->knet_handle, value);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
}
#endif
}
void totemknet_configure_log_level(void)
{
int logsys_log_mode;
int knet_log_mode = KNET_LOG_INFO;
uint8_t s;
int err;
if (!global_instance || !global_instance->knet_handle) {
return;
}
/* Reconfigure logging level */
logsys_log_mode = logsys_config_debug_get("KNET");
switch (logsys_log_mode) {
case LOGSYS_DEBUG_OFF:
knet_log_mode = KNET_LOG_INFO;
break;
case LOGSYS_DEBUG_ON:
knet_log_mode = KNET_LOG_DEBUG;
break;
case LOGSYS_DEBUG_TRACE:
#ifdef KNET_LOG_TRACE
knet_log_mode = KNET_LOG_TRACE;
#else
knet_log_mode = KNET_LOG_DEBUG;
#endif
break;
}
log_printf (LOGSYS_LEVEL_DEBUG, "totemknet setting log level %s", knet_log_get_loglevel_name(knet_log_mode));
err = 0;
for (s = 0; s<KNET_MAX_SUBSYSTEMS; s++) {
err = knet_log_set_loglevel(global_instance->knet_handle, s, knet_log_mode);
}
/* If one fails, they all fail. no point in issuing KNET_MAX_SUBSYSTEMS errors */
if (err) {
log_printf (LOGSYS_LEVEL_ERROR, "totemknet failed to set log level: %s", strerror(errno));
}
}
/* NOTE: this relies on the fact that totem_reload_notify() is called first */
static void totemknet_refresh_config(
int32_t event,
const char *key_name,
struct icmap_notify_value new_val,
struct icmap_notify_value old_val,
void *user_data)
{
uint8_t reloading;
int after_reload;
uint32_t link_no;
size_t num_nodes;
knet_node_id_t host_ids[KNET_MAX_HOST];
int i;
int err;
struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
ENTER();
/*
* If a full reload is in progress then don't do anything until it's done and
* can reconfigure it all atomically
*/
if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
return;
}
after_reload = (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0);
knet_set_access_list_config(instance);
if (strcmp(key_name, "totem.knet_pmtud_interval") == 0 || after_reload) {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %u",
instance->totem_config->knet_pmtud_interval);
err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
}
}
if (strcmp(key_name, "totem.knet_mtu") == 0 || after_reload) {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_mtu now %u", instance->totem_config->knet_mtu);
err = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud failed");
}
}
/* Configure link parameters for each node */
err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
if (err != 0) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
}
for (i=0; i<num_nodes; i++) {
int linkerr = 0;
for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
continue;
}
err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
instance->totem_config->interfaces[link_no].knet_ping_interval,
instance->totem_config->interfaces[link_no].knet_ping_timeout,
instance->totem_config->interfaces[link_no].knet_ping_precision);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
linkerr = err;
}
err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
instance->totem_config->interfaces[link_no].knet_pong_count);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
linkerr = err;
}
err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
instance->totem_config->interfaces[link_no].knet_link_priority);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
linkerr = err;
}
}
if (linkerr) {
icmap_set_string("config.reload_error_message", "Failed to set knet ping timers(2)");
}
}
/* Log levels get reconfigured from logconfig.c as that happens last in the reload */
LEAVE();
}
static void totemknet_add_config_notifications(struct totemknet_instance *instance)
{
icmap_track_t icmap_track_totem = NULL;
icmap_track_t icmap_track_reload = NULL;
ENTER();
icmap_track_add("totem.",
ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX,
totemknet_refresh_config,
instance,
&icmap_track_totem);
icmap_track_add("config.totemconfig_reload_in_progress",
ICMAP_TRACK_ADD | ICMAP_TRACK_MODIFY,
totemknet_refresh_config,
instance,
&icmap_track_reload);
LEAVE();
}
static int totemknet_is_crypto_enabled(const struct totemknet_instance *instance)
{
return (!(strcmp(instance->totem_config->crypto_cipher_type, "none") == 0 &&
strcmp(instance->totem_config->crypto_hash_type, "none") == 0));
}
static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
{
struct knet_handle_crypto_cfg crypto_cfg;
int res;
/* These have already been validated */
memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
crypto_cfg.private_key_len = instance->totem_config->private_key_len;
#ifdef HAVE_KNET_CRYPTO_RECONF
knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
crypto_cfg.crypto_model,
crypto_cfg.crypto_cipher_type,
crypto_cfg.crypto_hash_type,
instance->totem_config->crypto_index
);
/* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
if (!totemknet_is_crypto_enabled(instance)) {
res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
if (res) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
}
}
/* use_config will be called later when all nodes are synced */
res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
if (res == 0) {
/* Keep a copy in case it fails in future */
memcpy(&instance->last_good_crypto_cfg, &crypto_cfg, sizeof(crypto_cfg));
}
if (res == -1) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
goto exit_error;
}
if (res == -2) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
goto exit_error;
}
#else
knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
crypto_cfg.crypto_model,
crypto_cfg.crypto_cipher_type,
crypto_cfg.crypto_hash_type
);
res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
if (res == -1) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
goto exit_error;
}
if (res == -2) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
goto exit_error;
}
#endif
exit_error:
#ifdef HAVE_KNET_CRYPTO_RECONF
if (res) {
icmap_set_string("config.reload_error_message", "Failed to set crypto parameters");
/* Restore the old values in cmap & totem_config */
icmap_set_string("totem.crypto_cipher", instance->last_good_crypto_cfg.crypto_cipher_type);
icmap_set_string("totem.crypto_hash", instance->last_good_crypto_cfg.crypto_hash_type);
icmap_set_string("totem.crypto_model", instance->last_good_crypto_cfg.crypto_model);
memcpy(instance->totem_config->crypto_hash_type, instance->last_good_crypto_cfg.crypto_hash_type,
sizeof(instance->last_good_crypto_cfg.crypto_hash_type));
memcpy(instance->totem_config->crypto_cipher_type, instance->last_good_crypto_cfg.crypto_cipher_type,
sizeof(instance->last_good_crypto_cfg.crypto_cipher_type));
memcpy(instance->totem_config->crypto_model, instance->last_good_crypto_cfg.crypto_model,
sizeof(instance->last_good_crypto_cfg.crypto_model));
}
#endif
return res;
}
/*
* Create an instance
*/
int totemknet_initialize (
qb_loop_t *poll_handle,
void **knet_context,
struct totem_config *totem_config,
totemsrp_stats_t *stats,
void *context,
int (*deliver_fn) (
void *context,
const void *msg,
unsigned int msg_len,
const struct sockaddr_storage *system_from),
int (*iface_change_fn) (
void *context,
const struct totem_ip_address *iface_address,
unsigned int link_no),
void (*mtu_changed) (
void *context,
int net_mtu),
void (*target_set_completed) (
void *context))
{
struct totemknet_instance *instance;
char *tmp_str;
int8_t channel=0;
int allow_knet_handle_fallback=0;
int res;
int i;
instance = malloc (sizeof (struct totemknet_instance));
if (instance == NULL) {
return (-1);
}
totemknet_instance_initialize (instance);
instance->totem_config = totem_config;
/*
* Configure logging
*/
instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
instance->totemknet_log_level_error = totem_config->totem_logging_configuration.log_level_error;
instance->totemknet_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
instance->totemknet_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
instance->totemknet_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
instance->totemknet_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
instance->totemknet_log_printf = totem_config->totem_logging_configuration.log_printf;
instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
/*
* Initialize local variables for totemknet
*/
instance->our_nodeid = instance->totem_config->node_id;
for (i=0; i< INTERFACE_MAX; i++) {
totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
instance->my_ids[i].nodeid = instance->our_nodeid;
instance->ip_port[i] = totem_config->interfaces[i].ip_port;
/* Needed for totemsrp */
totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
}
instance->poll_handle = poll_handle;
instance->context = context;
instance->totemknet_deliver_fn = deliver_fn;
instance->totemknet_iface_change_fn = iface_change_fn;
instance->totemknet_mtu_changed = mtu_changed;
instance->totemknet_target_set_completed = target_set_completed;
instance->loopback_link = 0;
res = pipe(instance->logpipes);
if (res == -1) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
goto exit_error;
}
if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
goto exit_error;
}
if (icmap_get_string("system.allow_knet_handle_fallback", &tmp_str) == CS_OK) {
if (strcmp(tmp_str, "yes") == 0) {
allow_knet_handle_fallback = 1;
}
free(tmp_str);
}
#if defined(KNET_API_VER) && (KNET_API_VER == 2)
instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
#else
instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
#endif
if (allow_knet_handle_fallback && !instance->knet_handle && errno == ENAMETOOLONG) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
#if defined(KNET_API_VER) && (KNET_API_VER == 2)
instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
#else
instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
#endif
}
if (!instance->knet_handle) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
goto exit_error;
}
knet_set_access_list_config(instance);
res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
}
res = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_set failed");
}
#ifdef HAVE_KNET_SETPRIO_DSCP
res = knet_handle_setprio_dscp(instance->knet_handle, instance->totem_config->ip_dscp);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_setprio_dscp failed");
}
#endif
res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
}
res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
}
res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
}
res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
}
global_instance = instance;
/* Setup knet logging level */
totemknet_configure_log_level();
/* Get an fd into knet */
instance->knet_fd = 0;
#ifdef KNET_DATAFD_FLAG_RX_RETURN_INFO
res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel, KNET_DATAFD_FLAG_RX_RETURN_INFO);
#else
res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
#endif
if (res) {
knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
goto exit_error;
}
/* Enable crypto if requested */
#ifdef HAVE_KNET_CRYPTO_RECONF
if (totemknet_is_crypto_enabled(instance)) {
res = totemknet_set_knet_crypto(instance);
if (res == 0) {
res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
if (res) {
knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
goto exit_error;
}
} else {
knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
goto exit_error;
}
res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
if (res) {
knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
goto exit_error;
}
} else {
res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
if (res) {
knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
goto exit_error;
}
}
#else
if (totemknet_is_crypto_enabled(instance)) {
res = totemknet_set_knet_crypto(instance);
if (res) {
knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
goto exit_error;
}
}
#endif
/* Set up compression */
if (strcmp(totem_config->knet_compression_model, "none") != 0) {
/* Not fatal, but will log */
(void)totemknet_configure_compression(instance, totem_config);
}
knet_handle_setfwd(instance->knet_handle, 1);
instance->link_mode = KNET_LINK_POLICY_PASSIVE;
if (strcmp(instance->totem_config->link_mode, "active")==0) {
instance->link_mode = KNET_LINK_POLICY_ACTIVE;
}
if (strcmp(instance->totem_config->link_mode, "rr")==0) {
instance->link_mode = KNET_LINK_POLICY_RR;
}
for (i=0; i<INTERFACE_MAX; i++) {
instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
if (!instance->link_status[i]) {
goto exit_error;
}
}
qb_loop_poll_add (instance->poll_handle,
QB_LOOP_MED,
instance->logpipes[0],
POLLIN, instance, log_deliver_fn);
qb_loop_poll_add (instance->poll_handle,
QB_LOOP_HIGH,
instance->knet_fd,
POLLIN, instance, data_deliver_fn);
/*
* Upper layer isn't ready to receive message because it hasn't
* initialized yet. Add short timer to check the interfaces.
*/
qb_loop_timer_add (instance->poll_handle,
QB_LOOP_MED,
100*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
totemknet_start_merge_detect_timeout(instance);
/* Start listening for config changes */
totemknet_add_config_notifications(instance);
/* Add stats keys to icmap */
stats_knet_add_handle();
knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
*knet_context = instance;
return (0);
exit_error:
log_flush_messages(instance);
free(instance);
return (-1);
}
void *totemknet_buffer_alloc (void)
{
/* Need to have space for a message AND a struct mcast in case of encapsulated messages */
return malloc(KNET_MAX_PACKET_SIZE + 512);
}
void totemknet_buffer_release (void *ptr)
{
return free (ptr);
}
int totemknet_processor_count_set (
void *knet_context,
int processor_count)
{
return (0);
}
int totemknet_recv_flush (void *knet_context)
{
return (0);
}
int totemknet_send_flush (void *knet_context)
{
return (0);
}
int totemknet_token_send (
void *knet_context,
const void *msg,
unsigned int msg_len)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
return (res);
}
int totemknet_mcast_flush_send (
void *knet_context,
const void *msg,
unsigned int msg_len)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
mcast_sendmsg (instance, msg, msg_len, 0);
return (res);
}
int totemknet_mcast_noflush_send (
void *knet_context,
const void *msg,
unsigned int msg_len)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
mcast_sendmsg (instance, msg, msg_len, 1);
return (res);
}
extern int totemknet_iface_check (void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
return (res);
}
extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
}
int totemknet_token_target_set (
void *knet_context,
unsigned int nodeid)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
instance->token_target.nodeid = nodeid;
instance->totemknet_target_set_completed (instance->context);
return (res);
}
extern int totemknet_recv_mcast_empty (
void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
unsigned int res;
struct sockaddr_storage system_from;
struct msghdr msg_hdr;
struct iovec iov_recv;
struct pollfd ufd;
int nfds;
int msg_processed = 0;
iov_recv.iov_base = instance->iov_buffer;
iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
memset(&msg_hdr, 0, sizeof(msg_hdr));
msg_hdr.msg_name = &system_from;
msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
msg_hdr.msg_iov = &iov_recv;
msg_hdr.msg_iovlen = 1;
do {
ufd.fd = instance->knet_fd;
ufd.events = POLLIN;
nfds = poll (&ufd, 1, 0);
if (nfds == 1 && ufd.revents & POLLIN) {
res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
if (res != -1) {
msg_processed = 1;
} else {
msg_processed = -1;
}
}
} while (nfds == 1);
return (msg_processed);
}
int totemknet_iface_set (void *knet_context,
const struct totem_ip_address *local_addr,
unsigned short ip_port,
unsigned int iface_no)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
totemip_copy(&instance->my_ids[iface_no], local_addr);
knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
instance->ip_port[iface_no] = ip_port;
return 0;
}
int totemknet_member_add (
void *knet_context,
const struct totem_ip_address *local,
const struct totem_ip_address *member,
int link_no)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int err;
int port = instance->ip_port[link_no];
struct sockaddr_storage remote_ss;
struct sockaddr_storage local_ss;
int addrlen;
int i;
int host_found = 0;
knet_node_id_t host_ids[KNET_MAX_HOST];
size_t num_host_ids;
/* Only create 1 loopback link and use link 0 */
if (member->nodeid == instance->our_nodeid) {
if (!instance->loopback_link) {
link_no = 0;
instance->loopback_link = 1;
} else {
/* Already done */
return 0;
}
}
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
/* Only add the host if it doesn't already exist in knet */
err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
return -1;
}
for (i=0; i<num_host_ids; i++) {
if (host_ids[i] == member->nodeid) {
host_found = 1;
}
}
if (!host_found) {
err = knet_host_add(instance->knet_handle, member->nodeid);
if (err != 0 && errno != EEXIST) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
return -1;
}
} else {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
}
if (err == 0) {
if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
return -1;
}
}
memset(&local_ss, 0, sizeof(local_ss));
memset(&remote_ss, 0, sizeof(remote_ss));
/* Casts to remove const */
totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
if (member->nodeid == instance->our_nodeid) {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
KNET_TRANSPORT_LOOPBACK,
&local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
}
else {
err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
instance->totem_config->interfaces[link_no].knet_transport,
&local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
}
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
return -1;
}
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
instance->totem_config->interfaces[link_no].knet_link_priority);
err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
instance->totem_config->interfaces[link_no].knet_link_priority);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
}
/*
* Ping timeouts may be 0 here for a newly added interface (on a reload),
* so we leave this till later, it will get done in totemknet_refresh_config.
* For the initial startup, we are all preset and ready to go from here.
*/
if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
instance->totem_config->interfaces[link_no].knet_ping_interval,
instance->totem_config->interfaces[link_no].knet_ping_timeout,
instance->totem_config->interfaces[link_no].knet_ping_precision);
if (err) {
/* Flush logs before reporting this error so that the knet message prints before ours */
int saved_errno = errno;
log_flush_messages(instance);
errno = saved_errno;
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
icmap_set_string("config.reload_error_message", "Failed to set knet ping timers");
return -1;
}
err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
instance->totem_config->interfaces[link_no].knet_pong_count);
if (err) {
/* Flush logs before reporting this error so that the knet message prints before ours */
int saved_errno = errno;
log_flush_messages(instance);
errno = saved_errno;
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
icmap_set_string("config.reload_error_message", "Failed to set knet pong count");
return -1;
}
}
err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
if (err) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
return -1;
}
/* register stats */
stats_knet_add_member(member->nodeid, link_no);
return (0);
}
int totemknet_member_remove (
void *knet_context,
const struct totem_ip_address *token_target,
int link_no)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res;
uint8_t link_list[KNET_MAX_LINK];
size_t num_links;
knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
/* Don't remove the link with the loopback on it until we shut down */
if (token_target->nodeid == instance->our_nodeid) {
return 0;
}
/* Tidy stats */
stats_knet_del_member(token_target->nodeid, link_no);
/* Remove the link first */
res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
if (res != 0) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
return res;
}
res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
if (res != 0) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
return res;
}
/* If this is the last link, then remove the node */
res = knet_link_get_link_list(instance->knet_handle,
token_target->nodeid, link_list, &num_links);
if (res) {
return (0); /* not really failure */
}
if (num_links == 0) {
res = knet_host_remove(instance->knet_handle, token_target->nodeid);
}
return res;
}
int totemknet_member_list_rebind_ip (
void *knet_context)
{
return (0);
}
static int totemknet_configure_compression (
struct totemknet_instance *instance,
struct totem_config *totem_config)
{
struct knet_handle_compress_cfg compress_cfg;
int res = 0;
assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
compress_cfg.compress_level = totem_config->knet_compression_level;
res = knet_handle_compress(instance->knet_handle, &compress_cfg);
if (res) {
KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
}
return res;
}
int totemknet_reconfigure (
void *knet_context,
struct totem_config *totem_config)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res = 0;
(void)totemknet_configure_compression(instance, totem_config);
#ifdef HAVE_LIBNOZZLE
/* Set up nozzle device(s). Return code is ignored, because inability
* configure nozzle is not fatal problem, errors are logged and
* there is not much else we can do */
(void)setup_nozzle(instance);
#endif
if (totem_config->crypto_changed) {
/* Flip crypto_index */
totem_config->crypto_index = 3-totem_config->crypto_index;
res = totemknet_set_knet_crypto(instance);
if (res == 0) {
knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
totem_config->crypto_model,
totem_config->crypto_cipher_type,
totem_config->crypto_hash_type);
} else {
icmap_set_string("config.reload_error_message", "Failed to set knet crypto");
}
}
return (res);
}
int totemknet_crypto_reconfigure_phase (
void *knet_context,
struct totem_config *totem_config,
cfg_message_crypto_reconfig_phase_t phase)
{
#ifdef HAVE_KNET_CRYPTO_RECONF
int res;
int config_to_use;
int config_to_clear;
struct knet_handle_crypto_cfg crypto_cfg;
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
switch (phase) {
case CRYPTO_RECONFIG_PHASE_ACTIVATE:
config_to_use = totem_config->crypto_index;
if (!totemknet_is_crypto_enabled(instance)) {
config_to_use = 0; /* we are clearing it */
}
/* Enable the new config on this node */
res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
if (res == -1) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
}
break;
case CRYPTO_RECONFIG_PHASE_CLEANUP:
/*
* All nodes should now have the new config. clear the old one out
* OR disable crypto entirely if that's what the new config insists on.
*/
config_to_clear = 3-totem_config->crypto_index;
knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
strcpy(crypto_cfg.crypto_model, "none");
strcpy(crypto_cfg.crypto_cipher_type, "none");
strcpy(crypto_cfg.crypto_hash_type, "none");
res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
if (res == -1) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
}
if (res == -2) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
}
/* If crypto is enabled then disable all cleartext reception */
if (totemknet_is_crypto_enabled(instance)) {
res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
if (res) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
}
}
}
#endif
return 0;
}
void totemknet_stats_clear (
void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
(void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
}
/* For the stats module */
int totemknet_link_get_status (
knet_node_id_t node, uint8_t link_no,
struct knet_link_status *status)
{
int res;
int ret = CS_OK;
/* We are probably not using knet */
if (!global_instance) {
return CS_ERR_NOT_EXIST;
}
if (link_no >= INTERFACE_MAX) {
return CS_ERR_NOT_EXIST; /* Invalid link number */
}
res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
if (res) {
switch (errno) {
case EINVAL:
ret = CS_ERR_INVALID_PARAM;
break;
case EBUSY:
ret = CS_ERR_BUSY;
break;
case EDEADLK:
ret = CS_ERR_TRY_AGAIN;
break;
default:
ret = CS_ERR_LIBRARY;
break;
}
}
return (ret);
}
int totemknet_handle_get_stats (
struct knet_handle_stats *stats)
{
int res;
/* We are probably not using knet */
if (!global_instance) {
return CS_ERR_NOT_EXIST;
}
res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
if (res != 0) {
return (qb_to_cs_error(-errno));
}
return CS_OK;
}
static void timer_function_merge_detect_timeout (
void *data)
{
struct totemknet_instance *instance = (struct totemknet_instance *)data;
if (instance->merge_detect_messages_sent_before_timeout == 0) {
instance->send_merge_detect_message = 1;
}
instance->merge_detect_messages_sent_before_timeout = 0;
totemknet_start_merge_detect_timeout(instance);
}
static void totemknet_start_merge_detect_timeout(
void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
qb_loop_timer_add(instance->poll_handle,
QB_LOOP_MED,
instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_merge_detect_timeout,
&instance->timer_merge_detect_timeout);
}
static void totemknet_stop_merge_detect_timeout(
void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
qb_loop_timer_del(instance->poll_handle,
instance->timer_merge_detect_timeout);
}
static void log_flush_messages (void *knet_context)
{
struct pollfd pfd;
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int cont;
cont = 1;
while (cont) {
pfd.fd = instance->logpipes[0];
pfd.events = POLLIN;
pfd.revents = 0;
if ((poll(&pfd, 1, 0) > 0) &&
(pfd.revents & POLLIN) &&
(log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
cont = 1;
} else {
cont = 0;
}
}
}
#ifdef HAVE_LIBNOZZLE
#define NOZZLE_NAME "nozzle.name"
#define NOZZLE_IPADDR "nozzle.ipaddr"
#define NOZZLE_PREFIX "nozzle.ipprefix"
#define NOZZLE_MACADDR "nozzle.macaddr"
#define NOZZLE_CHANNEL 1
static char *get_nozzle_script_dir(void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
char filename[PATH_MAX + FILENAME_MAX + 1];
static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
int res;
const char *dirname_res;
/*
* Build script directory based on corosync.conf file location
*/
res = snprintf(filename, sizeof(filename), "%s",
corosync_get_config_file());
if (res >= sizeof(filename)) {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
return NULL;
}
dirname_res = dirname(filename);
res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
dirname_res, "updown.d");
if (res >= sizeof(updown_dirname)) {
knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
return NULL;
}
return updown_dirname;
}
/*
* Deliberately doesn't return the status as caller doesn't care.
* The result will be logged though
*/
static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
{
int res;
char *exec_string;
res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
if (res == -1 && errno != ENOENT) {
knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
} else if (res == -2) {
knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
}
}
/*
* Reparse IP address to add in our node ID
* IPv6 addresses must end in '::'
* IPv4 addresses must just be valid
* '/xx' lengths are optional for IPv6, mandatory for IPv4
*
* Returns the modified IP address as a string to pass into libnozzle
*/
static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
const char *input_addr,
const char *prefix, int nodeid,
char *output_addr, size_t output_len)
{
char *coloncolon;
int bits;
int max_prefix = 64;
uint32_t nodeid_mask;
uint32_t addr_mask;
uint32_t masked_nodeid;
struct in_addr *addr;
struct totem_ip_address totemip;
coloncolon = strstr(input_addr, "::");
if (!coloncolon) {
max_prefix = 30;
}
bits = atoi(prefix);
if (bits < 8 || bits > max_prefix) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
return -1;
}
/* IPv6 is easy */
if (coloncolon) {
memcpy(output_addr, input_addr, coloncolon-input_addr);
sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
return 0;
}
/* For IPv4 we need to parse the address into binary, mask off the required bits,
* add in the masked_nodeid and 'print' it out again
*/
nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
addr_mask = UINT32_MAX ^ nodeid_mask;
masked_nodeid = nodeid & nodeid_mask;
if (totemip_parse(&totemip, input_addr, AF_INET)) {
knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
return -1;
}
addr = (struct in_addr *)&totemip.addr;
addr->s_addr &= htonl(addr_mask);
addr->s_addr |= htonl(masked_nodeid);
inet_ntop(AF_INET, addr, output_addr, output_len);
return 0;
}
static int create_nozzle_device(void *knet_context, const char *name,
const char *ipaddr, const char *prefix,
const char *macaddr)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
char device_name[IFNAMSIZ+1];
size_t size = IFNAMSIZ;
int8_t channel = NOZZLE_CHANNEL;
nozzle_t nozzle_dev;
int nozzle_fd;
int res;
char *updown_dir;
char parsed_ipaddr[INET6_ADDRSTRLEN];
char mac[19];
memset(device_name, 0, size);
memset(&mac, 0, sizeof(mac));
strncpy(device_name, name, size);
updown_dir = get_nozzle_script_dir(knet_context);
knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
nozzle_dev = nozzle_open(device_name, size, updown_dir);
if (!nozzle_dev) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
return -1;
}
instance->nozzle_handle = nozzle_dev;
if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
goto out_clean;
}
if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
/* Prints its own errors */
goto out_clean;
}
knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
goto out_clean;
}
nozzle_fd = nozzle_get_fd(nozzle_dev);
knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
#ifdef KNET_DATAFD_FLAG_RX_RETURN_INFO
res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel, 0);
#else
res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
#endif
if (res != 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
goto out_clean;
}
run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
res = nozzle_set_up(nozzle_dev);
if (res != 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
goto out_clean;
}
run_nozzle_script(instance, NOZZLE_UP, "up");
return 0;
out_clean:
nozzle_close(nozzle_dev);
return -1;
}
static int remove_nozzle_device(void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
int res;
int datafd;
res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
if (res != 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
return -1;
}
res = knet_handle_remove_datafd(instance->knet_handle, datafd);
if (res != 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
return -1;
}
run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
res = nozzle_set_down(instance->nozzle_handle);
if (res != 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
return -1;
}
run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
res = nozzle_close(instance->nozzle_handle);
if (res != 0) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
return -1;
}
knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
return 0;
}
static void free_nozzle(struct totemknet_instance *instance)
{
free(instance->nozzle_name);
free(instance->nozzle_ipaddr);
free(instance->nozzle_prefix);
free(instance->nozzle_macaddr);
instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
instance->nozzle_macaddr = NULL;
}
static int setup_nozzle(void *knet_context)
{
struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
char *ipaddr_str = NULL;
char *name_str = NULL;
char *prefix_str = NULL;
char *macaddr_str = NULL;
char mac[32];
int name_res;
int macaddr_res;
int res = -1;
/*
* Return value ignored on purpose. icmap_get_string changes
* ipaddr_str/prefix_str only on success.
*/
(void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
(void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
name_res = icmap_get_string(NOZZLE_NAME, &name_str);
/* Is is being removed? */
if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
remove_nozzle_device(instance);
free_nozzle(instance);
goto out_free;
}
if (!name_str) {
/* no nozzle */
goto out_free;
}
if (!ipaddr_str) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
goto out_free;
}
if (!prefix_str) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
goto out_free;
}
if (macaddr_str && strlen(macaddr_str) != 17) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
goto out_free;
}
if (!macaddr_str) {
macaddr_str = (char*)"54:54:01:00:00:00";
}
if (instance->nozzle_name &&
(strcmp(name_str, instance->nozzle_name) == 0) &&
(strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
(strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
(instance->nozzle_macaddr == NULL ||
strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
/* Nothing has changed */
knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
goto out_free;
}
/* Add nodeid into MAC address */
memcpy(mac, macaddr_str, 12);
snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
instance->our_nodeid >> 8,
instance->our_nodeid & 0xFF);
knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
if (name_res == CS_OK && name_str) {
/* Reconfigure */
if (instance->nozzle_name) {
remove_nozzle_device(instance);
free_nozzle(instance);
}
res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
mac);
instance->nozzle_name = strdup(name_str);
instance->nozzle_ipaddr = strdup(ipaddr_str);
instance->nozzle_prefix = strdup(prefix_str);
instance->nozzle_macaddr = strdup(macaddr_str);
if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
!instance->nozzle_prefix) {
knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
/*
* This 'free' will cause a complete reconfigure of the device next time we reload
* but will also let the the current device keep working until then.
* remove_nozzle() only needs the, statically-allocated, nozzle_handle
*/
free_nozzle(instance);
}
}
out_free:
free(name_str);
free(ipaddr_str);
free(prefix_str);
if (macaddr_res == CS_OK) {
free(macaddr_str);
}
return res;
}
#endif // HAVE_LIBNOZZLE
config.h
INTERFACE_MAX
#define INTERFACE_MAX
Definition coroapi.h:88
nodeid
unsigned int nodeid
Definition coroapi.h:0
addr
unsigned char addr[TOTEMIP_ADDRLEN]
Definition coroapi.h:2
CS_PRI_NODE_ID
#define CS_PRI_NODE_ID
Definition corotypes.h:59
qb_to_cs_error
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
cs_error_t
cs_error_t
The cs_error_t enum.
Definition corotypes.h:98
CS_ERR_BUSY
@ CS_ERR_BUSY
Definition corotypes.h:108
CS_ERR_TRY_AGAIN
@ CS_ERR_TRY_AGAIN
Definition corotypes.h:104
CS_OK
@ CS_OK
Definition corotypes.h:99
CS_ERR_INVALID_PARAM
@ CS_ERR_INVALID_PARAM
Definition corotypes.h:105
CS_ERR_LIBRARY
@ CS_ERR_LIBRARY
Definition corotypes.h:100
CS_ERR_NOT_EXIST
@ CS_ERR_NOT_EXIST
Definition corotypes.h:110
util.h
corosync_exit_error
#define corosync_exit_error(err)
Definition exec/util.h:72
COROSYNC_DONE_MAINCONFIGREAD
@ COROSYNC_DONE_MAINCONFIGREAD
Definition exec/util.h:51
value
uint32_t value
Definition exec/votequorum.c:2
icmap.h
icmap_get_uint8
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition icmap.c:872
ICMAP_TRACK_MODIFY
#define ICMAP_TRACK_MODIFY
Definition icmap.h:78
ICMAP_TRACK_DELETE
#define ICMAP_TRACK_DELETE
Definition icmap.h:77
icmap_track_add
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition icmap.c:1163
icmap_set_string
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition icmap.c:631
ICMAP_TRACK_PREFIX
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition icmap.h:85
icmap_track_t
struct icmap_track * icmap_track_t
Track type.
Definition icmap.h:128
ICMAP_TRACK_ADD
#define ICMAP_TRACK_ADD
Definition icmap.h:76
icmap_get_string
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition icmap.c:860
CFG_INTERFACE_STATUS_MAX_LEN
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition ipc_cfg.h:43
logsys.h
LOGSYS_LEVEL_ERROR
#define LOGSYS_LEVEL_ERROR
Definition logsys.h:72
LEAVE
#define LEAVE
Definition logsys.h:334
log_printf
#define log_printf(level, format, args...)
Definition logsys.h:332
LOGSYS_LEVEL_INFO
#define LOGSYS_LEVEL_INFO
Definition logsys.h:75
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition logsys.h:71
logsys_config_debug_get
int logsys_config_debug_get(const char *subsys)
Return the debug flag for this subsys.
Definition logsys.c:780
LOGSYS_DEBUG_OFF
#define LOGSYS_DEBUG_OFF
Definition logsys.h:92
LOGSYS_DEBUG_TRACE
#define LOGSYS_DEBUG_TRACE
Definition logsys.h:94
LOGSYS_DEBUG_ON
#define LOGSYS_DEBUG_ON
Definition logsys.h:93
LOGSYS_LEVEL_WARNING
#define LOGSYS_LEVEL_WARNING
Definition logsys.h:73
_logsys_subsys_create
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition logsys.c:427
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition logsys.h:76
LOGSYS_LEVEL_TRACE
#define LOGSYS_LEVEL_TRACE
Definition logsys.h:77
ENTER
#define ENTER
Definition logsys.h:333
corosync_get_config_file
const char * corosync_get_config_file(void)
Definition main.c:212
main.h
user_data
void * user_data
Definition sam.c:127
sq.h
icmap_notify_value
Structure passed as new_value and old_value in change callback.
Definition icmap.h:91
totem_config
Definition totem.h:159
totem_config::crypto_model
char crypto_model[CONFIG_STRING_LEN_MAX]
Definition totem.h:224
totem_config::private_key_len
unsigned int private_key_len
Definition totem.h:177
totem_config::knet_mtu
unsigned int knet_mtu
Definition totem.h:170
totem_config::node_id
unsigned int node_id
Definition totem.h:167
totem_config::knet_compression_threshold
uint32_t knet_compression_threshold
Definition totem.h:236
totem_config::totem_logging_configuration
struct totem_logging_configuration totem_logging_configuration
Definition totem.h:208
totem_config::interfaces
struct totem_interface * interfaces
Definition totem.h:165
totem_config::crypto_changed
int crypto_changed
Definition totem.h:232
totem_config::merge_timeout
unsigned int merge_timeout
Definition totem.h:198
totem_config::ip_dscp
unsigned char ip_dscp
Definition totem.h:250
totem_config::knet_compression_level
int knet_compression_level
Definition totem.h:238
totem_config::net_mtu
unsigned int net_mtu
Definition totem.h:210
totem_config::knet_compression_model
char knet_compression_model[CONFIG_STRING_LEN_MAX]
Definition totem.h:234
totem_config::block_unlisted_ips
unsigned int block_unlisted_ips
Definition totem.h:246
totem_config::private_key
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition totem.h:175
totem_config::crypto_index
int crypto_index
Definition totem.h:230
totem_config::knet_pmtud_interval
unsigned int knet_pmtud_interval
Definition totem.h:169
totem_config::crypto_cipher_type
char crypto_cipher_type[CONFIG_STRING_LEN_MAX]
Definition totem.h:226
totem_config::link_mode
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition totem.h:206
totem_config::crypto_hash_type
char crypto_hash_type[CONFIG_STRING_LEN_MAX]
Definition totem.h:228
totem_interface::knet_ping_timeout
int knet_ping_timeout
Definition totem.h:93
totem_interface::knet_link_priority
int knet_link_priority
Definition totem.h:91
totem_interface::boundto
struct totem_ip_address boundto
Definition totem.h:84
totem_interface::ip_port
uint16_t ip_port
Definition totem.h:87
totem_interface::knet_ping_interval
int knet_ping_interval
Definition totem.h:92
totem_interface::configured
uint8_t configured
Definition totem.h:89
totem_interface::knet_ping_precision
int knet_ping_precision
Definition totem.h:94
totem_interface::knet_pong_count
int knet_pong_count
Definition totem.h:95
totem_interface::knet_transport
int knet_transport
Definition totem.h:96
totem_interface::bindnet
struct totem_ip_address bindnet
Definition totem.h:83
totem_ip_address
The totem_ip_address struct.
Definition coroapi.h:111
totem_ip_address::nodeid
unsigned int nodeid
Definition coroapi.h:112
totem_logging_configuration::log_subsys_id
int log_subsys_id
Definition totem.h:116
totem_logging_configuration::log_level_error
int log_level_error
Definition totem.h:111
totem_logging_configuration::log_level_notice
int log_level_notice
Definition totem.h:113
totem_logging_configuration::log_level_debug
int log_level_debug
Definition totem.h:114
totem_logging_configuration::log_level_warning
int log_level_warning
Definition totem.h:112
totem_logging_configuration::log_printf
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition totem.h:101
totem_message_header
Definition totem.h:126
totem_node_status
Definition totem.h:267
totem_node_status::onwire_max
uint8_t onwire_max
Definition totem.h:274
totem_node_status::reachable
uint8_t reachable
Definition totem.h:270
totem_node_status::remote
uint8_t remote
Definition totem.h:271
totem_node_status::nodeid
unsigned int nodeid
Definition totem.h:269
totem_node_status::onwire_min
uint8_t onwire_min
Definition totem.h:273
totem_node_status::onwire_ver
uint8_t onwire_ver
Definition totem.h:275
totem_node_status::external
uint8_t external
Definition totem.h:272
totem_node_status::link_status
struct knet_link_status link_status[KNET_MAX_LINK]
Definition totem.h:276
totemknet_instance
Definition totemknet.c:92
totemknet_instance::totemknet_subsys_id
int totemknet_subsys_id
Definition totemknet.c:135
totemknet_instance::totemknet_target_set_completed
void(* totemknet_target_set_completed)(void *context)
Definition totemknet.c:120
totemknet_instance::log_mutex
pthread_mutex_t log_mutex
Definition totemknet.c:177
totemknet_instance::logpipes
int logpipes[2]
Definition totemknet.c:174
totemknet_instance::totemknet_mtu_changed
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition totemknet.c:116
totemknet_instance::crypto_inst
struct crypto_instance * crypto_inst
Definition totemknet.c:93
totemknet_instance::totem_config
struct totem_config * totem_config
Definition totemknet.c:162
totemknet_instance::nozzle_prefix
char * nozzle_prefix
Definition totemknet.c:181
totemknet_instance::timer_netif_check_timeout
qb_loop_timer_handle timer_netif_check_timeout
Definition totemknet.c:166
totemknet_instance::knet_subsys_id
int knet_subsys_id
Definition totemknet.c:137
totemknet_instance::link_status
char * link_status[INTERFACE_MAX]
Definition totemknet.c:152
totemknet_instance::knet_fd
int knet_fd
Definition totemknet.c:175
totemknet_instance::totemknet_log_printf
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition totemknet.c:139
totemknet_instance::knet_handle
knet_handle_t knet_handle
Definition totemknet.c:99
totemknet_instance::ip_port
uint16_t ip_port[INTERFACE_MAX]
Definition totemknet.c:156
totemknet_instance::totemknet_iface_change_fn
int(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition totemknet.c:111
totemknet_instance::nozzle_handle
nozzle_t nozzle_handle
Definition totemknet.c:183
totemknet_instance::nozzle_macaddr
char * nozzle_macaddr
Definition totemknet.c:182
totemknet_instance::totemknet_log_level_debug
int totemknet_log_level_debug
Definition totemknet.c:133
totemknet_instance::last_good_crypto_cfg
struct knet_handle_crypto_cfg last_good_crypto_cfg
Definition totemknet.c:95
totemknet_instance::loopback_link
int loopback_link
Definition totemknet.c:160
totemknet_instance::token_target
struct totem_ip_address token_target
Definition totemknet.c:164
totemknet_instance::nozzle_ipaddr
char * nozzle_ipaddr
Definition totemknet.c:180
totemknet_instance::our_nodeid
int our_nodeid
Definition totemknet.c:158
totemknet_instance::timer_merge_detect_timeout
qb_loop_timer_handle timer_merge_detect_timeout
Definition totemknet.c:168
totemknet_instance::totemknet_log_level_warning
int totemknet_log_level_warning
Definition totemknet.c:129
totemknet_instance::my_ids
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition totemknet.c:154
totemknet_instance::totemknet_deliver_fn
int(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition totemknet.c:105
totemknet_instance::knet_context
void void * knet_context
Definition totemknet.c:148
totemknet_instance::iov_buffer
char iov_buffer[KNET_MAX_PACKET_SIZE+1]
Definition totemknet.c:150
totemknet_instance::merge_detect_messages_sent_before_timeout
unsigned int merge_detect_messages_sent_before_timeout
Definition totemknet.c:172
totemknet_instance::send_merge_detect_message
int send_merge_detect_message
Definition totemknet.c:170
totemknet_instance::totemknet_log_level_error
int totemknet_log_level_error
Definition totemknet.c:127
totemknet_instance::totemknet_log_level_security
int totemknet_log_level_security
Definition totemknet.c:125
totemknet_instance::poll_handle
qb_loop_t * poll_handle
Definition totemknet.c:97
totemknet_instance::link_mode
int link_mode
Definition totemknet.c:101
totemknet_instance::totemknet_log_level_notice
int totemknet_log_level_notice
Definition totemknet.c:131
totemknet_instance::context
void * context
Definition totemknet.c:103
totemknet_instance::nozzle_name
char * nozzle_name
Definition totemknet.c:179
totemsrp_stats_t
Definition totemstats.h:53
work_item
Definition totemknet.c:190
work_item::msg
const void * msg
Definition totemknet.c:191
work_item::msg_len
unsigned int msg_len
Definition totemknet.c:192
work_item::instance
struct totemknet_instance * instance
Definition totemknet.c:193
swab.h
__attribute__
typedef __attribute__
cfg_message_crypto_reconfig_phase_t
cfg_message_crypto_reconfig_phase_t
Definition totem.h:154
CRYPTO_RECONFIG_PHASE_CLEANUP
@ CRYPTO_RECONFIG_PHASE_CLEANUP
Definition totem.h:156
CRYPTO_RECONFIG_PHASE_ACTIVATE
@ CRYPTO_RECONFIG_PHASE_ACTIVATE
Definition totem.h:155
type
char type
Definition totem.h:2
totemip.h
totemip_parse
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition totemip.c:306
totemip_copy
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition totemip.c:123
totemip_totemip_to_sockaddr_convert
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition totemip.c:264
totemip_print
const char * totemip_print(const struct totem_ip_address *addr)
Definition totemip.c:256
totemknet_mcast_flush_send
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition totemknet.c:1442
totemknet_iface_set
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition totemknet.c:1538
totemknet_finalize
int totemknet_finalize(void *knet_context)
Definition totemknet.c:658
totemknet_recv_flush
int totemknet_recv_flush(void *knet_context)
Definition totemknet.c:1420
totemknet_member_list_rebind_ip
int totemknet_member_list_rebind_ip(void *knet_context)
Definition totemknet.c:1740
totemknet_buffer_alloc
void * totemknet_buffer_alloc(void)
Definition totemknet.c:1402
totemknet_processor_count_set
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition totemknet.c:1413
totemknet_mcast_noflush_send
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition totemknet.c:1455
global_instance
struct totemknet_instance * global_instance
Definition totemknet.c:188
NOZZLE_IPADDR
#define NOZZLE_IPADDR
Definition totemknet.c:1986
totemknet_buffer_release
void totemknet_buffer_release(void *ptr)
Definition totemknet.c:1408
totemknet_initialize
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, int(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), int(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Create an instance.
Definition totemknet.c:1133
totemknet_ifaces_get
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition totemknet.c:563
totemknet_member_add
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition totemknet.c:1555
NOZZLE_CHANNEL
#define NOZZLE_CHANNEL
Definition totemknet.c:1990
totemknet_crypto_set
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition totemknet.c:388
totemknet_member_remove
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition totemknet.c:1694
totemknet_token_send
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition totemknet.c:1430
NOZZLE_NAME
#define NOZZLE_NAME
Definition totemknet.c:1985
totemknet_link_get_status
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition totemknet.c:1868
OWN_INDEX_NONE
#define OWN_INDEX_NONE
Definition totemknet.c:482
totemknet_nodestatus_get
int totemknet_nodestatus_get(void *knet_context, unsigned int nodeid, struct totem_node_status *node_status)
Definition totemknet.c:485
totemknet_handle_get_stats
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition totemknet.c:1905
MSG_NOSIGNAL
#define MSG_NOSIGNAL
Definition totemknet.c:82
totemknet_stats_clear
void totemknet_stats_clear(void *knet_context)
Definition totemknet.c:1859
totemknet_send_flush
int totemknet_send_flush(void *knet_context)
Definition totemknet.c:1425
totemknet_net_mtu_adjust
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition totemknet.c:1479
NOZZLE_MACADDR
#define NOZZLE_MACADDR
Definition totemknet.c:1988
knet_log_printf
#define knet_log_printf(level, format, args...)
Definition totemknet.c:234
NOZZLE_PREFIX
#define NOZZLE_PREFIX
Definition totemknet.c:1987
totemknet_token_target_set
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition totemknet.c:1486
KNET_LOGSYS_PERROR
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition totemknet.c:250
totemknet_configure_log_level
void totemknet_configure_log_level(void)
Definition totemknet.c:884
totemknet_reconfigure
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition totemknet.c:1767
totemknet_crypto_reconfigure_phase
int totemknet_crypto_reconfigure_phase(void *knet_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
Definition totemknet.c:1800
libknet_log_printf
#define libknet_log_printf(level, format, args...)
Definition totemknet.c:242
totemknet_recv_mcast_empty
int totemknet_recv_mcast_empty(void *knet_context)
Definition totemknet.c:1500
totemknet_iface_check
int totemknet_iface_check(void *knet_context)
Definition totemknet.c:1469
totemknet.h
header
struct totem_message_header header
Definition totemsrp.c:0
system_from
struct srp_addr system_from
Definition totemsrp.c:1
totemstats.h
stats_knet_add_handle
void stats_knet_add_handle(void)
Definition stats.c:753
stats_knet_del_member
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition stats.c:740
stats_knet_add_member
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition stats.c:730