corosync  2.4.5
totemrrp.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2005 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <pthread.h>
40 #include <sys/mman.h>
41 #include <sys/types.h>
42 #include <sys/stat.h>
43 #include <sys/socket.h>
44 #include <netdb.h>
45 #include <sys/un.h>
46 #include <sys/ioctl.h>
47 #include <sys/param.h>
48 #include <netinet/in.h>
49 #include <arpa/inet.h>
50 #include <unistd.h>
51 #include <fcntl.h>
52 #include <stdlib.h>
53 #include <stdio.h>
54 #include <errno.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <limits.h>
60 
61 #include <corosync/sq.h>
62 #include <corosync/list.h>
63 #include <corosync/swab.h>
64 #include <qb/qbdefs.h>
65 #include <qb/qbloop.h>
66 #define LOGSYS_UTILS_ONLY 1
67 #include <corosync/logsys.h>
68 
69 #include "totemnet.h"
70 #include "totemrrp.h"
71 
72 void rrp_deliver_fn (
73  void *context,
74  const void *msg,
75  unsigned int msg_len);
76 
78  void *context,
79  const struct totem_ip_address *iface_addr);
80 
81 struct totemrrp_instance;
84  unsigned int *faulty;
85  unsigned int *token_recv_count;
86  unsigned int *mcast_recv_count;
87  unsigned char token[15000];
88  unsigned int token_len;
89  qb_loop_timer_handle timer_expired_token;
90  qb_loop_timer_handle timer_problem_decrementer;
92  unsigned int token_xmit_iface;
93  unsigned int msg_xmit_iface;
94 };
95 
98  unsigned int *faulty;
99  unsigned int *last_token_recv;
100  unsigned int *counter_problems;
101  unsigned char token[15000];
102  unsigned int token_len;
103  unsigned int last_token_seq;
104  qb_loop_timer_handle timer_expired_token;
105  qb_loop_timer_handle timer_problem_decrementer;
107 };
108 
109 struct rrp_algo {
110  const char *name;
111 
112  void * (*initialize) (
113  struct totemrrp_instance *rrp_instance,
114  int interface_count);
115 
116  void (*mcast_recv) (
117  struct totemrrp_instance *instance,
118  unsigned int iface_no,
119  void *context,
120  const void *msg,
121  unsigned int msg_len);
122 
124  struct totemrrp_instance *instance,
125  const void *msg,
126  unsigned int msg_len);
127 
129  struct totemrrp_instance *instance,
130  const void *msg,
131  unsigned int msg_len);
132 
133  void (*token_recv) (
134  struct totemrrp_instance *instance,
135  unsigned int iface_no,
136  void *context,
137  const void *msg,
138  unsigned int msg_len,
139  unsigned int token_seqid);
140 
141  void (*token_send) (
142  struct totemrrp_instance *instance,
143  const void *msg,
144  unsigned int msg_len);
145 
146  void (*recv_flush) (
147  struct totemrrp_instance *instance);
148 
149  void (*send_flush) (
150  struct totemrrp_instance *instance);
151 
152  void (*iface_check) (
153  struct totemrrp_instance *instance);
154 
156  struct totemrrp_instance *instance,
157  unsigned int processor_count);
158 
160  struct totemrrp_instance *instance,
161  struct totem_ip_address *token_target,
162  unsigned int iface_no);
163 
164  void (*ring_reenable) (
165  struct totemrrp_instance *instance,
166  unsigned int iface_no);
167 
169  struct totemrrp_instance *instance);
170 
171  int (*member_add) (
172  struct totemrrp_instance *instance,
173  const struct totem_ip_address *member,
174  unsigned int iface_no);
175 
176  int (*member_remove) (
177  struct totemrrp_instance *instance,
178  const struct totem_ip_address *member,
179  unsigned int iface_no);
180 
182  struct totemrrp_instance *instance,
183  enum totem_configuration_type configuration_type,
184  const struct srp_addr *member_list, size_t member_list_entries,
185  const struct srp_addr *left_list, size_t left_list_entries,
186  const struct srp_addr *joined_list, size_t joined_list_entries,
187  const struct memb_ring_id *ring_id);
188 };
189 
190 #define STATUS_STR_LEN 512
192  qb_loop_t *poll_handle;
193 
195 
197 
198  void *context;
199 
201 
203  void *context,
204  const void *msg,
205  unsigned int msg_len);
206 
208  void *context,
209  const struct totem_ip_address *iface_addr,
210  unsigned int iface_no);
211 
213  const void *msg,
214  unsigned int *seqid,
215  unsigned int *token_is);
216 
218  void *context);
219 
220  unsigned int (*totemrrp_msgs_missing) (void);
221 
222  /*
223  * Function and data used to log messages
224  */
226 
228 
230 
232 
234 
236 
238  int level,
239  int subsys,
240  const char *function,
241  const char *file,
242  int line,
243  const char *format, ...)__attribute__((format(printf, 6, 7)));
244 
245  void **net_handles;
246 
248 
250 
252 
254 
256 
258 
260 
262 };
263 
264 static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
265  unsigned int iface_no, int is_faulty);
266 
267 /*
268  * None Replication Forward Declerations
269  */
270 static void none_mcast_recv (
271  struct totemrrp_instance *instance,
272  unsigned int iface_no,
273  void *context,
274  const void *msg,
275  unsigned int msg_len);
276 
277 static void none_mcast_noflush_send (
278  struct totemrrp_instance *instance,
279  const void *msg,
280  unsigned int msg_len);
281 
282 static void none_mcast_flush_send (
283  struct totemrrp_instance *instance,
284  const void *msg,
285  unsigned int msg_len);
286 
287 static void none_token_recv (
288  struct totemrrp_instance *instance,
289  unsigned int iface_no,
290  void *context,
291  const void *msg,
292  unsigned int msg_len,
293  unsigned int token_seqid);
294 
295 static void none_token_send (
296  struct totemrrp_instance *instance,
297  const void *msg,
298  unsigned int msg_len);
299 
300 static void none_recv_flush (
301  struct totemrrp_instance *instance);
302 
303 static void none_send_flush (
304  struct totemrrp_instance *instance);
305 
306 static void none_iface_check (
307  struct totemrrp_instance *instance);
308 
309 static void none_processor_count_set (
310  struct totemrrp_instance *instance,
311  unsigned int processor_count_set);
312 
313 static void none_token_target_set (
314  struct totemrrp_instance *instance,
315  struct totem_ip_address *token_target,
316  unsigned int iface_no);
317 
318 static void none_ring_reenable (
319  struct totemrrp_instance *instance,
320  unsigned int iface_no);
321 
322 static int none_mcast_recv_empty (
323  struct totemrrp_instance *instance);
324 
325 static int none_member_add (
326  struct totemrrp_instance *instance,
327  const struct totem_ip_address *member,
328  unsigned int iface_no);
329 
330 static int none_member_remove (
331  struct totemrrp_instance *instance,
332  const struct totem_ip_address *member,
333  unsigned int iface_no);
334 
335 static void none_membership_changed (
336  struct totemrrp_instance *instance,
337  enum totem_configuration_type configuration_type,
338  const struct srp_addr *member_list, size_t member_list_entries,
339  const struct srp_addr *left_list, size_t left_list_entries,
340  const struct srp_addr *joined_list, size_t joined_list_entries,
341  const struct memb_ring_id *ring_id);
342 
343 /*
344  * Passive Replication Forward Declerations
345  */
346 static void *passive_instance_initialize (
347  struct totemrrp_instance *rrp_instance,
348  int interface_count);
349 
350 static void passive_mcast_recv (
351  struct totemrrp_instance *instance,
352  unsigned int iface_no,
353  void *context,
354  const void *msg,
355  unsigned int msg_len);
356 
357 static void passive_mcast_noflush_send (
358  struct totemrrp_instance *instance,
359  const void *msg,
360  unsigned int msg_len);
361 
362 static void passive_mcast_flush_send (
363  struct totemrrp_instance *instance,
364  const void *msg,
365  unsigned int msg_len);
366 
367 static void passive_monitor (
368  struct totemrrp_instance *rrp_instance,
369  unsigned int iface_no,
370  int is_token_recv_count);
371 
372 static void passive_token_recv (
373  struct totemrrp_instance *instance,
374  unsigned int iface_no,
375  void *context,
376  const void *msg,
377  unsigned int msg_len,
378  unsigned int token_seqid);
379 
380 static void passive_token_send (
381  struct totemrrp_instance *instance,
382  const void *msg,
383  unsigned int msg_len);
384 
385 static void passive_recv_flush (
386  struct totemrrp_instance *instance);
387 
388 static void passive_send_flush (
389  struct totemrrp_instance *instance);
390 
391 static void passive_iface_check (
392  struct totemrrp_instance *instance);
393 
394 static void passive_processor_count_set (
395  struct totemrrp_instance *instance,
396  unsigned int processor_count_set);
397 
398 static void passive_token_target_set (
399  struct totemrrp_instance *instance,
400  struct totem_ip_address *token_target,
401  unsigned int iface_no);
402 
403 static void passive_ring_reenable (
404  struct totemrrp_instance *instance,
405  unsigned int iface_no);
406 
407 static int passive_mcast_recv_empty (
408  struct totemrrp_instance *instance);
409 
410 static int passive_member_add (
411  struct totemrrp_instance *instance,
412  const struct totem_ip_address *member,
413  unsigned int iface_no);
414 
415 static int passive_member_remove (
416  struct totemrrp_instance *instance,
417  const struct totem_ip_address *member,
418  unsigned int iface_no);
419 
420 static void passive_membership_changed (
421  struct totemrrp_instance *instance,
422  enum totem_configuration_type configuration_type,
423  const struct srp_addr *member_list, size_t member_list_entries,
424  const struct srp_addr *left_list, size_t left_list_entries,
425  const struct srp_addr *joined_list, size_t joined_list_entries,
426  const struct memb_ring_id *ring_id);
427 
428 /*
429  * Active Replication Forward Definitions
430  */
431 static void *active_instance_initialize (
432  struct totemrrp_instance *rrp_instance,
433  int interface_count);
434 
435 static void active_mcast_recv (
436  struct totemrrp_instance *instance,
437  unsigned int iface_no,
438  void *context,
439  const void *msg,
440  unsigned int msg_len);
441 
442 static void active_mcast_noflush_send (
443  struct totemrrp_instance *instance,
444  const void *msg,
445  unsigned int msg_len);
446 
447 static void active_mcast_flush_send (
448  struct totemrrp_instance *instance,
449  const void *msg,
450  unsigned int msg_len);
451 
452 static void active_token_recv (
453  struct totemrrp_instance *instance,
454  unsigned int iface_no,
455  void *context,
456  const void *msg,
457  unsigned int msg_len,
458  unsigned int token_seqid);
459 
460 static void active_token_send (
461  struct totemrrp_instance *instance,
462  const void *msg,
463  unsigned int msg_len);
464 
465 static void active_recv_flush (
466  struct totemrrp_instance *instance);
467 
468 static void active_send_flush (
469  struct totemrrp_instance *instance);
470 
471 static void active_iface_check (
472  struct totemrrp_instance *instance);
473 
474 static void active_processor_count_set (
475  struct totemrrp_instance *instance,
476  unsigned int processor_count_set);
477 
478 static void active_token_target_set (
479  struct totemrrp_instance *instance,
480  struct totem_ip_address *token_target,
481  unsigned int iface_no);
482 
483 static void active_ring_reenable (
484  struct totemrrp_instance *instance,
485  unsigned int iface_no);
486 
487 static int active_mcast_recv_empty (
488  struct totemrrp_instance *instance);
489 
490 static int active_member_add (
491  struct totemrrp_instance *instance,
492  const struct totem_ip_address *member,
493  unsigned int iface_no);
494 
495 static int active_member_remove (
496  struct totemrrp_instance *instance,
497  const struct totem_ip_address *member,
498  unsigned int iface_no);
499 
500 static void active_membership_changed (
501  struct totemrrp_instance *instance,
502  enum totem_configuration_type configuration_type,
503  const struct srp_addr *member_list, size_t member_list_entries,
504  const struct srp_addr *left_list, size_t left_list_entries,
505  const struct srp_addr *joined_list, size_t joined_list_entries,
506  const struct memb_ring_id *ring_id);
507 
508 static void active_timer_expired_token_start (
510 
511 static void active_timer_expired_token_cancel (
513 
514 static void active_timer_problem_decrementer_start (
516 
517 static void active_timer_problem_decrementer_cancel (
519 
520 /*
521  * 0-5 reserved for totemsrp.c
522  */
523 #define MESSAGE_TYPE_RING_TEST_ACTIVE 6
524 #define MESSAGE_TYPE_RING_TEST_ACTIVATE 7
525 
526 #define ENDIAN_LOCAL 0xff22
527 
528 /*
529  * Rollover handling:
530  *
531  * ARR_SEQNO_START_TOKEN is the starting sequence number of last seen sequence
532  * for a token for active redundand ring. This should remain zero, unless testing
533  * overflow in which case 07fffff00 or 0xffffff00 are good starting values.
534  * It should be same as on defined in totemsrp.c
535  */
536 
537 #define ARR_SEQNO_START_TOKEN 0x0
538 
539 /*
540  * These can be used ot test different rollover points
541  * #define ARR_SEQNO_START_MSG 0xfffffe00
542  */
543 
544 /*
545  * Threshold value when recv_count for passive rrp should be adjusted.
546  * Set this value to some smaller for testing of adjusting proper
547  * functionality. Also keep in mind that this value must be smaller
548  * then rrp_problem_count_threshold
549  */
550 #define PASSIVE_RECV_COUNT_THRESHOLD (INT_MAX / 2)
551 
553  char type;
555  unsigned short endian_detector;
558 } __attribute__((packed));
562  void *context;
563  int iface_no;
564 };
565 
566 struct rrp_algo none_algo = {
567  .name = "none",
568  .initialize = NULL,
569  .mcast_recv = none_mcast_recv,
570  .mcast_noflush_send = none_mcast_noflush_send,
571  .mcast_flush_send = none_mcast_flush_send,
572  .token_recv = none_token_recv,
573  .token_send = none_token_send,
574  .recv_flush = none_recv_flush,
575  .send_flush = none_send_flush,
576  .iface_check = none_iface_check,
577  .processor_count_set = none_processor_count_set,
578  .token_target_set = none_token_target_set,
579  .ring_reenable = none_ring_reenable,
580  .mcast_recv_empty = none_mcast_recv_empty,
581  .member_add = none_member_add,
582  .member_remove = none_member_remove,
583  .membership_changed = none_membership_changed
584 };
585 
587  .name = "passive",
588  .initialize = passive_instance_initialize,
589  .mcast_recv = passive_mcast_recv,
590  .mcast_noflush_send = passive_mcast_noflush_send,
591  .mcast_flush_send = passive_mcast_flush_send,
592  .token_recv = passive_token_recv,
593  .token_send = passive_token_send,
594  .recv_flush = passive_recv_flush,
595  .send_flush = passive_send_flush,
596  .iface_check = passive_iface_check,
597  .processor_count_set = passive_processor_count_set,
598  .token_target_set = passive_token_target_set,
599  .ring_reenable = passive_ring_reenable,
600  .mcast_recv_empty = passive_mcast_recv_empty,
601  .member_add = passive_member_add,
602  .member_remove = passive_member_remove,
603  .membership_changed = passive_membership_changed
604 };
605 
607  .name = "active",
608  .initialize = active_instance_initialize,
609  .mcast_recv = active_mcast_recv,
610  .mcast_noflush_send = active_mcast_noflush_send,
611  .mcast_flush_send = active_mcast_flush_send,
612  .token_recv = active_token_recv,
613  .token_send = active_token_send,
614  .recv_flush = active_recv_flush,
615  .send_flush = active_send_flush,
616  .iface_check = active_iface_check,
617  .processor_count_set = active_processor_count_set,
618  .token_target_set = active_token_target_set,
619  .ring_reenable = active_ring_reenable,
620  .mcast_recv_empty = active_mcast_recv_empty,
621  .member_add = active_member_add,
622  .member_remove = active_member_remove,
623  .membership_changed = active_membership_changed
624 };
625 
626 struct rrp_algo *rrp_algos[] = {
627  &none_algo,
628  &passive_algo,
629  &active_algo
630 };
631 
632 #define RRP_ALGOS_COUNT 3
633 
634 #define log_printf(level, format, args...) \
635 do { \
636  rrp_instance->totemrrp_log_printf ( \
637  level, rrp_instance->totemrrp_subsys_id, \
638  __FUNCTION__, __FILE__, __LINE__, \
639  format, ##args); \
640 } while (0);
641 
642 static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
643  unsigned int iface_no, int is_faulty)
644 {
645  rrp_instance->stats.faulty[iface_no] = (is_faulty ? 1 : 0);
646 }
647 
648 static void test_active_msg_endian_convert(const struct message_header *in, struct message_header *out)
649 {
650  out->type = in->type;
651  out->encapsulated = in->encapsulated;
653  out->ring_number = swab32 (in->ring_number);
655 }
656 
657 static void timer_function_test_ring_timeout (void *context)
658 {
659  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
660  struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
661  unsigned int *faulty = NULL;
662  int iface_no = deliver_fn_context->iface_no;
663  struct message_header msg = {
665  .endian_detector = ENDIAN_LOCAL,
666  };
667 
668  if (strcmp(rrp_instance->totem_config->rrp_mode, "active") == 0)
669  faulty = ((struct active_instance *)(rrp_instance->rrp_algo_instance))->faulty;
670  if (strcmp(rrp_instance->totem_config->rrp_mode, "passive") == 0)
671  faulty = ((struct passive_instance *)(rrp_instance->rrp_algo_instance))->faulty;
672 
673  assert (faulty != NULL);
674 
675  if (faulty[iface_no] == 1) {
676  msg.ring_number = iface_no;
677  msg.nodeid_activator = rrp_instance->my_nodeid;
679  rrp_instance->net_handles[iface_no],
680  &msg, sizeof (struct message_header));
681  qb_loop_timer_add (rrp_instance->poll_handle,
682  QB_LOOP_MED,
683  rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
684  (void *)deliver_fn_context,
685  timer_function_test_ring_timeout,
686  &rrp_instance->timer_active_test_ring_timeout[iface_no]);
687  }
688 }
689 
690 /*
691  * None Replication Implementation
692  */
693 
694 static void none_mcast_recv (
695  struct totemrrp_instance *rrp_instance,
696  unsigned int iface_no,
697  void *context,
698  const void *msg,
699  unsigned int msg_len)
700 {
701  rrp_instance->totemrrp_deliver_fn (
702  context,
703  msg,
704  msg_len);
705 }
706 
707 static void none_mcast_flush_send (
708  struct totemrrp_instance *instance,
709  const void *msg,
710  unsigned int msg_len)
711 {
712  totemnet_mcast_flush_send (instance->net_handles[0], msg, msg_len);
713 }
714 
715 static void none_mcast_noflush_send (
716  struct totemrrp_instance *instance,
717  const void *msg,
718  unsigned int msg_len)
719 {
720  totemnet_mcast_noflush_send (instance->net_handles[0], msg, msg_len);
721 }
722 
723 static void none_token_recv (
724  struct totemrrp_instance *rrp_instance,
725  unsigned int iface_no,
726  void *context,
727  const void *msg,
728  unsigned int msg_len,
729  unsigned int token_seq)
730 {
731  rrp_instance->totemrrp_deliver_fn (
732  context,
733  msg,
734  msg_len);
735 }
736 
737 static void none_token_send (
738  struct totemrrp_instance *instance,
739  const void *msg,
740  unsigned int msg_len)
741 {
743  instance->net_handles[0],
744  msg, msg_len);
745 }
746 
747 static void none_recv_flush (struct totemrrp_instance *instance)
748 {
749  totemnet_recv_flush (instance->net_handles[0]);
750 }
751 
752 static void none_send_flush (struct totemrrp_instance *instance)
753 {
754  totemnet_send_flush (instance->net_handles[0]);
755 }
756 
757 static void none_iface_check (struct totemrrp_instance *instance)
758 {
759  totemnet_iface_check (instance->net_handles[0]);
760 }
761 
762 static void none_processor_count_set (
763  struct totemrrp_instance *instance,
764  unsigned int processor_count)
765 {
767  processor_count);
768 }
769 
770 static void none_token_target_set (
771  struct totemrrp_instance *instance,
772  struct totem_ip_address *token_target,
773  unsigned int iface_no)
774 {
775  totemnet_token_target_set (instance->net_handles[0], token_target);
776 }
777 
778 static void none_ring_reenable (
779  struct totemrrp_instance *instance,
780  unsigned int iface_no)
781 {
782  /*
783  * No operation
784  */
785 }
786 
787 static int none_mcast_recv_empty (
788  struct totemrrp_instance *instance)
789 {
790  int res;
791 
792  res = totemnet_recv_mcast_empty (instance->net_handles[0]);
793 
794  return (res);
795 }
796 
797 static int none_member_add (
798  struct totemrrp_instance *instance,
799  const struct totem_ip_address *member,
800  unsigned int iface_no)
801 {
802  int res;
803  res = totemnet_member_add (instance->net_handles[0], member);
804  return (res);
805 }
806 
807 static int none_member_remove (
808  struct totemrrp_instance *instance,
809  const struct totem_ip_address *member,
810  unsigned int iface_no)
811 {
812  int res;
813  res = totemnet_member_remove (instance->net_handles[0], member);
814  return (res);
815 }
816 
817 static void none_membership_changed (
818  struct totemrrp_instance *rrp_instance,
819  enum totem_configuration_type configuration_type,
820  const struct srp_addr *member_list, size_t member_list_entries,
821  const struct srp_addr *left_list, size_t left_list_entries,
822  const struct srp_addr *joined_list, size_t joined_list_entries,
823  const struct memb_ring_id *ring_id)
824 {
825  int i;
826 
827  for (i = 0; i < left_list_entries; i++) {
828  if (left_list->no_addrs < 1 ||
829  (left_list[i].addr[0].family != AF_INET && left_list[i].addr[0].family != AF_INET6)) {
830  log_printf(rrp_instance->totemrrp_log_level_error,
831  "Membership left list contains incorrect address. "
832  "This is sign of misconfiguration between nodes!");
833  } else {
834  totemnet_member_set_active(rrp_instance->net_handles[0],
835  &left_list[i].addr[0], 0);
836  }
837  }
838 
839  for (i = 0; i < joined_list_entries; i++) {
840  if (joined_list->no_addrs < 1 ||
841  (joined_list[i].addr[0].family != AF_INET && joined_list[i].addr[0].family != AF_INET6)) {
842  log_printf(rrp_instance->totemrrp_log_level_error,
843  "Membership join list contains incorrect address. "
844  "This is sign of misconfiguration between nodes!");
845  } else {
846  totemnet_member_set_active(rrp_instance->net_handles[0],
847  &joined_list[i].addr[0], 1);
848  }
849  }
850 }
851 
852 /*
853  * Passive Replication Implementation
854  */
855 void *passive_instance_initialize (
856  struct totemrrp_instance *rrp_instance,
857  int interface_count)
858 {
859  struct passive_instance *instance;
860  int i;
861 
862  instance = malloc (sizeof (struct passive_instance));
863  if (instance == 0) {
864  goto error_exit;
865  }
866  memset (instance, 0, sizeof (struct passive_instance));
867 
868  instance->faulty = malloc (sizeof (int) * interface_count);
869  if (instance->faulty == 0) {
870  free (instance);
871  instance = 0;
872  goto error_exit;
873  }
874  memset (instance->faulty, 0, sizeof (int) * interface_count);
875 
876  for (i = 0; i < interface_count; i++) {
877  stats_set_interface_faulty (rrp_instance, i, 0);
878  }
879 
880  instance->token_recv_count = malloc (sizeof (int) * interface_count);
881  if (instance->token_recv_count == 0) {
882  free (instance->faulty);
883  free (instance);
884  instance = 0;
885  goto error_exit;
886  }
887  memset (instance->token_recv_count, 0, sizeof (int) * interface_count);
888 
889  instance->mcast_recv_count = malloc (sizeof (int) * interface_count);
890  if (instance->mcast_recv_count == 0) {
891  free (instance->token_recv_count);
892  free (instance->faulty);
893  free (instance);
894  instance = 0;
895  goto error_exit;
896  }
897  memset (instance->mcast_recv_count, 0, sizeof (int) * interface_count);
898 
899 error_exit:
900  return ((void *)instance);
901 }
902 
903 static void timer_function_passive_token_expired (void *context)
904 {
905  struct passive_instance *passive_instance = (struct passive_instance *)context;
906  struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
907 
908  rrp_instance->totemrrp_deliver_fn (
909  passive_instance->totemrrp_context,
910  passive_instance->token,
911  passive_instance->token_len);
912 }
913 
914 /* TODO
915 static void timer_function_passive_problem_decrementer (void *context)
916 {
917 // struct passive_instance *passive_instance = (struct passive_instance *)context;
918 // struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
919 
920 }
921 */
922 
923 
924 static void passive_timer_expired_token_start (
925  struct passive_instance *passive_instance)
926 {
927  qb_loop_timer_add (
928  passive_instance->rrp_instance->poll_handle,
929  QB_LOOP_MED,
930  passive_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
931  (void *)passive_instance,
932  timer_function_passive_token_expired,
933  &passive_instance->timer_expired_token);
934 }
935 
936 static void passive_timer_expired_token_cancel (
937  struct passive_instance *passive_instance)
938 {
939  qb_loop_timer_del (
940  passive_instance->rrp_instance->poll_handle,
941  passive_instance->timer_expired_token);
942 }
943 
944 /*
945 static void passive_timer_problem_decrementer_start (
946  struct passive_instance *passive_instance)
947 {
948  qb_loop_timer_add (
949  QB_LOOP_MED,
950  passive_instance->rrp_instance->poll_handle,
951  passive_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
952  (void *)passive_instance,
953  timer_function_passive_problem_decrementer,
954  &passive_instance->timer_problem_decrementer);
955 }
956 
957 static void passive_timer_problem_decrementer_cancel (
958  struct passive_instance *passive_instance)
959 {
960  qb_loop_timer_del (
961  passive_instance->rrp_instance->poll_handle,
962  passive_instance->timer_problem_decrementer);
963 }
964 */
965 
966 /*
967  * Monitor function implementation from rrp paper.
968  * rrp_instance is passive rrp instance, iface_no is interface with received messgae/token and
969  * is_token_recv_count is boolean variable which donates if message is token (>1) or regular
970  * message (= 0)
971  */
972 static void passive_monitor (
973  struct totemrrp_instance *rrp_instance,
974  unsigned int iface_no,
975  int is_token_recv_count)
976 {
977  struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
978  unsigned int *recv_count;
979  unsigned int max;
980  unsigned int i;
981  unsigned int min_all, min_active;
982  unsigned int threshold;
983 
984  /*
985  * Monitor for failures
986  */
987  if (is_token_recv_count) {
988  recv_count = passive_instance->token_recv_count;
989  threshold = rrp_instance->totem_config->rrp_problem_count_threshold;
990  } else {
991  recv_count = passive_instance->mcast_recv_count;
992  threshold = rrp_instance->totem_config->rrp_problem_count_mcast_threshold;
993  }
994 
995  recv_count[iface_no] += 1;
996 
997  max = 0;
998  for (i = 0; i < rrp_instance->interface_count; i++) {
999  if (max < recv_count[i]) {
1000  max = recv_count[i];
1001  }
1002  }
1003 
1004  /*
1005  * Max is larger than threshold -> start adjusting process
1006  */
1008  min_all = min_active = recv_count[iface_no];
1009 
1010  for (i = 0; i < rrp_instance->interface_count; i++) {
1011  if (recv_count[i] < min_all) {
1012  min_all = recv_count[i];
1013  }
1014 
1015  if (passive_instance->faulty[i] == 0 &&
1016  recv_count[i] < min_active) {
1017  min_active = recv_count[i];
1018  }
1019  }
1020 
1021  if (min_all > 0) {
1022  /*
1023  * There is one or more faulty device with recv_count > 0
1024  */
1025  for (i = 0; i < rrp_instance->interface_count; i++) {
1026  recv_count[i] -= min_all;
1027  }
1028  } else {
1029  /*
1030  * No faulty device with recv_count > 0, adjust only active
1031  * devices
1032  */
1033  for (i = 0; i < rrp_instance->interface_count; i++) {
1034  if (passive_instance->faulty[i] == 0) {
1035  recv_count[i] -= min_active;
1036  }
1037  }
1038  }
1039 
1040  /*
1041  * Find again max
1042  */
1043  max = 0;
1044 
1045  for (i = 0; i < rrp_instance->interface_count; i++) {
1046  if (max < recv_count[i]) {
1047  max = recv_count[i];
1048  }
1049  }
1050  }
1051 
1052  for (i = 0; i < rrp_instance->interface_count; i++) {
1053  if ((passive_instance->faulty[i] == 0) &&
1054  (max - recv_count[i] > threshold)) {
1055  passive_instance->faulty[i] = 1;
1056 
1057  qb_loop_timer_add (rrp_instance->poll_handle,
1058  QB_LOOP_MED,
1059  rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
1060  rrp_instance->deliver_fn_context[i],
1061  timer_function_test_ring_timeout,
1062  &rrp_instance->timer_active_test_ring_timeout[i]);
1063 
1064  stats_set_interface_faulty (rrp_instance, i, passive_instance->faulty[i]);
1065 
1066  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1067  "Marking ringid %u interface %s FAULTY",
1068  i,
1069  totemnet_iface_print (rrp_instance->net_handles[i]));
1070  log_printf (
1071  rrp_instance->totemrrp_log_level_error,
1072  "%s",
1073  rrp_instance->status[i]);
1074  }
1075  }
1076 }
1077 
1078 static void passive_mcast_recv (
1079  struct totemrrp_instance *rrp_instance,
1080  unsigned int iface_no,
1081  void *context,
1082  const void *msg,
1083  unsigned int msg_len)
1084 {
1085  struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
1086 
1087  rrp_instance->totemrrp_deliver_fn (
1088  context,
1089  msg,
1090  msg_len);
1091 
1092  if (rrp_instance->totemrrp_msgs_missing() == 0 &&
1093  passive_instance->timer_expired_token) {
1094  /*
1095  * Delivers the last token
1096  */
1097  rrp_instance->totemrrp_deliver_fn (
1098  passive_instance->totemrrp_context,
1099  passive_instance->token,
1100  passive_instance->token_len);
1101  passive_timer_expired_token_cancel (passive_instance);
1102  }
1103 
1104  passive_monitor (rrp_instance, iface_no, 0);
1105 }
1106 
1107 static void passive_mcast_flush_send (
1108  struct totemrrp_instance *instance,
1109  const void *msg,
1110  unsigned int msg_len)
1111 {
1112  struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1113  int i = 0;
1114 
1115  do {
1116  passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
1117  i++;
1118  } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
1119 
1120  if (i > instance->interface_count) {
1121  /*
1122  * All interfaces are faulty. It's still needed to send mcast
1123  * message to local host so use first interface.
1124  */
1125  passive_instance->msg_xmit_iface = 0;
1126  }
1127 
1128  totemnet_mcast_flush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
1129 }
1130 
1131 static void passive_mcast_noflush_send (
1132  struct totemrrp_instance *instance,
1133  const void *msg,
1134  unsigned int msg_len)
1135 {
1136  struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1137  int i = 0;
1138 
1139  do {
1140  passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
1141  i++;
1142  } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
1143 
1144 
1145  if (i > instance->interface_count) {
1146  /*
1147  * All interfaces are faulty. It's still needed to send mcast
1148  * message to local host so use first interface.
1149  */
1150  passive_instance->msg_xmit_iface = 0;
1151  }
1152 
1153  totemnet_mcast_noflush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
1154 }
1155 
1156 static void passive_token_recv (
1157  struct totemrrp_instance *rrp_instance,
1158  unsigned int iface_no,
1159  void *context,
1160  const void *msg,
1161  unsigned int msg_len,
1162  unsigned int token_seq)
1163 {
1164  struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
1165 
1166  passive_instance->totemrrp_context = context; // this should be in totemrrp_instance ? TODO
1167 
1168  if (rrp_instance->totemrrp_msgs_missing() == 0) {
1169  rrp_instance->totemrrp_deliver_fn (
1170  context,
1171  msg,
1172  msg_len);
1173  } else {
1174  memcpy (passive_instance->token, msg, msg_len);
1175  passive_timer_expired_token_start (passive_instance);
1176 
1177  }
1178 
1179  passive_monitor (rrp_instance, iface_no, 1);
1180 }
1181 
1182 static void passive_token_send (
1183  struct totemrrp_instance *instance,
1184  const void *msg,
1185  unsigned int msg_len)
1186 {
1187  struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1188  int i = 0;
1189 
1190  do {
1191  passive_instance->token_xmit_iface = (passive_instance->token_xmit_iface + 1) % instance->interface_count;
1192  i++;
1193  } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->token_xmit_iface] == 1));
1194 
1195  if (i > instance->interface_count) {
1196  /*
1197  * All interfaces are faulty. It's still needed to send token
1198  * message to (potentionally) local host so use first interface.
1199  */
1200  passive_instance->msg_xmit_iface = 0;
1201  }
1202 
1204  instance->net_handles[passive_instance->token_xmit_iface],
1205  msg, msg_len);
1206 }
1207 
1208 static void passive_recv_flush (struct totemrrp_instance *instance)
1209 {
1210  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1211  unsigned int i;
1212 
1213  for (i = 0; i < instance->interface_count; i++) {
1214  if (rrp_algo_instance->faulty[i] == 0) {
1215 
1216  totemnet_recv_flush (instance->net_handles[i]);
1217  }
1218  }
1219 }
1220 
1221 static void passive_send_flush (struct totemrrp_instance *instance)
1222 {
1223  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1224  unsigned int i;
1225 
1226  for (i = 0; i < instance->interface_count; i++) {
1227  if (rrp_algo_instance->faulty[i] == 0) {
1228 
1229  totemnet_send_flush (instance->net_handles[i]);
1230  }
1231  }
1232 }
1233 
1234 static void passive_iface_check (struct totemrrp_instance *instance)
1235 {
1236  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1237  unsigned int i;
1238 
1239  for (i = 0; i < instance->interface_count; i++) {
1240  if (rrp_algo_instance->faulty[i] == 0) {
1241 
1242  totemnet_iface_check (instance->net_handles[i]);
1243  }
1244  }
1245 }
1246 
1247 static void passive_processor_count_set (
1248  struct totemrrp_instance *instance,
1249  unsigned int processor_count)
1250 {
1251  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1252  unsigned int i;
1253 
1254  for (i = 0; i < instance->interface_count; i++) {
1255  if (rrp_algo_instance->faulty[i] == 0) {
1256 
1258  processor_count);
1259  }
1260  }
1261 }
1262 
1263 static void passive_token_target_set (
1264  struct totemrrp_instance *instance,
1265  struct totem_ip_address *token_target,
1266  unsigned int iface_no)
1267 {
1268  totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1269 }
1270 
1271 static int passive_mcast_recv_empty (
1272  struct totemrrp_instance *instance)
1273 {
1274  int res;
1275  int msgs_emptied = 0;
1276  int i;
1277 
1278  for (i = 0; i < instance->interface_count; i++) {
1279  res = totemnet_recv_mcast_empty (instance->net_handles[i]);
1280  if (res == -1) {
1281  return (-1);
1282  }
1283  if (res == 1) {
1284  msgs_emptied = 1;
1285  }
1286  }
1287 
1288  return (msgs_emptied);
1289 }
1290 
1291 static int passive_member_add (
1292  struct totemrrp_instance *instance,
1293  const struct totem_ip_address *member,
1294  unsigned int iface_no)
1295 {
1296  int res;
1297  res = totemnet_member_add (instance->net_handles[iface_no], member);
1298  return (res);
1299 }
1300 
1301 static int passive_member_remove (
1302  struct totemrrp_instance *instance,
1303  const struct totem_ip_address *member,
1304  unsigned int iface_no)
1305 {
1306  int res;
1307  res = totemnet_member_remove (instance->net_handles[iface_no], member);
1308  return (res);
1309 }
1310 
1311 static void passive_membership_changed (
1312  struct totemrrp_instance *rrp_instance,
1313  enum totem_configuration_type configuration_type,
1314  const struct srp_addr *member_list, size_t member_list_entries,
1315  const struct srp_addr *left_list, size_t left_list_entries,
1316  const struct srp_addr *joined_list, size_t joined_list_entries,
1317  const struct memb_ring_id *ring_id)
1318 {
1319  int i;
1320  int interface;
1321 
1322  for (interface = 0; interface < rrp_instance->interface_count; interface++) {
1323  for (i = 0; i < left_list_entries; i++) {
1324  if (left_list->no_addrs < interface + 1 ||
1325  (left_list[i].addr[interface].family != AF_INET &&
1326  left_list[i].addr[interface].family != AF_INET6)) {
1327  log_printf(rrp_instance->totemrrp_log_level_error,
1328  "Membership left list contains incorrect address. "
1329  "This is sign of misconfiguration between nodes!");
1330  } else {
1331  totemnet_member_set_active(rrp_instance->net_handles[interface],
1332  &left_list[i].addr[interface], 0);
1333  }
1334  }
1335 
1336  for (i = 0; i < joined_list_entries; i++) {
1337  if (joined_list->no_addrs < interface + 1 ||
1338  (joined_list[i].addr[interface].family != AF_INET &&
1339  joined_list[i].addr[interface].family != AF_INET6)) {
1340  log_printf(rrp_instance->totemrrp_log_level_error,
1341  "Membership join list contains incorrect address. "
1342  "This is sign of misconfiguration between nodes!");
1343  } else {
1344  totemnet_member_set_active(rrp_instance->net_handles[interface],
1345  &joined_list[i].addr[interface], 1);
1346  }
1347  }
1348  }
1349 }
1350 
1351 static void passive_ring_reenable (
1352  struct totemrrp_instance *instance,
1353  unsigned int iface_no)
1354 {
1355  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1356  int i;
1357 
1358  memset (rrp_algo_instance->mcast_recv_count, 0, sizeof (unsigned int) *
1359  instance->interface_count);
1360  memset (rrp_algo_instance->token_recv_count, 0, sizeof (unsigned int) *
1361  instance->interface_count);
1362 
1363  if (iface_no == instance->interface_count) {
1364  memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1365  instance->interface_count);
1366  for (i = 0; i < instance->interface_count; i++) {
1367  stats_set_interface_faulty (instance, i, 0);
1368  }
1369  } else {
1370  rrp_algo_instance->faulty[iface_no] = 0;
1371  stats_set_interface_faulty (instance, iface_no, 0);
1372  }
1373 }
1374 
1375 /*
1376  * Active Replication Implementation
1377  */
1378 void *active_instance_initialize (
1379  struct totemrrp_instance *rrp_instance,
1380  int interface_count)
1381 {
1382  struct active_instance *instance;
1383  int i;
1384 
1385  instance = malloc (sizeof (struct active_instance));
1386  if (instance == 0) {
1387  goto error_exit;
1388  }
1389  memset (instance, 0, sizeof (struct active_instance));
1390 
1391  instance->faulty = malloc (sizeof (int) * interface_count);
1392  if (instance->faulty == 0) {
1393  free (instance);
1394  instance = 0;
1395  goto error_exit;
1396  }
1397  memset (instance->faulty, 0, sizeof (unsigned int) * interface_count);
1398 
1399  for (i = 0; i < interface_count; i++) {
1400  stats_set_interface_faulty (rrp_instance, i, 0);
1401  }
1402 
1403  instance->last_token_recv = malloc (sizeof (int) * interface_count);
1404  if (instance->last_token_recv == 0) {
1405  free (instance->faulty);
1406  free (instance);
1407  instance = 0;
1408  goto error_exit;
1409  }
1410  memset (instance->last_token_recv, 0, sizeof (unsigned int) * interface_count);
1411 
1412  instance->counter_problems = malloc (sizeof (int) * interface_count);
1413  if (instance->counter_problems == 0) {
1414  free (instance->last_token_recv);
1415  free (instance->faulty);
1416  free (instance);
1417  instance = 0;
1418  goto error_exit;
1419  }
1420  memset (instance->counter_problems, 0, sizeof (unsigned int) * interface_count);
1421 
1422  instance->timer_expired_token = 0;
1423 
1424  instance->timer_problem_decrementer = 0;
1425 
1426  instance->rrp_instance = rrp_instance;
1427 
1428  instance->last_token_seq = ARR_SEQNO_START_TOKEN - 1;
1429 
1430 error_exit:
1431  return ((void *)instance);
1432 }
1433 static void timer_function_active_problem_decrementer (void *context)
1434 {
1435  struct active_instance *active_instance = (struct active_instance *)context;
1436  struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
1437  unsigned int problem_found = 0;
1438  unsigned int i;
1439 
1440  for (i = 0; i < rrp_instance->interface_count; i++) {
1441  if (active_instance->counter_problems[i] > 0) {
1442  problem_found = 1;
1443  active_instance->counter_problems[i] -= 1;
1444  if (active_instance->counter_problems[i] == 0) {
1445  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1446  "ring %d active with no faults", i);
1447  } else {
1448  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1449  "Decrementing problem counter for iface %s to [%d of %d]",
1450  totemnet_iface_print (rrp_instance->net_handles[i]),
1451  active_instance->counter_problems[i],
1452  rrp_instance->totem_config->rrp_problem_count_threshold);
1453  }
1454  log_printf (
1455  rrp_instance->totemrrp_log_level_warning,
1456  "%s",
1457  rrp_instance->status[i]);
1458  }
1459  }
1460  if (problem_found) {
1461  active_timer_problem_decrementer_start (active_instance);
1462  } else {
1463  active_instance->timer_problem_decrementer = 0;
1464  }
1465 }
1466 
1467 static void timer_function_active_token_expired (void *context)
1468 {
1469  struct active_instance *active_instance = (struct active_instance *)context;
1470  struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
1471  unsigned int i;
1472 
1473  for (i = 0; i < rrp_instance->interface_count; i++) {
1474  if (active_instance->last_token_recv[i] == 0) {
1475  active_instance->counter_problems[i] += 1;
1476 
1477  if (active_instance->timer_problem_decrementer == 0) {
1478  active_timer_problem_decrementer_start (active_instance);
1479  }
1480  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1481  "Incrementing problem counter for seqid %d iface %s to [%d of %d]",
1482  active_instance->last_token_seq,
1483  totemnet_iface_print (rrp_instance->net_handles[i]),
1484  active_instance->counter_problems[i],
1485  rrp_instance->totem_config->rrp_problem_count_threshold);
1486  log_printf (
1487  rrp_instance->totemrrp_log_level_warning,
1488  "%s",
1489  rrp_instance->status[i]);
1490  }
1491  }
1492  for (i = 0; i < rrp_instance->interface_count; i++) {
1493  if (active_instance->counter_problems[i] >= rrp_instance->totem_config->rrp_problem_count_threshold &&
1494  active_instance->faulty[i] == 0) {
1495  active_instance->faulty[i] = 1;
1496 
1497  qb_loop_timer_add (rrp_instance->poll_handle,
1498  QB_LOOP_MED,
1499  rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
1500  rrp_instance->deliver_fn_context[i],
1501  timer_function_test_ring_timeout,
1502  &rrp_instance->timer_active_test_ring_timeout[i]);
1503 
1504  stats_set_interface_faulty (rrp_instance, i, active_instance->faulty[i]);
1505 
1506  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1507  "Marking seqid %d ringid %u interface %s FAULTY",
1508  active_instance->last_token_seq,
1509  i,
1510  totemnet_iface_print (rrp_instance->net_handles[i]));
1511  log_printf (
1512  rrp_instance->totemrrp_log_level_error,
1513  "%s",
1514  rrp_instance->status[i]);
1515  active_timer_problem_decrementer_cancel (active_instance);
1516  }
1517  }
1518 
1519  rrp_instance->totemrrp_deliver_fn (
1520  active_instance->totemrrp_context,
1521  active_instance->token,
1522  active_instance->token_len);
1523 }
1524 
1525 static void active_timer_expired_token_start (
1526  struct active_instance *active_instance)
1527 {
1528  qb_loop_timer_add (
1529  active_instance->rrp_instance->poll_handle,
1530  QB_LOOP_MED,
1531  active_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
1532  (void *)active_instance,
1533  timer_function_active_token_expired,
1534  &active_instance->timer_expired_token);
1535 }
1536 
1537 static void active_timer_expired_token_cancel (
1538  struct active_instance *active_instance)
1539 {
1540  qb_loop_timer_del (
1541  active_instance->rrp_instance->poll_handle,
1542  active_instance->timer_expired_token);
1543 }
1544 
1545 static void active_timer_problem_decrementer_start (
1546  struct active_instance *active_instance)
1547 {
1548  qb_loop_timer_add (
1549  active_instance->rrp_instance->poll_handle,
1550  QB_LOOP_MED,
1551  active_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
1552  (void *)active_instance,
1553  timer_function_active_problem_decrementer,
1554  &active_instance->timer_problem_decrementer);
1555 }
1556 
1557 static void active_timer_problem_decrementer_cancel (
1558  struct active_instance *active_instance)
1559 {
1560  qb_loop_timer_del (
1561  active_instance->rrp_instance->poll_handle,
1562  active_instance->timer_problem_decrementer);
1563  active_instance->timer_problem_decrementer = 0;
1564 }
1565 
1566 
1567 /*
1568  * active replication
1569  */
1570 static void active_mcast_recv (
1571  struct totemrrp_instance *instance,
1572  unsigned int iface_no,
1573  void *context,
1574  const void *msg,
1575  unsigned int msg_len)
1576 {
1577  instance->totemrrp_deliver_fn (
1578  context,
1579  msg,
1580  msg_len);
1581 }
1582 
1583 static void active_mcast_flush_send (
1584  struct totemrrp_instance *instance,
1585  const void *msg,
1586  unsigned int msg_len)
1587 {
1588  int i;
1589  int msg_sent;
1590  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1591 
1592  msg_sent = 0;
1593 
1594  for (i = 0; i < instance->interface_count; i++) {
1595  if (rrp_algo_instance->faulty[i] == 0) {
1596  msg_sent = 1;
1597  totemnet_mcast_flush_send (instance->net_handles[i], msg, msg_len);
1598  }
1599  }
1600 
1601  if (!msg_sent) {
1602  /*
1603  * All interfaces are faulty. It's still needed to send mcast
1604  * message to local host so use first interface.
1605  */
1606  totemnet_mcast_flush_send (instance->net_handles[0], msg, msg_len);
1607  }
1608 }
1609 
1610 static void active_mcast_noflush_send (
1611  struct totemrrp_instance *instance,
1612  const void *msg,
1613  unsigned int msg_len)
1614 {
1615  int i;
1616  int msg_sent;
1617  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1618 
1619  msg_sent = 0;
1620 
1621  for (i = 0; i < instance->interface_count; i++) {
1622  if (rrp_algo_instance->faulty[i] == 0) {
1623  msg_sent = 1;
1624  totemnet_mcast_noflush_send (instance->net_handles[i], msg, msg_len);
1625  }
1626  }
1627 
1628  if (!msg_sent) {
1629  /*
1630  * All interfaces are faulty. It's still needed to send mcast
1631  * message to local host so use first interface.
1632  */
1633  totemnet_mcast_noflush_send (instance->net_handles[0], msg, msg_len);
1634  }
1635 }
1636 
1637 static void active_token_recv (
1638  struct totemrrp_instance *rrp_instance,
1639  unsigned int iface_no,
1640  void *context,
1641  const void *msg,
1642  unsigned int msg_len,
1643  unsigned int token_seq)
1644 {
1645  int i;
1646  struct active_instance *active_instance = (struct active_instance *)rrp_instance->rrp_algo_instance;
1647 
1648  active_instance->totemrrp_context = context;
1649  if (sq_lt_compare (active_instance->last_token_seq, token_seq)) {
1650  memcpy (active_instance->token, msg, msg_len);
1651  active_instance->token_len = msg_len;
1652  for (i = 0; i < rrp_instance->interface_count; i++) {
1653  active_instance->last_token_recv[i] = 0;
1654  }
1655 
1656  active_instance->last_token_recv[iface_no] = 1;
1657  active_timer_expired_token_start (active_instance);
1658  }
1659 
1660  /*
1661  * This doesn't follow spec because the spec assumes we will know
1662  * when token resets occur.
1663  */
1664  active_instance->last_token_seq = token_seq;
1665 
1666  if (token_seq == active_instance->last_token_seq) {
1667  active_instance->last_token_recv[iface_no] = 1;
1668  for (i = 0; i < rrp_instance->interface_count; i++) {
1669  if ((active_instance->last_token_recv[i] == 0) &&
1670  active_instance->faulty[i] == 0) {
1671  return; /* don't deliver token */
1672  }
1673  }
1674  active_timer_expired_token_cancel (active_instance);
1675 
1676  rrp_instance->totemrrp_deliver_fn (
1677  context,
1678  msg,
1679  msg_len);
1680  }
1681 }
1682 
1683 static void active_token_send (
1684  struct totemrrp_instance *instance,
1685  const void *msg,
1686  unsigned int msg_len)
1687 {
1688  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1689  int i;
1690  int msg_sent;
1691 
1692  msg_sent = 0;
1693 
1694  for (i = 0; i < instance->interface_count; i++) {
1695  if (rrp_algo_instance->faulty[i] == 0) {
1696  msg_sent = 1;
1698  instance->net_handles[i],
1699  msg, msg_len);
1700 
1701  }
1702  }
1703 
1704  if (!msg_sent) {
1705  /*
1706  * All interfaces are faulty. It's still needed to send token
1707  * message to (potentionally) local host so use first interface.
1708  */
1710  instance->net_handles[0],
1711  msg, msg_len);
1712 
1713  }
1714 }
1715 
1716 static void active_recv_flush (struct totemrrp_instance *instance)
1717 {
1718  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1719  unsigned int i;
1720 
1721  for (i = 0; i < instance->interface_count; i++) {
1722  if (rrp_algo_instance->faulty[i] == 0) {
1723 
1724  totemnet_recv_flush (instance->net_handles[i]);
1725  }
1726  }
1727 }
1728 
1729 static void active_send_flush (struct totemrrp_instance *instance)
1730 {
1731  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1732  unsigned int i;
1733 
1734  for (i = 0; i < instance->interface_count; i++) {
1735  if (rrp_algo_instance->faulty[i] == 0) {
1736 
1737  totemnet_send_flush (instance->net_handles[i]);
1738  }
1739  }
1740 }
1741 
1742 static int active_member_add (
1743  struct totemrrp_instance *instance,
1744  const struct totem_ip_address *member,
1745  unsigned int iface_no)
1746 {
1747  int res;
1748  res = totemnet_member_add (instance->net_handles[iface_no], member);
1749  return (res);
1750 }
1751 
1752 static int active_member_remove (
1753  struct totemrrp_instance *instance,
1754  const struct totem_ip_address *member,
1755  unsigned int iface_no)
1756 {
1757  int res;
1758  res = totemnet_member_remove (instance->net_handles[iface_no], member);
1759  return (res);
1760 }
1761 
1762 static void active_membership_changed (
1763  struct totemrrp_instance *rrp_instance,
1764  enum totem_configuration_type configuration_type,
1765  const struct srp_addr *member_list, size_t member_list_entries,
1766  const struct srp_addr *left_list, size_t left_list_entries,
1767  const struct srp_addr *joined_list, size_t joined_list_entries,
1768  const struct memb_ring_id *ring_id)
1769 {
1770  int i;
1771  int interface;
1772 
1773  for (interface = 0; interface < rrp_instance->interface_count; interface++) {
1774  for (i = 0; i < left_list_entries; i++) {
1775  if (left_list->no_addrs < interface + 1 ||
1776  (left_list[i].addr[interface].family != AF_INET &&
1777  left_list[i].addr[interface].family != AF_INET6)) {
1778  log_printf(rrp_instance->totemrrp_log_level_error,
1779  "Membership left list contains incorrect address. "
1780  "This is sign of misconfiguration between nodes!");
1781  } else {
1782  totemnet_member_set_active(rrp_instance->net_handles[interface],
1783  &left_list[i].addr[interface], 0);
1784  }
1785  }
1786 
1787  for (i = 0; i < joined_list_entries; i++) {
1788  if (joined_list->no_addrs < interface + 1 ||
1789  (joined_list[i].addr[interface].family != AF_INET &&
1790  joined_list[i].addr[interface].family != AF_INET6)) {
1791  log_printf(rrp_instance->totemrrp_log_level_error,
1792  "Membership join list contains incorrect address. "
1793  "This is sign of misconfiguration between nodes!");
1794  } else {
1795  totemnet_member_set_active(rrp_instance->net_handles[interface],
1796  &joined_list[i].addr[interface], 1);
1797  }
1798  }
1799  }
1800 }
1801 
1802 static void active_iface_check (struct totemrrp_instance *instance)
1803 {
1804  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1805  unsigned int i;
1806 
1807  for (i = 0; i < instance->interface_count; i++) {
1808  if (rrp_algo_instance->faulty[i] == 0) {
1809 
1810  totemnet_iface_check (instance->net_handles[i]);
1811  }
1812  }
1813 }
1814 
1815 static void active_processor_count_set (
1816  struct totemrrp_instance *instance,
1817  unsigned int processor_count)
1818 {
1819  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1820  unsigned int i;
1821 
1822  for (i = 0; i < instance->interface_count; i++) {
1823  if (rrp_algo_instance->faulty[i] == 0) {
1824 
1826  processor_count);
1827  }
1828  }
1829 }
1830 
1831 static void active_token_target_set (
1832  struct totemrrp_instance *instance,
1833  struct totem_ip_address *token_target,
1834  unsigned int iface_no)
1835 {
1836  totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1837 }
1838 
1839 static int active_mcast_recv_empty (
1840  struct totemrrp_instance *instance)
1841 {
1842  int res;
1843  int msgs_emptied = 0;
1844  int i;
1845 
1846  for (i = 0; i < instance->interface_count; i++) {
1847  res = totemnet_recv_mcast_empty (instance->net_handles[i]);
1848  if (res == -1) {
1849  return (-1);
1850  }
1851  if (res == 1) {
1852  msgs_emptied = 1;
1853  }
1854  }
1855 
1856  return (msgs_emptied);
1857 }
1858 
1859 static void active_ring_reenable (
1860  struct totemrrp_instance *instance,
1861  unsigned int iface_no)
1862 {
1863  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1864  int i;
1865 
1866  if (iface_no == instance->interface_count) {
1867  memset (rrp_algo_instance->last_token_recv, 0, sizeof (unsigned int) *
1868  instance->interface_count);
1869  memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1870  instance->interface_count);
1871  memset (rrp_algo_instance->counter_problems, 0, sizeof (unsigned int) *
1872  instance->interface_count);
1873 
1874  for (i = 0; i < instance->interface_count; i++) {
1875  stats_set_interface_faulty (instance, i, 0);
1876  }
1877  } else {
1878  rrp_algo_instance->last_token_recv[iface_no] = 0;
1879  rrp_algo_instance->faulty[iface_no] = 0;
1880  rrp_algo_instance->counter_problems[iface_no] = 0;
1881 
1882  stats_set_interface_faulty (instance, iface_no, 0);
1883  }
1884 }
1885 
1886 static void totemrrp_instance_initialize (struct totemrrp_instance *instance)
1887 {
1888  memset (instance, 0, sizeof (struct totemrrp_instance));
1889 }
1890 
1891 static int totemrrp_algorithm_set (
1892  struct totem_config *totem_config,
1893  struct totemrrp_instance *instance)
1894 {
1895  unsigned int res = -1;
1896  unsigned int i;
1897 
1898  for (i = 0; i < RRP_ALGOS_COUNT; i++) {
1899  if (strcmp (totem_config->rrp_mode, rrp_algos[i]->name) == 0) {
1900  instance->rrp_algo = rrp_algos[i];
1901  if (rrp_algos[i]->initialize) {
1902  instance->rrp_algo_instance = rrp_algos[i]->initialize (
1903  instance,
1904  totem_config->interface_count);
1905  }
1906  res = 0;
1907  break;
1908  }
1909  }
1910  for (i = 0; i < totem_config->interface_count; i++) {
1911  instance->status[i] = malloc (STATUS_STR_LEN+1);
1912  snprintf (instance->status[i], STATUS_STR_LEN,
1913  "ring %d active with no faults", i);
1914  }
1915  return (res);
1916 }
1917 
1919  void *context,
1920  const void *msg,
1921  unsigned int msg_len)
1922 {
1923  unsigned int token_seqid;
1924  unsigned int token_is;
1925 
1926  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1927  struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
1928  const struct message_header *hdr = msg;
1929  struct message_header tmp_msg, activate_msg;
1930 
1931  memset(&tmp_msg, 0, sizeof(struct message_header));
1932  memset(&activate_msg, 0, sizeof(struct message_header));
1933 
1934  rrp_instance->totemrrp_token_seqid_get (
1935  msg,
1936  &token_seqid,
1937  &token_is);
1938 
1939  if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVE) {
1940  log_printf (
1941  rrp_instance->totemrrp_log_level_debug,
1942  "received message requesting test of ring now active");
1943 
1944  if (hdr->endian_detector != ENDIAN_LOCAL) {
1945  test_active_msg_endian_convert(hdr, &tmp_msg);
1946  hdr = &tmp_msg;
1947  }
1948 
1949  if (hdr->nodeid_activator == rrp_instance->my_nodeid) {
1950  /*
1951  * Send an activate message
1952  */
1953  activate_msg.type = MESSAGE_TYPE_RING_TEST_ACTIVATE;
1954  activate_msg.endian_detector = ENDIAN_LOCAL;
1955  activate_msg.ring_number = hdr->ring_number;
1956  activate_msg.nodeid_activator = rrp_instance->my_nodeid;
1958  rrp_instance->net_handles[deliver_fn_context->iface_no],
1959  &activate_msg, sizeof (struct message_header));
1960  } else {
1961  /*
1962  * Send a ring test message
1963  */
1965  rrp_instance->net_handles[deliver_fn_context->iface_no],
1966  msg, msg_len);
1967  }
1968  } else
1969  if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVATE) {
1970 
1971  if (hdr->endian_detector != ENDIAN_LOCAL) {
1972  test_active_msg_endian_convert(hdr, &tmp_msg);
1973  hdr = &tmp_msg;
1974  }
1975 
1976  log_printf (
1977  rrp_instance->totemrrp_log_level_debug,
1978  "Received ring test activate message for ring %d sent by node %u",
1979  hdr->ring_number,
1980  hdr->nodeid_activator);
1981 
1982  if (rrp_instance->stats.faulty[deliver_fn_context->iface_no]) {
1983  log_printf (rrp_instance->totemrrp_log_level_notice,
1984  "Automatically recovered ring %d", hdr->ring_number);
1985  }
1986 
1987  totemrrp_ring_reenable (rrp_instance, deliver_fn_context->iface_no);
1988  if (hdr->nodeid_activator != rrp_instance->my_nodeid) {
1990  rrp_instance->net_handles[deliver_fn_context->iface_no],
1991  msg, msg_len);
1992  }
1993  } else
1994  if (token_is) {
1995  /*
1996  * Deliver to the token receiver for this rrp algorithm
1997  */
1998  rrp_instance->rrp_algo->token_recv (
1999  rrp_instance,
2000  deliver_fn_context->iface_no,
2001  deliver_fn_context->context,
2002  msg,
2003  msg_len,
2004  token_seqid);
2005  } else {
2006  /*
2007  * Deliver to the mcast receiver for this rrp algorithm
2008  */
2009  rrp_instance->rrp_algo->mcast_recv (
2010  rrp_instance,
2011  deliver_fn_context->iface_no,
2012  deliver_fn_context->context,
2013  msg,
2014  msg_len);
2015  }
2016 }
2017 
2019  void *context,
2020  const struct totem_ip_address *iface_addr)
2021 {
2022  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
2023 
2024  deliver_fn_context->instance->my_nodeid = iface_addr->nodeid;
2025  deliver_fn_context->instance->totemrrp_iface_change_fn (
2026  deliver_fn_context->context,
2027  iface_addr,
2028  deliver_fn_context->iface_no);
2029 }
2030 
2032  void *rrp_context)
2033 {
2034  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2035  int i;
2036 
2037  for (i = 0; i < instance->interface_count; i++) {
2038  totemnet_finalize (instance->net_handles[i]);
2039  }
2040  free (instance->net_handles);
2041  free (instance);
2042  return (0);
2043 }
2044 
2045 static void rrp_target_set_completed (void *context)
2046 {
2047  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
2048 
2049  deliver_fn_context->instance->totemrrp_target_set_completed (deliver_fn_context->context);
2050 }
2051 
2052 /*
2053  * Totem Redundant Ring interface
2054  * depends on poll abstraction, POSIX, IPV4
2055  */
2056 
2057 /*
2058  * Create an instance
2059  */
2061  qb_loop_t *poll_handle,
2062  void **rrp_context,
2063  struct totem_config *totem_config,
2064  totemsrp_stats_t *stats,
2065  void *context,
2066 
2067  void (*deliver_fn) (
2068  void *context,
2069  const void *msg,
2070  unsigned int msg_len),
2071 
2072  void (*iface_change_fn) (
2073  void *context,
2074  const struct totem_ip_address *iface_addr,
2075  unsigned int iface_no),
2076 
2077  void (*token_seqid_get) (
2078  const void *msg,
2079  unsigned int *seqid,
2080  unsigned int *token_is),
2081 
2082  unsigned int (*msgs_missing) (void),
2083 
2084  void (*target_set_completed) (void *context))
2085 {
2086  struct totemrrp_instance *instance;
2087  unsigned int res;
2088  int i;
2089 
2090  instance = malloc (sizeof (struct totemrrp_instance));
2091  if (instance == 0) {
2092  return (-1);
2093  }
2094 
2095  totemrrp_instance_initialize (instance);
2096 
2097  instance->totem_config = totem_config;
2098  stats->rrp = &instance->stats;
2099  instance->stats.interface_count = totem_config->interface_count;
2100  instance->stats.faulty = calloc(instance->stats.interface_count, sizeof(uint8_t));
2101 
2102  res = totemrrp_algorithm_set (
2103  instance->totem_config,
2104  instance);
2105  if (res == -1) {
2106  goto error_destroy;
2107  }
2108 
2109  /*
2110  * Configure logging
2111  */
2119 
2120  instance->interfaces = totem_config->interfaces;
2121 
2122  instance->poll_handle = poll_handle;
2123 
2124  instance->totemrrp_deliver_fn = deliver_fn;
2125 
2126  instance->totemrrp_iface_change_fn = iface_change_fn;
2127 
2128  instance->totemrrp_token_seqid_get = token_seqid_get;
2129 
2130  instance->totemrrp_target_set_completed = target_set_completed;
2131 
2132  instance->totemrrp_msgs_missing = msgs_missing;
2133 
2134  instance->interface_count = totem_config->interface_count;
2135 
2136  instance->net_handles = malloc (sizeof (void *) * totem_config->interface_count);
2137 
2138  instance->context = context;
2139 
2140  instance->poll_handle = poll_handle;
2141 
2142 
2143  for (i = 0; i < totem_config->interface_count; i++) {
2144  struct deliver_fn_context *deliver_fn_context;
2145 
2146  deliver_fn_context = malloc (sizeof (struct deliver_fn_context));
2147  assert (deliver_fn_context);
2148  deliver_fn_context->instance = instance;
2149  deliver_fn_context->context = context;
2150  deliver_fn_context->iface_no = i;
2151  instance->deliver_fn_context[i] = (void *)deliver_fn_context;
2152 
2153  res = totemnet_initialize (
2154  poll_handle,
2155  &instance->net_handles[i],
2156  totem_config,
2157  stats,
2158  i,
2159  (void *)deliver_fn_context,
2162  rrp_target_set_completed);
2163 
2164  if (res == -1) {
2165  goto error_destroy;
2166  }
2167 
2168  totemnet_net_mtu_adjust (instance->net_handles[i], totem_config);
2169  }
2170 
2171  *rrp_context = instance;
2172  return (0);
2173 
2174 error_destroy:
2175  free (instance);
2176  return (res);
2177 }
2178 
2179 void *totemrrp_buffer_alloc (void *rrp_context)
2180 {
2181  struct totemrrp_instance *instance = rrp_context;
2182  assert (instance != NULL);
2183  return totemnet_buffer_alloc (instance->net_handles[0]);
2184 }
2185 
2186 void totemrrp_buffer_release (void *rrp_context, void *ptr)
2187 {
2188  struct totemrrp_instance *instance = rrp_context;
2189  assert (instance != NULL);
2190  totemnet_buffer_release (instance->net_handles[0], ptr);
2191 }
2192 
2194  void *rrp_context,
2195  unsigned int processor_count)
2196 {
2197  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2198  instance->rrp_algo->processor_count_set (instance, processor_count);
2199 
2200  instance->processor_count = processor_count;
2201 
2202  return (0);
2203 }
2204 
2206  void *rrp_context,
2207  struct totem_ip_address *addr,
2208  unsigned int iface_no)
2209 {
2210  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2211  instance->rrp_algo->token_target_set (instance, addr, iface_no);
2212 
2213  return (0);
2214 }
2215 int totemrrp_recv_flush (void *rrp_context)
2216 {
2217  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2218 
2219  instance->rrp_algo->recv_flush (instance);
2220 
2221  return (0);
2222 }
2223 
2224 int totemrrp_send_flush (void *rrp_context)
2225 {
2226  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2227  instance->rrp_algo->send_flush (instance);
2228 
2229  return (0);
2230 }
2231 
2233  void *rrp_context,
2234  const void *msg,
2235  unsigned int msg_len)
2236 {
2237  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2238  instance->rrp_algo->token_send (instance, msg, msg_len);
2239 
2240  return (0);
2241 }
2242 
2244  void *rrp_context,
2245  const void *msg,
2246  unsigned int msg_len)
2247 {
2248  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2249  int res = 0;
2250 
2251 // TODO this needs to return the result
2252  instance->rrp_algo->mcast_flush_send (instance, msg, msg_len);
2253 
2254  return (res);
2255 }
2256 
2258  void *rrp_context,
2259  const void *msg,
2260  unsigned int msg_len)
2261 {
2262  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2263  /*
2264  * merge detects go out through mcast_flush_send so it is safe to
2265  * flush these messages if we are only one processor. This avoids
2266  * an encryption/hmac and decryption/hmac
2267  */
2268  if (instance->processor_count > 1) {
2269 
2270 // TODO this needs to return the result
2271  instance->rrp_algo->mcast_noflush_send (instance, msg, msg_len);
2272  }
2273 
2274  return (0);
2275 }
2276 
2277 int totemrrp_iface_check (void *rrp_context)
2278 {
2279  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2280 
2281  instance->rrp_algo->iface_check (instance);
2282 
2283  return (0);
2284 }
2285 
2287  void *rrp_context,
2288  char ***status,
2289  unsigned int *iface_count)
2290 {
2291  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2292  *status = instance->status;
2293 
2294  if (iface_count) {
2295  *iface_count = instance->interface_count;
2296  }
2297 
2298  return (0);
2299 }
2300 
2302  void *rrp_context,
2303  const char *cipher_type,
2304  const char *hash_type)
2305 {
2306  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2307  int res;
2308 
2309  res = totemnet_crypto_set(instance->net_handles[0], cipher_type, hash_type);
2310 
2311  return (res);
2312 }
2313 
2314 
2315 /*
2316  * iface_no indicates the interface number [0, ..., interface_count-1] of the
2317  * specific ring which will be reenabled. We specify iface_no == interface_count
2318  * means reenabling all the rings.
2319  */
2321  void *rrp_context,
2322  unsigned int iface_no)
2323 {
2324  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2325  int res = 0;
2326  unsigned int i;
2327 
2328  instance->rrp_algo->ring_reenable (instance, iface_no);
2329 
2330  if (iface_no == instance->interface_count) {
2331  for (i = 0; i < instance->interface_count; i++) {
2332  snprintf (instance->status[i], STATUS_STR_LEN,
2333  "ring %d active with no faults", i);
2334  }
2335  } else {
2336  snprintf (instance->status[iface_no], STATUS_STR_LEN,
2337  "ring %d active with no faults", iface_no);
2338  }
2339 
2340  return (res);
2341 }
2342 
2344  void *rrp_context)
2345 {
2346  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2347  int res;
2348 
2349  res = instance->rrp_algo->mcast_recv_empty (instance);
2350 
2351  return (res);
2352 }
2353 
2355  void *rrp_context,
2356  const struct totem_ip_address *member,
2357  int iface_no)
2358 {
2359  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2360  int res;
2361 
2362  res = instance->rrp_algo->member_add (instance, member, iface_no);
2363 
2364  return (res);
2365 }
2366 
2368  void *rrp_context,
2369  const struct totem_ip_address *member,
2370  int iface_no)
2371 {
2372  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2373  int res;
2374 
2375  res = instance->rrp_algo->member_remove (instance, member, iface_no);
2376 
2377  return (res);
2378 }
2379 
2381  void *rrp_context,
2382  enum totem_configuration_type configuration_type,
2383  const struct srp_addr *member_list, size_t member_list_entries,
2384  const struct srp_addr *left_list, size_t left_list_entries,
2385  const struct srp_addr *joined_list, size_t joined_list_entries,
2386  const struct memb_ring_id *ring_id)
2387 {
2388  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2389 
2390  instance->rrp_algo->membership_changed (instance,
2391  configuration_type,
2392  member_list, member_list_entries,
2393  left_list, left_list_entries,
2394  joined_list, joined_list_entries,
2395  ring_id);
2396 }