C18 MFC never recovers from removal of static route to source

pimd/CAVEATS

@@ -134,4 +134,28 @@ C16 AMT Draft (mboned-auto-multicast) is not supported.
 
 C17 SNMP / RFC 5060 (PIM MIB) is not supported.
 
+C18 MFC never recovers from removal of static route to source
+
+    # route add -host 1.2.3.4 gw 192.168.56.10
+    Before removal:
+    quagga-pimd-router# sh ip mroute
+    Source          Group           Proto Input iVifI Output oVifI TTL Uptime
+    1.2.3.4         232.1.2.3       I     eth1      3 eth0       2   1 00:00:36
+
+    # route del -host 1.2.3.4 gw 192.168.56.10
+    After removal: sh ip mroute --> empty output
+
+    # route add -host 1.2.3.4 gw 192.168.56.10
+    After the route is restored: sh ip mroute --> never recovers (empty output)
+
+    At this point, "no ip pim ssm" on the upstream interface (eth0) crashes pimd:
+
+    2014/02/14 16:30:14 PIM: ifmembership_set: (S,G)=(1.2.3.4,232.1.2.3) membership now is NOINFO on interface eth0
+    2014/02/14 16:30:14 PIM: pim_ifchannel_update_assert_tracking_desired: AssertTrackingDesired(1.2.3.4,232.1.2.3,eth0) changed from 1 to 0
+    2014/02/14 16:30:14 PIM: pim_zebra.c del_oif: nonexistent protocol mask 2 removed OIF eth0 (vif_index=2, min_ttl=0) from channel (S,G)=(1.2.3.4,232.1.2.3)
+    2014/02/14 16:30:14 PIM: pim_ifchannel_update_could_assert: CouldAssert(1.2.3.4,232.1.2.3,eth0) changed from 1 to 0
+    2014/02/14 16:30:14 PIM: pim_ifchannel_update_my_assert_metric: my_assert_metric(1.2.3.4,232.1.2.3,eth0) changed from 0,0,0,10.0.2.15 to 1,4294967295,4294967295,0.0.0.0
+    2014/02/14 16:30:14 PIM: pim_zebra.c del_oif: nonexistent protocol mask 1 removed OIF eth0 (vif_index=2, min_ttl=0) from channel (S,G)=(1.2.3.4,232.1.2.3)
+    2014/02/14 16:30:14 PIM: Assertion `!IGMP_SOURCE_TEST_FORWARDING(source->source_flags)' failed in file pim_igmpv3.c, line 412, function igmp_source_delete
+
 -x-

pimd/COMMANDS

@@ -47,7 +47,9 @@ verification commands:
 debug commands:
        clear ip interfaces		Reset interfaces
        clear ip igmp interfaces		Reset IGMP interfaces
+       clear ip mroute                  Reset multicast routes
        clear ip pim interfaces		Reset PIM interfaces
+       clear ip pim oil			Rescan PIM OIL (output interface list)
        debug igmp			IGMP protocol activity
        debug mroute			PIM interaction with kernel MFC cache
        debug pim			PIM protocol activity

pimd/pim_cmd.c

@@ -49,6 +49,7 @@
 #include "pim_rpf.h"
 #include "pim_macro.h"
 #include "pim_ssmpingd.h"
+#include "pim_zebra.h"
 
 static struct cmd_node pim_global_node = {
   PIM_NODE,
@@ -1097,6 +1098,25 @@ static void show_rpf_refresh_stats(struct vty *vty, time_t now)
 	  refresh_uptime, VTY_NEWLINE);
 }
 
+static void show_scan_oil_stats(struct vty *vty, time_t now)
+{
+  char uptime_scan_oil[10];
+  char uptime_mroute_add[10];
+  char uptime_mroute_del[10];
+
+  pim_time_uptime_begin(uptime_scan_oil, sizeof(uptime_scan_oil), now, qpim_scan_oil_last);
+  pim_time_uptime_begin(uptime_mroute_add, sizeof(uptime_mroute_add), now, qpim_mroute_add_last);
+  pim_time_uptime_begin(uptime_mroute_del, sizeof(uptime_mroute_del), now, qpim_mroute_del_last);
+
+  vty_out(vty,
+          "Scan OIL - Last: %s  Events: %lld%s"
+          "MFC Add  - Last: %s  Events: %lld%s"
+          "MFC Del  - Last: %s  Events: %lld%s",
+          uptime_scan_oil,   (long long) qpim_scan_oil_events,   VTY_NEWLINE,
+          uptime_mroute_add, (long long) qpim_mroute_add_events, VTY_NEWLINE,
+          uptime_mroute_del, (long long) qpim_mroute_del_events, VTY_NEWLINE);
+}
+
 static void pim_show_rpf(struct vty *vty)
 {
   struct listnode     *up_node;
@@ -1576,6 +1596,57 @@ DEFUN (clear_ip_igmp_interfaces,
   return CMD_SUCCESS;
 }
 
+static void mroute_add_all()
+{
+  struct listnode    *node;
+  struct channel_oil *c_oil;
+
+  for (ALL_LIST_ELEMENTS_RO(qpim_channel_oil_list, node, c_oil)) {
+    if (pim_mroute_add(&c_oil->oil)) {
+      /* just log warning */
+      char source_str[100];
+      char group_str[100];
+      pim_inet4_dump("<source?>", c_oil->oil.mfcc_origin, source_str, sizeof(source_str));
+      pim_inet4_dump("<group?>", c_oil->oil.mfcc_mcastgrp, group_str, sizeof(group_str));
+      zlog_warn("%s %s: (S,G)=(%s,%s) failure writing MFC",
+                __FILE__, __PRETTY_FUNCTION__,
+                source_str, group_str);
+    }
+  }
+}
+
+static void mroute_del_all()
+{
+  struct listnode    *node;
+  struct channel_oil *c_oil;
+
+  for (ALL_LIST_ELEMENTS_RO(qpim_channel_oil_list, node, c_oil)) {
+    if (pim_mroute_del(&c_oil->oil)) {
+      /* just log warning */
+      char source_str[100];
+      char group_str[100];
+      pim_inet4_dump("<source?>", c_oil->oil.mfcc_origin, source_str, sizeof(source_str));
+      pim_inet4_dump("<group?>", c_oil->oil.mfcc_mcastgrp, group_str, sizeof(group_str));
+      zlog_warn("%s %s: (S,G)=(%s,%s) failure clearing MFC",
+                __FILE__, __PRETTY_FUNCTION__,
+                source_str, group_str);
+    }
+  }
+}
+
+DEFUN (clear_ip_mroute,
+       clear_ip_mroute_cmd,
+       "clear ip mroute",
+       CLEAR_STR
+       IP_STR
+       "Reset multicast routes\n")
+{
+  mroute_del_all();
+  mroute_add_all();
+
+  return CMD_SUCCESS;
+}
+
 DEFUN (clear_ip_pim_interfaces,
        clear_ip_pim_interfaces_cmd,
        "clear ip pim interfaces",
@@ -1589,6 +1660,19 @@ DEFUN (clear_ip_pim_interfaces,
   return CMD_SUCCESS;
 }
 
+DEFUN (clear_ip_pim_oil,
+       clear_ip_pim_oil_cmd,
+       "clear ip pim oil",
+       CLEAR_STR
+       IP_STR
+       CLEAR_IP_PIM_STR
+       "Rescan PIM OIL (output interface list)\n")
+{
+  pim_scan_oil();
+
+  return CMD_SUCCESS;
+}
+
 DEFUN (show_ip_igmp_interface,
        show_ip_igmp_interface_cmd,
        "show ip igmp interface",
@@ -2026,6 +2110,10 @@ DEFUN (show_ip_multicast,
 
   show_rpf_refresh_stats(vty, now);
 
+  vty_out(vty, "%s", VTY_NEWLINE);
+
+  show_scan_oil_stats(vty, now);
+
   show_multicast_interfaces(vty);
   
   return CMD_SUCCESS;
@@ -2258,44 +2346,6 @@ DEFUN (show_ip_ssmpingd,
   return CMD_SUCCESS;
 }
 
-static void mroute_add_all()
-{
-  struct listnode    *node;
-  struct channel_oil *c_oil;
-
-  for (ALL_LIST_ELEMENTS_RO(qpim_channel_oil_list, node, c_oil)) {
-    if (pim_mroute_add(&c_oil->oil)) {
-      /* just log warning */
-      char source_str[100];
-      char group_str[100]; 
-      pim_inet4_dump("<source?>", c_oil->oil.mfcc_origin, source_str, sizeof(source_str));
-      pim_inet4_dump("<group?>", c_oil->oil.mfcc_mcastgrp, group_str, sizeof(group_str));
-      zlog_warn("%s %s: (S,G)=(%s,%s) failure writing MFC",
-		__FILE__, __PRETTY_FUNCTION__,
-		source_str, group_str);
-    }
-  }
-}
-
-static void mroute_del_all()
-{
-  struct listnode    *node;
-  struct channel_oil *c_oil;
-
-  for (ALL_LIST_ELEMENTS_RO(qpim_channel_oil_list, node, c_oil)) {
-    if (pim_mroute_del(&c_oil->oil)) {
-      /* just log warning */
-      char source_str[100];
-      char group_str[100]; 
-      pim_inet4_dump("<source?>", c_oil->oil.mfcc_origin, source_str, sizeof(source_str));
-      pim_inet4_dump("<group?>", c_oil->oil.mfcc_mcastgrp, group_str, sizeof(group_str));
-      zlog_warn("%s %s: (S,G)=(%s,%s) failure clearing MFC",
-		__FILE__, __PRETTY_FUNCTION__,
-		source_str, group_str);
-    }
-  }
-}
-
 DEFUN (ip_multicast_routing,
        ip_multicast_routing_cmd,
        PIM_CMD_IP_MULTICAST_ROUTING,
@@ -4268,7 +4318,9 @@ void pim_cmd_init()
 
   install_element (ENABLE_NODE, &clear_ip_interfaces_cmd);
   install_element (ENABLE_NODE, &clear_ip_igmp_interfaces_cmd);
+  install_element (ENABLE_NODE, &clear_ip_mroute_cmd);
   install_element (ENABLE_NODE, &clear_ip_pim_interfaces_cmd);
+  install_element (ENABLE_NODE, &clear_ip_pim_oil_cmd);
 
   install_element (ENABLE_NODE, &show_ip_igmp_interface_cmd);
   install_element (ENABLE_NODE, &show_ip_igmp_join_cmd);

pimd/pim_mroute.c

@@ -399,6 +399,9 @@ int pim_mroute_add(struct mfcctl *mc)
 {
   int err;
 
+  qpim_mroute_add_last = pim_time_monotonic_sec();
+  ++qpim_mroute_add_events;
+
   if (PIM_MROUTE_IS_DISABLED) {
     zlog_warn("%s: global multicast is disabled",
 	      __PRETTY_FUNCTION__);
@@ -424,6 +427,9 @@ int pim_mroute_del(struct mfcctl *mc)
 {
   int err;
 
+  qpim_mroute_del_last = pim_time_monotonic_sec();
+  ++qpim_mroute_del_events;
+
   if (PIM_MROUTE_IS_DISABLED) {
     zlog_warn("%s: global multicast is disabled",
 	      __PRETTY_FUNCTION__);

pimd/pim_zebra.c

@@ -353,12 +353,15 @@ static void scan_upstream_rpf_cache()
   
 }
 
-static void scan_oil()
+void pim_scan_oil()
 {
   struct listnode    *node;
   struct listnode    *nextnode;
   struct channel_oil *c_oil;
 
+  qpim_scan_oil_last = pim_time_monotonic_sec();
+  ++qpim_scan_oil_events;
+
   for (ALL_LIST_ELEMENTS(qpim_channel_oil_list, node, nextnode, c_oil)) {
     int old_vif_index;
     int input_iface_vif_index = fib_lookup_if_vif_index(c_oil->oil.mfcc_origin);
@@ -445,7 +448,7 @@ static int on_rpf_cache_refresh(struct thread *t)
   scan_upstream_rpf_cache();
 
   /* update kernel multicast forwarding cache (MFC) */
-  scan_oil();
+  pim_scan_oil();
 
   qpim_rpf_cache_refresh_last = pim_time_monotonic_sec();
   ++qpim_rpf_cache_refresh_events;

pimd/pim_zebra.h

@@ -28,6 +28,8 @@
 
 void pim_zebra_init(void);
 
+void pim_scan_oil(void);
+
 void igmp_anysource_forward_start(struct igmp_group *group);
 void igmp_anysource_forward_stop(struct igmp_group *group);
 

pimd/pimd.c

@@ -61,6 +61,12 @@ int64_t                   qpim_rpf_cache_refresh_last =  0;
 struct in_addr            qpim_inaddr_any;
 struct list              *qpim_ssmpingd_list = 0;
 struct in_addr            qpim_ssmpingd_group_addr;
+int64_t                   qpim_scan_oil_events = 0;
+int64_t                   qpim_scan_oil_last = 0;
+int64_t                   qpim_mroute_add_events = 0;
+int64_t                   qpim_mroute_add_last = 0;
+int64_t                   qpim_mroute_del_events = 0;
+int64_t                   qpim_mroute_del_last = 0;
 
 static void pim_free()
 {

pimd/pimd.h

@@ -90,6 +90,12 @@ int64_t                   qpim_rpf_cache_refresh_last;
 struct in_addr            qpim_inaddr_any;
 struct list              *qpim_ssmpingd_list; /* list of struct ssmpingd_sock */
 struct in_addr            qpim_ssmpingd_group_addr;
+int64_t                   qpim_scan_oil_events;
+int64_t                   qpim_scan_oil_last;
+int64_t                   qpim_mroute_add_events;
+int64_t                   qpim_mroute_add_last;
+int64_t                   qpim_mroute_del_events;
+int64_t                   qpim_mroute_del_last;
 
 #define PIM_JP_HOLDTIME (qpim_t_periodic * 7 / 2)