fixed support for --report-notes and added --report-packets, solving issues EricssonResearch#165 and EricssonResearch#141

Author: jariarkko

.gitignore

@@ -20,5 +20,6 @@ spindump_test.out
 install_manifest.txt
 test/*.out.pre
 test/*.out
+test/*.cmd
 src/*.out
 build

Usage.md

@@ -83,9 +83,16 @@ Sets the tool to either report or not report individual spin bit values or spin
     --report-qr-loss
     --not-report-qr-loss
 
-Sets the tool to either report or not report round-trip loss rates or unidirectional loss rates for QUIC connections based on two experimental algorithms, when using the --textual mode. The default is to not report packet loss rates.
+ Sets the tool to either report or not report round-trip loss rates or unidirectional loss rates for QUIC connections based on two experimental algorithms, when using the --textual mode. The default is to not report packet loss rates.
 
-    --debug
+    --report-packets
+    --no-report-packets
+    --report-notes
+    --no-report-notes
+
+The first two options set the tool to either report statistics on every packet or not. Note that --report-spins also sets reporting for all packets, but only for QUIC flows. The second two options set the tool to either report the textual "note" field on JSON or textual outputs or not.
+
+    --debug 
     --no-debug
     --deepdebug
     --no-deepdebug

doc/api/analyzer.md

@@ -145,7 +145,7 @@ Called whenever there's a spin bit value in a QUIC connection from the responder
 
 ### spindump_analyze_event_newpacket
 
-Called whenever there's a new packet.
+Called whenever there's a new packet but no other callback was made (e.g., no new RTT measurements were made).
 
 ### spindump_analyze_event_firstresponsepacket
 

src/spindump_analyze.c

@@ -348,7 +348,11 @@ spindump_analyze_process_handlers(struct spindump_analyze* state,
     if ((handler->eventmask & event) != 0) {
       spindump_assert(spindump_analyze_max_handlers == spindump_connection_max_handlers);
       spindump_assert(i < spindump_connection_max_handlers);
-      spindump_deepdebugf("calling handler %x (%lx)", handler->eventmask, (unsigned long)handler->function);
+      state->stats->analyzerHandlerCalls++;
+      spindump_deepdebugf("calling %uth handler %x (%lx)",
+                          state->stats->analyzerHandlerCalls,
+                          handler->eventmask,
+                          (unsigned long)handler->function);
       (*(handler->function))(state,
                              handler->handlerData,
                              &connection->handlerConnectionDatas[i],
@@ -451,10 +455,19 @@ spindump_analyze_process(struct spindump_analyze* state,
   //
 
   spindump_assert(state != 0);
+  spindump_assert(state->stats != 0);
   spindump_assert(packet != 0);
   spindump_assert(spindump_packet_isvalid(packet));
   spindump_assert(p_connection != 0);
 
+  //
+  // Store a count of events before processing this packet
+  //
+  
+  packet->analyzerHandlerCalls = state->stats->analyzerHandlerCalls;
+  spindump_deepdebugf("initialized handler counter to %u for spindump_analyze_process",
+                      packet->analyzerHandlerCalls);
+  
   //
   // Switch based on type of L2
   //
@@ -699,21 +712,27 @@ spindump_analyze_process_pakstats(struct spindump_analyze* state,
                                       connection);
   }
 
-  spindump_analyze_process_handlers(state,
-                                    spindump_analyze_event_newpacket,
-                                    packet,
-                                    connection);
-
-        if (ecnCe) {
-                spindump_analyze_process_handlers(state,
+  if (ecnCe) {
+    spindump_analyze_process_handlers(state,
                                       fromResponder ? spindump_analyze_event_responderecnce :
-                                                                                                                                                        spindump_analyze_event_initiatorecnce,
+                                      spindump_analyze_event_initiatorecnce,
                                       packet,
                                       connection);
-        }
-
+  }
 
+  //
+  // Call the generic "new packet" handler unless some other handler was called earlier
+  //
 
+  spindump_deepdebugf("considering whether to call newpacket handler %u == %u?",
+                      packet->analyzerHandlerCalls,state->stats->analyzerHandlerCalls);
+  if (packet->analyzerHandlerCalls == state->stats->analyzerHandlerCalls) {
+    spindump_analyze_process_handlers(state,
+                                      spindump_analyze_event_newpacket,
+                                      packet,
+                                      connection);
+  }
+  
   //
   // Loop through any possible aggregated connections this connection
   // belongs to, and report the same measurement udpates there.

src/spindump_analyze_event.c

@@ -78,6 +78,10 @@ static void
 spindump_analyze_processevent_qrloss_measurement(struct spindump_analyze* state,
                                                  const struct spindump_event* event,
                                                  struct spindump_connection** p_connection);
+static void
+spindump_analyze_processevent_packet(struct spindump_analyze* state,
+                                     const struct spindump_event* event,
+                                     struct spindump_connection** p_connection);
 static int
 spindump_analyze_event_parseicmpsessionid(const struct spindump_event* event,
                                           uint16_t* p_peerId);
@@ -177,6 +181,9 @@ spindump_analyze_processevent(struct spindump_analyze* state,
   case spindump_event_type_rtloss_measurement:
     spindump_analyze_processevent_rtloss_measurement(state, event, p_connection);
     break;
+  case spindump_event_type_packet:
+    spindump_analyze_processevent_packet(state, event, p_connection);
+    break;
   default:
     spindump_errorf("invalid event type %u", event->eventType);
     return;
@@ -885,6 +892,35 @@ spindump_analyze_processevent_qrloss_measurement(struct spindump_analyze* state,
   spindump_analyze_event_updateinfo(state,*p_connection,event);
 }
 
+//
+// Process an event of type "packet" from another instance of Spindump
+// somewhere else. Update statistics and make any other necessary
+// changes in the local database of connections.
+//
+// The parameter state is the analyzer data structure, event is the
+// incoming event, and p_connection is an output parameter, in the end
+// pointing to either 0 if no affected connection could be identified,
+// or a pointer to the connection object from the connection table of
+// the analyzer.
+//
+
+static void
+spindump_analyze_processevent_packet(struct spindump_analyze* state,
+                                     const struct spindump_event* event,
+                                     struct spindump_connection** p_connection) {
+  
+  *p_connection = spindump_analyze_processevent_find_connection(state,event);
+  if (*p_connection == 0) return;
+  
+  //
+  // Did we find a connection in the end? If yes, update other
+  // information (statistics, state) from the event to the connection
+  // object.
+  //
+
+  spindump_analyze_event_updateinfo(state,*p_connection,event);
+}
+
 //
 // Find the already existing connection pointed to by the
 // event. Return that connection object, or 0 if not found. If the

src/spindump_event.c

@@ -96,6 +96,7 @@ spindump_event_type_tostring(enum spindump_event_type type) {
   case spindump_event_type_ecn_congestion_event: return("ecnce");
   case spindump_event_type_rtloss_measurement: return("rtloss");
   case spindump_event_type_qrloss_measurement: return("qrloss");
+  case spindump_event_type_packet: return("packet");
   default:
     spindump_errorf("invalid event type");
     return("UNKNOWN");
@@ -183,6 +184,8 @@ spindump_event_equal(const struct spindump_event* event1,
     if (strcmp(event1->u.qrlossMeasurement.qLoss, event2->u.qrlossMeasurement.qLoss) != 0) return(0);
     if (strcmp(event1->u.qrlossMeasurement.rLoss, event2->u.qrlossMeasurement.rLoss) != 0) return(0);
     break;
+  case spindump_event_type_packet:
+    break;
   default:
     spindump_errorf("unrecognised event type %u", event1->eventType);
     return(0);

src/spindump_event.h

@@ -40,7 +40,8 @@ enum spindump_event_type {
   spindump_event_type_spin_value = 6,
   spindump_event_type_ecn_congestion_event = 7,
   spindump_event_type_rtloss_measurement = 8,
-  spindump_event_type_qrloss_measurement = 9
+  spindump_event_type_qrloss_measurement = 9,
+  spindump_event_type_packet = 10
 };
 
 enum spindump_direction {

src/spindump_event_parser_json.c

@@ -57,7 +57,10 @@ spindump_event_parser_json_parse_aux_rtloss_measurement(const struct spindump_js
                                                         struct spindump_event* event);
 static int
 spindump_event_parser_json_parse_aux_qrloss_measurement(const struct spindump_json_value* json,
-                                                          struct spindump_event* event);
+                                                        struct spindump_event* event);
+static int
+spindump_event_parser_json_parse_aux_packet(const struct spindump_json_value* json,
+                                            struct spindump_event* event);
 
 //
 // Actual code --------------------------------------------------------------------------------
@@ -225,6 +228,12 @@ spindump_event_parser_json_parse(const struct spindump_json_value* json,
     }
     break;
 
+  case spindump_event_type_packet:
+    if (!spindump_event_parser_json_parse_aux_packet(json,event)) {
+      return(0);
+    }
+    break;
+    
   default:
     spindump_errorf("Invalid event type %u", event->eventType);
     return(0);
@@ -479,7 +488,7 @@ spindump_event_parser_json_parse_aux_rtloss_measurement(const struct spindump_js
 
 static int
 spindump_event_parser_json_parse_aux_qrloss_measurement(const struct spindump_json_value* json,
-                                                struct spindump_event* event) {
+                                                        struct spindump_event* event) {
   const struct spindump_json_value* whoField = spindump_json_value_getfield("Who",json);
   const struct spindump_json_value* qField = spindump_json_value_getfield("Q_loss",json);
   const struct spindump_json_value* rField = spindump_json_value_getfield("R_loss",json);
@@ -511,6 +520,16 @@ spindump_event_parser_json_parse_aux_qrloss_measurement(const struct spindump_js
   return(1);
 }
 
+//
+// Parse the record about a new packet. It has no extra information.
+//
+
+static int
+spindump_event_parser_json_parse_aux_packet(const struct spindump_json_value* json,
+                                            struct spindump_event* event) {
+  return(1);
+}
+
 //
 // Take an event description in the input parameter "event", and print
 // it out as a JSON-formatted Spindump event. The printed version will
@@ -656,6 +675,9 @@ spindump_event_parser_json_print(const struct spindump_event* event,
     addtobuffer2(", \"R_loss\": \"%s\"", event->u.qrlossMeasurement.rLoss);
     break;
 
+  case spindump_event_type_packet:
+    break;
+    
   default:
     spindump_errorf("invalid event type");
   }

src/spindump_event_parser_text.c

@@ -183,6 +183,9 @@ spindump_event_parser_text_print(const struct spindump_event* event,
     }
     break;
 
+  case spindump_event_type_packet:
+    break;
+    
   default:
     spindump_errorf("invalid event type");
   }

src/spindump_eventformatter.c

@@ -67,6 +67,7 @@ spindump_eventformatter_initialize(struct spindump_analyze* analyzer,
                                    int reportSpinFlips,
                                    int reportRtLoss,
                                    int reportQrLoss,
+                                   int reportPackets,
                                    int reportNotes,
                                    int anonymizeLeft,
                                    int anonymizeRight,
@@ -92,6 +93,7 @@ spindump_eventformatter_initialize(struct spindump_analyze* analyzer,
                                    int reportSpinFlips,
                                    int reportRtLoss,
                                    int reportQrLoss,
+                                   int reportPackets,
                                    int reportNotes,
                                    int anonymizeLeft,
                                    int anonymizeRight,
@@ -128,6 +130,7 @@ spindump_eventformatter_initialize(struct spindump_analyze* analyzer,
   formatter->reportSpinFlips = reportSpinFlips;
   formatter->reportRtLoss = reportRtLoss;
   formatter->reportQrLoss = reportQrLoss;
+  formatter->reportPackets = reportPackets;
   formatter->reportNotes = reportNotes;
   formatter->anonymizeLeft = anonymizeLeft;
   formatter->anonymizeRight = anonymizeRight;
@@ -165,6 +168,7 @@ spindump_eventformatter_initialize_file(struct spindump_analyze* analyzer,
                                         int reportSpinFlips,
                                         int reportRtLoss,
                                         int reportQrLoss,
+                                        int reportPackets,
                                         int reportNotes,
                                         int anonymizeLeft,
                                         int anonymizeRight,
@@ -183,6 +187,7 @@ spindump_eventformatter_initialize_file(struct spindump_analyze* analyzer,
                                                                                  reportSpinFlips,
                                                                                  reportRtLoss,
                                                                                  reportQrLoss,
+                                                                                 reportPackets,
                                                                                  reportNotes,
                                                                                  anonymizeLeft,
                                                                                  anonymizeRight,
@@ -223,6 +228,7 @@ spindump_eventformatter_initialize_remote(struct spindump_analyze* analyzer,
                                           int reportSpinFlips,
                                           int reportRtLoss,
                                           int reportQrLoss,
+                                          int reportPackets,
                                           int reportNotes,
                                           int anonymizeLeft,
                                           int anonymizeRight,
@@ -242,6 +248,7 @@ spindump_eventformatter_initialize_remote(struct spindump_analyze* analyzer,
                                                                                  reportSpinFlips,
                                                                                  reportRtLoss,
                                                                                  reportQrLoss,
+                                                                                 reportPackets,
                                                                                  reportNotes,
                                                                                  anonymizeLeft,
                                                                                  anonymizeRight,
@@ -503,7 +510,7 @@ spindump_eventformatter_measurement_one(struct spindump_analyze* state,
   // Sanity checks
   //
 
-  spindump_deepdeepdebugf("spindump_eventformatter_measurement_one called for event %u", event);
+  spindump_deepdeepdebugf("spindump_eventformatter_measurement_one handler called for event %u", event);
 
   spindump_assert(state != 0);
   spindump_assert(handlerData != 0);
@@ -611,8 +618,13 @@ spindump_eventformatter_measurement_one(struct spindump_analyze* state,
     break;
 
   case spindump_analyze_event_newpacket:
-    spindump_deepdeepdebugf("point 5x");
-    return;
+    spindump_deepdeepdebugf("point 5x %d in eventformatter handler", formatter->reportPackets);
+    if (formatter->reportPackets) {
+      eventType = spindump_event_type_packet;
+      break;
+    } else {
+      return;
+    }
 
   case spindump_analyze_event_initiatorecnce:
     spindump_deepdeepdebugf("point 5k");
@@ -669,7 +681,7 @@ spindump_eventformatter_measurement_one(struct spindump_analyze* state,
   spindump_connections_getnetworks(connection,&initiatorAddress,&responderAddress);
   const char* notes = 0;
   char notesbuf[sizeof(eventobj.notes)];
-  spindump_deepdeepdebugf("reportNotes in eventformatter = %u", formatter->reportNotes);
+  spindump_deepdeepdebugf("reportPackets and -Notes in eventformatter = %u %u", formatter->reportPackets, formatter->reportNotes);
   if (formatter->reportNotes) {
     spindump_connection_report_brief_notefieldval(connection,sizeof(notesbuf),notesbuf);
     notes = &notesbuf[0];
@@ -863,6 +875,9 @@ spindump_eventformatter_measurement_one(struct spindump_analyze* state,
     sprintf(eventobj.u.qrlossMeasurement.rLoss, "%.3f", connection->rLossesFrom2to1*100);
     break;
 
+  case spindump_analyze_event_newpacket:
+    break;
+    
   default:
     return;
 

src/spindump_eventformatter.h

@@ -69,6 +69,7 @@ struct spindump_eventformatter {
   int reportSpinFlips;
   int reportRtLoss;
   int reportQrLoss;
+  int reportPackets;
   int reportNotes;
   int anonymizeLeft;
   int anonymizeRight;
@@ -93,6 +94,7 @@ spindump_eventformatter_initialize_file(struct spindump_analyze* analyzer,
                                         int reportSpinFlips,
                                         int reportRtLoss,
                                         int reportQrLoss,
+                                        int reportPackets,
                                         int reportNotes,
                                         int anonymizeLeft,
                                         int anonymizeRight,
@@ -110,6 +112,7 @@ spindump_eventformatter_initialize_remote(struct spindump_analyze* analyzer,
                                           int reportSpinFlips,
                                           int reportRtLoss,
                                           int reportQrLoss,
+                                          int reportPackets,
                                           int reportNotes,
                                           int anonymizeLeft,
                                           int anonymizeRight,