[#956] allow instances to go 25% over their hard max

conf/carbon.conf.example

@@ -472,8 +472,11 @@ DESTINATIONS = 127.0.0.1:2004
 
 # This is the maximum number of datapoints that can be queued up
 # for a single destination. Once this limit is hit, we will
-# stop accepting new data if USE_FLOW_CONTROL is True, otherwise
-# we will drop any subsequently received datapoints.
+# stop accepting new data if USE_FLOW_CONTROL is True. In-flight and
+# internally-generated datapoints will still be processed, and data
+# will still be dropped if MAX_QUEUE_SIZE_HARD_PCT * MAX_QUEUE_SIZE
+# is hit. If USE_FLOW_CONTROL is False, metrics are immediately dropped
+# after MAX_QUEUE_SIZE, and MAX_QUEUE_SIZE_HARD_PCT is unused.
 MAX_QUEUE_SIZE = 10000
 
 # This is the factor that the queue must be empty before it will accept
@@ -485,6 +488,11 @@ MAX_QUEUE_SIZE = 10000
 # even before the relay incrementally clears more of the queue
 QUEUE_LOW_WATERMARK_PCT = 0.8
 
+# This is the factor of the max length of a queue before data will be dropped
+# with USE_FLOW_CONTROL enabled. When incoming data is paused, in-flight data
+# is still processed, which can send a queue slightly over the configured max.
+MAX_QUEUE_SIZE_HARD_PCT = 1.25
+
 # Set this to False to drop datapoints when any send queue (sending datapoints
 # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
 # default) then sockets over which metrics are received will temporarily stop accepting
@@ -619,8 +627,11 @@ REPLICATION_FACTOR = 1
 
 # This is the maximum number of datapoints that can be queued up
 # for a single destination. Once this limit is hit, we will
-# stop accepting new data if USE_FLOW_CONTROL is True, otherwise
-# we will drop any subsequently received datapoints.
+# stop accepting new data if USE_FLOW_CONTROL is True. In-flight and
+# internally-generated datapoints will still be processed, and data
+# will still be dropped if MAX_QUEUE_SIZE_HARD_PCT * MAX_QUEUE_SIZE
+# is hit. If USE_FLOW_CONTROL is False, metrics are immediately dropped
+# after MAX_QUEUE_SIZE, and MAX_QUEUE_SIZE_HARD_PCT is unused.
 MAX_QUEUE_SIZE = 10000
 
 # This is the factor that the queue must be empty before it will accept
@@ -632,6 +643,11 @@ MAX_QUEUE_SIZE = 10000
 # even before the relay incrementally clears more of the queue
 QUEUE_LOW_WATERMARK_PCT = 0.8
 
+# This is the factor of the max length of a queue before data will be dropped
+# with USE_FLOW_CONTROL enabled. When incoming data is paused, in-flight data
+# is still processed, which can send a queue slightly over the configured max.
+MAX_QUEUE_SIZE_HARD_PCT = 1.25
+
 # Set this to False to drop datapoints when any send queue (sending datapoints
 # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
 # default) then sockets over which metrics are received will temporarily stop accepting

lib/carbon/client.py

@@ -35,6 +35,10 @@
 
 
 SEND_QUEUE_LOW_WATERMARK = settings.MAX_QUEUE_SIZE * settings.QUEUE_LOW_WATERMARK_PCT
+if settings.USE_FLOW_CONTROL:
+    SEND_QUEUE_HARD_MAX = settings.MAX_QUEUE_SIZE * settings.MAX_QUEUE_SIZE_HARD_PCT
+else:
+    SEND_QUEUE_HARD_MAX = settings.MAX_QUEUE_SIZE
 
 
 class CarbonClientProtocol(object):
@@ -350,7 +354,10 @@ def sendDatapoint(self, metric, datapoint):
     if self.queueSize >= settings.MAX_QUEUE_SIZE:
       if not self.queueFull.called:
         self.queueFull.callback(self.queueSize)
-      instrumentation.increment(self.fullQueueDrops)
+      if self.queueSize < SEND_QUEUE_HARD_MAX:
+        self.enqueue(metric, datapoint)
+      else:
+        instrumentation.increment(self.fullQueueDrops)
     else:
       self.enqueue(metric, datapoint)
 

lib/carbon/conf.py

@@ -70,6 +70,7 @@
   FORWARD_ALL=True,
   MAX_QUEUE_SIZE=10000,
   QUEUE_LOW_WATERMARK_PCT=0.8,
+  MAX_QUEUE_SIZE_HARD_PCT=1.25,
   TIME_TO_DEFER_SENDING=0.0001,
   ENABLE_AMQP=False,
   AMQP_METRIC_NAME_IN_BODY=False,