#19 Turn off low success rate web log alerts

tasks/apache.yml

@@ -27,3 +27,10 @@
     owner: 'netdata'
     group: 'netdata'
     mode: '660'
+
+- name: "Install Custom Alerts"
+  template:
+    src: 'health/web_log.conf'
+    dest: '/etc/netdata/health.d/web_log.conf'
+  notify:
+    - "Restart NetData"

tasks/install.yml

@@ -40,7 +40,7 @@
 
 - name: "Install Custom Alerts"
   template:
-    src: '{{ item }}.conf'
+    src: 'health/{{ item }}.conf'
     dest: '/etc/netdata/health.d/{{ item }}.conf'
   with_items:
     - 'fluentd_buffer'

templates/health/web_log.conf

+# make sure we can collect web log data
+
+template: last_collected_secs
+      on: web_log.response_codes
+families: *
+    calc: $now - $last_collected_t
+   units: seconds ago
+   every: 10s
+    warn: $this > (($status >= $WARNING)  ? ($update_every) : ( 5 * $update_every))
+    crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
+   delay: down 5m multiplier 1.5 max 1h
+    info: number of seconds since the last successful data collection
+      to: webmaster
+
+
+# -----------------------------------------------------------------------------
+# high level response code alarms
+
+# the following alarms trigger only when there are enough data.
+# we assume there are enough data when:
+#
+#  $1m_requests > 120
+#
+# i.e. when there are at least 120 requests during the last minute
+
+template: 1m_requests
+      on: web_log.response_statuses
+families: *
+  lookup: sum -1m unaligned
+    calc: ($this == 0)?(1):($this)
+   units: requests
+   every: 10s
+    info: the sum of all HTTP requests over the last minute
+
+template: 1m_successful
+      on: web_log.response_statuses
+families: *
+  lookup: sum -1m unaligned of successful_requests
+    calc: $this * 100 / $1m_requests
+   units: %
+   every: 10s
+    warn: ($1m_requests > 120) ? ($this < (($status >= $WARNING ) ? ( 95 ) : ( 85 )) ) : ( 0 )
+    crit: ($1m_requests > 120) ? ($this < (($status == $CRITICAL) ? ( 85 ) : ( 75 )) ) : ( 0 )
+   delay: up 2m down 15m multiplier 1.5 max 1h
+    info: the ratio of successful HTTP responses (1xx, 2xx, 304) over the last minute
+      to: silent
+
+template: 1m_redirects
+      on: web_log.response_statuses
+families: *
+  lookup: sum -1m unaligned of redirects
+    calc: $this * 100 / $1m_requests
+   units: %
+   every: 10s
+    warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING ) ? (  1 ) : ( 20 )) ) : ( 0 )
+    crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 20 ) : ( 30 )) ) : ( 0 )
+   delay: up 2m down 15m multiplier 1.5 max 1h
+    info: the ratio of HTTP redirects (3xx except 304) over the last minute
+      to: webmaster
+
+template: 1m_bad_requests
+      on: web_log.response_statuses
+families: *
+  lookup: sum -1m unaligned of bad_requests
+    calc: $this * 100 / $1m_requests
+   units: %
+   every: 10s
+    warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING)  ? ( 10 ) : ( 30 )) ) : ( 0 )
+    crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 30 ) : ( 50 )) ) : ( 0 )
+   delay: up 2m down 15m multiplier 1.5 max 1h
+    info: the ratio of HTTP bad requests (4xx) over the last minute
+      to: webmaster
+
+template: 1m_internal_errors
+      on: web_log.response_statuses
+families: *
+  lookup: sum -1m unaligned of server_errors
+    calc: $this * 100 / $1m_requests
+   units: %
+   every: 10s
+    warn: ($1m_requests > 120) ? ($this > (($status >= $WARNING)  ? ( 1 ) : ( 2 )) ) : ( 0 )
+    crit: ($1m_requests > 120) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 )
+   delay: up 2m down 15m multiplier 1.5 max 1h
+    info: the ratio of HTTP internal server errors (5xx), over the last minute
+      to: webmaster
+
+# unmatched lines
+
+# the following alarms trigger only when there are enough data.
+# we assume there are enough data when:
+#
+#  $1m_total_requests > 120
+#
+# i.e. when there are at least 120 requests during the last minute
+
+template: 1m_total_requests
+      on: web_log.response_codes
+families: *
+  lookup: sum -1m unaligned
+    calc: ($this == 0)?(1):($this)
+   units: requests
+   every: 10s
+    info: the sum of all HTTP requests over the last minute
+
+template: 1m_unmatched
+on: web_log.response_codes
+families: *
+  lookup: sum -1m unaligned of unmatched
+    calc: $this * 100 / $1m_total_requests
+   units: %
+   every: 10s
+    warn: ($1m_total_requests > 120) ? ($this > 1) : ( 0 )
+    crit: ($1m_total_requests > 120) ? ($this > 5) : ( 0 )
+   delay: up 1m down 5m multiplier 1.5 max 1h
+    info: the ratio of unmatched lines, over the last minute
+      to: webmaster
+
+# -----------------------------------------------------------------------------
+# web slow
+
+# the following alarms trigger only when there are enough data.
+# we assume there are enough data when:
+#
+#  $1m_requests > 120
+#
+# i.e. when there are at least 120 requests during the last minute
+
+template: 10m_response_time
+      on: web_log.response_time
+families: *
+  lookup: average -10m unaligned of avg
+   units: ms
+   every: 30s
+    info: the average time to respond to HTTP requests, over the last 10 minutes
+
+template: web_slow
+      on: web_log.response_time
+families: *
+  lookup: average -1m unaligned of avg
+   units: ms
+   every: 10s
+   green: 500
+     red: 1000
+    warn: ($1m_requests > 120) ? ($this > $green && $this > ($10m_response_time * 2) ) : ( 0 )
+    crit: ($1m_requests > 120) ? ($this > $red   && $this > ($10m_response_time * 4) ) : ( 0 )
+   delay: down 15m multiplier 1.5 max 1h
+    info: the average time to respond to HTTP requests, over the last 1 minute
+      to: webmaster
+
+# -----------------------------------------------------------------------------
+# web too many or too few requests
+
+# the following alarms trigger only when there are enough data.
+# we assume there are enough data when:
+#
+#  $5m_successful_old > 120
+#
+# i.e. when there were at least 120 requests during the 5 minutes starting
+#      at -10m and ending at -5m
+
+template: 5m_successful_old
+      on: web_log.response_statuses
+families: *
+  lookup: average -5m at -5m unaligned of successful_requests
+   units: requests/s
+   every: 30s
+    info: average rate of successful HTTP requests over the last 5 minutes
+
+template: 5m_successful
+      on: web_log.response_statuses
+families: *
+  lookup: average -5m unaligned of successful_requests
+   units: requests/s
+   every: 30s
+    info: average successful HTTP requests over the last 5 minutes
+
+template: 5m_requests_ratio
+      on: web_log.response_codes
+families: *
+    calc: ($5m_successful_old > 0)?($5m_successful * 100 / $5m_successful_old):(100)
+   units: %
+   every: 30s
+    warn: ($5m_successful_old > 120) ? ($this > 200 OR $this < 50) : (0)
+    crit: ($5m_successful_old > 120) ? ($this > 400 OR $this < 25) : (0)
+   delay: down 15m multiplier 1.5 max 1h
+    info: the percentage of successful web requests over the last 5 minutes, \
+          compared with the previous 5 minutes \
+          (clear notification for this alarm will not be sent)
+      to: webmaster