Enhance blacklist function (by ip, by referer and by user-agent)

defaults/main.yml

 default_proxy: ''
 proxy_default_backend: ''
 proxy_certificates: []
-proxy_blacklist_ips: []
 proxy_timeout_connect: '5s'
 proxy_timeout_client: '20s'
 proxy_timeout_server: '45s'
 proxy_redirect_aliase: false
+proxy_blacklist:
+  ip:
+    - '146.185.176.158'
+    - '162.243.9.72'
+    - '173.199.114.0/24'
+    - '173.199.115.0/24'
+    - '173.199.115.112/29'
+    - '173.199.116.0/24'
+    - '173.199.117.0/24'
+    - '173.199.118.0/24'
+    - '173.199.119.0/24'
+    - '173.199.120.0/24'
+    - '182.50.130.0/24'
+    - '188.92.74.0/24'
+    - '195.239.0/24'
+    - '198.186.190.0/23'
+    - '198.186.192.0/23'
+    - '198.186.194.0/24'
+    - '208.167.230.0/24'
+    - '209.222.12.0/24'
+    - '210.171.3.0/24'
+    - '212.100.254.105'
+    - '212.113.0.0/24'
+    - '212.113.32.0/21'
+    - '212.113.37.0/24'
+    - '213.186.0.0/24'
+    - '213.186.96.0/19'
+    - '46.137.98.159'
+    - '5.10.83.0/24'
+    - '5.10.83.0/25'
+    - '5.9.0.0/24'
+    - '5.9.104.0/24'
+    - '50.112.126.117'
+    - '54.232.100.158'
+    - '54.235.220.243'
+    - '54.249.240.15'
+    - '54.251.45.250'
+    - '54.252.97.95'
+    - '69.42.83.0/24'
+  referer:
+    - 'best-seo-solution.com'
+    - 'best-seo-offer.com'
+    - 'buttons-for-website.com'
+    - 'buttons-for-your-website.com'
+    - 'semalt.com'
+    - '7makemoneyonline.com'
+  agent:
+    - 'AhrefsBot'
+    - 'Ahrefs'
+    - 'rogerbot'
+    - 'MJ12bot'
+    - 'majestic12'
+    - 'MJ12'
+    - 'SiteBot'
+    - 'Semrush'
+    - 'CCBot'
+    - '80legs'
+    - 'Sogou'
+    - 'DigExt'
+    - 'spbot'
+    - 'ia_archiver'
+    - 'Rankivabot'
+    - 'DBLBot'
+    - 'libw'
+    - 'Java'
+    - 'Voil'
+    - 'Twice'
+    - 'Sogou'
+    - 'psbot'
+    - 'Exabot'
+    - 'boitho'
+    - 'ajSitemap'
+    - 'Rankivabot'
+    - 'SeznamBot'
+    - 'DBLBot'
+    - 'Ezooms'
+    - 'Ezooms/1.0'
+    - 'exabot'
+    - 'dotbot'
+    - 'gigabot'

tasks/configure.yml

@@ -2,58 +2,69 @@
 # file: roles/haproxy/tasks/configure.yml
 
 - name: "Proxy | Install SSL certificates"
-  copy: src={{inventory_dir}}/files/ssl/{{item.file}}
-        dest=/etc/haproxy/certs
+  copy:
+    src='{{inventory_dir}}/files/ssl/{{item.file}}'
+    dest='/etc/haproxy/certs'
   with_items: '{{ proxy_certificates }}'
   when: not item.letsencrypt|default(false)
   notify: 'Proxy | Restart HAProxy'
 
 - name: "Proxy | Create host lists"
-  template: src=host_list
-            dest=/etc/haproxy/{{item}}.list
-            owner=root
-            group=root
-            mode=644
+  template:
+    src='host_list'
+    dest='/etc/haproxy/{{item}}.list'
+    owner='root'
+    group='root'
+    mode='644'
   with_items: '{{ groups.all }}'
   notify: 'Proxy | Restart HAProxy'
 
 - name: "Proxy | Create host ssl lists"
-  template: src=host_ssl_list
-            dest=/etc/haproxy/{{item}}.ssl.list
-            owner=root
-            group=root
-            mode=644
+  template:
+    src='host_ssl_list'
+    dest='/etc/haproxy/{{item}}.ssl.list'
+    owner='root'
+    group='root'
+    mode='644'
   with_items: '{{ groups.all }}'
   notify: 'Proxy | Restart HAProxy'
 
 - name: "Proxy | Create empty crm lists files"
-  file: dest=/etc/haproxy/{{item}}.crm.list
-        owner=root
-        group=root
-        mode=644
-        state=touch
+  file:
+    dest='/etc/haproxy/{{item}}.crm.list'
+    owner='root'
+    group='root'
+    mode='644'
+    state='touch'
   with_items: '{{ groups.all }}'
   changed_when: false
 
 - name: "Proxy | Create config file"
-  template: src=haproxy_cfg
-            dest=/etc/haproxy/haproxy.cfg
-            owner=root
-            group=root
-            mode=644
+  template:
+    src='haproxy_cfg'
+    dest='/etc/haproxy/haproxy.cfg'
+    owner='root'
+    group='root'
+    mode='644'
   notify: 'Proxy | Restart HAProxy'
 
 - name: "Proxy | Install update php script"
-  copy: src=etc_haproxy_update_update_php
-        dest=/etc/haproxy/update/update.php
-        owner=root
-        group=root
-        mode=444
+  copy:
+    src='etc_haproxy_update_update_php'
+    dest='/etc/haproxy/update/update.php'
+    owner='root'
+    group='root'
+    mode='444'
 
-- name: "Proxy | Update blacklist"
-  template: src=blacklist
-            dest=/etc/haproxy/blacklist
-            owner=root
-            group=root
-            mode=644
+- name: "Proxy | Update blacklists"
+  template:
+    src='{{ item }}'
+    dest='/etc/haproxy/{{ item }}'
+    owner='root'
+    group='root'
+    mode='644'
+  with_items:
+    - 'blacklist.ip'
+    - 'blacklist.referer'
+    - 'blacklist.agent'
   notify: 'Proxy | Restart HAProxy'

templates/blacklist

-{% for host in groups['all'] %}
-{% for line in hostvars[host].proxy_blacklist_ips|default([]) %}
-{{line}}
-{% endfor %}
-{% endfor %}

templates/blacklist.agent

+{% for line in hostvars[host].proxy_blacklist.agent|default([]) %}
+{{line}}
+{% endfor %}

templates/blacklist.ip

+{% for line in hostvars[host].proxy_blacklist.ip|default([]) %}
+{{line}}
+{% endfor %}

templates/blacklist.referer

+{% for line in hostvars[host].proxy_blacklist.referer|default([]) %}
+{{line}}
+{% endfor %}

templates/haproxy_cfg

@@ -52,8 +52,12 @@ userlist kibana
 
 frontend http_in
   bind *:80
-  acl blockedip src  -i -f /etc/haproxy/blacklist
+  acl blockedip src -f /etc/haproxy/blacklist.ip
   http-request deny if blockedip
+  acl blockedreferer hdr_sub(referer) -i -f /etc/haproxy/blacklist.referer
+  http-request deny if blockedreferer
+  acl blockedagent hdr_sub(user-agent) -i -f /etc/haproxy/blacklist.agent
+  http-request deny if blockedagent
 {% for host in groups['all'] %}
 {% for redirect in hostvars[host].proxy_redirect|default([]) %}
   redirect prefix {{ redirect.protocol|default('https') }}://{{redirect.to}} code 301 if { hdr(host) -i -n {{redirect.from}} }
@@ -92,8 +96,12 @@ frontend http_in
 
 frontend https_in_{{ cert.ip }}
   bind {{ cert.ip }}:443 ssl crt /etc/haproxy/certs/{{ cert.file }} no-sslv3
-  acl blockedip src  -i -f /etc/haproxy/blacklist
+  acl blockedip src -f /etc/haproxy/blacklist.ip
   http-request deny if blockedip
+  acl blockedreferer hdr_sub(referer) -i -f /etc/haproxy/blacklist.referer
+  http-request deny if blockedreferer
+  acl blockedagent hdr_sub(user-agent) -i -f /etc/haproxy/blacklist.agent
+  http-request deny if blockedagent
 {% for host in groups['all'] %}
 {% for redirect in hostvars[host].proxy_redirect|default([]) %}
   redirect prefix {{ redirect.protocol|default('https') }}://{{redirect.to}} code 301 if { hdr(host) -i -n {{redirect.from}} }