Implement global Apache access control

defaults/main.yml

@@ -18,3 +18,87 @@ apache_cache:
   bytype:
     type: 'text/html'
     default: 'A900'
+apache_global_redirect:
+  agents:
+    - 'AhrefsBot'
+    - 'Ahrefs'
+    - 'rogerbot'
+    - 'MJ12bot'
+    - 'majestic12'
+    - 'MJ12'
+    - 'SiteBot'
+    - 'Semrush'
+    - 'CCBot'
+    - '80legs'
+    - 'Sogou'
+    - 'DigExt'
+    - 'spbot'
+    - 'ia_archiver'
+    - '.*Rankivabot.*'
+    - '.*DBLBot.*'
+    - '.*libw.*'
+    - '.*Java.*'
+    - '.*Voil.*'
+    - '.*Twice.*'
+    - '.*Sogou.*'
+    - '.*psbot.*'
+    - '.*Exabot.*'
+    - '.*boitho.*'
+    - '.*ajSitemap.*'
+    - '.*Rankivabot.*'
+    - '.*SeznamBot.*'
+    - '.*DBLBot.*'
+    - 'Ezooms'
+    - 'Ezooms/1.0'
+    - 'exabot'
+    - 'dotbot'
+    - 'gigabot'
+  referer:
+    - 'buttons-for-website\.com'
+    - 'semalt\.com'
+  blocked_ips:
+    - '173.199.114.0/24'
+    - '173.199.115.0/24'
+    - '173.199.116.0/24'
+    - '173.199.117.0/24'
+    - '173.199.118.0/24'
+    - '173.199.119.0/24'
+    - '173.199.120.0/24'
+    - '213.186.96.0/19'
+    - '212.113.32.0/21'
+    - '173.199.115.112/29'
+    - '5.10.83.0/25'
+    - '5.10.83.'
+    - '210.171.3.'
+    - '5.9.104.'
+    - '5.9.0.'
+    - '213.186.'
+    - '195.239.'
+    - '173.199.120.'
+    - '173.199.114.'
+    - '173.199.115.'
+    - '173.199.116.'
+    - '173.199.117.'
+    - '173.199.118.'
+    - '173.199.119.'
+    - '208.167.230.'
+    - '209.222.12.'
+    - '188.92.74.'
+    - '69.42.83.'
+    - '213.186.'
+    - '212.113.37.'
+    - '182.50.130.'
+    - '212.113.'
+    - '54.235.220.243'
+    - '50.112.126.117'
+    - '162.243.9.72'
+    - '54.251.45.250'
+    - '54.252.97.95'
+    - '54.249.240.15'
+    - '54.232.100.158'
+    - '146.185.176.158'
+    - '46.137.98.159'
+    - '212.100.254.105'
+    - '198.186.194.0/24'
+    - '198.186.190.0/23'
+    - '198.186.192.0/23'

tasks/apache.yml

@@ -43,6 +43,16 @@
     line='ExtendedStatus On'
   notify: "Apache | Restart Apache"
 
+- name: "Apache | Create Global Redirect Configuration"
+  template:
+    src='etc-apache2-conf-available-global-redirect'
+    dest='/etc/apache2/conf-available/global-redirect.conf'
+    owner='root'
+    group='root'
+    mode='644'
+  notify:
+    - "Apache | Restart Apache"
+
 - name: "Apache | Install SSL certificates"
   copy:
     src={{inventory_dir}}/files/ssl/{{item.1.file}}

templates/etc-apache2-conf-available-global-redirect

+Options +FollowSymLinks
+RewriteEngine on
+
+{% for agent in apache_global_redirect.agents|default([]) %}
+RewriteCond %{HTTP_USER_AGENT} "{{ agent }}" [NC,OR]
+{% endfor %}
+RewriteCond %{HTTP_USER_AGENT} "dummyagent" [NC]
+RewriteRule ^.* - [F,L]
+
+# TODO: Make this work globally in Apache 2.4 and 2.2
+#Order allow,deny
+#allow from all
+#{% for ip in apache_global_redirect.blocked_ips|default([]) %}
+#deny from {{ ip }}
+#{% endfor %}
+
+{% for referer in apache_global_redirect.referer|default([]) %}
+RewriteCond %{HTTP_REFERER} ^([^.]+.)*{{ referer }} [NC,OR]
+{% endfor %}
+RewriteCond %{HTTP_REFERER} ^([^.]+.)*dummy\.referer [NC]
+RewriteRule .* - [F,L]

templates/etc-apache2-sites-available-default

 <VirtualHost *:80>
+  Include /etc/apache2/conf-available/global-redirect.conf
+
   ServerAdmin {{apache_server_admin}}
 
   DocumentRoot /var/www{{apache_server_default_root}}
@@ -50,6 +52,8 @@
 {% for alias in apache_server_default_aliases_redirect %}
 
 <VirtualHost *:80>
+  Include /etc/apache2/conf-available/global-redirect.conf
+
   ServerName {{alias.from}}
 {% if alias.secure is defined %}
   ServerAlias {{alias.to}}

templates/etc-apache2-sites-available-default-ssl

@@ -2,6 +2,8 @@ NameVirtualHost *:443
 {% for certs in apache_certificates %}
 
 <VirtualHost _default_:443>
+  Include /etc/apache2/conf-available/global-redirect.conf
+
 {% if certs.domain is defined %}
   ServerName {{ certs.domain }}
 {% endif %}
@@ -58,6 +60,8 @@ NameVirtualHost *:443
 {% for alias in certs.redirects %}
 
 <VirtualHost _default_:443>
+  Include /etc/apache2/conf-available/global-redirect.conf
+
   ServerName {{alias.from}}
   Redirect 301 / https://{{alias.to}}/
 {% for cert in certs %}

templates/etc-apache2-sites-available-vhost

 <VirtualHost *:80>
+  Include /etc/apache2/conf-available/global-redirect.conf
+
   ServerAdmin {{apache_server_admin}}
   ServerName  {{item.domain}}
   ServerAlias {{item.alias}}