Previously, I am exploring the possiblity to use BunnyCDN + Shield for some kind of protection. But seems like their basic plan also do not really cover advanced bots.
So I am now exploring another way by using fail2ban.
May I have your comments on these 2 jail settings? (I’ll modify the apache log path later)
- Bot blocker
# Fail2Ban filter for blocking aggressive bots and scanners
# Place this file in: /etc/fail2ban/filter.d/apache-bot-blocker.conf
[INCLUDES]
# overwrite with apache-common.local if _apache_error_client is incorrect.
before = apache-common.conf
[Definition]
# Match aggressive bot patterns
failregex = ^<HOST> -.*"-" "(|-|Unknown|.*bot.*|.*crawler.*|.*spider.*|.*scraper.*)" .*$
^<HOST> -.*"[^"]*" [0-9]+ [0-9]+ "[^"]*" ".*(python|curl|wget|scrapy|mechanize|phantomjs).*"
# Ignore legitimate crawlers (optional - comment out if you want to block all)
ignoreregex = (?i).*(Googlebot|bingbot|YandexBot|DuckDuckBot|Baiduspider|Slurp|Sogou|Exabot|facebot|Applebot|Twitterbot|LinkedInBot|PetalBot).*
datepattern = %%d/%%b/%%Y:%%H:%%M:%%S %%z
[Init]
# Maximum lines to buffer for multi-line regex
maxlines = 1
journalmatch = _SYSTEMD_UNIT=apache2.service
# Fail2Ban jail configuration for Apache bot blocking
# Place this in: /etc/fail2ban/jail.d/apache-bot-blocker.conf
[apache-bot-blocker]
enabled = true
port = all
filter = apache-bot-blocker
logpath = /var/log/apache*/domains/*com.log
# Ban after 8 bot requests (lenient to avoid false positives)
maxretry = 8
# Within 2 minutes
findtime = 120
# Ban for 60 minutes (1 hour)
bantime = 3600
# Permanent ban after 3 violations
bantime.increment = true
bantime.maxtime = 604800
bantime.factor = 2
# Action: Use HestiaCP's firewall management
action = hestia[name=WEB]
# sendmail-whois[name=apache-bot, [email protected]]
- Error-catcher
# Fail2Ban filter for catching all non-success HTTP responses
# Place this file in: /etc/fail2ban/filter.d/apache-error-catcher.conf
[INCLUDES]
# overwrite with apache-common.local if _apache_error_client is incorrect.
before = apache-common.conf
[Definition]
# Catch all GET/POST/HEAD requests with error status codes
failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*" (400|401|403|404|405|500) .*$
^<HOST> -.*"(GET|POST).*(eval\(|base64|<script|script>|union.*select|insert.*into|delete.*from).*" .*$
^<HOST> -.*"\\x[0-9A-Fa-f]{2}.*" .*$
ignoreregex =
datepattern = %%d/%%b/%%Y:%%H:%%M:%%S %%z
[Init]
# Maximum lines to buffer for multi-line regex
maxlines = 1
journalmatch = _SYSTEMD_UNIT=apache2.service
# Fail2Ban jail configuration for Apache error catcher
# Place this in: /etc/fail2ban/jail.d/apache-error-catcher.conf
[apache-error-catcher]
enabled = true
port = all
filter = apache-error-catcher
logpath = /var/log/apache*/domains/*com.log
# Ban after 5 suspicious requests
maxretry = 5
# Within 60 seconds
findtime = 60
# Ban for 1 hour
bantime = 3600
# Permanent ban after 3 violations
bantime.increment = true
bantime.maxtime = 604800
bantime.factor = 2
# Action: Use HestiaCP's firewall management
action = hestia[name=WEB]