Случилось так, что у одного из сайтов появилось какое-то «левое» зеркало, которое сильно подбило позиции в поисковике. Я думал что домен просто направили на IP того сервера, на который ссылался домен. Прописал в конфиг виртуального хоста строки, запрещающие доступ к сайту по IP адресу
if ($host !~ ^(example.com|www.example.com)$ ) { return 403; }
Но не помогло, так как выяснилось что сайт на «левом» зеркале парсился PHP скриптами через CURL. Сразу возникла идея просто запретить доступ по User Agent, но он был изменен на GoogleBot/2.1 , и если его забанить то и поисковые боты не смогут попасть на сайт. Осталось только 1 решение, сравнивать User Agent и IP адрес, с которого был запрос, и если это реальные IP адреса гуглобота, то пропускать, иначе отдавать ошибку 403. В конфиг nginx добавляем в секцию http следующие строки ( это все IP адреса и подсети, которые использует гуглобот и которые мне удалось найти, если есть чем дополнить пишите в коментариях ):
geo $googlebotip { default 0; 64.18.0.0/20 1; 64.233.160.0/19 1; 66.102.0.0/20 1; 66.249.80.0/20 1; 72.14.192.0/18 1; 74.125.0.0/16 1; 108.177.8.0/21 1; 172.217.0.0/19 1; 173.194.0.0/16 1; 207.126.144.0/20 1; 209.85.128.0/17 1; 216.58.192.0/19 1; 216.239.32.0/19 1; 203.208.60.0/24 1; 66.249.64.0/20 1; 72.14.199.0/24 1; 209.85.238.0/24 1; 66.249.90.0/24 1; 66.249.91.0/24 1; 66.249.92.0/24 1; 2001:4860:4000::/36 1; 2404:6800:4000::/36 1; 2607:f8b0:4000::/36 1; 2800:3f0:4000::/36 1; 2a00:1450:4000::/36 1; 2c0f:fb50:4000::/36 1; 2001:4860:4801:1::/64 1; 2001:4860:4801:2::/64 1; 2001:4860:4801:3::/64 1; 2001:4860:4801:4::/64 1; 2001:4860:4801:5::/64 1; 2001:4860:4801:6::/64 1; 2001:4860:4801:7::/64 1; 2001:4860:4801:8::/64 1; 2001:4860:4801:9::/64 1; 2001:4860:4801:a::/64 1; 2001:4860:4801:b::/64 1; 2001:4860:4801:c::/64 1; 2001:4860:4801:d::/64 1; 2001:4860:4801:e::/64 1; 2001:4860:4801:2001::/64 1; 2001:4860:4801:2002::/64 1; 104.132.0.0/21 1; 104.132.12.0/24 1; 104.132.128.0/24 1; 104.132.129.0/24 1; 104.132.13.0/26 1; 104.132.13.112/28 1; 104.132.13.128/25 1; 104.132.13.64/27 1; 104.132.13.96/28 1; 104.132.130.0/24 1; 104.132.131.0/24 1; 104.132.132.0/24 1; 104.132.133.0/24 1; 104.132.134.0/24 1; 104.132.135.0/24 1; 104.132.136.0/23 1; 104.132.138.0/24 1; 104.132.139.0/24 1; 104.132.14.0/23 1; 104.132.140.0/24 1; 104.132.141.0/26 1; 104.132.141.112/28 1; 104.132.141.128/25 1; 104.132.141.64/27 1; 104.132.141.96/28 1; 104.132.142.0/24 1; 104.132.143.0/24 1; 104.132.144.0/24 1; 104.132.145.0/24 1; 104.132.146.0/24 1; 104.132.147.0/24 1; 104.132.148.0/23 1; 104.132.150.0/24 1; 104.132.151.0/24 1; 104.132.152.0/24 1; 104.132.153.0/24 1; 104.132.154.0/23 1; 104.132.156.0/24 1; 104.132.157.0/24 1; 104.132.158.0/24 1; 104.132.159.0/24 1; 104.132.16.0/24 1; 104.132.160.0/24 1; 104.132.161.0/24 1; 104.132.162.0/24 1; 104.132.163.0/24 1; 104.132.164.0/23 1; 104.132.166.0/24 1; 104.132.167.0/24 1; 104.132.168.0/24 1; 104.132.169.0/24 1; 104.132.17.0/26 1; 104.132.17.112/28 1; 104.132.17.128/25 1; 104.132.17.64/27 1; 104.132.17.96/28 1; 104.132.170.0/24 1; 104.132.171.0/24 1; 104.132.172.0/22 1; 104.132.176.0/23 1; 104.132.178.0/24 1; 104.132.179.0/24 1; 104.132.18.0/24 1; 104.132.180.0/24 1; 104.132.181.0/24 1; 104.132.182.0/24 1; 104.132.183.0/24 1; 104.132.184.0/24 1; 104.132.185.0/24 1; 104.132.186.0/24 1; 104.132.187.0/24 1; 104.132.188.0/24 1; 104.132.189.0/24 1; 104.132.19.0/24 1; 104.132.190.0/23 1; 104.132.192.0/22 1; 104.132.196.0/24 1; 104.132.197.0/24 1; 104.132.198.0/23 1; 104.132.20.0/24 1; 104.132.200.0/23 1; 104.132.202.0/24 1; 104.132.203.0/24 1; 104.132.204.0/24 1; 104.132.205.0/24 1; 104.132.206.0/23 1; 104.132.208.0/24 1; 104.132.209.0/24 1; 104.132.21.0/26 1; 104.132.21.112/28 1; 104.132.21.128/25 1; 104.132.21.64/27 1; 104.132.21.96/28 1; 104.132.210.0/23 1; 104.132.212.0/22 1; 104.132.216.0/21 1; 104.132.22.0/24 1; 104.132.224.0/19 1; 104.132.23.0/24 1; 104.132.24.0/26 1; 104.132.24.128/25 1; 104.132.24.64/26 1; 104.132.25.0/24 1; 104.132.26.0/24 1; 104.132.27.0/24 1; 104.132.28.0/24 1; 104.132.29.0/24 1; 104.132.30.0/23 1; 104.132.32.0/24 1; 104.132.33.0/24 1; 104.132.34.0/24 1; 104.132.35.0/24 1; 104.132.36.0/22 1; 104.132.40.0/21 1; 104.132.48.0/22 1; 104.132.52.0/23 1; 104.132.54.0/24 1; 104.132.55.0/24 1; 104.132.56.0/21 1; 104.132.64.0/18 1; 104.132.8.0/22 1; 104.133.0.0/17 1; 104.133.128.0/18 1; 104.133.192.0/19 1; 104.133.224.0/20 1; 104.133.240.0/21 1; 104.133.248.0/24 1; 104.133.249.0/24 1; 104.133.250.0/23 1; 104.133.252.0/22 1; 104.134.0.0/16 1; 104.135.0.0/17 1; 104.135.128.0/18 1; 104.135.192.0/19 1; 104.135.224.0/19 1; 104.154.0.0/15 1; 104.196.0.0/15 1; 104.198.0.0/16 1; 104.199.0.0/17 1; 104.199.128.0/20 1; 104.199.144.0/23 1; 104.199.146.0/24 1; 104.199.147.0/24 1; 104.199.148.0/22 1; 104.199.152.0/21 1; 104.199.160.0/19 1; 104.199.192.0/18 1; 107.167.160.0/19 1; 107.178.192.0/18 1; 108.170.192.0/20 1; 108.170.208.0/21 1; 108.170.216.0/24 1; 108.170.217.0/25 1; 108.170.217.128/28 1; 108.170.217.160/27 1; 108.170.217.192/26 1; 108.170.218.0/23 1; 108.170.220.0/22 1; 108.170.224.0/19 1; 108.177.0.0/17 1; 108.59.80.0/24 1; 108.59.81.0/27 1; 108.59.82.0/23 1; 108.59.84.0/22 1; 108.59.88.0/22 1; 108.59.92.0/27 1; 108.59.92.128/26 1; 108.59.92.192/27 1; 108.59.92.96/27 1; 108.59.93.0/27 1; 108.59.93.192/26 1; 108.59.93.32/29 1; 108.59.93.40/31 1; 108.59.93.43/32 1; 108.59.93.44/30 1; 108.59.93.48/28 1; 108.59.93.64/26 1; 108.59.94.0/28 1; 108.59.94.128/26 1; 108.59.94.16/29 1; 108.59.94.192/28 1; 108.59.94.208/29 1; 108.59.94.240/28 1; 108.59.94.32/27 1; 108.59.94.64/26 1; 108.59.95.0/24 1; 12.216.80.0/24 1; 12.234.149.240/29 1; 125.16.7.72/30 1; 125.17.82.112/30 1; 128.177.109.0/26 1; 128.177.119.128/25 1; 128.177.163.0/25 1; 130.211.0.0/16 1; 142.250.0.0/15 1; 146.148.0.0/17 1; 162.216.148.0/22 1; 162.222.176.0/21 1; 172.102.8.0/21 1; 172.217.0.0/16 1; 172.253.0.0/16 1; 173.194.0.0/18 1; 173.194.100.0/22 1; 173.194.104.0/21 1; 173.194.112.0/20 1; 173.194.128.0/17 1; 173.194.64.0/19 1; 173.194.96.0/24 1; 173.194.97.0/24 1; 173.194.98.0/24 1; 173.194.99.0/24 1; 173.255.112.0/22 1; 173.255.116.0/25 1; 173.255.116.128/26 1; 173.255.116.192/27 1; 173.255.117.128/25 1; 173.255.117.32/27 1; 173.255.117.64/26 1; 173.255.118.0/23 1; 173.255.120.0/24 1; 173.255.121.0/25 1; 173.255.121.128/26 1; 173.255.122.128/26 1; 173.255.122.64/26 1; 173.255.123.0/24 1; 173.255.124.0/27 1; 173.255.124.128/29 1; 173.255.124.144/28 1; 173.255.124.160/27 1; 173.255.124.192/27 1; 173.255.124.232/29 1; 173.255.124.240/29 1; 173.255.124.32/28 1; 173.255.124.48/29 1; 173.255.124.64/26 1; 173.255.125.0/27 1; 173.255.125.128/25 1; 173.255.125.72/29 1; 173.255.125.80/28 1; 173.255.125.96/27 1; 173.255.126.0/23 1; 180.87.33.64/26 1; 192.104.160.0/23 1; 192.158.28.0/22 1; 192.178.0.0/15 1; 195.16.45.144/29 1; 198.108.100.192/28 1; 199.192.112.0/25 1; 199.192.112.128/26 1; 199.192.112.192/27 1; 199.192.112.224/29 1; 199.192.113.0/25 1; 199.192.113.128/27 1; 199.192.113.176/28 1; 199.192.113.192/26 1; 199.192.114.0/25 1; 199.192.114.192/26 1; 199.192.115.0/28 1; 199.192.115.128/25 1; 199.192.115.80/28 1; 199.192.115.96/27 1; 199.223.232.0/21 1; 203.222.167.144/28 1; 206.160.135.240/28 1; 207.223.160.0/20 1; 208.184.125.240/28 1; 208.21.209.0/28 1; 208.44.48.240/29 1; 208.46.199.160/29 1; 209.185.108.128/25 1; 213.155.151.128/26 1; 213.200.103.128/26 1; 213.200.99.192/26 1; 216.109.75.80/28 1; 216.136.145.128/27 1; 216.239.32.0/24 1; 216.239.33.0/29 1; 216.239.33.104/29 1; 216.239.33.112/28 1; 216.239.33.128/25 1; 216.239.33.16/28 1; 216.239.33.32/29 1; 216.239.33.40/29 1; 216.239.33.48/28 1; 216.239.33.64/27 1; 216.239.33.8/29 1; 216.239.33.96/29 1; 216.239.34.0/24 1; 216.239.35.0/24 1; 216.239.36.0/23 1; 216.239.38.0/24 1; 216.239.39.0/24 1; 216.239.40.0/22 1; 216.239.44.0/23 1; 216.239.46.0/23 1; 216.239.48.0/22 1; 216.239.52.0/23 1; 216.239.54.0/24 1; 216.239.55.0/28 1; 216.239.55.128/27 1; 216.239.55.16/29 1; 216.239.55.160/29 1; 216.239.55.168/29 1; 216.239.55.176/28 1; 216.239.55.192/26 1; 216.239.55.24/29 1; 216.239.55.32/27 1; 216.239.55.64/26 1; 216.239.56.0/21 1; 216.252.220.0/22 1; 216.33.229.144/29 1; 216.33.229.160/29 1; 216.34.7.176/28 1; 216.74.130.48/28 1; 216.74.153.0/27 1; 217.118.234.96/28 1; 23.236.48.0/20 1; 23.251.128.0/19 1; 4.3.2.0/24 1; 41.206.188.128/26 1; 61.246.190.124/30 1; 61.246.224.136/30 1; 63.158.137.224/29 1; 63.161.156.0/24 1; 63.166.17.128/25 1; 63.226.245.56/29 1; 63.237.119.112/29 1; 63.88.22.0/23 1; 64.124.98.104/29 1; 64.233.160.0/23 1; 64.233.162.0/24 1; 64.233.163.0/24 1; 64.233.164.0/22 1; 64.233.168.0/21 1; 64.233.176.0/20 1; 64.41.146.208/28 1; 64.41.221.192/28 1; 64.68.64.64/26 1; 64.68.80.0/20 1; 64.71.148.240/29 1; 64.9.224.0/19 1; 65.167.144.64/28 1; 65.170.13.0/28 1; 65.171.1.144/28 1; 65.216.183.0/24 1; 65.220.13.0/24 1; 66.102.0.0/21 1; 66.102.12.0/23 1; 66.102.14.0/25 1; 66.102.14.128/30 1; 66.102.14.132/31 1; 66.102.14.134/31 1; 66.102.14.136/29 1; 66.102.14.144/28 1; 66.102.14.160/27 1; 66.102.14.192/26 1; 66.102.15.0/24 1; 66.102.8.0/22 1; 66.227.77.144/29 1; 66.249.80.0/23 1; 66.249.82.0/24 1; 66.249.83.0/24 1; 66.249.84.0/23 1; 66.249.86.0/23 1; 66.249.88.0/21 1; 67.148.177.136/29 1; 70.32.128.0/22 1; 70.32.132.0/23 1; 70.32.136.0/21 1; 70.32.144.0/20 1; 72.14.192.0/19 1; 72.14.224.0/22 1; 72.14.228.0/23 1; 72.14.230.0/29 1; 72.14.230.104/29 1; 72.14.230.112/28 1; 72.14.230.128/25 1; 72.14.230.16/29 1; 72.14.230.24/29 1; 72.14.230.32/29 1; 72.14.230.40/29 1; 72.14.230.48/29 1; 72.14.230.56/29 1; 72.14.230.64/30 1; 72.14.230.68/30 1; 72.14.230.72/29 1; 72.14.230.8/29 1; 72.14.230.80/28 1; 72.14.230.96/29 1; 72.14.231.0/29 1; 72.14.231.104/30 1; 72.14.231.108/30 1; 72.14.231.112/29 1; 72.14.231.120/31 1; 72.14.231.122/31 1; 72.14.231.124/30 1; 72.14.231.128/25 1; 72.14.231.16/29 1; 72.14.231.24/29 1; 72.14.231.32/29 1; 72.14.231.40/29 1; 72.14.231.48/29 1; 72.14.231.56/29 1; 72.14.231.64/29 1; 72.14.231.72/29 1; 72.14.231.8/29 1; 72.14.231.80/28 1; 72.14.231.96/29 1; 72.14.232.0/21 1; 72.14.240.0/24 1; 72.14.241.0/29 1; 72.14.241.128/25 1; 72.14.241.16/29 1; 72.14.241.24/29 1; 72.14.241.32/29 1; 72.14.241.40/29 1; 72.14.241.48/28 1; 72.14.241.64/26 1; 72.14.241.8/29 1; 72.14.242.0/23 1; 72.14.244.0/22 1; 72.14.248.0/21 1; 74.125.0.0/20 1; 74.125.112.0/22 1; 74.125.116.0/22 1; 74.125.120.0/22 1; 74.125.124.0/22 1; 74.125.128.0/19 1; 74.125.16.0/24 1; 74.125.160.0/20 1; 74.125.17.0/24 1; 74.125.176.0/22 1; 74.125.18.0/28 1; 74.125.18.128/28 1; 74.125.18.144/29 1; 74.125.18.152/29 1; 74.125.18.16/29 1; 74.125.18.160/27 1; 74.125.18.192/28 1; 74.125.18.208/29 1; 74.125.18.216/29 1; 74.125.18.224/27 1; 74.125.18.24/29 1; 74.125.18.32/27 1; 74.125.18.64/28 1; 74.125.18.80/29 1; 74.125.18.88/29 1; 74.125.18.96/27 1; 74.125.180.0/24 1; 74.125.181.0/24 1; 74.125.182.0/23 1; 74.125.184.0/22 1; 74.125.188.0/24 1; 74.125.189.0/24 1; 74.125.19.0/24 1; 74.125.190.0/24 1; 74.125.191.0/24 1; 74.125.192.0/18 1; 74.125.20.0/22 1; 74.125.24.0/21 1; 74.125.32.0/20 1; 74.125.48.0/21 1; 74.125.56.0/27 1; 74.125.56.128/26 1; 74.125.56.192/28 1; 74.125.56.208/28 1; 74.125.56.224/27 1; 74.125.56.32/29 1; 74.125.56.40/29 1; 74.125.56.48/28 1; 74.125.56.64/26 1; 74.125.57.0/28 1; 74.125.57.128/30 1; 74.125.57.132/30 1; 74.125.57.136/29 1; 74.125.57.144/28 1; 74.125.57.16/29 1; 74.125.57.160/28 1; 74.125.57.176/28 1; 74.125.57.192/26 1; 74.125.57.24/29 1; 74.125.57.32/28 1; 74.125.57.48/28 1; 74.125.57.64/28 1; 74.125.57.80/29 1; 74.125.57.88/29 1; 74.125.57.96/27 1; 74.125.58.0/24 1; 74.125.59.0/25 1; 74.125.59.128/26 1; 74.125.59.192/27 1; 74.125.59.224/28 1; 74.125.59.240/28 1; 74.125.60.0/29 1; 74.125.60.104/29 1; 74.125.60.112/29 1; 74.125.60.120/29 1; 74.125.60.128/29 1; 74.125.60.136/29 1; 74.125.60.144/29 1; 74.125.60.152/29 1; 74.125.60.16/29 1; 74.125.60.160/29 1; 74.125.60.168/29 1; 74.125.60.176/30 1; 74.125.60.180/30 1; 74.125.60.184/29 1; 74.125.60.192/29 1; 74.125.60.200/29 1; 74.125.60.208/28 1; 74.125.60.224/27 1; 74.125.60.24/29 1; 74.125.60.32/27 1; 74.125.60.64/29 1; 74.125.60.72/29 1; 74.125.60.8/29 1; 74.125.60.80/29 1; 74.125.60.88/29 1; 74.125.60.96/29 1; 74.125.61.0/29 1; 74.125.61.104/30 1; 74.125.61.108/30 1; 74.125.61.112/29 1; 74.125.61.120/29 1; 74.125.61.128/29 1; 74.125.61.136/29 1; 74.125.61.144/30 1; 74.125.61.148/30 1; 74.125.61.152/29 1; 74.125.61.16/29 1; 74.125.61.160/29 1; 74.125.61.168/29 1; 74.125.61.176/29 1; 74.125.61.184/29 1; 74.125.61.192/29 1; 74.125.61.200/29 1; 74.125.61.208/31 1; 74.125.61.210/31 1; 74.125.61.212/30 1; 74.125.61.216/29 1; 74.125.61.224/29 1; 74.125.61.232/31 1; 74.125.61.234/31 1; 74.125.61.236/30 1; 74.125.61.24/29 1; 74.125.61.240/28 1; 74.125.61.32/28 1; 74.125.61.48/29 1; 74.125.61.56/29 1; 74.125.61.64/29 1; 74.125.61.72/29 1; 74.125.61.8/29 1; 74.125.61.80/29 1; 74.125.61.88/29 1; 74.125.61.96/29 1; 74.125.62.0/24 1; 74.125.63.0/24 1; 74.125.64.0/19 1; 74.125.96.0/20 1; 77.109.131.208/28 1; 77.67.50.32/27 1; 8.34.208.0/25 1; 8.34.208.128/29 1; 8.34.208.144/28 1; 8.34.208.160/27 1; 8.34.208.192/26 1; 8.34.209.0/24 1; 8.34.210.0/23 1; 8.34.212.0/22 1; 8.34.216.0/24 1; 8.34.217.0/28 1; 8.34.217.128/25 1; 8.34.217.24/29 1; 8.34.217.32/27 1; 8.34.217.64/26 1; 8.34.218.0/23 1; 8.34.220.0/22 1; 8.35.192.0/23 1; 8.35.194.0/24 1; 8.35.195.0/25 1; 8.35.195.128/28 1; 8.35.195.160/27 1; 8.35.195.192/26 1; 8.35.196.0/22 1; 8.35.200.0/21 1; 8.6.48.0/21 1; 8.8.4.0/24 1; 8.8.8.0/24 1; 80.149.20.0/25 1; 80.239.168.192/26 1; 85.182.250.0/25 1; 85.182.250.128/26 1; }
Далее в конфиг виртуального хоста прописываем
if ($googlebotip = 0) { set $GoogleBAD A; }
if ($http_user_agent ~ "Googlebot" ) { set $GoogleBAD "${GoogleBAD}B"; }
if ($GoogleBAD = AB) { return 403; }
Перезапускаем Nginx, и видим что на «левом» зеркале ошибка 403, но при этом поисковые боты Google нормально ходят по сайту, не выдавая никаких ошибок. Данный метод также подойдет при DDoS атаке с измененным User Agent под гуглобота. На этом все
|