Merge branch 'main' into improve_nginx_regex

public/robots.txt

@@ -3,12 +3,10 @@
 # Using: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/blob/master/robots.txt/robots.txt
 ### Version Information for nginx-ultimate-bad-bot-blocker #
 ###################################################
-### Version: V4.2024.07.4631
-### Updated: Thu Jul 18 22:01:11 UTC 2024
-### Bad Bot Count: 663
+### Version: V4.2025.01.4984
+### Updated: Sun Jan 19 10:06:56 UTC 2025
+### Bad Bot Count: 675
 ###################################################
-### Version Information ##
-
 User-agent: 01h4x.com
 Disallow:/
 User-agent: 360Spider
@@ -97,6 +95,8 @@ User-agent: Backlink-Ceck
 Disallow:/
 User-agent: BacklinkCrawler
 Disallow:/
+User-agent: BacklinksExtendedBot
+Disallow:/
 User-agent: Badass
 Disallow:/
 User-agent: Bandit
@@ -363,8 +363,6 @@ User-agent: Go-Ahead-Got-It
 Disallow:/
 User-agent: GoZilla
 Disallow:/
-User-agent: Google-Extended
-Disallow:/
 User-agent: Gotit
 Disallow:/
 User-agent: GrabNet
@@ -813,6 +811,8 @@ User-agent: SEOkicks
 Disallow:/
 User-agent: SEOkicks-Robot
 Disallow:/
+User-agent: SEOlyt
+Disallow:/
 User-agent: SEOlyticsCrawler
 Disallow:/
 User-agent: SEOprofiler
@@ -853,10 +853,22 @@ User-agent: Semrush
 Disallow:/
 User-agent: SemrushBot
 Disallow:/
+User-agent: SemrushBot-BA
+Disallow:/
+User-agent: SemrushBot-FT
+Disallow:/
+User-agent: SemrushBot-OCOB
+Disallow:/
+User-agent: SemrushBot-SI
+Disallow:/
+User-agent: SemrushBot-SWA
+Disallow:/
 User-agent: SentiBot
 Disallow:/
 User-agent: SenutoBot
 Disallow:/
+User-agent: SeoCherryBot
+Disallow:/
 User-agent: SeoSiteCheckup
 Disallow:/
 User-agent: SeobilityBot
@@ -867,6 +879,8 @@ User-agent: Shodan
 Disallow:/
 User-agent: Siphon
 Disallow:/
+User-agent: SiteAuditBot
+Disallow:/
 User-agent: SiteCheckerBotCrawler
 Disallow:/
 User-agent: SiteExplorer
@@ -915,8 +929,14 @@ User-agent: Spanner
 Disallow:/
 User-agent: Spbot
 Disallow:/
+User-agent: Spider_Bot
+Disallow:/
+User-agent: Spider_Bot/3.0
+Disallow:/
 User-agent: Spinn3r
 Disallow:/
+User-agent: SplitSignalBot
+Disallow:/
 User-agent: SputnikBot
 Disallow:/
 User-agent: Sqlmap
@@ -1209,7 +1229,7 @@ User-agent: crawler4j
 Disallow:/
 User-agent: dataforseo.com
 Disallow:/
-User-agent: DataForSeoBot
+User-agent: dataforseobot
 Disallow:/
 User-agent: demandbase-bot
 Disallow:/
@@ -1219,6 +1239,8 @@ User-agent: eCatch
 Disallow:/
 User-agent: evc-batch
 Disallow:/
+User-agent: everyfeed-spider
+Disallow:/
 User-agent: facebookscraper
 Disallow:/
 User-agent: gopher
@@ -1337,6 +1359,8 @@ User-agent: zgrab
 Disallow:/
 
 # Extra bad bots
+User-agent: Google-Extended
+Disallow: /
 User-agent: PerplexityBot
 Disallow: /
 User-agent: OmigiliBot