clean up and robots changes

config/gunicorn_config.py

@@ -35,8 +35,6 @@ timeout = 300
 #max_requests = 0
 #max_requests_jitter = 0
 
-# Enable connection keepalive:
-#keepalive = 60
 
 # Logging ---------------------------------------------------------------------
 # Access log

gracedb/static/robots.txt

@@ -10,6 +10,7 @@
 # visible too. 
 
 User-agent: *
+Crawl-delay: 1024
 Disallow: /api/
 Disallow: /apiweb/
 Disallow: /superevents/
@@ -21,3 +22,131 @@ Disallow: /alerts/
 Disallow: /Shibboleth.sso/
 Disallow: /shibboleth-sp/
 Disallow: /login/
+
+#Turn off bad bots. 
+User-agent: Slurp
+Disallow: /
+
+#Block bots
+User-agent: A6-Indexer
+Disallow: /
+
+User-agent: AlphaSeoBot
+Disallow: /
+
+User-agent: AlphaSeoBot-SA
+Disallow: /
+
+# RDH, 08.19.19: I really don't want to block Applebot, but for now, I am. It is crawling us too much
+User-agent: Applebot
+Disallow: /
+
+User-agent: AspiegelBot
+Disallow: /
+
+User-agent: barkrowler
+Disallow: /
+
+# RDH, 05.13.20: I really don't want to block bing, but for now, I am. It is also already in htaccess rules
+User-agent: bingbot/2.0
+Disallow: /
+
+User-agent: Blackboard Safeassign
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: crawler4j
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+# RDH, 06.30.21: Very temporary to get some relief.
+User-Agent: Googlebot
+Disallow: /
+
+User-agent: Gigabot
+Disallow: /
+
+User-agent: LieBaoFast
+Disallow: /
+
+User-agent: MauiBot
+Disallow:/
+
+User-agent: MauiBot (crawler.feedback+wc@gmail.com) 
+Disallow: /
+
+User-agent: MegaIndex.ru/2.0
+Disallow: /
+
+User-agent: MqqBrowser
+Disallow:/
+
+User-agent: Nimbostratus-Bot/v1.3.2
+Disallow: /
+
+User-agent: Qwant-news
+Disallow: /
+
+User-agent: Qwantify
+Disallow: /
+
+User-agent: Seekport Crawler
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: SemrushBot-SA
+Disallow: / 
+
+User-agent: SeznamBot
+Disallow: /
+
+User-agent: SputnikBot/2.3
+Disallow:/
+
+User-agent: The Knowledge AI 
+Disallow:/
+
+User-agent: TinyTestBot
+Disallow: /
+
+User-agent: TurnitinBot
+Disallow: /
+
+User-agent: UCBrowser
+Disallow: / 
+
+User-agent: yacybot
+Disallow: / 
+
+User-agent: YandexBot
+Disallow: / 
+
+User-agent: Yeti
+Disallow: / 
+
+User-agent: YisouSpider
+Disallow: / 
+
+User-agent: Facebot Twitterbot
+Disallow: /
+
+User-agent: FacebookExternalHit
+Disallow: /
+
+User-agent: Mattermost-Bot
+Disallow: / 
+
+User-agent: AhrefsBot
+Disallow: / 
+
+User-agent: ZumBot
+Disallow: /