From 93b1e4433d81bb1a851002bf611b2e098bb68af5 Mon Sep 17 00:00:00 2001 From: tuxifreund Date: Fri, 22 Jan 2021 22:28:12 +0100 Subject: [PATCH 1/3] Update robots.txt --- inyoka_theme_ubuntuusers/static/robots.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/inyoka_theme_ubuntuusers/static/robots.txt b/inyoka_theme_ubuntuusers/static/robots.txt index 2592201de..25718cdc4 100644 --- a/inyoka_theme_ubuntuusers/static/robots.txt +++ b/inyoka_theme_ubuntuusers/static/robots.txt @@ -1,10 +1,8 @@ User-agent: * Crawl-delay: 10 -Disallow: /search Disallow: /_image? Disallow: /Benutzer/ Disallow: /users/ -Disallow: /*?flavour=mobile User-agent: BecomeBot Disallow: / From 54edfb1b6bdb19f8fa36508bd4a54a045768eabf Mon Sep 17 00:00:00 2001 From: KaiserBarbarossa Date: Sat, 23 Jan 2021 09:27:41 +0100 Subject: [PATCH 2/3] Allow BecomeBot, Yahoo and Baidu --- inyoka_theme_ubuntuusers/static/robots.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/inyoka_theme_ubuntuusers/static/robots.txt b/inyoka_theme_ubuntuusers/static/robots.txt index 25718cdc4..08b0b8f44 100644 --- a/inyoka_theme_ubuntuusers/static/robots.txt +++ b/inyoka_theme_ubuntuusers/static/robots.txt @@ -3,12 +3,3 @@ Crawl-delay: 10 Disallow: /_image? Disallow: /Benutzer/ Disallow: /users/ - -User-agent: BecomeBot -Disallow: / - -User-agent: Yahoo! Slurp -Disallow: / - -User-agent: Baiduspider -Disallow: / From 63a63190bb4a3c7a8d49346546d9089f78813b3a Mon Sep 17 00:00:00 2001 From: KaiserBarbarossa Date: Sat, 23 Jan 2021 09:29:52 +0100 Subject: [PATCH 3/3] Disallow mobile flavour again --- inyoka_theme_ubuntuusers/static/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/inyoka_theme_ubuntuusers/static/robots.txt b/inyoka_theme_ubuntuusers/static/robots.txt index 08b0b8f44..b31b094b3 100644 --- a/inyoka_theme_ubuntuusers/static/robots.txt +++ b/inyoka_theme_ubuntuusers/static/robots.txt @@ -3,3 +3,4 @@ Crawl-delay: 10 Disallow: /_image? Disallow: /Benutzer/ Disallow: /users/ +Disallow: /*?flavour=mobile