From cb8d301c7ea4fb0ffbc7747bec2c93b7a2886c8f Mon Sep 17 00:00:00 2001 From: Douglas Muth Date: Sat, 12 May 2018 12:43:04 -0400 Subject: [PATCH] Added robots.txt --- robots.txt | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 robots.txt diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..24589da --- /dev/null +++ b/robots.txt @@ -0,0 +1,62 @@ +# +# robots.txt +# +# This file is to prevent the crawling and indexing of certain parts +# of your site by web crawlers and spiders run by sites like Yahoo! +# and Google. By telling these "robots" where not to go on your site, +# you save bandwidth and server resources. +# +# This file will be ignored unless it is at the root of your host: +# Used: http://example.com/robots.txt +# Ignored: http://example.com/site/robots.txt +# +# For more information about the robots.txt standard, see: +# http://www.robotstxt.org/wc/robots.html +# +# For syntax checking, see: +# http://www.sxw.org.uk/computing/robots/check.html + +User-agent: * +Crawl-delay: 10 + +# +# From http://www.gotfuturama.com/Multimedia/AsciiArt/dispatch.cgi?target=Bender1 +# +# QR +# #t(@ +# @R +# @S +# RSQ +# @K@ +# #KQ7 +# RSQ@ +# @%GC(7Q/ +# #@K%C6QKCCC6@ +# (@%CC3%C3CC%CCCC6Q +# GCCC%C3CC66CCCsCCC@ +# sQ3C6CCCC%CCC3CCCCC%C7 +# @6%CCCCCOC6CsCCC66O%s@ +# GsCCC36CC6CC6%%CC%CC%@ +# GC36C3s%CCCCsCC%C%C%CsG +# sCCC666CCCCC6%CC%CCsRG%#7 +# C6%C63CRQGC3C77%7(t((t((7Q^ +# sC3CC@C((t((7QG#K#@@@@@@@@G7Q +# S6CCO#(t((@%/%O@@@@@@O(//R@@sC +# QsCC6st(%@ CG @@s@ +# @OCCC%((%R/C@ RC @@7@ +# @OO3s@(((@/ (R/ @#R( +# @s3CCCQC(t%%GQRQQQ##QGCGC((7R +# GGCCCsCCCG36SGQKSSSGsss +# Gs6C6CCCCCCCCCCs6CC6CC3 +# /@CCC3Cs3GSQQQ#QRGQS@@Q +# @3%C@@(@///%///(((( +# @CCOR/(@(O%O%%(((((( +# GCCs@%(@///%////(((C +# CCCCG@(@%%CQ%%%%O(/#C +# sC3C%CCGGQQ#QQGGG6CC#@ +# @((%@#3CC3%C3CC%CCCCsQC((Q7 +# #Qt((t((7%Q#R@QK@Q#QG37((t((7@ +# %3((t(((((((t((t((t((t(tt((t(t((t@ +# %7t((t((t((((tt((((((((t(tt(t((t(((#C +# +