gitlab_medici/old/html/robots.txt @ f851ac73
| 1 | 6daefa8c | Petr Lukašík | # exclude help system from robots |
|---|---|---|---|
| 2 | User-agent: * |
||
| 3 | Disallow: /manual/ |
||
| 4 | Disallow: /manual-2.0/ |
||
| 5 | Disallow: /manual-1.3/ |
||
| 6 | Disallow: /addon-modules/ |
||
| 7 | Disallow: /doc/ |
||
| 8 | Disallow: /images/ |
||
| 9 | # the next line is a spam bot trap, for grepping the logs. you should _really_ change this to something else... |
||
| 10 | Disallow: /all_our_e-mail_addresses |
||
| 11 | # same idea here... |
||
| 12 | Disallow: /admin/ |
||
| 13 | # but allow htdig to index our doc-tree |
||
| 14 | #User-agent: htdig |
||
| 15 | #Disallow: |
||
| 16 | # disallow stress test |
||
| 17 | user-agent: stress-agent |
||
| 18 | Disallow: / |