# htdig
User-agent:  htdig
Disallow: 

# W3C Link checker on medmedia
User-agent:  w3c-checklink
Disallow: 

# go away
User-agent: *
Disallow: /