robots.txt
User-agent: Mediapartners-Google*
User-agent: *
Disallow:
User-agent: *
Disallow: /Images/
User-agent: *
Disallow: /Includes/
...
User-agent: *
Disallow:
User-agent: *
Disallow: /Images/
User-agent: *
Disallow: /Includes/
...
urllist.txt
...