# All robots will spider the domain
User-agent: *
Disallow:

# Disallow directory /cgi-bin/
User-agent: *
Disallow: /cgi-bin/

# Disallow directory /config/
User-agent: *
Disallow: /config/

# Disallow directory /css/
User-agent: *
Disallow: /css/

# Disallow directory /data/
User-agent: *
Disallow: /data/

# Disallow directory /files/
User-agent: *
Disallow: /files/

# Disallow directory /images/
User-agent: *
Disallow: /scripts/