User-agent: Scrapy
Allow: /

# Reference for best practices: https://yoast.com/wordpress-robots-txt-example/ and https://wordpress.org/support/topic/no-logo-in-google-search-results/#post-12399896

User-agent: *
Allow: /
Disallow: *?wptheme
Disallow: /?p=*
Disallow: /?s=
Disallow: /search/

# Reference why to allow some part of the admin: https://webmasters.stackexchange.com/questions/110123/should-the-admin-ajax-php-file-in-my-wordpress-site-be-allow-or-disallowed-in#comment160387_110123
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php