User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /
User-agent: *
Disallow: /demo/
Disallow: /mailbox/
Disallow: /OLD_SITE/
Disallow: /wp-admin/
Disallow: /wp-admin/admin-ajax.php
Disallow: /wp-content/
Disallow: /wp-includes/
disallow: /*?parameter=
disallow: /*?wordfence_lh=
#Define website sitemaps
Sitemap: http://mobiosolutions.com/sitemap_index.xml
Sitemap: http://mobiosolutions.com/page-sitemap.xml