# Allow major search engines User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / # Block known heavy / unwanted bots (AI scrapers + aggressive crawlers) User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Bytespider Disallow: / User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Amazonbot Disallow: / # Default rules for all other bots User-agent: * # Block sensitive & attack endpoints Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-json/ # Block hidden/system files Disallow: /.env Disallow: /.git Disallow: /config/ Disallow: /vendor/ # Block query spam (optional but helpful) Disallow: /*?*utm_ Disallow: /*?*ref= # Allow normal indexing Allow: /