# robots.txt for wangxubin.site # Allow all search engines to crawl the site User-agent: * Allow: / # Disallow admin and testing pages Disallow: /admin-visitor-stats.html Disallow: /life Disallow: /migrate-to-gist.html Disallow: /source/img/scenery/ Disallow: /resources/research_work_draft/ Disallow: /resources/cv/ Disallow: /security/ Disallow: /performance/ Disallow: /*test*.html Disallow: /*debug*.html Disallow: /button-*.html Disallow: /css-conflict-detector.html Disallow: /ios-*.html Disallow: /tier-*.html Disallow: /unified-*.html # Allow crawling of images and resources Allow: /source/img/ Allow: /source/css/ Allow: /source/js/ # Sitemap location Sitemap: https://www.wangxubin.site/sitemap.xml # Crawl-delay for respectful crawling Crawl-delay: 1 # Specific rules for common search engines User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Baiduspider Allow: /