finally finish all website check in.
This commit is contained in:
24
public/robots.txt
Normal file
24
public/robots.txt
Normal file
@@ -0,0 +1,24 @@
|
||||
# Jiao77.cn - 个人学术报告网站
|
||||
# 允许所有搜索引擎抓取
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
# 网站地图位置
|
||||
Sitemap: http://jiao77.cn/sitemap.xml
|
||||
|
||||
# 不允许抓取的路径(模板和示例页面)
|
||||
Disallow: /report/template/
|
||||
|
||||
# 主要搜索引擎配置
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
User-agent: Baiduspider
|
||||
Allow: /
|
||||
|
||||
# 爬取延迟(秒)- 避免过度请求
|
||||
Crawl-delay: 1
|
||||
Reference in New Issue
Block a user