{
  "schemaVersion": "1.2",
  "type": "tutorial-evidence-record",
  "language": "zh-CN",
  "generatedAt": "2026-09-19",
  "publisher": {
    "id": "https://zenleak.cn/#organization",
    "name": "ZenLeak GEO 学院",
    "editorialName": "ZenLeak 编辑部",
    "contact": "geo@allintwo.cloud"
  },
  "policies": {
    "sourceRelationship": "Page sources provide general context. Explicit claim evidence includes a source excerpt, locator, relation, review date and limits; unmapped claims do not imply one-to-one source support.",
    "rankingDisclaimer": "Inclusion in this evidence layer does not guarantee search ranking, indexing, citation, recommendation, or adoption by an AI system."
  },
  "record": {
    "id": "ai-crawler-governance",
    "title": "AI 爬虫治理：分开管理搜索索引、答案代理与模型训练",
    "url": "https://zenleak.cn/technical/ai-crawler-governance/",
    "path": "/technical/ai-crawler-governance/",
    "evidenceUrl": "https://zenleak.cn/evidence/#ai-crawler-governance",
    "representations": {
      "json": "https://zenleak.cn/evidence/ai-crawler-governance.json",
      "markdown": "https://zenleak.cn/evidence/ai-crawler-governance.md"
    },
    "category": "技术配置",
    "contentType": "guide",
    "status": "reviewed",
    "statusLabel": "已复核",
    "datePublished": "2026-09-17",
    "dateModified": "2026-09-17",
    "reviewedAt": "2026-09-17",
    "evidenceReviewedAt": "2026-09-19",
    "claims": [
      {
        "id": "ai-crawler-governance-claim-1",
        "text": "AI 相关机器人并非一种用途：搜索索引、用户触发访问、答案搜索取材和模型训练可能使用不同标识与控制方式。",
        "url": "https://zenleak.cn/evidence/#ai-crawler-governance-claim-1",
        "articleUrl": "https://zenleak.cn/technical/ai-crawler-governance/#ai-crawler-governance-claim-1",
        "evidence": []
      },
      {
        "id": "ai-crawler-governance-claim-2",
        "text": "治理策略应先决定业务目标，再按官方文档核对 User-Agent、IP 验证和 robots 规则；不能只凭请求字符串认定真实身份。",
        "url": "https://zenleak.cn/evidence/#ai-crawler-governance-claim-2",
        "articleUrl": "https://zenleak.cn/technical/ai-crawler-governance/#ai-crawler-governance-claim-2",
        "evidence": []
      },
      {
        "id": "ai-crawler-governance-claim-3",
        "text": "允许抓取不等于授权所有用途，也不保证被采用；robots.txt 不是访问控制或合同替代，敏感内容必须使用认证与权限机制。",
        "url": "https://zenleak.cn/evidence/#ai-crawler-governance-claim-3",
        "articleUrl": "https://zenleak.cn/technical/ai-crawler-governance/#ai-crawler-governance-claim-3",
        "evidence": [
          {
            "tutorialId": "ai-crawler-governance",
            "claimText": "允许抓取不等于授权所有用途，也不保证被采用；robots.txt 不是访问控制或合同替代，敏感内容必须使用认证与权限机制。",
            "relation": "context",
            "sourceUrl": "https://www.rfc-editor.org/rfc/rfc9309#section-3",
            "sourceTitle": "RFC 9309 · Security Considerations",
            "locator": "Section 3",
            "excerpt": "The Robots Exclusion Protocol is not a substitute for valid content security measures.",
            "checkedAt": "2026-09-19",
            "limits": "直接支持不能用 robots 替代安全措施；用途授权、合同和采用概率超出 RFC 的结论范围。"
          }
        ]
      }
    ],
    "sources": [
      {
        "label": "OpenAI crawlers",
        "url": "https://platform.openai.com/docs/bots",
        "publisher": "OpenAI"
      },
      {
        "label": "Google crawler user agents",
        "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
        "publisher": "Google Search Central"
      },
      {
        "label": "Robots Exclusion Protocol (RFC 9309)",
        "url": "https://www.rfc-editor.org/rfc/rfc9309",
        "publisher": "IETF"
      }
    ],
    "prerequisites": [
      {
        "id": "crawl-render-index",
        "title": "抓取、渲染、归一与索引：页面不可见问题的完整诊断链",
        "url": "https://zenleak.cn/technical/crawl-render-index/"
      },
      {
        "id": "robots-sitemap-llms",
        "title": "robots、sitemap 和 llms.txt：GEO 技术基础配置",
        "url": "https://zenleak.cn/technical/robots-sitemap-llms/"
      }
    ],
    "related": [
      {
        "id": "indexing-vs-ai-citation",
        "title": "百度已经收录，为什么 AI 仍然不引用？从抓取、索引到答案引用的完整链路",
        "url": "https://zenleak.cn/technical/indexing-vs-ai-citation/"
      },
      {
        "id": "china-ai-search-geo",
        "title": "国内 AI 搜索如何获取网页信息：DeepSeek、Kimi、豆包、通义千问、智谱对比",
        "url": "https://zenleak.cn/content/china-ai-search-geo/"
      },
      {
        "id": "robots-sitemap-llms",
        "title": "robots、sitemap 和 llms.txt：GEO 技术基础配置",
        "url": "https://zenleak.cn/technical/robots-sitemap-llms/"
      },
      {
        "id": "crawl-render-index",
        "title": "抓取、渲染、归一与索引：页面不可见问题的完整诊断链",
        "url": "https://zenleak.cn/technical/crawl-render-index/"
      },
      {
        "id": "root-domain-launch",
        "title": "主域名上线怎么优化：ZenLeak 根域名发布清单",
        "url": "https://zenleak.cn/technical/root-domain-launch/"
      },
      {
        "id": "ai-citation-test",
        "title": "AI 引用测试怎么做：建立 GEO 实验记录",
        "url": "https://zenleak.cn/experiments/ai-citation-test/"
      }
    ]
  }
}