搜索引擎爬虫日志分析引擎文档：Nginx/CDN 日志接入、UA 解析与 rDNS 验证

log_format seo_crawler_json escape=json '{'
  '"timestamp":"$time_iso8601",'
  '"client_ip":"$http_x_forwarded_for",'
  '"remote_addr":"$remote_addr",'
  '"host":"$host",'
  '"method":"$request_method",'
  '"uri":"$request_uri",'
  '"status":$status,'
  '"bytes_sent":$body_bytes_sent,'
  '"request_time":$request_time,'
  '"upstream_time":"$upstream_response_time",'
  '"user_agent":"$http_user_agent",'
  '"referer":"$http_referer"'
'}';

map $http_user_agent $is_search_crawler {
  default 0;
  ~*(Baiduspider|bingbot|Googlebot|Sogou|YisouSpider|Bytespider) 1;
}

access_log /var/log/nginx/search_crawlers.json.log seo_crawler_json if=$is_search_crawler;