AI crawler tracking

import { reportCrawl } from '@productlense/ai-crawl';
import { env } from '$env/dynamic/private';

export async function handle({ event, resolve }) {
  const response = await resolve(event);
  const report = reportCrawl(
    { token: env.PL_BOT_TOKEN },
    { path: event.url.pathname, userAgent: event.request.headers.get('user-agent'), status: response.status },
  );
  // On Cloudflare, keep the report alive after the response is sent (ctx, or context on older adapters)
  const ctx = event.platform?.ctx ?? event.platform?.context;
  ctx?.waitUntil?.(report);
  return response;
}

"""Reports AI crawler visits to ProductLense. Needs only the standard library."""
import json
import os
import re
import threading
import urllib.request
from urllib.parse import urlsplit

ENDPOINT = "https://api.productlense.app/api/crawl"
AI_BOTS = re.compile(r"ChatGPT-User|Claude-User|Perplexity-User|MistralAI-User|DuckAssistBot|Meta-ExternalFetcher|Gemini-Deep-Research|cohere-ai|OAI-SearchBot|Claude-SearchBot|PerplexityBot|Googlebot|Bingbot|Applebot|DuckDuckBot|YouBot|Bravebot|GPTBot|ClaudeBot|anthropic-ai|Claude-Web|CCBot|Google-Extended|GoogleOther|Bytespider|Meta-ExternalAgent|FacebookBot|Amazonbot|cohere-training-data-crawler|Diffbot|AI2Bot|ImagesiftBot|Omgilibot|PanguBot|Timpibot", re.IGNORECASE)


def _send(body, token):
    request = urllib.request.Request(
        ENDPOINT,
        data=json.dumps(body).encode(),
        method="POST",
        headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
    )
    try:
        urllib.request.urlopen(request, timeout=2).close()
    except Exception:
        pass  # never let ProductLense affect your site


def report_crawl(path, user_agent, status=None):
    """Call once per request. Does nothing for people; for crawlers it sends in the background."""
    token = os.environ.get("PL_BOT_TOKEN")
    if not token or not user_agent or not AI_BOTS.search(user_agent[:1024]):
        return
    body = {"path": urlsplit(path).path or "/", "ua": user_agent}
    if status:
        body["status"] = status
    threading.Thread(target=_send, args=(body, token), daemon=True).start()

require "json"
require "net/http"

class ProductLenseCrawlers # Rack middleware: reports AI crawlers after your app answers
  ENDPOINT = URI("https://api.productlense.app/api/crawl")
  AI_BOTS = /ChatGPT-User|Claude-User|Perplexity-User|MistralAI-User|DuckAssistBot|Meta-ExternalFetcher|Gemini-Deep-Research|cohere-ai|OAI-SearchBot|Claude-SearchBot|PerplexityBot|Googlebot|Bingbot|Applebot|DuckDuckBot|YouBot|Bravebot|GPTBot|ClaudeBot|anthropic-ai|Claude-Web|CCBot|Google-Extended|GoogleOther|Bytespider|Meta-ExternalAgent|FacebookBot|Amazonbot|cohere-training-data-crawler|Diffbot|AI2Bot|ImagesiftBot|Omgilibot|PanguBot|Timpibot/i

  def initialize(app)
    @app = app
  end

  def call(env)
    status, headers, body = @app.call(env)
    report(env, status)
    [status, headers, body]
  end

  private

  def report(env, status)
    token = ENV["PL_BOT_TOKEN"]
    ua = env["HTTP_USER_AGENT"].to_s
    return if token.to_s.empty? || !AI_BOTS.match?(ua[0, 1024])

    payload = { path: env["PATH_INFO"].to_s.empty? ? "/" : env["PATH_INFO"], ua: ua, status: status.to_i }.to_json
    Thread.new do
      Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: true, open_timeout: 2, read_timeout: 2) do |http|
        http.post(ENDPOINT.path, payload, "Authorization" => "Bearer #{token}", "Content-Type" => "application/json")
      end
    rescue StandardError
      nil # never let ProductLense affect your site
    end
  rescue StandardError
    nil
  end
end

require_relative "../lib/productlense_crawlers"

module YourApp
  class Application < Rails::Application
    config.middleware.use ProductLenseCrawlers
  end
end

import type { Config, Context } from '@netlify/edge-functions';

const AI_BOTS = /ChatGPT-User|Claude-User|Perplexity-User|MistralAI-User|DuckAssistBot|Meta-ExternalFetcher|Gemini-Deep-Research|cohere-ai|OAI-SearchBot|Claude-SearchBot|PerplexityBot|Googlebot|Bingbot|Applebot|DuckDuckBot|YouBot|Bravebot|GPTBot|ClaudeBot|anthropic-ai|Claude-Web|CCBot|Google-Extended|GoogleOther|Bytespider|Meta-ExternalAgent|FacebookBot|Amazonbot|cohere-training-data-crawler|Diffbot|AI2Bot|ImagesiftBot|Omgilibot|PanguBot|Timpibot/i;

export default async (request: Request, context: Context) => {
  const response = await context.next();
  const ua = request.headers.get('user-agent') ?? '';
  const token = Netlify.env.get('PL_BOT_TOKEN');
  if (token && AI_BOTS.test(ua.slice(0, 1024))) {
    const report = fetch('https://api.productlense.app/api/crawl', {
      method: 'POST',
      headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
      body: JSON.stringify({ path: new URL(request.url).pathname, ua, status: response.status }),
    }).catch(() => {});
    context.waitUntil?.(report); // keeps the report alive after the response
  }
  return response;
};

export const config: Config = {
  path: '/*',
  excludedPath: ['/*.css', '/*.js', '/*.png', '/*.jpg', '/*.svg', '/*.webp', '/*.woff2', '/*.ico'],
};

map $http_user_agent $pl_ai_bot {
    default 0;
    "~*(ChatGPT-User|Claude-User|Perplexity-User|MistralAI-User|DuckAssistBot|Meta-ExternalFetcher|Gemini-Deep-Research|cohere-ai|OAI-SearchBot|Claude-SearchBot|PerplexityBot|Googlebot|Bingbot|Applebot|DuckDuckBot|YouBot|Bravebot|GPTBot|ClaudeBot|anthropic-ai|Claude-Web|CCBot|Google-Extended|GoogleOther|Bytespider|Meta-ExternalAgent|FacebookBot|Amazonbot|cohere-training-data-crawler|Diffbot|AI2Bot|ImagesiftBot|Omgilibot|PanguBot|Timpibot)" 1;
}

location / {
    mirror /_pl_crawl;
    mirror_request_body off;
    # ...your existing configuration
}

location = /_pl_crawl {
    internal;
    if ($pl_ai_bot = 0) { return 204; }
    proxy_method POST;
    proxy_pass_request_body off;
    proxy_set_header Content-Length "";
    proxy_set_header Authorization "Bearer plbot_YOUR_TOKEN";
    proxy_set_header X-PL-Path $request_uri;
    proxy_set_header User-Agent $http_user_agent;
    proxy_connect_timeout 1s;
    proxy_read_timeout 2s;
    proxy_ssl_server_name on;
    proxy_pass https://api.productlense.app/api/crawl;
}

const AI_BOTS = /ChatGPT-User|Claude-User|Perplexity-User|MistralAI-User|DuckAssistBot|Meta-ExternalFetcher|Gemini-Deep-Research|cohere-ai|OAI-SearchBot|Claude-SearchBot|PerplexityBot|Googlebot|Bingbot|Applebot|DuckDuckBot|YouBot|Bravebot|GPTBot|ClaudeBot|anthropic-ai|Claude-Web|CCBot|Google-Extended|GoogleOther|Bytespider|Meta-ExternalAgent|FacebookBot|Amazonbot|cohere-training-data-crawler|Diffbot|AI2Bot|ImagesiftBot|Omgilibot|PanguBot|Timpibot/i;

export default {
  async fetch(request, env, ctx) {
    const response = await fetch(request); // your site, unchanged
    const ua = request.headers.get('user-agent') || '';
    if (env.PL_BOT_TOKEN && AI_BOTS.test(ua.slice(0, 1024))) {
      ctx.waitUntil(
        fetch('https://api.productlense.app/api/crawl', {
          method: 'POST',
          headers: { Authorization: `Bearer ${env.PL_BOT_TOKEN}`, 'Content-Type': 'application/json' },
          body: JSON.stringify({ path: new URL(request.url).pathname, ua, status: response.status }),
        }).catch(() => {}),
      );
    }
    return response;
  },
};