idx-cli/crates/idx-core/src/curl_impersonate.rs

use std::io::Read;
use std::process::{Command, Output, Stdio};

use crate::error::IdxError;

const CANDIDATES: &[&str] = &[
    "curl_chrome142",
    "curl_chrome136",
    "curl_chrome133a",
    "curl_chrome131",
    "curl_chrome124",
    "curl_chrome120",
    "curl_chrome116",
];
const OVERRIDE_ENV: &str = "IDX_CURL_IMPERSONATE_BIN";

pub fn chrome_curl_binary() -> Result<String, IdxError> {
    if let Ok(value) = std::env::var(OVERRIDE_ENV) {
        let trimmed = value.trim();
        if !trimmed.is_empty() {
            return Ok(trimmed.to_string());
        }
    }

    CANDIDATES
        .iter()
        .copied()
        .find(|bin| Command::new(bin).arg("--version").output().is_ok())
        .map(str::to_string)
        .ok_or_else(|| {
            IdxError::Http(format!(
                "no curl_chrome* binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
            ))
        })
}

pub fn run(stage: &str, args: &[&str]) -> Result<Output, IdxError> {
    let owned_args: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
    run_owned(stage, &owned_args)
}

pub fn run_owned(stage: &str, args: &[String]) -> Result<Output, IdxError> {
    let binary = chrome_curl_binary()?;
    run_binary(stage, &binary, args)
}

/// Browser profiles tried, in order, when a site's bot protection rejects
/// some TLS fingerprints but not others (IDX's Cloudflare rules block
/// `curl_chrome142` on some pages while Firefox/Safari profiles pass).
const FALLBACK_PROFILES: &[&str] = &[
    "curl_chrome142",
    "curl_firefox144",
    "curl_chrome136",
    "curl_firefox135",
    "curl_safari184",
    "curl_chrome131",
    "curl_chrome124",
    "curl_chrome120",
    "curl_chrome116",
];
const FALLBACK_ROUNDS: usize = 2;

/// Run curl with each available impersonation profile until `accept` approves
/// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only
/// profile used. Returns the first accepted output, or the last error.
pub fn run_owned_with_fallback(
    stage: &str,
    args: &[String],
    accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
) -> Result<Output, IdxError> {
    let profiles: Vec<String> = match std::env::var(OVERRIDE_ENV) {
        Ok(value) if !value.trim().is_empty() => vec![value.trim().to_string()],
        _ => FALLBACK_PROFILES
            .iter()
            .copied()
            .filter(|bin| Command::new(bin).arg("--version").output().is_ok())
            .map(str::to_string)
            .collect(),
    };
    if profiles.is_empty() {
        return Err(IdxError::Http(format!(
            "no curl-impersonate binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate"
        )));
    }

    let mut last_error = None;
    for _ in 0..FALLBACK_ROUNDS {
        for binary in &profiles {
            match run_binary(stage, binary, args) {
                Ok(output) => match accept(&output.stdout) {
                    Ok(()) => return Ok(output),
                    Err(err) => last_error = Some(err),
                },
                Err(err) => last_error = Some(err),
            }
        }
    }
    Err(last_error.expect("at least one profile was tried"))
}

/// Every curl run gives up after this long, so a stalled server cannot hang
/// a command or an MCP call.
const MAX_TIME_SECS: &str = "600";
/// Largest response kept in memory (IDX report files are a few MB).
const MAX_RESPONSE_BYTES: usize = 256 * 1024 * 1024;

fn run_binary(stage: &str, binary: &str, args: &[String]) -> Result<Output, IdxError> {
    run_binary_limited(stage, binary, args, MAX_RESPONSE_BYTES)
}

/// `--max-filesize` cannot stop a response without a length, nor (before
/// curl 8.20) a small compressed one that decodes to more, so stdout is
/// also capped while it is read: past `limit`, curl is killed and reaped.
fn run_binary_limited(
    stage: &str,
    binary: &str,
    args: &[String],
    limit: usize,
) -> Result<Output, IdxError> {
    let max_filesize = limit.to_string();
    let mut child = Command::new(binary)
        .args(["--max-time", MAX_TIME_SECS, "--max-filesize", max_filesize.as_str()])
        .args(args)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .map_err(|e| {
        if e.kind() == std::io::ErrorKind::NotFound {
            return IdxError::Http(format!(
                "curl-impersonate binary '{binary}' not found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
            ));
        }
        IdxError::Http(format!("failed to run {binary} for {stage}: {e}"))
    })?;

    // stderr drains on its own thread so a chatty curl cannot block on it.
    let mut stderr_pipe = child.stderr.take().expect("stderr is piped");
    let stderr_reader = std::thread::spawn(move || {
        let mut stderr = Vec::new();
        let _ = stderr_pipe.read_to_end(&mut stderr);
        stderr
    });
    let mut stdout = Vec::new();
    let mut stdout_pipe = child.stdout.take().expect("stdout is piped");
    let read = (&mut stdout_pipe)
        .take(limit as u64 + 1)
        .read_to_end(&mut stdout);
    if read.is_err() || stdout.len() > limit {
        // The curl_* profiles are scripts that run curl without exec, so the
        // kill reaches only the wrapper; curl itself still holds stderr. Do
        // not wait for it: closing stdout ends it on its next write (or
        // --max-time ends a stalled one), and the stderr thread exits with it.
        drop(stdout_pipe);
        let _ = child.kill();
        let _ = child.wait();
        drop(stderr_reader);
        return Err(match read {
            Err(e) => IdxError::Http(format!(
                "failed to read {stage} response from {binary}: {e}"
            )),
            Ok(_) => IdxError::Http(format!("{stage} response exceeds {limit} bytes")),
        });
    }
    let status = child
        .wait()
        .map_err(|e| IdxError::Http(format!("failed to wait for {binary} for {stage}: {e}")))?;
    let stderr = stderr_reader.join().unwrap_or_default();
    let output = Output {
        status,
        stdout,
        stderr,
    };

    if output.status.success() {
        return Ok(output);
    }

    let stderr = String::from_utf8_lossy(&output.stderr);
    let detail = stderr.trim();
    Err(IdxError::Http(format!(
        "{stage} {binary} failed (status {}): {}",
        output.status,
        if detail.is_empty() {
            "no output"
        } else {
            detail
        }
    )))
}

#[cfg(all(test, unix))]
mod tests {
    use super::*;

    #[test]
    fn output_past_the_limit_fails_at_once_behind_a_profile_wrapper() {
        // Like the curl_* profile scripts, the wrapper runs "curl" without
        // exec; this one writes too much, then stalls holding stderr open.
        let dir = std::env::temp_dir().join(format!("idx-curl-{}", fastrand::u64(..)));
        std::fs::create_dir_all(&dir).unwrap();
        let fake = dir.join("curl");
        std::fs::write(
            &fake,
            "#!/bin/sh\n( head -c 8192 /dev/zero; sleep 30 )\ntrue\n",
        )
        .unwrap();
        let mut perms = std::fs::metadata(&fake).unwrap().permissions();
        std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
        std::fs::set_permissions(&fake, perms).unwrap();

        let started = std::time::Instant::now();
        let err = run_binary_limited("test", fake.to_str().unwrap(), &[], 4096).unwrap_err();

        assert!(err.to_string().contains("exceeds 4096 bytes"), "{err}");
        assert!(started.elapsed() < std::time::Duration::from_secs(10));
        std::fs::remove_dir_all(&dir).unwrap();
    }
}