From ca4d3d09c2b4fb73547348fb3f5fdd168af859c4 Mon Sep 17 00:00:00 2001 From: 0xrsydn Date: Sat, 31 Oct 2026 23:58:21 +0700 Subject: [PATCH] feat(stockbit): one browser sign-in in idx-core for the CLI and MCP `stockbit::signin` is now the single sign-in flow. `idx stockbit login` waits on it in the foreground; the `stockbit_login` MCP tool runs it on a thread and `stockbit_status` reports `sign_in.state`, so an agent knows when to stop polling. Auto uses a browser unless the shell is over SSH or has no display, and falls back to the local page only when the browser cannot start. The browser way opens Stockbit's sign-in page in a profile only idx uses and reads the session cookie over DevTools. What a real sign-in showed: - The cookie is host-only on stockbit.com, so deleting by the assumed .stockbit.com removed nothing and left live tokens in the profile. It is now deleted by its own domain and path, with every other tab closed first, and read back to confirm. - A signed-out page writes the cookie without tokens; it no longer counts. - A profile belongs to one browser, so it records the executable that made it. - On macOS the config directory depends on XDG_CONFIG_HOME, so a shell and an app started from the Dock disagreed; both default locations are now checked. - A thread does not outlive main, so idx-mcp closes a waiting browser when stdin closes or on SIGTERM or Ctrl-C. Progress goes to stderr, which keeps `-o json` to one document on stdout. Google refused its own sign-in in this window on 2026-10-31, most likely over the debugging flag; password sign-in works, and the docs and roadmap say so. Claude-Session: https://claude.ai/code/session_01RqPPVaoFxLToGAXXBEoCtH --- .gitignore | 3 + Cargo.lock | 25 + ROADMAP.md | 16 +- crates/idx-cli/src/cli/stockbit.rs | 191 +-- crates/idx-cli/tests/cli.rs | 18 + crates/idx-core/src/api/stockbit/auth.rs | 103 +- crates/idx-core/src/api/stockbit/browser.rs | 1381 ++++++++++++++++++ crates/idx-core/src/api/stockbit/loopback.rs | 333 ++++- crates/idx-core/src/api/stockbit/mod.rs | 26 + crates/idx-core/src/api/stockbit/signin.rs | 425 ++++++ crates/idx-mcp/Cargo.toml | 2 +- crates/idx-mcp/src/main.rs | 42 +- crates/idx-mcp/src/params.rs | 33 + crates/idx-mcp/src/server.rs | 76 +- crates/idx-mcp/tests/golden/tools.json | 23 +- crates/idx-mcp/tests/mcp.rs | 101 +- docs/ARCHITECTURE.md | 67 +- flake.nix | 6 +- web/src/content/docs/cli/reference.md | 15 +- web/src/content/docs/cli/stockbit.md | 117 +- web/src/content/docs/mcp/tools.md | 12 +- 21 files changed, 2801 insertions(+), 214 deletions(-) create mode 100644 crates/idx-core/src/api/stockbit/browser.rs create mode 100644 crates/idx-core/src/api/stockbit/signin.rs diff --git a/.gitignore b/.gitignore index 2cc5a6b..b877eb3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ out.txt # Cloudflare CLI account cache .cloudflare/ + +# macOS Finder metadata +.DS_Store diff --git a/Cargo.lock b/Cargo.lock index 41c6c3b..653be63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1135,6 +1135,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1765,6 +1776,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -1967,8 +1988,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", + "libc", + "mio", "pin-project-lite", + "signal-hook-registry", "tokio-macros", + "windows-sys 0.61.2", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index 52140a7..dc20fd7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,10 +14,18 @@ history lives in git. - Multi-period financial statements (today `financials` is single-period) - Order book depth without a login: no free source exists (IDX sells delayed depth to vendors); `stocks orderbook` needs the user's Stockbit login. -- Stockbit login follow-ups: a browser capture for `idx stockbit login` - (Chrome DevTools, as fzn0x/idx does) instead of pasting the cookie, - intraday OHLCV from the session's `1d` chart, and observing the session - API's open-market state name (assumed `STATE_NAME_MARKET_OPEN`). +- Stockbit login follow-ups: intraday OHLCV from the session's `1d` chart; + observing the session API's open-market state name (assumed + `STATE_NAME_MARKET_OPEN`; `STATE_NAME_MARKET_CLOSED` seen 2026-10-29); an + opt-in daily refresh so a session outlives the refresh token's week, once + it is known whether Stockbit caps a session's total life; and the sign-in + response (`/login/v6/*`, read over DevTools' `Fetch`) as a second signal + beside the `credentialStorage` cookie, should Stockbit rename or move it. +- Google sign-in in idx's browser window: Google refused it on 2026-10-31, + most likely over `--remote-debugging-port`. First confirm by trying Google + in a plain window on a throwaway profile; if that works, sign in with no + debugging flag and reopen the profile with it, on a blank page, to read the + cookie. - Observe Stockbit's `market_hour.status` in the pre-open (08:45-09:00) and the closing auction (15:50-16:00) before `--watch` or alerts build on it; `open`, `break` and `close` are known, and the bid/offer gate hides the book diff --git a/crates/idx-cli/src/cli/stockbit.rs b/crates/idx-cli/src/cli/stockbit.rs index 0fa2cde..991d1b6 100644 --- a/crates/idx-cli/src/cli/stockbit.rs +++ b/crates/idx-cli/src/cli/stockbit.rs @@ -4,7 +4,7 @@ use clap::{Args, Subcommand}; use crate::output::text::terminal_safe; use crate::output::{OutputFormat, json}; -use idx_core::api::stockbit::{self, auth::SessionStatus, auth::TokenInfo}; +use idx_core::api::stockbit::{self, auth::SessionStatus, auth::TokenInfo, signin}; use idx_core::error::IdxError; use idx_core::runtime; @@ -14,10 +14,9 @@ use idx_core::runtime; long_about = "Your own Stockbit session, for `stocks orderbook`, live `--provider stockbit` \ quotes and Stockbit daily history. Opt-in and personal: idx uses your account's tokens, \ stored in the config directory with owner-only permissions, and never logs them. Stockbit \ -rotates the refresh token on every refresh, so a browser sharing the session is logged out \ -once idx refreshes: log in through a separate browser profile for idx. Stockbit's terms \ -allow personal, non-commercial use.", - after_help = "Examples:\n idx stockbit login # paste the credentialStorage cookie\n pbpaste | idx stockbit login\n idx stockbit status\n idx stockbit logout" +rotates the refresh token on every refresh, so one session has one owner: `login` signs in \ +through a browser profile only idx uses. Stockbit's terms allow personal, non-commercial use.", + after_help = "Examples:\n idx stockbit login # sign in through a browser\n idx stockbit login --no-browser # local page: bookmarklet or paste\n pbpaste | idx stockbit login # the credentialStorage cookie on stdin\n idx stockbit status\n idx stockbit logout" )] pub struct StockbitCmd { #[command(subcommand)] @@ -27,20 +26,23 @@ pub struct StockbitCmd { #[derive(Debug, Subcommand)] pub enum StockbitSubcommand { #[command( - about = "Connect a session in your browser, or from stdin", - long_about = "In a terminal, opens a local page on 127.0.0.1 with two one-click ways to \ -hand idx your Stockbit session: a bookmarklet that reads stockbit.com's cookie in your own \ -browser, or a read-the-clipboard button. Without a browser (or with --no-browser), the page \ -URL is printed instead. With stdin piped, it reads the value of stockbit.com's \ -`credentialStorage` cookie (DevTools, Application, Cookies) or a refresh token from stdin, \ -as before. Either way idx checks the session with one request. Servers can set \ -IDX_STOCKBIT_REFRESH_TOKEN instead." + about = "Sign in through your browser, or from stdin", + long_about = "In a terminal, idx opens Chrome (or Chromium, Edge, Brave, Helium) on \ +Stockbit's sign-in page with a profile only idx uses. Sign in there: idx reads the session \ +cookie over the DevTools protocol, checks it with one request, takes it out of that profile, \ +and closes the window. Your password, one-time codes and any reCAPTCHA stay in the browser. \ +A stored session that still works means nothing to do: idx exits at once. Over SSH, without \ +a display, without such a browser, or with --no-browser, idx serves a local page on \ +127.0.0.1 instead, with a bookmarklet that reads the cookie in your own browser and a \ +clipboard button. With stdin piped, it reads the value of stockbit.com's \ +`credentialStorage` cookie (DevTools, Application, Cookies) or a refresh token from stdin. \ +Servers can set IDX_STOCKBIT_REFRESH_TOKEN instead; IDX_STOCKBIT_BROWSER picks the browser." )] Login { - /// Print the page URL instead of opening a browser + /// Use the local page instead of a browser sign-in #[arg(long)] no_browser: bool, - /// Fixed port for the local page (for `ssh -L`); default: any free one + /// Port for the local page (for `ssh -L`); default 7878, or any free one #[arg(long)] port: Option, }, @@ -50,27 +52,29 @@ IDX_STOCKBIT_REFRESH_TOKEN instead." Logout, } -/// How long the local page waits for the session. -const LOGIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); - pub fn handle(cmd: &StockbitCmd, format: &OutputFormat) -> Result<(), IdxError> { match &cmd.command { StockbitSubcommand::Login { no_browser, port } => { - let status = if std::io::stdin().is_terminal() { - match login_in_browser(*port, !no_browser) { - Ok(()) => stockbit::session_status()?, - Err(err) => { - runtime::warn(format!("browser login unavailable ({err}); paste instead")); - stockbit::login(&read_paste()?)? - } - } + let interactive = std::io::stdin().is_terminal(); + // A session that still works means there is nothing to sign in to. + let reuse = interactive && stockbit::verify_session().is_ok(); + let status = if reuse { + stockbit::session_status()? + } else if interactive { + sign_in(*no_browser, *port, format)? } else { stockbit::login(&read_paste()?)? }; if format.is_json() { return json::print_json(&status); } - println!("Stored and checked your Stockbit session."); + if reuse { + println!( + "Already signed in to Stockbit; use `idx stockbit logout` to switch accounts." + ); + } else { + println!("Stored and checked your Stockbit session."); + } print_status(&status); } StockbitSubcommand::Status => { @@ -104,73 +108,74 @@ pub fn handle(cmd: &StockbitCmd, format: &OutputFormat) -> Result<(), IdxError> Ok(()) } -/// Serves the local login page and waits for a session it can verify. -/// -/// The page is only reachable from this machine. Over SSH the browser is -/// usually somewhere else, so idx picks a stable port, skips opening a -/// browser here, and prints the forward to run on the machine with the -/// browser. -fn login_in_browser(port: Option, open: bool) -> Result<(), IdxError> { - let ssh = std::env::var("SSH_CONNECTION").ok(); - // A stable port makes the printed `ssh -L` match what the page serves. - let page = stockbit::loopback::LoginPage::bind(port.unwrap_or(if ssh.is_some() { - DEFAULT_SSH_PORT - } else { - 0 - }))?; - println!("Open this page to connect Stockbit: {}", page.url()); - match &ssh { - Some(connection) => print_ssh_hint(connection, page.port()), - None if open => open_in_browser(&page.url()), - None => {} - } - page.wait(LOGIN_TIMEOUT, |pasted| { - stockbit::login(pasted) - .map(|_| ()) - .map_err(|err| err.to_string()) - }) -} - -/// Port the page uses over SSH, so `ssh -L 7878:…:7878` matches it. -const DEFAULT_SSH_PORT: u16 = 7878; - -/// The forward to run where the browser is, from `SSH_CONNECTION` -/// (`client_ip client_port server_ip server_port`). -fn print_ssh_hint(connection: &str, port: u16) { - let fields: Vec<&str> = connection.split_whitespace().collect(); - let target = match (std::env::var("USER").ok(), fields.get(2)) { - (Some(user), Some(server)) => format!("{user}@{server}"), - (None, Some(server)) => (*server).to_string(), - _ => "".to_string(), +/// Runs idx-core's sign-in in the foreground, telling the user on stderr +/// where to go. When neither the browser nor the page can start, the cookie +/// can still be pasted. +fn sign_in( + no_browser: bool, + port: Option, + format: &OutputFormat, +) -> Result { + let options = signin::Options { + method: if no_browser { + signin::Method::Page + } else { + signin::Method::Auto + }, + port, + open_page: !no_browser, }; - runtime::warn( - "this shell is over SSH: the page listens on the remote machine, so open it from the \ - machine with your browser", - ); - println!( - "\nFrom the machine you are sitting at, run (and keep it open):\n \ - ssh -L {port}:127.0.0.1:{port} {target}\n\ - then open the URL above there.\n" - ); + let sign_in = match signin::SignIn::start(&options) { + Ok(sign_in) => sign_in, + Err(err) => { + runtime::warn(format!("cannot start a sign-in ({err}); paste instead")); + return stockbit::login(&read_paste()?); + } + }; + describe(sign_in.started(), format); + sign_in.wait() } -/// Opens `url` with the desktop's default browser, best effort. -fn open_in_browser(url: &str) { - #[cfg(target_os = "macos")] - let (program, args) = ("open", vec![url]); - #[cfg(target_os = "windows")] - let (program, args) = ("cmd", vec!["/C", "start", "", url]); - #[cfg(all(unix, not(target_os = "macos")))] - let (program, args) = ("xdg-open", vec![url]); - let opened = std::process::Command::new(program) - .args(args) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn() - .is_ok(); - if !opened { - runtime::warn(format!("could not open a browser; open {url} yourself")); +/// Where to sign in. These lines are the instructions, so they print even +/// with `--quiet`; on stderr, so stdout keeps only the result. +fn describe(started: &signin::Started, format: &OutputFormat) { + let say = |message: String| { + if format.is_json() { + eprintln!( + "{}", + serde_json::json!({ "level": "info", "message": message }) + ); + } else { + eprintln!("{}", terminal_safe(&message)); + } + }; + match started { + signin::Started::Browser { browser } => say(format!( + "Opened {browser} on Stockbit's sign-in page. Sign in there; idx closes the window \ + when you are done." + )), + signin::Started::Page { + url, + forward, + remote, + reason, + .. + } => { + if let Some(reason) = reason + && !remote + { + runtime::warn(format!("{reason}; using the local page")); + } + say(format!("Open this page to connect Stockbit: {url}")); + if *remote { + say(format!( + "This shell is over SSH, so the page is on the remote machine. From the \ + machine with your browser, run (and keep it open):\n {forward}\nthen open \ + the URL above there." + )); + } + } + signin::Started::Existing => {} } } @@ -194,7 +199,9 @@ fn read_paste() -> Result { } if raw.trim().is_empty() { return Err(IdxError::InvalidInput( - "nothing pasted: pipe or paste the credentialStorage cookie value or a refresh token" + "nothing on stdin: run `idx stockbit login` in a terminal to sign in through a \ + browser, pipe the credentialStorage cookie (`pbpaste | ssh idx stockbit \ + login`), or set IDX_STOCKBIT_REFRESH_TOKEN" .to_string(), )); } diff --git a/crates/idx-cli/tests/cli.rs b/crates/idx-cli/tests/cli.rs index e8902ef..c086316 100644 --- a/crates/idx-cli/tests/cli.rs +++ b/crates/idx-cli/tests/cli.rs @@ -2197,6 +2197,24 @@ fn stockbit_login_stores_a_private_session_and_logout_removes_it() { assert!(!session.exists()); } +/// An agent on a headless box gets told how to log in, not "nothing pasted". +#[test] +fn stockbit_login_with_empty_stdin_says_how_to_log_in() { + let root = test_env_dir("stockbit-login-empty"); + let output = bin_with_root(&root) + .env("IDX_USE_MOCK_PROVIDER", "1") + .args(["-o", "json", "stockbit", "login"]) + .output() + .expect("run login"); + assert!(!output.status.success()); + let error: Value = serde_json::from_slice(&output.stderr).expect("JSON error"); + assert_eq!(error["code"], "INVALIDINPUT"); + let message = error["message"].as_str().unwrap(); + for hint in ["idx stockbit login", "IDX_STOCKBIT_REFRESH_TOKEN"] { + assert!(message.contains(hint), "{message}"); + } +} + #[test] fn invalid_quote_ttl_env_returns_non_zero() { test_bin("invalid-quote-ttl") diff --git a/crates/idx-core/src/api/stockbit/auth.rs b/crates/idx-core/src/api/stockbit/auth.rs index 93da93b..a20f13e 100644 --- a/crates/idx-core/src/api/stockbit/auth.rs +++ b/crates/idx-core/src/api/stockbit/auth.rs @@ -94,6 +94,57 @@ pub struct SessionStatus { pub session_file: String, } +/// Where the session and idx's browser profile live: beside `config.toml`, +/// or in the other default location when that is where they are. +/// +/// On macOS the config directory depends on the environment: a shell that +/// sets `XDG_CONFIG_HOME=~/.config` and an app started from the Dock, which +/// does not, resolve to different places, and the CLI and an MCP client must +/// still find one session. A custom `XDG_CONFIG_HOME` has no other location, +/// so a run isolated with one never reaches the user's session. +pub fn session_dir() -> Result { + let config = crate::config::config_path()?; + let primary = config + .parent() + .ok_or_else(|| IdxError::ConfigError("config path has no directory".to_string()))? + .to_path_buf(); + let other = if cfg!(target_os = "macos") { + std::env::var_os("HOME").and_then(|home| other_default(&primary, Path::new(&home))) + } else { + None + }; + Ok(choose_dir(primary, other)) +} + +/// The second default config location, when `primary` is one of the two. +fn other_default(primary: &Path, home: &Path) -> Option { + let dot_config = home.join(".config/idx"); + let library = home.join("Library/Application Support/idx"); + if primary == dot_config { + Some(library) + } else if primary == library { + Some(dot_config) + } else { + None + } +} + +/// The directory holding a session, else a browser profile, else `primary`. +fn choose_dir(primary: PathBuf, other: Option) -> PathBuf { + let Some(other) = other else { + return primary; + }; + for entry in [SESSION_FILE, super::browser::PROFILE_DIR] { + if primary.join(entry).exists() { + return primary; + } + if other.join(entry).exists() { + return other; + } + } + primary +} + /// Reads, writes and refreshes the session. #[derive(Debug, Clone)] pub struct SessionStore { @@ -102,19 +153,15 @@ pub struct SessionStore { } impl SessionStore { - /// The session beside `config.toml`, seeded from the environment. + /// The session in `session_dir`, seeded from the environment. pub fn open() -> Result { - let config = crate::config::config_path()?; - let dir = config - .parent() - .ok_or_else(|| IdxError::ConfigError("config path has no directory".to_string()))?; let env = |name| { std::env::var(name) .map(|v| normalize_token(&v)) .unwrap_or_default() }; Ok(Self::at( - dir.to_path_buf(), + session_dir()?, Session { access: env(ENV_ACCESS), refresh: env(ENV_REFRESH), @@ -393,6 +440,50 @@ mod tests { use super::*; + /// The two default locations pair up; anything else stands alone, so + /// a run with its own `XDG_CONFIG_HOME` cannot reach the user's session. + #[test] + fn only_the_two_default_locations_pair_up() { + let home = Path::new("/Users/me"); + let dot_config = home.join(".config/idx"); + let library = home.join("Library/Application Support/idx"); + assert_eq!(other_default(&dot_config, home), Some(library.clone())); + assert_eq!(other_default(&library, home), Some(dot_config)); + assert_eq!(other_default(Path::new("/tmp/run/idx"), home), None); + } + + /// A session anywhere beats a profile, a profile beats nothing, and with + /// neither the primary location is used. + #[test] + fn the_session_is_found_in_either_location() { + let root = std::env::temp_dir().join(format!( + "idx-session-dir-{}-{}", + std::process::id(), + fastrand::u32(..) + )); + let (primary, other) = (root.join("primary"), root.join("other")); + std::fs::create_dir_all(&primary).unwrap(); + std::fs::create_dir_all(&other).unwrap(); + let choose = || choose_dir(primary.clone(), Some(other.clone())); + + assert_eq!(choose(), primary); + assert_eq!(choose_dir(primary.clone(), None), primary); + + std::fs::create_dir(other.join(super::super::browser::PROFILE_DIR)).unwrap(); + assert_eq!(choose(), other, "a profile says where the user signs in"); + + std::fs::write(primary.join(SESSION_FILE), "{}").unwrap(); + assert_eq!(choose(), primary, "a session beats a profile"); + + std::fs::remove_file(primary.join(SESSION_FILE)).unwrap(); + std::fs::write(other.join(SESSION_FILE), "{}").unwrap(); + assert_eq!(choose(), other); + + std::fs::write(primary.join(SESSION_FILE), "{}").unwrap(); + assert_eq!(choose(), primary, "with two, the primary one wins"); + let _ = std::fs::remove_dir_all(&root); + } + fn fixture(name: &str) -> String { std::fs::read_to_string(format!( "{}/tests/fixtures/{name}", diff --git a/crates/idx-core/src/api/stockbit/browser.rs b/crates/idx-core/src/api/stockbit/browser.rs new file mode 100644 index 0000000..51f7a9c --- /dev/null +++ b/crates/idx-core/src/api/stockbit/browser.rs @@ -0,0 +1,1381 @@ +//! Signing in to Stockbit inside a real browser, with no DevTools and no +//! copying. +//! +//! Stockbit publishes no OAuth client a third party can register, and its own +//! sign-in is a first-party flow guarded by reCAPTCHA, so a CLI cannot ask for +//! a token directly. What it can do is hand the sign-in to the browser that +//! already knows how to do it, then read the cookie that browser ends up with: +//! +//! 1. Launch a Chromium-family browser on Stockbit's sign-in page with a +//! profile directory only idx uses and `--remote-debugging-port=0`. Chrome +//! writes the port it picked and the WebSocket path to +//! `/DevToolsActivePort`. +//! 2. Connect to that endpoint over WebSocket and ask for the cookies. +//! 3. Keep asking until the `credentialStorage` cookie holds tokens `verify` +//! accepts, take it out of the profile, then close the browser. +//! +//! What Stockbit's web app does with that cookie (read from its own bundle, +//! 2026-10-29): it is the app's persisted credential store, written by +//! script, host-only on `stockbit.com`, for 30 days, as percent-encoded +//! `{"state":{"access","refresh","user"},"version":0}`. Every way in (password, +//! Google or Apple, a new device's one-time code, account recovery) and the +//! app's own `POST /login/refresh` end in that one store, so the cookie is +//! the single place a finished sign-in shows up. A signed-out page writes the +//! store too, without tokens, which is why a cookie counts only once +//! `auth::session_from_paste` finds tokens in it. +//! +//! The password, the one-time code and any reCAPTCHA stay in the browser and +//! on Stockbit's page: idx never sees them. The cookie travels in memory only, +//! and only as far as `verify`, which stores the two tokens. +//! +//! A dedicated profile is required by the browser (Chrome 136 and later refuse +//! remote debugging on the default profile) and wanted anyway: idx's session +//! never mixes with the user's everyday browsing. + +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, TcpStream}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use base64::Engine; +use serde_json::{Value, json}; + +use crate::error::IdxError; + +/// The cookie Stockbit keeps its session in. +const COOKIE: &str = "credentialStorage"; +/// The page the browser opens, which is Stockbit's own sign-in. +const SIGN_IN_URL: &str = "https://stockbit.com/login"; +/// Profile directory idx keeps its own browser session in, beside +/// `config.toml`. +pub const PROFILE_DIR: &str = "stockbit-browser"; +/// How long the browser may take to publish its DevTools endpoint. +const START_TIMEOUT: Duration = Duration::from_secs(20); +/// Pause between cookie reads. +const POLL: Duration = Duration::from_millis(500); +/// Socket timeout for one DevTools message. +const IO_TIMEOUT: Duration = Duration::from_secs(10); +/// How long the browser gets to quit after `Browser.close`. +const EXIT_GRACE: Duration = Duration::from_secs(3); +/// Largest DevTools message accepted. +const MAX_MESSAGE: u64 = 8 * 1024 * 1024; + +/// Names the browser idx starts, ahead of any it finds. +pub const ENV_BROWSER: &str = "IDX_STOCKBIT_BROWSER"; +/// File in the profile naming the executable that last opened it. A profile +/// belongs to one browser: its cookies are encrypted with that browser's own +/// key, and an older Chromium may refuse a profile a newer one wrote. +const MADE_BY: &str = "idx-browser"; + +/// A Chromium-family browser idx can drive over the DevTools protocol. +pub struct Browser { + /// What to call the browser in messages (`Google Chrome`). + pub name: String, + /// Its executable. + pub path: PathBuf, +} + +impl Browser { + /// The browser to open `profile` with: the one `IDX_STOCKBIT_BROWSER` + /// names, else the one that opened that profile before, else the first + /// supported browser installed here. + /// + /// Under the mock provider only the variable counts, so a test never + /// opens a browser it did not ask for. + pub fn find(profile: &Path) -> Result { + if let Some(path) = std::env::var_os(ENV_BROWSER) { + let path = PathBuf::from(path); + if !path.is_file() { + return Err(IdxError::ConfigError(format!( + "{ENV_BROWSER} is {}, which is not a file", + path.display() + ))); + } + return Ok(Self::at(path)); + } + #[cfg(feature = "testing")] + if crate::api::mock::enabled() { + return Err(IdxError::Unsupported(format!( + "no browser under the mock provider; set {ENV_BROWSER}" + ))); + } + if let Some(browser) = Self::recorded(profile) { + return Ok(browser); + } + candidates() + .into_iter() + .find(|(_, path)| path.is_file()) + .map(|(name, path)| Self { + name: name.to_string(), + path, + }) + .ok_or_else(|| { + IdxError::Unsupported(format!( + "no Chrome, Chromium, Edge, Brave or Helium found; set {ENV_BROWSER} to a \ + Chromium-family browser" + )) + }) + } + + /// A browser known only by its executable. + fn at(path: PathBuf) -> Self { + let name = path + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()) + .unwrap_or_else(|| "the browser".to_string()); + Self { name, path } + } + + /// The browser that last opened `profile`, if it is still installed. + fn recorded(profile: &Path) -> Option { + let raw = std::fs::read_to_string(profile.join(MADE_BY)).ok()?; + let path = PathBuf::from(raw.trim()); + path.is_file().then(|| Self::at(path)) + } + + /// Opens the browser on Stockbit's sign-in page and connects to it. + /// + /// Returns once the browser answers over DevTools, so a browser that will + /// not start is reported here, before anyone waits on it. + pub fn open(&self, profile: &Path) -> Result { + let launched = self.launch(profile)?; + let devtools = connect(profile, &|| launched.is_running())?; + Ok(Window { launched, devtools }) + } + + /// Starts the browser on its sign-in page. + fn launch(&self, profile: &Path) -> Result { + std::fs::create_dir_all(profile) + .map_err(|e| IdxError::Io(format!("cannot create idx's browser profile: {e}")))?; + owner_only(profile); + // A port file left by an earlier run would be read as this run's. + let _ = std::fs::remove_file(profile.join("DevToolsActivePort")); + let child = Command::new(&self.path) + .args(self.args(profile)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| IdxError::Io(format!("cannot open {}: {e}", self.name)))?; + // Best effort: without it, the next run searches again. + if let Some(path) = self.path.to_str() { + let _ = std::fs::write(profile.join(MADE_BY), path); + } + Ok(Launched { + child: std::cell::RefCell::new(child), + }) + } + + /// Arguments that keep idx's sign-in in its own profile and publish a + /// DevTools port instead of a fixed one. + fn args(&self, profile: &Path) -> Vec { + vec![ + format!("--user-data-dir={}", profile.display()), + "--remote-debugging-port=0".to_string(), + "--no-first-run".to_string(), + "--no-default-browser-check".to_string(), + "--disable-search-engine-choice-screen".to_string(), + SIGN_IN_URL.to_string(), + ] + } +} + +/// A browser window idx opened on Stockbit's sign-in page. +pub struct Window { + launched: Launched, + devtools: DevTools, +} + +impl Window { + /// Waits for a session `verify` accepts, then closes the browser. + /// + /// A session already in idx's profile that still verifies is taken at + /// once. One `verify` refuses is reported once and then left alone: + /// signing in again in the open window replaces it. Setting `cancel` + /// ends the wait and closes the browser too. + pub fn wait( + mut self, + timeout: Duration, + cancel: &AtomicBool, + verify: &dyn Fn(&str) -> Result<(), IdxError>, + ) -> Result<(), IdxError> { + let launched = &self.launched; + let result = wait_for_session(&mut self.devtools, timeout, verify, &|| { + launched.is_running() && !cancel.load(Ordering::Relaxed) + }); + self.launched.exit(EXIT_GRACE); + result + } +} + +/// Connects to the browser that is starting on `profile`. +fn connect(profile: &Path, alive: &dyn Fn() -> bool) -> Result { + let (port, path) = read_endpoint(profile, START_TIMEOUT, alive)?; + DevTools::connect(port, &path) +} + +/// Waits on a running browser for a session `verify` accepts. +/// +/// Apart from `Window` so tests can drive the wait against a stand-in +/// DevTools endpoint without launching a browser. +fn wait_for_session( + devtools: &mut DevTools, + timeout: Duration, + verify: &dyn Fn(&str) -> Result<(), IdxError>, + alive: &dyn Fn() -> bool, +) -> Result<(), IdxError> { + let deadline = Instant::now() + timeout; + let mut rejected: Option = None; + let mut last_error: Option = None; + // The window is watched so closing it ends the wait instead of running + // out the clock; the first polls may not see a page yet. + let mut saw_window = false; + let mut misses = 0; + loop { + match devtools.session_cookie() { + Ok(Some(cookie)) if rejected.as_deref() != Some(cookie.value.as_str()) => { + match verify(&cookie.value) { + Ok(()) => { + // The session now lives in idx's own file. Take the + // copy out of the browser, but leave Stockbit's + // device cookies in place. + if let Err(err) = devtools.forget_session(&cookie) { + crate::runtime::warn(format!( + "could not remove the session cookie from idx's browser \ + profile ({err}); deleting that profile folder removes it" + )); + } + devtools.close(); + return Ok(()); + } + Err(err) => { + crate::runtime::warn(format!( + "Stockbit refused the session in idx's browser ({err}); sign in \ + again in that window" + )); + rejected = Some(cookie.value); + } + } + } + // A later read that works clears an earlier failure, so the + // timeout message names only what is still wrong. + Ok(_) => last_error = None, + Err(err) => last_error = Some(err.to_string()), + } + if !alive() { + devtools.close(); + return Err(IdxError::Io( + "the browser exited before idx got a session; run `idx stockbit login` again" + .to_string(), + )); + } + match devtools.has_page() { + Ok(true) => { + saw_window = true; + misses = 0; + } + Ok(false) => { + misses += 1; + if saw_window && misses >= 2 { + devtools.close(); + return Err(IdxError::Io( + "the Stockbit sign-in window was closed before the session arrived; \ + run `idx stockbit login` again" + .to_string(), + )); + } + } + Err(err) => last_error = Some(err.to_string()), + } + if Instant::now() >= deadline { + devtools.close(); + let mut message = format!( + "the Stockbit sign-in timed out after {}s and idx closed its browser window; \ + re-run `idx stockbit login`, or add --no-browser for the local page", + timeout.as_secs() + ); + if let Some(err) = last_error { + message.push_str(&format!(" (last DevTools error: {err})")); + } + return Err(IdxError::Io(message)); + } + std::thread::sleep(POLL); + } +} + +/// Where idx keeps its own browser profile. +pub fn profile_dir() -> Result { + Ok(super::auth::session_dir()?.join(PROFILE_DIR)) +} + +/// Browsers to try, in order, for this platform. +fn candidates() -> Vec<(&'static str, PathBuf)> { + let mut found = Vec::new(); + #[cfg(target_os = "macos")] + { + let homes: Vec = ["/Applications", "/System/Applications"] + .iter() + .map(PathBuf::from) + .chain(std::env::var_os("HOME").map(|home| PathBuf::from(home).join("Applications"))) + .collect(); + let apps = [ + ( + "Google Chrome", + "Google Chrome.app/Contents/MacOS/Google Chrome", + ), + ("Chromium", "Chromium.app/Contents/MacOS/Chromium"), + ( + "Microsoft Edge", + "Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + ), + ("Brave", "Brave Browser.app/Contents/MacOS/Brave Browser"), + ("Helium", "Helium.app/Contents/MacOS/Helium"), + ( + "Google Chrome Canary", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + ), + ]; + for home in homes { + for (name, rel) in apps { + found.push((name, home.join(rel))); + } + } + } + #[cfg(target_os = "windows")] + { + let roots: Vec = ["PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA"] + .iter() + .filter_map(std::env::var_os) + .map(PathBuf::from) + .collect(); + for root in roots { + for (name, rel) in [ + ("Google Chrome", r"Google\Chrome\Application\chrome.exe"), + ("Chromium", r"Chromium\Application\chrome.exe"), + ("Microsoft Edge", r"Microsoft\Edge\Application\msedge.exe"), + ( + "Brave", + r"BraveSoftware\Brave-Browser\Application\brave.exe", + ), + ] { + found.push((name, root.join(rel))); + } + } + } + #[cfg(all(unix, not(target_os = "macos")))] + { + let names = [ + ("Google Chrome", "google-chrome"), + ("Google Chrome", "google-chrome-stable"), + ("Chromium", "chromium"), + ("Chromium", "chromium-browser"), + ("Microsoft Edge", "microsoft-edge"), + ("Brave", "brave-browser"), + ]; + let dirs: Vec = std::env::var_os("PATH") + .map(|path| std::env::split_paths(&path).collect()) + .unwrap_or_default(); + for (name, exe) in names { + for dir in &dirs { + found.push((name, dir.join(exe))); + } + } + } + found +} + +/// A browser process idx started and must not leave behind. +struct Launched { + // A `RefCell` because the wait needs liveness through an immutable + // borrow; the process is only touched from one thread. + child: std::cell::RefCell, +} + +impl Launched { + /// Whether the browser is still running. + fn is_running(&self) -> bool { + matches!(self.child.borrow_mut().try_wait(), Ok(None)) + } + + /// Gives the browser `grace` to quit after `Browser.close`, then kills it. + fn exit(&self, grace: Duration) { + let deadline = Instant::now() + grace; + while Instant::now() < deadline { + if !self.is_running() { + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + } +} + +impl Drop for Launched { + fn drop(&mut self) { + let child = self.child.get_mut(); + let _ = child.kill(); + let _ = child.wait(); + } +} + +/// Reads `/DevToolsActivePort`: the port Chrome picked, then the +/// WebSocket path to the browser endpoint. +/// +/// A browser that exits before writing it has usually handed the window to +/// one already running on the same profile, which idx cannot drive. +fn read_endpoint( + profile: &Path, + timeout: Duration, + alive: &dyn Fn() -> bool, +) -> Result<(u16, String), IdxError> { + let file = profile.join("DevToolsActivePort"); + let deadline = Instant::now() + timeout; + loop { + if let Ok(raw) = std::fs::read_to_string(&file) + && let Some((port, path)) = parse_endpoint(&raw) + { + return Ok((port, path)); + } + if !alive() { + return Err(IdxError::Io( + "the browser exited as it started; if a window for idx is already open, close \ + it and try again" + .to_string(), + )); + } + if Instant::now() >= deadline { + return Err(IdxError::Io(format!( + "the browser did not publish a DevTools port within {}s; if a window for idx is \ + already open, close it and try again", + timeout.as_secs() + ))); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// The two lines of a `DevToolsActivePort` file. +fn parse_endpoint(raw: &str) -> Option<(u16, String)> { + let mut lines = raw.lines(); + let port = lines.next()?.trim().parse::().ok()?; + let path = lines.next()?.trim(); + (port != 0 && path.starts_with('/')).then(|| (port, path.to_string())) +} + +/// Stockbit's session cookie as the browser holds it: the value, and the +/// domain and path that name it for deletion. +struct SessionCookie { + value: String, + domain: String, + path: String, +} + +/// The session cookie among DevTools' cookies, if the browser has one with +/// tokens in it; the store a signed-out page writes does not count. +fn session_cookie(cookies: &Value) -> Option { + cookies + .as_array()? + .iter() + .filter(|cookie| cookie["name"] == COOKIE) + .filter_map(|cookie| { + let domain = cookie["domain"].as_str()?; + is_stockbit(domain).then(|| SessionCookie { + value: cookie["value"].as_str().unwrap_or_default().to_string(), + domain: domain.to_string(), + path: cookie["path"].as_str().unwrap_or("/").to_string(), + }) + }) + .find(|cookie| super::auth::session_from_paste(&cookie.value).is_some()) +} + +/// `stockbit.com` or a subdomain, as a cookie's domain names it (host-only +/// cookies have no leading dot). +fn is_stockbit(domain: &str) -> bool { + let host = domain.strip_prefix('.').unwrap_or(domain); + host == "stockbit.com" || host.ends_with(".stockbit.com") +} + +/// A DevTools protocol connection: one WebSocket, one request at a time. +struct DevTools { + stream: TcpStream, + next_id: u64, +} + +impl DevTools { + fn connect(port: u16, path: &str) -> Result { + let io = |e: std::io::Error| IdxError::Io(format!("browser DevTools: {e}")); + let mut stream = TcpStream::connect((Ipv4Addr::LOCALHOST, port)).map_err(io)?; + stream.set_read_timeout(Some(IO_TIMEOUT)).map_err(io)?; + stream.set_write_timeout(Some(IO_TIMEOUT)).map_err(io)?; + let key = base64::engine::general_purpose::STANDARD.encode(random_bytes()); + let request = format!( + "GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nUpgrade: websocket\r\n\ + Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n" + ); + stream.write_all(request.as_bytes()).map_err(io)?; + let head = read_head(&mut stream)?; + if !head.starts_with("HTTP/1.1 101") { + return Err(IdxError::Io(format!( + "browser DevTools refused the connection: {}", + head.lines().next().unwrap_or_default() + ))); + } + Ok(Self { stream, next_id: 0 }) + } + + /// The `credentialStorage` cookie the browser holds, if it has tokens. + fn session_cookie(&mut self) -> Result, IdxError> { + let result = self.call("Storage.getCookies", json!({}))?; + Ok(session_cookie(&result["cookies"])) + } + + /// Asks the browser to quit; it usually closes the socket instead of + /// answering, which is not an error here. + fn close(&mut self) { + let _ = self + .stream + .set_read_timeout(Some(Duration::from_millis(500))); + let _ = self.call("Browser.close", json!({})); + } + + /// Whether the browser still has a window open. + fn has_page(&mut self) -> Result { + Ok(!self.pages()?.is_empty()) + } + + /// Removes the session cookie from the profile, keeping the rest, and + /// checks it is gone. + /// + /// Stockbit's script holds the session in memory, writes it back to the + /// cookie, and syncs it across its tabs, so no Stockbit page may be open + /// when the cookie goes: idx opens a blank tab of its own, closes every + /// other one, and deletes from there. The cookie is named by the domain + /// and path the browser reported, since Stockbit sets it host-only. + fn forget_session(&mut self, cookie: &SessionCookie) -> Result<(), IdxError> { + let created = self.call("Target.createTarget", json!({ "url": "about:blank" }))?; + let blank = created["targetId"] + .as_str() + .ok_or_else(|| IdxError::Io("browser DevTools opened no tab".to_string()))? + .to_string(); + for target in self.pages()? { + if target != blank { + self.call("Target.closeTarget", json!({ "targetId": target }))?; + } + } + let attached = self.call( + "Target.attachToTarget", + json!({ "targetId": blank, "flatten": true }), + )?; + let session = attached["sessionId"] + .as_str() + .ok_or_else(|| IdxError::Io("browser DevTools attached no session".to_string()))? + .to_string(); + self.call_in( + Some(&session), + "Network.deleteCookies", + json!({ "name": COOKIE, "domain": cookie.domain, "path": cookie.path }), + )?; + if self.session_cookie()?.is_some() { + return Err(IdxError::Io( + "the session cookie was still there after deleting it".to_string(), + )); + } + Ok(()) + } + + /// The ids of the browser's open tabs. + fn pages(&mut self) -> Result, IdxError> { + let targets = self.call("Target.getTargets", json!({}))?; + Ok(targets["targetInfos"] + .as_array() + .map(|infos| { + infos + .iter() + .filter(|target| target["type"] == "page") + .filter_map(|target| target["targetId"].as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default()) + } + + /// One request and its reply, skipping events. + fn call(&mut self, method: &str, params: Value) -> Result { + self.call_in(None, method, params) + } + + /// A request in a session (a page attached with `flatten`), or in the + /// browser itself when `session` is `None`. + fn call_in( + &mut self, + session: Option<&str>, + method: &str, + params: Value, + ) -> Result { + self.next_id += 1; + let id = self.next_id; + let mut message = json!({ "id": id, "method": method, "params": params }); + if let Some(session) = session { + message["sessionId"] = json!(session); + } + let io = |e: std::io::Error| IdxError::Io(format!("browser DevTools: {e}")); + self.stream + .write_all(&frame(1, message.to_string().as_bytes())) + .map_err(io)?; + loop { + let payload = self.read_message()?; + let reply: Value = serde_json::from_slice(&payload) + .map_err(|e| IdxError::Io(format!("browser DevTools: {e}")))?; + if reply["id"].as_u64() != Some(id) { + continue; + } + if let Some(error) = reply.get("error") { + return Err(IdxError::Io(format!( + "browser DevTools {method}: {}", + error["message"].as_str().unwrap_or("failed") + ))); + } + return Ok(reply.get("result").cloned().unwrap_or(Value::Null)); + } + } + + /// One whole message, joining fragments and answering pings. + fn read_message(&mut self) -> Result, IdxError> { + let io = |e: std::io::Error| IdxError::Io(format!("browser DevTools: {e}")); + let mut buffer = Vec::new(); + loop { + let (fin, opcode, payload) = read_frame(&mut self.stream)?; + match opcode { + // ping: answer so the browser keeps the socket open. + 0x9 => self.stream.write_all(&frame(0xA, &payload)).map_err(io)?, + // pong: nothing to do. + 0xA => continue, + 0x8 => { + return Err(IdxError::Io( + "browser DevTools closed the connection".to_string(), + )); + } + // text, binary, or a continuation of the message in hand. + 0x0 => buffer.extend_from_slice(&payload), + 0x1 | 0x2 => { + buffer.clear(); + buffer.extend_from_slice(&payload); + } + _ => continue, + } + if fin { + return Ok(buffer); + } + } + } +} + +/// Reads the HTTP response head, up to the empty line. +fn read_head(stream: &mut TcpStream) -> Result { + let io = |e: std::io::Error| IdxError::Io(format!("browser DevTools: {e}")); + let mut raw = Vec::new(); + let mut byte = [0u8; 1]; + while !raw.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).map_err(io)? == 0 { + return Err(IdxError::Io( + "browser DevTools closed the connection".to_string(), + )); + } + raw.push(byte[0]); + if raw.len() > 8192 { + return Err(IdxError::Io( + "browser DevTools sent an oversized handshake".to_string(), + )); + } + } + Ok(String::from_utf8_lossy(&raw).into_owned()) +} + +/// A masked client frame: clients must mask, servers must not. +fn frame(opcode: u8, payload: &[u8]) -> Vec { + let mut out = Vec::with_capacity(payload.len() + 14); + out.push(0x80 | opcode); + let mask = 0x80; + match payload.len() { + len if len < 126 => out.push(mask | len as u8), + len if len <= u16::MAX as usize => { + out.push(mask | 126); + out.extend_from_slice(&(len as u16).to_be_bytes()); + } + len => { + out.push(mask | 127); + out.extend_from_slice(&(len as u64).to_be_bytes()); + } + } + let key = fastrand::u32(..).to_be_bytes(); + out.extend_from_slice(&key); + out.extend( + payload + .iter() + .enumerate() + .map(|(index, byte)| byte ^ key[index % 4]), + ); + out +} + +/// One frame: `(fin, opcode, payload)`. +fn read_frame(reader: &mut impl Read) -> Result<(bool, u8, Vec), IdxError> { + let io = |e: std::io::Error| IdxError::Io(format!("browser DevTools: {e}")); + let mut head = [0u8; 2]; + reader.read_exact(&mut head).map_err(io)?; + let fin = head[0] & 0x80 != 0; + let opcode = head[0] & 0x0F; + let masked = head[1] & 0x80 != 0; + let length = match head[1] & 0x7F { + 126 => { + let mut bytes = [0u8; 2]; + reader.read_exact(&mut bytes).map_err(io)?; + u16::from_be_bytes(bytes) as u64 + } + 127 => { + let mut bytes = [0u8; 8]; + reader.read_exact(&mut bytes).map_err(io)?; + u64::from_be_bytes(bytes) + } + short => u64::from(short), + }; + if length > MAX_MESSAGE { + return Err(IdxError::Io(format!( + "browser DevTools sent an oversized message ({length} bytes)" + ))); + } + let key = if masked { + let mut key = [0u8; 4]; + reader.read_exact(&mut key).map_err(io)?; + Some(key) + } else { + None + }; + let mut payload = vec![0u8; length as usize]; + reader.read_exact(&mut payload).map_err(io)?; + if let Some(key) = key { + for (index, byte) in payload.iter_mut().enumerate() { + *byte ^= key[index % 4]; + } + } + Ok((fin, opcode, payload)) +} + +fn random_bytes() -> [u8; 16] { + let mut bytes = [0u8; 16]; + for chunk in bytes.chunks_mut(4) { + chunk.copy_from_slice(&fastrand::u32(..).to_be_bytes()); + } + bytes +} + +/// Owner-only permissions on the profile, which holds Stockbit's cookie. +fn owner_only(path: &Path) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)); + } + #[cfg(not(unix))] + let _ = path; +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + use std::net::TcpListener; + + use super::*; + + /// A masked client frame round-trips through the reader. + #[test] + fn frames_round_trip() { + for payload in [b"".as_slice(), b"hi", &[b'x'; 300], &[b'y'; 70_000]] { + let raw = frame(1, payload); + assert_eq!(raw[1] & 0x80, 0x80, "client frames are masked"); + let (fin, opcode, decoded) = read_frame(&mut Cursor::new(raw)).unwrap(); + assert!(fin); + assert_eq!(opcode, 1); + assert_eq!(decoded, payload); + } + } + + /// The port file is two lines, and anything else is not an endpoint. + #[test] + fn endpoint_needs_a_port_and_a_path() { + assert_eq!( + parse_endpoint("9222\n/devtools/browser/abc\n"), + Some((9222, "/devtools/browser/abc".to_string())) + ); + assert_eq!(parse_endpoint("0\n/devtools/browser/abc"), None); + assert_eq!(parse_endpoint("9222\n"), None); + assert_eq!(parse_endpoint(""), None); + } + + /// What a signed-out Stockbit page stores: the credential store with no + /// tokens (constructed from the app's `partialize`, which drops unset + /// fields). + const SIGNED_OUT: &str = "%7B%22state%22%3A%7B%7D%2C%22version%22%3A0%7D"; + + /// A `credentialStorage` value with tokens in it. + fn session_value() -> String { + std::fs::read_to_string(format!( + "{}/tests/fixtures/stockbit_exodus_credential_storage_cookie_constructed.txt", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + .trim() + .to_string() + } + + /// Only Stockbit's own cookie counts, and only once it holds tokens. + #[test] + fn only_a_stockbit_cookie_with_tokens_counts() { + let session = session_value(); + let cookies = json!([ + {"name": COOKIE, "domain": ".stockbit.example", "path": "/", "value": session}, + {"name": COOKIE, "domain": "notstockbit.com", "path": "/", "value": session}, + {"name": "other", "domain": "stockbit.com", "path": "/", "value": session}, + {"name": COOKIE, "domain": "stockbit.com", "path": "/", "value": SIGNED_OUT}, + {"name": COOKIE, "domain": "stockbit.com", "path": "/", "value": session}, + ]); + let found = session_cookie(&cookies).expect("the session cookie"); + assert_eq!(found.value, session); + assert_eq!( + found.domain, "stockbit.com", + "host-only, as Stockbit sets it" + ); + assert_eq!(found.path, "/"); + + for value in [SIGNED_OUT, ""] { + let lone = json!([{"name": COOKIE, "domain": "stockbit.com", "value": value}]); + assert!( + session_cookie(&lone).is_none(), + "{value:?} is not a session" + ); + } + assert!(session_cookie(&json!([])).is_none()); + assert!(is_stockbit(".stockbit.com") && is_stockbit("exodus.stockbit.com")); + } + + /// The browser must not share the user's profile, and must pick its port. + #[test] + fn launch_keeps_its_own_profile_and_port() { + let browser = Browser { + name: "test".to_string(), + path: PathBuf::from("/bin/true"), + }; + let args = browser.args(Path::new("/tmp/idx-profile")); + assert!(args.contains(&"--user-data-dir=/tmp/idx-profile".to_string())); + assert!(args.contains(&"--remote-debugging-port=0".to_string())); + assert_eq!(args.last().map(String::as_str), Some(SIGN_IN_URL)); + } + + /// A whole sign-in against a stand-in browser: the session is handed to + /// `verify`, every Stockbit tab is closed before the cookie is deleted by + /// its own (host-only) domain, the cookie is gone afterwards, and the + /// browser is asked to close. + #[test] + fn a_sign_in_takes_the_cookie_and_removes_it_from_the_profile() { + let profile = temp_dir("sign-in"); + let browser = FakeBrowser::start(Fake::default()); + write_endpoint(&profile, browser.port); + let mut devtools = connect(&profile, &|| true).unwrap(); + let seen = std::cell::RefCell::new(Vec::new()); + wait_for_session( + &mut devtools, + Duration::from_secs(5), + &|pasted| { + seen.borrow_mut().push(pasted.to_string()); + Ok(()) + }, + &|| true, + ) + .unwrap(); + assert_eq!(seen.borrow().as_slice(), [session_value()]); + + let log = browser.finish(); + let at = |name: &str| log.methods.iter().position(|method| method == name); + assert!( + at("Target.closeTarget") < at("Network.deleteCookies"), + "Stockbit's tab closes before the cookie goes: {:?}", + log.methods + ); + assert_eq!(log.closed, ["T1"], "the sign-in tab, not idx's blank one"); + assert!(!log.cookie_left, "the session cookie is out of the profile"); + assert_eq!( + log.methods.last().map(String::as_str), + Some("Browser.close") + ); + let _ = std::fs::remove_dir_all(&profile); + } + + /// A cookie that survives the delete is an error, not a silent success. + #[test] + fn a_cookie_that_survives_the_delete_is_reported() { + let profile = temp_dir("sticky"); + let browser = FakeBrowser::start(Fake { + sticky: true, + ..Fake::default() + }); + write_endpoint(&profile, browser.port); + let mut devtools = connect(&profile, &|| true).unwrap(); + let cookie = devtools.session_cookie().unwrap().expect("a session"); + let err = devtools.forget_session(&cookie).unwrap_err(); + assert!(err.to_string().contains("still there"), "{err}"); + devtools.close(); + assert!(browser.finish().cookie_left); + let _ = std::fs::remove_dir_all(&profile); + } + + /// The store a signed-out page writes is not offered to `verify`. + #[test] + fn a_signed_out_page_is_not_a_session() { + let profile = temp_dir("signed-out"); + let browser = FakeBrowser::start(Fake { + cookie: Some(SIGNED_OUT.to_string()), + ..Fake::default() + }); + write_endpoint(&profile, browser.port); + let mut devtools = connect(&profile, &|| true).unwrap(); + let checks = std::cell::Cell::new(0); + let err = wait_for_session( + &mut devtools, + Duration::from_millis(300), + &|_| { + checks.set(checks.get() + 1); + Ok(()) + }, + &|| true, + ) + .unwrap_err(); + assert!(err.to_string().contains("timed out"), "{err}"); + assert_eq!(checks.get(), 0); + browser.finish(); + let _ = std::fs::remove_dir_all(&profile); + } + + /// A session idx refuses is reported once, and the wait goes on. + #[test] + fn a_refused_cookie_is_reported_once_and_the_wait_continues() { + let profile = temp_dir("refused"); + let browser = FakeBrowser::start(Fake::default()); + write_endpoint(&profile, browser.port); + let mut devtools = connect(&profile, &|| true).unwrap(); + let checks = std::cell::Cell::new(0); + let err = wait_for_session( + &mut devtools, + Duration::from_millis(300), + &|_| { + checks.set(checks.get() + 1); + Err(IdxError::AuthError("expired".to_string())) + }, + &|| true, + ) + .unwrap_err(); + assert!(err.to_string().contains("timed out"), "{err}"); + assert_eq!(checks.get(), 1, "one refusal per distinct cookie"); + browser.finish(); + let _ = std::fs::remove_dir_all(&profile); + } + + /// Closing the window ends the wait at once, not at the timeout. + #[test] + fn closing_the_window_ends_the_wait() { + let profile = temp_dir("closed"); + let browser = FakeBrowser::start(Fake { + cookie: None, + pages_for: 1, + ..Fake::default() + }); + write_endpoint(&profile, browser.port); + let mut devtools = connect(&profile, &|| true).unwrap(); + let started = Instant::now(); + let err = wait_for_session(&mut devtools, Duration::from_secs(30), &|_| Ok(()), &|| { + true + }) + .unwrap_err(); + assert!(err.to_string().contains("window was closed"), "{err}"); + assert!( + started.elapsed() < Duration::from_secs(5), + "waited {:?}", + started.elapsed() + ); + browser.finish(); + let _ = std::fs::remove_dir_all(&profile); + } + + /// A browser that quits ends the wait too. + #[test] + fn a_browser_that_quits_ends_the_wait() { + let profile = temp_dir("quit"); + let browser = FakeBrowser::start(Fake { + cookie: None, + ..Fake::default() + }); + write_endpoint(&profile, browser.port); + let mut devtools = connect(&profile, &|| true).unwrap(); + let err = wait_for_session(&mut devtools, Duration::from_secs(30), &|_| Ok(()), &|| { + false + }) + .unwrap_err(); + assert!(err.to_string().contains("browser exited"), "{err}"); + browser.finish(); + let _ = std::fs::remove_dir_all(&profile); + } + + /// Writes a stand-in browser executable into `dir`. + #[cfg(unix)] + fn fake_executable(dir: &Path, body: &str) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + + let script = dir.join("fake-browser.sh"); + std::fs::write(&script, format!("#!/bin/sh\n{body}")).unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap(); + script + } + + /// Launch, endpoint, WebSocket, verify, close: one sign-in end to end, + /// with a stand-in executable in place of the browser. + #[cfg(unix)] + #[test] + fn a_sign_in_starts_the_browser_and_closes_it() { + let profile = temp_dir("launch"); + let server = FakeBrowser::start(Fake::default()); + + // Writes the port file where the browser would, then waits to be + // closed. + let path = fake_executable( + &profile, + &format!( + "for arg in \"$@\"; do\n\ + case \"$arg\" in --user-data-dir=*) dir=\"${{arg#--user-data-dir=}}\" ;; esac\n\ + done\n\ + printf '%s\\n%s\\n' {port} /devtools/browser/test > \"$dir/DevToolsActivePort\"\n\ + sleep 30\n", + port = server.port + ), + ); + let browser = Browser { + name: "fake".to_string(), + path, + }; + let started = Instant::now(); + browser + .open(&profile) + .unwrap() + .wait(Duration::from_secs(5), &AtomicBool::new(false), &|_| Ok(())) + .unwrap(); + assert_eq!( + server.finish().methods.last().map(String::as_str), + Some("Browser.close") + ); + // The stand-in sleeps for 30 s; the sign-in must not wait for it. + assert!( + started.elapsed() < Duration::from_secs(10), + "the sign-in waited {:?}", + started.elapsed() + ); + let _ = std::fs::remove_dir_all(&profile); + } + + /// Cancelling ends the wait and closes the browser, so a server that + /// is exiting leaves no window behind. + #[cfg(unix)] + #[test] + fn cancelling_closes_the_browser() { + let profile = temp_dir("cancel"); + let server = FakeBrowser::start(Fake { + cookie: None, + ..Fake::default() + }); + let path = fake_executable( + &profile, + &format!( + "for arg in \"$@\"; do\n\ + case \"$arg\" in --user-data-dir=*) dir=\"${{arg#--user-data-dir=}}\" ;; esac\n\ + done\n\ + printf '%s\\n%s\\n' {port} /devtools/browser/test > \"$dir/DevToolsActivePort\"\n\ + sleep 30\n", + port = server.port + ), + ); + let browser = Browser { + name: "fake".to_string(), + path, + }; + let window = browser.open(&profile).unwrap(); + let cancel = std::sync::Arc::new(AtomicBool::new(false)); + let waiter = { + let cancel = cancel.clone(); + std::thread::spawn(move || window.wait(Duration::from_secs(30), &cancel, &|_| Ok(()))) + }; + std::thread::sleep(Duration::from_millis(200)); + let started = Instant::now(); + cancel.store(true, Ordering::Relaxed); + assert!(waiter.join().unwrap().is_err()); + assert!( + started.elapsed() < Duration::from_secs(6), + "cancelling took {:?}", + started.elapsed() + ); + assert_eq!( + server.finish().methods.last().map(String::as_str), + Some("Browser.close") + ); + let _ = std::fs::remove_dir_all(&profile); + } + + /// The browser that opened a profile is named in it, and is the one used + /// again; one no longer installed is not. + #[cfg(unix)] + #[test] + fn a_profile_remembers_its_browser() { + let profile = temp_dir("made-by"); + let path = fake_executable(&profile, "exit 0\n"); + assert!(Browser::recorded(&profile).is_none()); + let browser = Browser { + name: "Named".to_string(), + path: path.clone(), + }; + // The launch records it even when the browser then goes away. + assert!(browser.open(&profile).is_err()); + let again = Browser::recorded(&profile).expect("the recorded browser"); + assert_eq!(again.path, path); + assert_eq!(again.name, "fake-browser"); + + std::fs::remove_file(&path).unwrap(); + assert!(Browser::recorded(&profile).is_none()); + let _ = std::fs::remove_dir_all(&profile); + } + + /// A browser that hands its window to one already running exits at + /// once; that is reported then, not after the start timeout. + #[cfg(unix)] + #[test] + fn a_browser_that_exits_at_launch_fails_fast() { + let profile = temp_dir("handoff"); + let browser = Browser { + name: "fake".to_string(), + path: fake_executable(&profile, "exit 0\n"), + }; + let started = Instant::now(); + let err = browser.open(&profile).err().expect("no window"); + assert!(err.to_string().contains("exited as it started"), "{err}"); + assert!(started.elapsed() < Duration::from_secs(5)); + let _ = std::fs::remove_dir_all(&profile); + } + + /// The real thing: launches the browser named by `IDX_STOCKBIT_BROWSER` + /// and checks the wait ends on the timeout, not on a broken connection, + /// which is what proves the browser answered `Storage.getCookies`. + /// + /// Ignored by default: it opens a browser window and loads Stockbit's + /// page, so it needs a machine with a browser and a network. + #[test] + #[ignore = "opens a real browser; set IDX_STOCKBIT_BROWSER"] + fn a_real_browser_answers_the_devtools_requests() { + let profile = temp_dir("real"); + let Ok(browser) = Browser::find(&profile) else { + eprintln!("IDX_STOCKBIT_BROWSER is not set; nothing to launch"); + return; + }; + let err = browser + .open(&profile) + .unwrap() + .wait(Duration::from_secs(25), &AtomicBool::new(false), &|_| { + panic!("a fresh profile holds no session to check") + }) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("timed out"), "{message}"); + assert!( + !message.contains("last DevTools error"), + "DevTools never answered: {message}" + ); + let _ = std::fs::remove_dir_all(&profile); + } + + /// How a stand-in browser behaves. + struct Fake { + /// The session cookie's value, if it holds one. + cookie: Option, + /// `Target.getTargets` reports pages for this many calls, then none. + pages_for: usize, + /// The cookie survives `Network.deleteCookies`. + sticky: bool, + } + + impl Default for Fake { + fn default() -> Self { + Self { + cookie: Some(session_value()), + pages_for: usize::MAX, + sticky: false, + } + } + } + + /// What a stand-in browser was asked, and what it was left holding. + struct Log { + methods: Vec, + /// Tabs closed with `Target.closeTarget`. + closed: Vec, + cookie_left: bool, + } + + /// A stand-in for a browser's DevTools endpoint: one WebSocket, scripted + /// answers, and the methods it was asked for. + struct FakeBrowser { + port: u16, + handle: Option>, + } + + impl FakeBrowser { + fn start(fake: Fake) -> Self { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let handle = std::thread::spawn(move || serve(listener, fake)); + Self { + port, + handle: Some(handle), + } + } + + fn finish(mut self) -> Log { + self.handle.take().unwrap().join().unwrap() + } + } + + /// Stockbit's cookie lives host-only on `stockbit.com`, and, as in + /// Chromium, a delete must name that exact domain and path. + fn serve(listener: TcpListener, fake: Fake) -> Log { + const DOMAIN: &str = "stockbit.com"; + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let head = read_head(&mut stream).unwrap(); + assert!(head.contains("Upgrade: websocket"), "{head}"); + stream + .write_all(b"HTTP/1.1 101 Switching Protocols\r\n\r\n") + .unwrap(); + let mut cookie = fake.cookie.clone(); + let mut pages = vec!["T1".to_string()]; + let mut log = Log { + methods: Vec::new(), + closed: Vec::new(), + cookie_left: false, + }; + let mut target_calls = 0; + while let Ok(payload) = read_message(&mut stream) { + let request: Value = serde_json::from_slice(&payload).unwrap(); + let method = request["method"].as_str().unwrap_or_default().to_string(); + let params = &request["params"]; + log.methods.push(method.clone()); + if method == "Browser.close" { + let reply = json!({ "id": request["id"], "result": {} }); + let _ = stream.write_all(&server_frame(reply.to_string().as_bytes())); + break; + } + let result = match method.as_str() { + "Storage.getCookies" => { + let cookies: Vec = cookie + .iter() + .map(|value| { + json!({ "name": COOKIE, "domain": DOMAIN, "path": "/", "value": value }) + }) + .collect(); + json!({ "cookies": cookies }) + } + "Target.getTargets" => { + target_calls += 1; + let infos: Vec = if target_calls <= fake.pages_for { + pages + .iter() + .map(|id| json!({ "targetId": id, "type": "page", "url": SIGN_IN_URL })) + .collect() + } else { + Vec::new() + }; + json!({ "targetInfos": infos }) + } + "Target.createTarget" => { + pages.push("BLANK".to_string()); + json!({ "targetId": "BLANK" }) + } + "Target.closeTarget" => { + let id = params["targetId"].as_str().unwrap_or_default().to_string(); + pages.retain(|page| *page != id); + log.closed.push(id); + json!({ "success": true }) + } + "Target.attachToTarget" => json!({ "sessionId": "S1" }), + "Network.deleteCookies" => { + assert_eq!(request["sessionId"], "S1", "deleted from a tab"); + if !fake.sticky + && params["name"] == COOKIE + && params["domain"] == DOMAIN + && params["path"] == "/" + { + cookie = None; + } + json!({}) + } + _ => json!({}), + }; + let reply = json!({ "id": request["id"], "result": result }); + if stream + .write_all(&server_frame(reply.to_string().as_bytes())) + .is_err() + { + break; + } + } + log.cookie_left = cookie.is_some(); + log + } + + /// One unmasked server frame. + fn server_frame(payload: &[u8]) -> Vec { + let mut out = vec![0x81]; + match payload.len() { + len if len < 126 => out.push(len as u8), + len => { + out.push(126); + out.extend_from_slice(&(len as u16).to_be_bytes()); + } + } + out.extend_from_slice(payload); + out + } + + fn read_message(stream: &mut TcpStream) -> Result, IdxError> { + let mut buffer = Vec::new(); + loop { + let (fin, _, payload) = read_frame(stream)?; + buffer.extend_from_slice(&payload); + if fin { + return Ok(buffer); + } + } + } + + fn write_endpoint(profile: &Path, port: u16) { + std::fs::write( + profile.join("DevToolsActivePort"), + format!("{port}\n/devtools/browser/test\n"), + ) + .unwrap(); + } + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "idx-browser-{name}-{}-{}", + std::process::id(), + fastrand::u32(..) + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } +} diff --git a/crates/idx-core/src/api/stockbit/loopback.rs b/crates/idx-core/src/api/stockbit/loopback.rs index bb7bbac..b154df3 100644 --- a/crates/idx-core/src/api/stockbit/loopback.rs +++ b/crates/idx-core/src/api/stockbit/loopback.rs @@ -67,15 +67,15 @@ impl LoginPage { self.port } - /// Serves until `verify` accepts a paste or `timeout` passes. `verify` - /// returns the error text to show on the page when the paste is refused. + /// Serves until `verify` accepts a paste or `timeout` passes. The error + /// `verify` returns for a refused paste is shown on the page. /// /// Each connection gets its own thread: browsers open idle preconnects, /// and answering them one at a time used to hold the page back for the /// read timeout on every one. pub fn wait(&self, timeout: Duration, verify: F) -> Result<(), IdxError> where - F: Fn(&str) -> Result<(), String> + Send + Sync + 'static, + F: Fn(&str) -> Result<(), IdxError> + Send + Sync + 'static, { let (done_tx, done_rx) = std::sync::mpsc::channel::<()>(); let verify = std::sync::Arc::new(verify); @@ -136,12 +136,38 @@ window.open('{target}#t='+encodeURIComponent(m[1]))}})()" } } +/// The forward to run where the browser is, for the page on `port`: +/// `ssh -L :127.0.0.1: @`. +/// +/// The target comes from `SSH_CONNECTION` (`client_ip client_port server_ip +/// server_port`) when idx runs inside an SSH session, else from `USER` and +/// `HOSTNAME`, else it is left for the reader to fill in. +pub fn ssh_forward(port: u16) -> String { + let target = ssh_target().unwrap_or_else(|| "".to_string()); + format!("ssh -L {port}:127.0.0.1:{port} {target}") +} + +fn ssh_target() -> Option { + if let Ok(connection) = std::env::var("SSH_CONNECTION") { + let fields: Vec<&str> = connection.split_whitespace().collect(); + if let Some(server) = fields.get(2) { + return Some(match std::env::var("USER") { + Ok(user) => format!("{user}@{server}"), + Err(_) => (*server).to_string(), + }); + } + } + let user = std::env::var("USER").ok()?; + let host = std::env::var("HOSTNAME").ok()?; + Some(format!("{user}@{host}")) +} + /// Answers one request; true when a paste was accepted. fn serve_once( mut stream: TcpStream, page_path: &str, page: &str, - verify: &dyn Fn(&str) -> Result<(), String>, + verify: &dyn Fn(&str) -> Result<(), IdxError>, ) -> bool { let _ = stream.set_read_timeout(Some(IO_TIMEOUT)); let _ = stream.set_write_timeout(Some(IO_TIMEOUT)); @@ -168,8 +194,8 @@ fn serve_once( ); return true; } - Err(message) => { - let body = serde_json::json!({ "ok": false, "message": message }); + Err(err) => { + let body = serde_json::json!({ "ok": false, "message": err.to_string() }); respond( &mut stream, 200, @@ -278,76 +304,240 @@ fn respond(stream: &mut TcpStream, status: u16, content_type: &str, body: &[u8]) const PAGE: &str = r#" + Connect Stockbit to idx -
-

Connect Stockbit to idx

-

One step. idx stores the session on this machine only, and never sends it anywhere but Stockbit.

- -
-

1. Open stockbit.com and make sure you are logged in.

+
+
+ +idxStockbit
-
-

2. While on stockbit.com, click this bookmarklet. It reads your Stockbit session cookie in your own browser and hands it to this page.

-

Send my Stockbit session

-

If your browser blocks it, drag the button to your bookmarks bar and click it there.

-
+
+

Connect your Stockbit account

+

idx then reads live quotes and the order book with your own session. The session stays on this machine.

+
-
-

Or copy it by hand. On stockbit.com: DevTools, Application, Cookies, https://stockbit.com, copy the credentialStorage value, then paste it here.

- -
+
+
    +
  1. + +
    +

    Sign in to Stockbit

    +

    Open stockbit.com in this browser and sign in. If the tab is already signed in, go to the next step.

    +Open stockbit.com +
    +
    2. +
  2. + +
    +

    Send the session to idx

    +

    While the Stockbit tab is open, click this button. It reads the session cookie in your own browser.

    +Send my Stockbit session +

    If the browser blocks it, drag the button to your bookmarks bar, open stockbit.com, then click it there.

    +
    +
    3. +
+ +
+Or paste the cookie by hand +
+
    +
  1. On stockbit.com, open DevTools with F12.
    2. +
  2. Go to Application, then Cookies, then https://stockbit.com.
    3. +
  3. Copy the value of credentialStorage and paste it below.
    4. +
+ +
+
-

-

idx keeps only the two tokens Stockbit issues, in a file only you can read. Stockbit replaces the refresh token each time it is used, so a browser sharing this session will be signed out once idx refreshes it. This page closes when you finish, or after a few minutes.

+ +
+ + + +
+

idx keeps only the two tokens Stockbit issues, in a file only you can read. Stockbit replaces the refresh token each time it is used, so a browser that shares this session signs out once idx refreshes it.

+

This page stops listening after a few minutes.

+
@@ -394,7 +584,7 @@ mod tests { let handle = std::thread::spawn(move || { let verify = |paste: &str| match paste.trim() { "good" => Ok(()), - _ => Err("not a Stockbit session".to_string()), + _ => Err(IdxError::InvalidInput("not a Stockbit session".to_string())), }; page.wait(Duration::from_secs(10), verify) }); @@ -419,12 +609,39 @@ mod tests { handle.join().unwrap().expect("accepted"); } + /// The CSP allows no fetch of its own, so a stray asset would fail + /// silently; the page also has to keep the two entry points the docs name. + #[test] + fn the_page_is_self_contained_and_keeps_both_entry_points() { + let page = LoginPage::bind(0).unwrap().page(); + for forbidden in ["