tui: add /validate subcommand with high‑risk web validation

- Add /validate slash command
- Plan + execute validations (Playwright MCP preference, curl, python)
- Pre‑validation account setup: auto‑register or manual fallback (opens login)
- Persist credentials in context/validation/credentials.json (usernames logged)
- Update bugs.md and report with validation results + transcript

Also adds Playwright tool support, inline python execution, and UI logs.
This commit is contained in:
kh.ai
2025-11-01 11:19:40 -07:00
parent ae5150c37a
commit 883a108624
6 changed files with 716 additions and 4 deletions

View File

@@ -416,6 +416,9 @@ impl App {
AppEvent::SecurityReviewScopeResolved { paths } => {
self.chat_widget.on_security_review_scope_resolved(paths);
}
AppEvent::OpenRegistrationPrompt { url, responder } => {
self.chat_widget.show_registration_prompt(url, responder);
}
AppEvent::SecurityReviewCommandStatus {
id,
summary,

View File

@@ -132,6 +132,13 @@ pub(crate) enum AppEvent {
responder: oneshot::Sender<bool>,
},
/// Prompt the user to register at least two accounts and paste credentials.
/// The responder receives `Some(raw_input)` when the user submits text, or `None` if dismissed.
OpenRegistrationPrompt {
url: Option<String>,
responder: oneshot::Sender<Option<String>>,
},
/// Notify that the security review scope has been resolved to specific paths.
SecurityReviewScopeResolved {
paths: Vec<String>,

View File

@@ -508,6 +508,7 @@ struct SecurityReviewFollowUpState {
}
#[allow(dead_code)]
#[derive(Clone)]
struct SecurityReviewArtifactsState {
repo_root: PathBuf,
snapshot_path: PathBuf,
@@ -1689,6 +1690,79 @@ impl ChatWidget {
SlashCommand::SecReview => {
self.open_security_review_popup();
}
SlashCommand::Validate => {
// Web/API validation for high-risk findings from the last review
if self.bottom_pane.is_task_running() || self.security_review_context.is_some() {
self.add_error_message(
"Cannot run /validate while a task is in progress.".to_string(),
);
self.request_redraw();
return;
}
let Some(artifacts) = self.security_review_artifacts.clone() else {
self.add_error_message(
"No security review results to validate. Run /secreview first.".to_string(),
);
self.request_redraw();
return;
};
self.bottom_pane.set_task_running(true);
self.bottom_pane
.update_status_header("Validating findings — preparing".to_string());
let provider = self.config.model_provider.clone();
let auth = self.auth_manager.auth();
let model = self.config.model.clone();
let tx = self.app_event_tx.clone();
let repo_path = self.config.cwd.clone();
tokio::spawn(async move {
use crate::security_review::run_web_validation;
match run_web_validation(
repo_path,
artifacts.snapshot_path.clone(),
artifacts.bugs_path.clone(),
artifacts.report_path.clone(),
artifacts.report_html_path.clone(),
provider,
auth,
model,
Some(tx.clone()),
)
.await
{
Ok(_) => {
tx.send(AppEvent::SecurityReviewLog(
"Validation complete; report updated.".to_string(),
));
}
Err(err) => {
tx.send(AppEvent::SecurityReviewLog(format!(
"Validation failed: {}",
err.message
)));
}
}
// Clear the in-progress flag regardless of outcome.
tx.send(AppEvent::SecurityReviewComplete {
result: crate::security_review::SecurityReviewResult {
findings_summary: String::new(),
bug_summary_table: None,
bugs: Vec::new(),
bugs_path: artifacts.bugs_path,
report_path: artifacts.report_path,
report_html_path: artifacts.report_html_path,
snapshot_path: artifacts.snapshot_path,
metadata_path: artifacts.metadata_path,
api_overview_path: artifacts.api_overview_path,
classification_json_path: artifacts.classification_json_path,
classification_table_path: artifacts.classification_table_path,
logs: vec![],
token_usage: codex_core::protocol::TokenUsage::default(),
},
});
});
}
SlashCommand::Model => {
self.open_model_popup();
}
@@ -2913,6 +2987,65 @@ impl ChatWidget {
}
}
pub(crate) fn show_registration_prompt(
&mut self,
url: Option<String>,
responder: tokio::sync::oneshot::Sender<Option<String>>,
) {
if let Some(link) = url.as_ref() {
// Try to open the URL in the default browser.
let link_clone = link.clone();
tokio::spawn(async move {
#[cfg(target_os = "macos")]
{
let _ = tokio::process::Command::new("open")
.arg(&link_clone)
.status()
.await;
}
#[cfg(all(unix, not(target_os = "macos")))]
{
let _ = tokio::process::Command::new("xdg-open")
.arg(&link_clone)
.status()
.await;
}
#[cfg(target_os = "windows")]
{
let _ = tokio::process::Command::new("rundll32.exe")
.arg("url.dll,FileProtocolHandler")
.arg(&link_clone)
.status()
.await;
}
});
}
let hint = if let Some(link) = url.as_ref() {
format!(
"We attempted to auto-register accounts but failed. A login/signup page was opened: {link}\n\nRegister at least two test accounts, then paste credentials as `user:pass, user2:pass2` and press Enter."
)
} else {
"We attempted to auto-register accounts but failed. Register at least two test accounts, then paste credentials as `user:pass, user2:pass2` and press Enter.".to_string()
};
use std::sync::Arc;
use std::sync::Mutex;
let responder_cell = Arc::new(Mutex::new(Some(responder)));
let responder_cell_clone = responder_cell;
let view = CustomPromptView::new(
"Register test accounts".to_string(),
hint,
None,
Box::new(move |input: String| {
if let Some(tx) = responder_cell_clone.lock().ok().and_then(|mut g| g.take()) {
let _ = tx.send(Some(input));
}
}),
);
self.bottom_pane.show_view(Box::new(view));
}
pub(crate) fn on_security_review_log(&mut self, message: String) {
if let Some(ctx) = self.security_review_context.as_mut() {
// Drop overly verbose heartbeat for bug analysis; header already shows progress.

View File

@@ -87,6 +87,47 @@ Produce newline-delimited JSON (NDJSON), one object per classified data type wit
# Output
Emit only NDJSON lines. Each JSON object must contain exactly the keys listed above (no arrays, extra keys, or prose).
"#;
// Validation plan prompts
pub(crate) const VALIDATION_PLAN_SYSTEM_PROMPT: &str = "You are an application security engineer planning minimal, safe validations for high-risk findings. Respond ONLY with JSON Lines as requested; do not include markdown or prose.";
pub(crate) const VALIDATION_PLAN_PROMPT_TEMPLATE: &str = r#"
Before any checks, create two test accounts if the app requires login. Prefer a short Python script that calls a signup endpoint or automates the registration form headlessly. If this is not feasible, return a `manual` instruction with a `login_url`.
Then select ONLY high-risk findings to validate. For each, choose the minimal tool and target:
- Use the Playwright MCP tool for web_browser checks (supply a reachable URL in `target`).
- Use tool "curl" for network_api checks (supply full URL in `target`).
- Use tool "python" only if a short, non-destructive PoC is essential (include inline script text in `script`).
Rules:
- Keep requests minimal and non-destructive; no state-changing actions.
- Prefer headless checks (e.g., page loads, HTTP status, presence of a marker string).
- Max 5 requests total; prioritize Critical/High severity or lowest risk_rank.
Context (findings):
{findings}
Output format (one JSON object per line, no fences):
- For account setup (emit at most one line): {"id_kind":"setup","action":"register|manual","login_url":"<string, optional>","tool":"python|manual","script":"<string, optional>"}
- For validations: {"id_kind":"risk_rank|summary_id","id_value":<int>,"tool":"playwright|curl|python","target":"<string, optional>","script":"<string, optional>"}
"#;
// Account setup planning (standalone, used when needed)
pub(crate) const VALIDATION_ACCOUNTS_SYSTEM_PROMPT: &str = "You plan how to create two test accounts for a typical web app. Respond ONLY with JSON Lines; no prose.";
pub(crate) const VALIDATION_ACCOUNTS_PROMPT_TEMPLATE: &str = r#"
Goal: ensure two test accounts exist prior to validation. Prefer a short Python script that registers accounts via HTTP or a headless flow; otherwise return a manual login URL.
Constraints:
- The script must be non-destructive and idempotent.
- Print credentials to stdout as JSON: {"accounts":[{"username":"...","password":"..."},{"username":"...","password":"..."}]}.
- If you cannot identify a safe automated path, return a single JSON line: {"action":"manual","login_url":"https://..."}.
Context (findings):
{findings}
Output format (one JSON object per line, no fences):
- Automated: {"action":"register","tool":"python","login_url":"<string, optional>","script":"<python script>"}
- Manual: {"action":"manual","login_url":"<string>"}
"#;
pub(crate) const MARKDOWN_OUTPUT_GUARD: &str = "\n# Output Guard (strict)\n - Output only the final markdown content requested.\n - Do not include goal, analysis, planning, chain-of-thought, or step lists.\n - Do not echo prompt sections like \"Task\", \"Steps\", \"Output\", or \"Important\".\n - Do not include any XML/angle-bracket blocks (e.g., <...> inputs) in the output.\n - Do not wrap the entire response in code fences; use code fences only for code snippets.\n - Do not include apologies, disclaimers, or references to being an AI model.\n";
pub(crate) const MARKDOWN_FIX_SYSTEM_PROMPT: &str = "You are a meticulous technical editor. Polish markdown formatting while preserving the original security analysis content. Focus on fixing numbering, bullet spacing, code fences, and diagram syntax without adding or removing information.";
pub(crate) const SPEC_COMBINE_PROMPT_TEMPLATE: &str = "You previously generated specification drafts for the following code locations:\n{project_locations}\n\nDraft content (each draft may include an \"API Entry Points\" section summarizing externally exposed interfaces):\n{spec_drafts}\n\nTask: merge these drafts into one comprehensive specification that describes the entire project. Remove duplication, keep terminology consistent, and ensure the final document reads as a single report that preserves API coverage. Follow the template exactly and return only markdown.\n\nNon-negotiable requirements:\n- Carry forward every concrete security-relevant fact, list, table, code block, and data classification entry from the drafts unless it is an exact duplicate.\n- When multiple drafts contribute to the same template section, include the union of their paragraphs and bullet points. If details differ, keep both and attribute them with inline labels such as `(from {location_label})` rather than dropping information.\n- Preserve API entry points verbatim (including tables) and incorporate them into the appropriate section without shortening columns.\n- Keep all identifiers (component names, queue names, environment variables, secrets, external services, metric names) exactly as written; do not rename or generalize.\n- Follow the template's structure exactly: populate every section, create the requested subsections, and include the explicit `Sources:` lines and bullet styles. Do not leave the instructional text in place or drop mandatory sections.\n- Populate the \"Relevant Source Files\" section with bullet points that reference each draft's location label and any concrete file paths mentioned in the drafts.\n- Ensure the \"Data Classification\" section exists even when the drafts were sparse; aggregate and preserve every classification detail there.\n- If multiple drafts contain tabular data (APIs, components, data classification), merge rows from all drafts and maintain duplicates when the sources disagree so the consumer can reconcile manually.\n- Do not introduce new speculation or remove nuance from mitigations, caveats, or risk descriptions provided in the drafts. Err on the side of length; the final document should be at least as detailed as the most verbose draft.\n\n# Available tools\n- READ: respond with `READ: <relative path>#Lstart-Lend` (range optional) to open code or draft files. Use paths relative to the repository root.\n- GREP_FILES: respond with `GREP_FILES: {\"pattern\": \"...\", \"include\": \"*.rs\", \"path\": \"subdir\", \"limit\": 200}` to list files whose contents match.\nEmit at most one tool command in a single message and wait for the tool output before continuing. Prefer READ for prose context; SEARCH is not available during this step.\n\nTemplate:\n{combined_template}\n";
@@ -97,7 +138,7 @@ You triage directories for a security review specification. Only choose director
- Limit the selection to the most critical directories (ideally 3-8).
Respond with a newline-separated list containing only the directory paths chosen from the provided list. Respond with `ALL` if every directory should be included. Do not add quotes or extra commentary.
"#;
pub(crate) const SPEC_MARKDOWN_TEMPLATE: &str = "# Project Specification\n- Location: {target_label}\n- Prepared by: {model_name}\n- Date: {date}\n- In-scope paths:\n```\n{project_locations}\n```\n\n## Overview\nSummarize the product or service, primary users, and the business problem it solves. Highlight the most security relevant entry points.\n\n## Architecture Summary\nDescribe the high-level system architecture, major services, data stores, and external integrations. Include a concise mermaid flowchart when it improves clarity.\n\n## Components\nList 5-8 major components. For each, note the role, responsibilities, key dependencies, and security-critical behavior.\n\n## Business Flows\nDocument up to 5 important flows (CRUD, external integrations, workflow orchestration). For each flow capture triggers, main steps, data touched, and security notes. Include a short mermaid sequence diagram if helpful.\n\n## Tech Stack\nCapture languages, frameworks, and infrastructure used by each major component. Tabulate runtimes, key libraries, storage technologies, and deployment targets.\n\n## Authentication\nExplain how principals authenticate, token lifecycles, libraries used, and how secrets are managed.\n\n## Authorization\nDescribe the authorization model, enforcement points, privileged roles, and escalation paths.\n\n## Data Classification\nIdentify sensitive data types handled by the project and where they are stored or transmitted.\n\n## Infrastructure and Deployment\nSummarize infrastructure-as-code, runtime platforms, and configuration or secret handling that affects security posture.\n\n## API Entry Points\nList externally reachable interfaces (HTTP/gRPC endpoints, message queues, CLIs, SDK methods) and how they handle security.\n\n### Server APIs\nProvide a markdown table with the exact columns:\n- endpoint path\n- authN method\n- authZ type\n- request parameters\n- example request (params, body, or method)\n- code location\n- parsing/validation logic\nIf the project exposes no server APIs, write `- None identified.` instead of a table.\n\n### Client APIs (optional)\nInclude a markdown table when the project ships an SDK, CLI, or other callable client surface. Columns:\n- api name (module.func or Class.method)\n- module/package\n- summary\n- parameters (omit if noisy)\n- returns (omit if noisy)\n- stability (public/official/internal)\n- code location\nIf there is no public client surface, state `- None.`\n";
pub(crate) const SPEC_MARKDOWN_TEMPLATE: &str = "# Project Specification\n- Location: {target_label}\n- Prepared by: {model_name}\n- Date: {date}\n- In-scope paths:\n```\n{project_locations}\n```\n\n## Overview\nSummarize the product or service, primary users, and the business problem it solves. Highlight the most security relevant entry points.\n\n## Architecture Summary\nDescribe the high-level system architecture, major services, data stores, and external integrations. Include a concise mermaid flowchart when it improves clarity. If the specification uses more than one mermaid diagram, add a `title Component request flow` line (with a descriptive label) inside each diagram so the rendered report shows distinct titles.\n\n## Components\nList 5-8 major components. For each, note the role, responsibilities, key dependencies, and security-critical behavior.\n\n## Business Flows\nDocument up to 5 important flows (CRUD, external integrations, workflow orchestration). For each flow capture triggers, main steps, data touched, and security notes. Include a short mermaid sequence diagram if helpful.\n\n## Tech Stack\nCapture languages, frameworks, and infrastructure used by each major component. Tabulate runtimes, key libraries, storage technologies, and deployment targets.\n\n## Authentication\nExplain how principals authenticate, token lifecycles, libraries used, and how secrets are managed.\n\n## Authorization\nDescribe the authorization model, enforcement points, privileged roles, and escalation paths.\n\n## Data Classification\nIdentify sensitive data types handled by the project and where they are stored or transmitted.\n\n## Infrastructure and Deployment\nSummarize infrastructure-as-code, runtime platforms, and configuration or secret handling that affects security posture.\n\n## API Entry Points\nList externally reachable interfaces (HTTP/gRPC endpoints, message queues, CLIs, SDK methods) and how they handle security.\n\n### Server APIs\nProvide a markdown table with the exact columns:\n- endpoint path\n- authN method\n- authZ type\n- request parameters\n- example request (params, body, or method)\n- code location\n- parsing/validation logic\nIf the project exposes no server APIs, write `- None identified.` instead of a table.\n\n### Client APIs (optional)\nInclude a markdown table when the project ships an SDK, CLI, or other callable client surface. Columns:\n- api name (module.func or Class.method)\n- module/package\n- summary\n- parameters (omit if noisy)\n- returns (omit if noisy)\n- stability (public/official/internal)\n- code location\nIf there is no public client surface, state `- None.`\n";
pub(crate) const SPEC_COMBINED_MARKDOWN_TEMPLATE: &str = r#"# Project Specification
Provide a 23 sentence executive overview summarizing the system's purpose, primary users, and the highest-value assets or flows that matter for security.
@@ -108,6 +149,7 @@ List bullet points for the key files and directories covered by the drafts. Use
Provide a concise overview of how control and data move through the system, highlighting major services, external dependencies, and trust boundaries.
Include exactly one overarching mermaid diagram here that captures the end-to-end flow (no per-component or sequence diagrams in this section).
Move any detailed or per-component diagrams to the relevant component subsections below.
If the specification contains additional mermaid diagrams, add a `title Component request flow` line (with a descriptive label) inside each diagram so the rendered report labels them distinctly.
End with a `Sources:` line enumerating the files or modules that support this description.
## Core Components

View File

@@ -1053,6 +1053,7 @@ pub(crate) enum BugIdentifier {
pub(crate) enum BugVerificationTool {
Curl,
Python,
Playwright,
}
impl BugVerificationTool {
@@ -1060,6 +1061,7 @@ impl BugVerificationTool {
match self {
BugVerificationTool::Curl => "curl",
BugVerificationTool::Python => "python",
BugVerificationTool::Playwright => "playwright",
}
}
}
@@ -1070,6 +1072,7 @@ pub(crate) struct BugVerificationRequest {
pub tool: BugVerificationTool,
pub target: Option<String>,
pub script_path: Option<PathBuf>,
pub script_inline: Option<String>,
}
#[derive(Clone, Debug)]
@@ -1079,6 +1082,7 @@ pub(crate) struct BugVerificationBatchRequest {
pub report_path: Option<PathBuf>,
pub report_html_path: Option<PathBuf>,
pub repo_path: PathBuf,
pub work_dir: PathBuf,
pub requests: Vec<BugVerificationRequest>,
}
@@ -8317,7 +8321,11 @@ fn build_bugs_markdown(
fix_mermaid_blocks(&combined)
}
async fn execute_bug_command(plan: BugCommandPlan, repo_path: PathBuf) -> BugCommandResult {
async fn execute_bug_command(
plan: BugCommandPlan,
repo_path: PathBuf,
work_dir: PathBuf,
) -> BugCommandResult {
let mut logs: Vec<String> = Vec::new();
let label = if let Some(rank) = plan.risk_rank {
format!("#{rank} {}", plan.title)
@@ -8395,7 +8403,41 @@ async fn execute_bug_command(plan: BugCommandPlan, repo_path: PathBuf) -> BugCom
}
}
BugVerificationTool::Python => {
let Some(script_path) = plan.request.script_path.as_ref() else {
let script_path_owned: Option<PathBuf> =
if let Some(path) = plan.request.script_path.as_ref() {
Some(path.clone())
} else if let Some(code) = plan.request.script_inline.as_ref() {
let _ = tokio_fs::create_dir_all(&work_dir).await;
let file_name = if let Some(rank) = plan.risk_rank {
format!("bug_rank_{rank}.py")
} else {
format!("bug_{}.py", plan.summary_id)
};
let temp_path = work_dir.join(file_name);
if let Err(err) = tokio_fs::write(&temp_path, code.as_bytes()).await {
validation.status = BugValidationStatus::Failed;
validation.summary = Some(format!(
"Failed to write inline python to {}: {err}",
temp_path.display()
));
logs.push(format!(
"{}: failed to write python script {}: {err}",
label,
temp_path.display()
));
validation.run_at = Some(OffsetDateTime::now_utc());
return BugCommandResult {
index: plan.index,
validation,
logs,
};
}
Some(temp_path)
} else {
None
};
let Some(script_path) = script_path_owned.as_ref() else {
validation.status = BugValidationStatus::Failed;
validation.summary = Some("Missing python script path".to_string());
logs.push(format!("{label}: no python script provided"));
@@ -8462,6 +8504,74 @@ async fn execute_bug_command(plan: BugCommandPlan, repo_path: PathBuf) -> BugCom
}
}
}
BugVerificationTool::Playwright => {
let Some(target) = plan.request.target.clone().filter(|t| !t.is_empty()) else {
validation.status = BugValidationStatus::Failed;
validation.summary = Some("Missing target URL".to_string());
logs.push(format!("{label}: no target URL provided for playwright"));
validation.run_at = Some(OffsetDateTime::now_utc());
return BugCommandResult {
index: plan.index,
validation,
logs,
};
};
let _ = tokio_fs::create_dir_all(&work_dir).await;
let file_stem = if let Some(rank) = plan.risk_rank {
format!("bug_rank_{rank}")
} else {
format!("bug_{}", plan.summary_id)
};
let screenshot_path = work_dir.join(format!("{file_stem}.png"));
let mut command = Command::new("npx");
command
.arg("--yes")
.arg("playwright")
.arg("screenshot")
.arg(&target)
.arg(&screenshot_path)
.current_dir(&repo_path);
match command.output().await {
Ok(output) => {
let duration = start.elapsed();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let success = output.status.success();
validation.status = if success {
BugValidationStatus::Passed
} else {
BugValidationStatus::Failed
};
let duration_label = fmt_elapsed_compact(duration.as_secs());
if success {
validation.summary = Some(format!(
"Saved screenshot to {} · {duration_label}",
display_path_for(&screenshot_path, &repo_path)
));
} else {
let summary_line = summarize_process_output(success, &stdout, &stderr);
validation.summary = Some(format!("{summary_line} · {duration_label}"));
}
let primary = if success { &stdout } else { &stderr };
let trimmed = primary.trim();
if !trimmed.is_empty() {
validation.output_snippet =
Some(truncate_text(trimmed, VALIDATION_OUTPUT_GRAPHEMES));
}
logs.push(format!(
"{}: playwright exited with status {}",
label, output.status
));
}
Err(err) => {
validation.status = BugValidationStatus::Failed;
validation.summary = Some(format!("Failed to run playwright: {err}"));
logs.push(format!("{label}: failed to run playwright: {err}"));
}
}
}
}
validation.run_at = Some(OffsetDateTime::now_utc());
@@ -8521,10 +8631,14 @@ pub(crate) async fn verify_bugs(
});
}
// Ensure work dir exists for artifacts/scripts
let _ = tokio_fs::create_dir_all(&batch.work_dir).await;
let mut command_results: Vec<BugCommandResult> = Vec::new();
let mut futures = futures::stream::iter(plans.into_iter().map(|plan| {
let repo_path = batch.repo_path.clone();
async move { execute_bug_command(plan, repo_path).await }
let work_dir = batch.work_dir.clone();
async move { execute_bug_command(plan, repo_path, work_dir).await }
}))
.buffer_unordered(8)
.collect::<Vec<_>>()
@@ -8603,6 +8717,415 @@ pub(crate) async fn verify_bugs(
Ok(BugVerificationOutcome { bugs, logs })
}
#[derive(Debug, Deserialize)]
struct AccountPlanItem {
action: String,
#[serde(default)]
login_url: Option<String>,
#[serde(default)]
tool: Option<String>,
#[serde(default)]
script: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
struct AccountsOutputJson {
accounts: Vec<AccountPair>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct AccountPair {
username: String,
password: String,
}
fn parse_accounts_inline(text: &str) -> Vec<AccountPair> {
// Accept formats like: user:pass, user2:pass2
let mut out = Vec::new();
for chunk in text.split(',') {
let part = chunk.trim();
if part.is_empty() {
continue;
}
if let Some((u, p)) = part.split_once(':') {
let u = u.trim();
let p = p.trim();
if !u.is_empty() && !p.is_empty() {
out.push(AccountPair {
username: u.to_string(),
password: p.to_string(),
});
}
}
}
out
}
async fn write_accounts(work_dir: &Path, creds: &[AccountPair]) -> Result<PathBuf, String> {
let path = work_dir.join("credentials.json");
let json = serde_json::to_vec_pretty(&AccountsOutputJson {
accounts: creds.to_vec(),
})
.map_err(|e| e.to_string())?;
tokio_fs::write(&path, json)
.await
.map_err(|e| format!("Failed to write {}: {e}", path.display()))?;
Ok(path)
}
#[allow(clippy::too_many_arguments)]
async fn setup_accounts(
client: &Client,
provider: &ModelProviderInfo,
auth: &Option<CodexAuth>,
model: &str,
snapshot: &SecurityReviewSnapshot,
work_dir: &Path,
progress_sender: Option<AppEventSender>,
) -> Result<Option<Vec<AccountPair>>, String> {
if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(
"Preparing test accounts for validation...".to_string(),
));
}
let findings = build_validation_findings_context(snapshot);
let prompt = VALIDATION_ACCOUNTS_PROMPT_TEMPLATE.replace("{findings}", &findings);
let metrics = Arc::new(ReviewMetrics::default());
let response = call_model(
client,
provider,
auth,
model,
VALIDATION_ACCOUNTS_SYSTEM_PROMPT,
&prompt,
metrics,
0.0,
)
.await
.map_err(|e| format!("Account planning failed: {e}"))?;
let mut chosen: Option<AccountPlanItem> = None;
for line in response.text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
if let Ok(item) = serde_json::from_str::<AccountPlanItem>(trimmed) {
chosen = Some(item);
break;
}
}
let Some(plan) = chosen else {
return Ok(None);
};
if plan.action.eq_ignore_ascii_case("register")
&& plan.tool.as_deref().unwrap_or("") == "python"
&& plan.script.as_ref().is_some()
{
// Write and run inline script
let _ = tokio_fs::create_dir_all(work_dir).await;
let script_path = work_dir.join("register_accounts.py");
let code = plan.script.as_ref().unwrap();
tokio_fs::write(&script_path, code.as_bytes())
.await
.map_err(|e| format!("Failed to write {}: {e}", script_path.display()))?;
let mut cmd = Command::new("python");
cmd.arg(&script_path);
if let Some(url) = plan.login_url.as_ref() {
cmd.arg(url);
}
let output = cmd
.output()
.await
.map_err(|e| format!("Failed to run python: {e}"))?;
let success = output.status.success();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !success {
if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(format!(
"Account registration failed: {}",
summarize_process_output(false, &stdout, &stderr)
)));
}
return Ok(None);
}
// Try to parse JSON from stdout
let creds = if let Ok(json) = serde_json::from_str::<AccountsOutputJson>(stdout.trim()) {
json.accounts
} else {
// best-effort: parse user:pass lines
let pairs = parse_accounts_inline(stdout.trim());
if pairs.len() >= 2 { pairs } else { Vec::new() }
};
if creds.len() < 2 {
return Ok(None);
}
return Ok(Some(creds));
}
// Manual fallback
if let Some(tx) = progress_sender.as_ref() {
let (resp_tx, resp_rx) = oneshot::channel();
tx.send(AppEvent::OpenRegistrationPrompt {
url: plan.login_url.clone(),
responder: resp_tx,
});
if let Ok(Some(input)) = resp_rx.await {
let creds = parse_accounts_inline(&input);
if creds.len() >= 2 {
return Ok(Some(creds));
}
}
}
Ok(None)
}
#[derive(Debug, Deserialize)]
struct ValidationPlanItem {
id_kind: String,
id_value: usize,
tool: String,
#[serde(default)]
target: Option<String>,
#[serde(default)]
script: Option<String>,
}
fn is_high_risk(bug: &SecurityReviewBug) -> bool {
let sev = bug.severity.to_ascii_lowercase();
if sev.contains("critical") || sev.contains("high") {
return true;
}
if let Some(rank) = bug.risk_rank {
return rank <= 5;
}
false
}
fn build_validation_findings_context(snapshot: &SecurityReviewSnapshot) -> String {
let mut selected: Vec<&BugSnapshot> = snapshot
.bugs
.iter()
.filter(|b| is_high_risk(&b.bug))
.collect();
// Keep to a reasonable number to bound prompt size
selected.sort_by_key(|b| b.bug.risk_rank.unwrap_or(usize::MAX));
if selected.len() > 6 {
selected.truncate(6);
}
let mut out = String::new();
for item in selected {
let rank = item
.bug
.risk_rank
.map(|r| format!("#{r}"))
.unwrap_or_else(|| "N/A".to_string());
let types = if item.bug.verification_types.is_empty() {
"[]".to_string()
} else {
format!("{:?}", item.bug.verification_types)
};
// Include the original markdown so the model can infer concrete targets
let _ = writeln!(
&mut out,
"- id_kind: {}\n id_value: {}\n risk_rank: {}\n title: {}\n severity: {}\n verification_types: {}\n details:\n{}\n---\n",
if item.bug.risk_rank.is_some() {
"risk_rank"
} else {
"summary_id"
},
item.bug.risk_rank.unwrap_or(item.bug.summary_id),
rank,
item.bug.title,
item.bug.severity,
types,
indent_block(&item.original_markdown, 2)
);
}
out
}
fn indent_block(s: &str, spaces: usize) -> String {
let pad = " ".repeat(spaces);
s.lines()
.map(|l| format!("{pad}{l}"))
.collect::<Vec<_>>()
.join("\n")
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn run_web_validation(
repo_path: PathBuf,
snapshot_path: PathBuf,
bugs_path: PathBuf,
report_path: Option<PathBuf>,
report_html_path: Option<PathBuf>,
provider: ModelProviderInfo,
auth: Option<CodexAuth>,
model: String,
progress_sender: Option<AppEventSender>,
) -> Result<(), BugVerificationFailure> {
// Load snapshot
let bytes = tokio_fs::read(&snapshot_path)
.await
.map_err(|e| BugVerificationFailure {
message: format!("Failed to read {}: {e}", snapshot_path.display()),
logs: vec![],
})?;
let snapshot: SecurityReviewSnapshot =
serde_json::from_slice(&bytes).map_err(|e| BugVerificationFailure {
message: format!("Failed to parse {}: {e}", snapshot_path.display()),
logs: vec![],
})?;
let client = Client::new();
if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(
"Planning web/API validation for high-risk findings...".to_string(),
));
}
// Ensure we have test accounts before validation
let work_dir = snapshot_path
.parent()
.map(|p| p.join("validation"))
.unwrap_or_else(|| repo_path.join(".codex_validation"));
let _ = tokio_fs::create_dir_all(&work_dir).await;
if let Some(creds) = setup_accounts(
&client,
&provider,
&auth,
&model,
&snapshot,
&work_dir,
progress_sender.clone(),
)
.await
.map_err(|e| BugVerificationFailure {
message: e,
logs: vec![],
})? {
let path = write_accounts(&work_dir, &creds)
.await
.map_err(|e| BugVerificationFailure {
message: e,
logs: vec![],
})?;
if let Some(tx) = progress_sender.as_ref() {
let names: Vec<String> = creds.iter().map(|p| p.username.clone()).collect();
tx.send(AppEvent::SecurityReviewLog(format!(
"Registered {} test accounts: {} (stored in {})",
creds.len(),
names.join(", "),
display_path_for(&path, &repo_path)
)));
}
} else if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(
"Proceeding without auto-registered accounts; user may have registered manually."
.to_string(),
));
}
// Build prompt
let findings = build_validation_findings_context(&snapshot);
let prompt = VALIDATION_PLAN_PROMPT_TEMPLATE.replace("{findings}", &findings);
let metrics = Arc::new(ReviewMetrics::default());
let response = call_model(
&client,
&provider,
&auth,
&model,
VALIDATION_PLAN_SYSTEM_PROMPT,
&prompt,
metrics.clone(),
0.0,
)
.await
.map_err(|err| BugVerificationFailure {
message: format!("Validation planning failed: {err}"),
logs: vec![],
})?;
let mut logs: Vec<String> = Vec::new();
if let Some(reasoning) = response.reasoning.as_ref() {
for line in reasoning
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
{
let truncated = truncate_text(line, MODEL_REASONING_LOG_MAX_GRAPHEMES);
let msg = format!("Model reasoning: {truncated}");
if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(msg.clone()));
}
logs.push(msg);
}
}
let mut requests: Vec<BugVerificationRequest> = Vec::new();
for line in response.text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let parsed: ValidationPlanItem = match serde_json::from_str(trimmed) {
Ok(v) => v,
Err(_) => continue,
};
if parsed.id_kind == "setup" {
// Already handled by setup_accounts() pre-step; skip.
continue;
}
let id = match parsed.id_kind.as_str() {
"risk_rank" => BugIdentifier::RiskRank(parsed.id_value),
"summary_id" => BugIdentifier::SummaryId(parsed.id_value),
_ => continue,
};
let tool = match parsed.tool.to_ascii_lowercase().as_str() {
"playwright" => BugVerificationTool::Playwright,
"curl" => BugVerificationTool::Curl,
"python" => BugVerificationTool::Python,
_ => continue,
};
requests.push(BugVerificationRequest {
id,
tool,
target: parsed.target.clone(),
script_path: None,
script_inline: parsed.script.clone(),
});
}
let batch = BugVerificationBatchRequest {
snapshot_path,
bugs_path,
report_path,
report_html_path,
repo_path,
work_dir,
requests,
};
if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(
"Executing validation checks...".to_string(),
));
}
let outcome = verify_bugs(batch).await?;
for line in outcome.logs {
if let Some(tx) = progress_sender.as_ref() {
tx.send(AppEvent::SecurityReviewLog(line.clone()));
}
}
Ok(())
}
#[derive(Debug, Clone)]
struct ModelCallOutput {
text: String,

View File

@@ -17,6 +17,8 @@ pub enum SlashCommand {
Review,
#[strum(serialize = "secreview")]
SecReview,
/// Validate high-risk findings from the last security review
Validate,
New,
Init,
Compact,
@@ -40,6 +42,7 @@ impl SlashCommand {
SlashCommand::Compact => "summarize conversation to prevent hitting the context limit",
SlashCommand::Review => "review my current changes and find issues",
SlashCommand::SecReview => "run an AppSec security review over the repo",
SlashCommand::Validate => "validate high-risk findings (web + api)",
SlashCommand::Undo => "restore the workspace to the last Codex snapshot",
SlashCommand::Quit => "exit Codex",
SlashCommand::Diff => "show git diff (including untracked files)",
@@ -71,6 +74,7 @@ impl SlashCommand {
| SlashCommand::Approvals
| SlashCommand::Review
| SlashCommand::SecReview
| SlashCommand::Validate
| SlashCommand::Logout => false,
SlashCommand::Diff
| SlashCommand::Mention