Matt
039ac6f9d3
Add Proxmox Backup Server datastore management: overview, datastore detail, download/prune/verify/GC dialogs, usePbs hook, backend commands, and tests. Fix macOS keychain re-writes failing with 'item already exists': replace keyring 3 with keyring-core plus per-platform stores (macOS Keychain, Windows Credential Manager, Linux keyutils), recover by deleting the stale item and retrying once, and surface actionable messages for locked keychains.
1097 lines
43 KiB
Rust
1097 lines
43 KiB
Rust
//! LIVE READ-ONLY integration test harness.
|
|
//!
|
|
//! Drives the app's real HTTP stack against a real Proxmox cluster to verify
|
|
//! every read endpoint and cluster connectivity. All tests are `#[ignore]`d
|
|
//! and only run with `--ignored`; they read `LIVE_PVE_URL` / `LIVE_PVE_USER` /
|
|
//! `LIVE_PVE_PASS` from the environment and skip (with an explanation) when
|
|
//! `LIVE_PVE_PASS` is unset.
|
|
//!
|
|
//! HARD RULE: no state-changing HTTP request is ever issued against Proxmox.
|
|
//! The only non-GET request is the authentication handshake
|
|
//! `POST /api2/json/access/ticket`, which only issues a session ticket.
|
|
//!
|
|
//! The OS keyring is deliberately never touched: `set_session_ticket` injects
|
|
//! the password-mode session into the in-memory connection state, so
|
|
//! `login_with_password`/`connect` (which write to the keyring) are NOT used.
|
|
//!
|
|
//! Run with:
|
|
//! TMPDIR=/home/user/.cache/clustri-build-tmp \
|
|
//! LIVE_PVE_URL=https://10.0.0.22:8006 LIVE_PVE_USER=ro_opencode@pve \
|
|
//! LIVE_PVE_PASS='...' \
|
|
//! cargo test --test live -- --ignored --nocapture
|
|
|
|
use std::collections::BTreeSet;
|
|
use std::time::Duration;
|
|
|
|
use clustri::{
|
|
api_request, derive_node_url, AuthContext, AuthMode, ConnectionConfig, ConnectionManager,
|
|
EndpointConfig, ServerType,
|
|
};
|
|
use reqwest::Method;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Environment + shared helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Reads the live environment. Returns `None` (with an explanation printed)
|
|
/// when `LIVE_PVE_PASS` is unset or empty so the test can skip cleanly.
|
|
fn live_env() -> Option<(String, String, String)> {
|
|
let pass = std::env::var("LIVE_PVE_PASS").ok().filter(|p| !p.is_empty())?;
|
|
let url = std::env::var("LIVE_PVE_URL")
|
|
.unwrap_or_else(|_| "https://10.0.0.22:8006".to_string());
|
|
let user = std::env::var("LIVE_PVE_USER")
|
|
.unwrap_or_else(|_| "ro_opencode@pve".to_string());
|
|
Some((url, user, pass))
|
|
}
|
|
|
|
fn skip_reason() -> String {
|
|
"LIVE_PVE_PASS env var is not set — skipping live test. Set LIVE_PVE_PASS (and optionally LIVE_PVE_URL / LIVE_PVE_USER) to run it.".to_string()
|
|
}
|
|
|
|
/// Prints a section label to stderr (visible with `--nocapture`).
|
|
fn label(msg: &str) {
|
|
eprintln!("\n========== {} ==========", msg);
|
|
}
|
|
|
|
/// Builds the same permissive HTTP client the app uses at the transport layer
|
|
/// (self-signed Proxmox certs are the norm; trust is an application-level
|
|
/// concern that does not apply to a throwaway read-only harness).
|
|
fn live_client() -> Result<reqwest::Client, String> {
|
|
reqwest::Client::builder()
|
|
.danger_accept_invalid_certs(true)
|
|
.connect_timeout(Duration::from_secs(10))
|
|
.timeout(Duration::from_secs(45))
|
|
.build()
|
|
.map_err(|e| format!("failed to build HTTP client: {}", e))
|
|
}
|
|
|
|
/// Performs the password authentication handshake against the real server:
|
|
/// `POST {url}/api2/json/access/ticket` with a form-encoded body. Returns
|
|
/// `(ticket, csrf_token)`. This mirrors the app's own `login_with_password`
|
|
/// transport (which is unusable here only because it also writes to the
|
|
/// keyring). This is the ONLY state-adjacent request in the harness and it
|
|
/// only issues a session ticket.
|
|
async fn login(url: &str, user: &str, pass: &str) -> Result<(String, String), String> {
|
|
let client = live_client()?;
|
|
let body = url::form_urlencoded::Serializer::new(String::new())
|
|
.append_pair("username", user)
|
|
.append_pair("password", pass)
|
|
.finish();
|
|
let login_url = format!("{}/api2/json/access/ticket", url.trim_end_matches('/'));
|
|
let response = client
|
|
.post(&login_url)
|
|
.header(reqwest::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
|
.body(body)
|
|
.send()
|
|
.await
|
|
.map_err(|e| format!("login POST failed: {}", e))?;
|
|
let status = response.status();
|
|
let body: serde_json::Value = response
|
|
.json()
|
|
.await
|
|
.map_err(|e| format!("login response parse failed: {}", e))?;
|
|
if !status.is_success() {
|
|
return Err(format!("login failed (HTTP {}): {}", status, body));
|
|
}
|
|
let ticket = body["data"]["ticket"]
|
|
.as_str()
|
|
.ok_or_else(|| format!("no ticket in login response: {}", body))?
|
|
.to_string();
|
|
let csrf = body["data"]["CSRFPreventionToken"]
|
|
.as_str()
|
|
.unwrap_or("")
|
|
.to_string();
|
|
Ok((ticket, csrf))
|
|
}
|
|
|
|
/// Builds a `ConnectionManager` with one password-mode connection (`"conn"`,
|
|
/// `accept_untrusted: true`) and injects the session ticket in memory via
|
|
/// `set_session_ticket` (no keyring). Returns the manager, the temp dir
|
|
/// (kept alive so the connections file stays valid), and the raw ticket.
|
|
async fn manager_with_session(
|
|
url: &str,
|
|
user: &str,
|
|
pass: &str,
|
|
) -> Result<(ConnectionManager, tempfile::TempDir, String), String> {
|
|
let (ticket, csrf) = login(url, user, pass).await?;
|
|
let dir = tempfile::tempdir().map_err(|e| format!("tempdir failed: {}", e))?;
|
|
let path = dir.path().join("connections.json");
|
|
let mut manager = ConnectionManager::new();
|
|
let config = ConnectionConfig {
|
|
id: "conn".to_string(),
|
|
name: "conn".to_string(),
|
|
primary: EndpointConfig {
|
|
url: url.trim_end_matches('/').to_string(),
|
|
node: None,
|
|
token: None,
|
|
},
|
|
fallbacks: vec![],
|
|
cert_fingerprint: None,
|
|
trusted: false,
|
|
accept_untrusted: true,
|
|
status: "disconnected".to_string(),
|
|
cluster_name: None,
|
|
is_cluster: false,
|
|
auth_mode: "password".to_string(),
|
|
username: Some(user.to_string()),
|
|
nodes: vec![],
|
|
cluster_id: None,
|
|
server_type: "pve".to_string(),
|
|
};
|
|
manager
|
|
.add_connection(config, &path)
|
|
.await
|
|
.map_err(|e| format!("add_connection failed: {}", e))?;
|
|
manager
|
|
.set_session_ticket("conn", &ticket, &csrf)
|
|
.await
|
|
.map_err(|e| format!("set_session_ticket failed: {}", e))?;
|
|
Ok((manager, dir, ticket))
|
|
}
|
|
|
|
/// A password-mode `AuthContext` carrying the login ticket (cookie auth).
|
|
fn ticket_auth(ticket: &str) -> AuthContext {
|
|
AuthContext {
|
|
mode: AuthMode::Password,
|
|
token: None,
|
|
ticket: Some(ticket.to_string()),
|
|
csrf_token: None,
|
|
server_type: ServerType::Pve,
|
|
}
|
|
}
|
|
|
|
/// Fetches the raw (unparsed) VM/container config object for printing the
|
|
/// verbatim disk / net value strings.
|
|
async fn raw_config(
|
|
url: &str,
|
|
ticket: &str,
|
|
node: &str,
|
|
vmid: u32,
|
|
vm_type: &str,
|
|
) -> Result<serde_json::Value, String> {
|
|
let client = live_client()?;
|
|
let path = format!("/nodes/{}/{}/{}/config", node, vm_type, vmid);
|
|
api_request(&client, url, Method::GET, &path, &ticket_auth(ticket), &[], None)
|
|
.await
|
|
.map_err(|e| e.to_string())
|
|
}
|
|
|
|
/// Raw authenticated GET that keeps the HTTP status and body even on a
|
|
/// non-success response, so parameter-verification failures can be inspected
|
|
/// verbatim. Cookie auth mirrors `api_request` password mode.
|
|
async fn raw_fetch(
|
|
url: &str,
|
|
ticket: &str,
|
|
path: &str,
|
|
query: &[(&str, &str)],
|
|
) -> Result<(reqwest::StatusCode, serde_json::Value), String> {
|
|
let client = live_client()?;
|
|
let mut target = format!("{}/api2/json{}", url.trim_end_matches('/'), path);
|
|
if !query.is_empty() {
|
|
let q = query
|
|
.iter()
|
|
.map(|(k, v)| format!("{}={}", k, v))
|
|
.collect::<Vec<_>>()
|
|
.join("&");
|
|
target.push('?');
|
|
target.push_str(&q);
|
|
}
|
|
let response = client
|
|
.get(&target)
|
|
.header(reqwest::header::COOKIE, format!("PVEAuthCookie={}", ticket))
|
|
.send()
|
|
.await
|
|
.map_err(|e| format!("GET {} failed: {}", target, e))?;
|
|
let status = response.status();
|
|
let body: serde_json::Value = response
|
|
.json()
|
|
.await
|
|
.unwrap_or_else(|_| serde_json::Value::Null);
|
|
Ok((status, body))
|
|
}
|
|
|
|
/// Prints a compact view of a raw JSON value: the `data` payload when the
|
|
/// standard `{ "data": ... }` envelope is present (an array becomes a count +
|
|
/// sample; an object prints in full), otherwise the whole envelope so error
|
|
/// bodies (`errors` / `message`) stay visible.
|
|
fn print_raw(label: &str, status: reqwest::StatusCode, body: &serde_json::Value) {
|
|
let payload = match body.get("data") {
|
|
Some(serde_json::Value::Null) | None => body.clone(),
|
|
Some(inner) => inner.clone(),
|
|
};
|
|
let text = serde_json::to_string(&payload).unwrap_or_default();
|
|
match &payload {
|
|
serde_json::Value::Array(items) => {
|
|
eprintln!(" {} -> HTTP {} array ({} entries)", label, status, items.len());
|
|
for item in items.iter().take(3) {
|
|
eprintln!(" {}", serde_json::to_string(item).unwrap_or_default());
|
|
}
|
|
if items.len() > 3 {
|
|
eprintln!(" ... ({} more)", items.len() - 3);
|
|
}
|
|
}
|
|
serde_json::Value::Object(_) => {
|
|
eprintln!(" {} -> HTTP {}: {}", label, status, text);
|
|
}
|
|
other => eprintln!(" {} -> HTTP {}: {}", label, status, other),
|
|
}
|
|
}
|
|
|
|
/// The entries of a raw response's `data` field when it is an array.
|
|
fn data_array(body: &serde_json::Value) -> Vec<serde_json::Value> {
|
|
body.get("data")
|
|
.and_then(|d| d.as_array())
|
|
.cloned()
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn is_disk_key(key: &str) -> bool {
|
|
let bus_key = |bus: &str| {
|
|
key.strip_prefix(bus)
|
|
.map_or(false, |suffix| !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()))
|
|
};
|
|
["scsi", "virtio", "ide", "sata", "nvme"].iter().any(|bus| bus_key(bus))
|
|
|| key == "rootfs"
|
|
|| bus_key("mp") // LXC mount points (mp0, mp1, ...)
|
|
}
|
|
|
|
fn is_net_key(key: &str) -> bool {
|
|
key.strip_prefix("net")
|
|
.map_or(false, |suffix| !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit()))
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 1: every read endpoint parses against real data
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn live_all_read_endpoints_parse() {
|
|
let Some((url, user, pass)) = live_env() else {
|
|
eprintln!("{}", skip_reason());
|
|
return;
|
|
};
|
|
eprintln!("Live target: {} (user {})", url, user);
|
|
let (manager, _dir, ticket) = match manager_with_session(&url, &user, &pass).await {
|
|
Ok(v) => v,
|
|
Err(e) => panic!("setup failed: {}", e),
|
|
};
|
|
|
|
let mut failures: Vec<String> = Vec::new();
|
|
let mut first_online_node: Option<String> = None;
|
|
|
|
// ---- get_nodes --------------------------------------------------------
|
|
label("get_nodes");
|
|
match manager.get_nodes("conn").await {
|
|
Ok(nodes) => {
|
|
for n in &nodes {
|
|
eprintln!(
|
|
" node={:<10} status={:<8} cpu={:<6} maxcpu={:<4} uptime={}",
|
|
n.node, n.status, n.cpu, n.maxcpu, n.uptime
|
|
);
|
|
}
|
|
first_online_node = nodes.iter().find(|n| n.status == "online").map(|n| n.node.clone());
|
|
eprintln!(" first_online_node = {:?}", first_online_node);
|
|
}
|
|
Err(e) => failures.push(format!("get_nodes: {}", e)),
|
|
}
|
|
|
|
// ---- get_cluster_status ------------------------------------------------
|
|
label("get_cluster_status");
|
|
let mut cluster_nodes_len = 0usize;
|
|
let mut cluster_status_ok = false;
|
|
match manager.get_cluster_status("conn").await {
|
|
Ok(status) => {
|
|
cluster_status_ok = true;
|
|
let nodes = status.nodes.as_ref();
|
|
cluster_nodes_len = nodes.map(|n| n.len()).unwrap_or(0);
|
|
eprintln!(" cluster name={:?} id={:?} node_count={}", status.name, status.id, cluster_nodes_len);
|
|
for cn in nodes.into_iter().flatten() {
|
|
eprintln!(
|
|
" node name={:<10} nodeid={:<4} online={} local={:?} ip={:?}",
|
|
cn.name, cn.nodeid, cn.online, cn.local, cn.ip
|
|
);
|
|
}
|
|
}
|
|
Err(e) => {
|
|
failures.push(format!("get_cluster_status: {}", e));
|
|
eprintln!(" ERROR: {} (the endpoint was denied / errored)", e);
|
|
}
|
|
}
|
|
if cluster_status_ok {
|
|
assert!(
|
|
cluster_nodes_len >= 2,
|
|
"this is a real cluster: expected >= 2 nodes in /cluster/status, got {}",
|
|
cluster_nodes_len
|
|
);
|
|
} else {
|
|
eprintln!(
|
|
" [note] /cluster/status was not readable for this user, so the '>= 2 cluster nodes' assertion was NOT evaluated"
|
|
);
|
|
}
|
|
|
|
// ---- get_vms -----------------------------------------------------------
|
|
label("get_vms");
|
|
let mut vms = Vec::new();
|
|
match manager.get_vms("conn").await {
|
|
Ok(list) => {
|
|
eprintln!(" total VMs: {}", list.len());
|
|
eprintln!(" {:<6} {:<5} {:<9} {:<8} {}", "vmid", "type", "status", "node", "name");
|
|
for v in &list {
|
|
eprintln!(
|
|
" {:<6} {:<5} {:<9} {:<8} {:?}",
|
|
v.vmid, v.r#type, v.status, v.node, v.name
|
|
);
|
|
}
|
|
vms = list;
|
|
}
|
|
Err(e) => failures.push(format!("get_vms: {}", e)),
|
|
}
|
|
|
|
// ---- get_storage ---------------------------------------------------------
|
|
label("get_storage");
|
|
let mut storages = Vec::new();
|
|
match manager.get_storage("conn").await {
|
|
Ok(list) => {
|
|
eprintln!(" total storages: {}", list.len());
|
|
for s in &list {
|
|
eprintln!(
|
|
" storage={:<20} type={:<6} content={:<30} enabled={} active={} shared={} node={}",
|
|
s.storage, s.r#type, s.content, s.enabled, s.active, s.shared, s.node
|
|
);
|
|
}
|
|
storages = list;
|
|
}
|
|
Err(e) => failures.push(format!("get_storage: {}", e)),
|
|
}
|
|
if storages.is_empty() {
|
|
eprintln!(" [note] no storages are visible to this user — per-storage content/detail checks below will be skipped");
|
|
}
|
|
|
|
// ---- per-storage content + detail -----------------------------------------
|
|
label("per-storage content & detail");
|
|
for s in &storages {
|
|
// Probe each storage through its OWN node (the `node` field of the
|
|
// storage list entry). Node-local storages are only servable by their
|
|
// owning node — the first-online-node path fails with "storage not
|
|
// available on node". Shared storages respond from any node.
|
|
let node = if s.node.is_empty() {
|
|
first_online_node.as_deref().unwrap_or("")
|
|
} else {
|
|
s.node.as_str()
|
|
};
|
|
if node.is_empty() {
|
|
failures.push(format!(
|
|
"get_storage_content({}): no node available for storage",
|
|
s.storage
|
|
));
|
|
failures.push(format!(
|
|
"get_storage_detail({}): no node available for storage detail",
|
|
s.storage
|
|
));
|
|
continue;
|
|
}
|
|
// Content via the storage's own node.
|
|
match manager
|
|
.get_storage_content("conn", &s.storage, Some(node))
|
|
.await
|
|
{
|
|
Ok(items) => {
|
|
eprintln!(" storage={:<20} content(Some({})) -> {} entries", s.storage, node, items.len());
|
|
if items.is_empty() {
|
|
eprintln!(" (empty)");
|
|
}
|
|
}
|
|
Err(e) => {
|
|
failures.push(format!("get_storage_content({}, Some({})): {}", s.storage, node, e))
|
|
}
|
|
}
|
|
// Detail via the storage's own node.
|
|
match manager.get_storage_detail("conn", node, &s.storage).await {
|
|
Ok(detail) => {
|
|
eprintln!(
|
|
" storage={:<20} detail({}) -> storage={} type={} content={} enabled={} active={} shared={} used={} total={} avail={}",
|
|
s.storage,
|
|
node,
|
|
detail.storage,
|
|
detail.r#type,
|
|
detail.content,
|
|
detail.enabled,
|
|
detail.active,
|
|
detail.shared,
|
|
detail.used,
|
|
detail.total,
|
|
detail.avail
|
|
);
|
|
}
|
|
Err(e) => {
|
|
failures.push(format!("get_storage_detail({}, {}): {}", s.storage, node, e));
|
|
// Raw capture: the exact field shape the struct mis-parses.
|
|
let path = format!("/nodes/{}/storage/{}/status", node, s.storage);
|
|
match raw_fetch(&url, &ticket, &path, &[]).await {
|
|
Ok((status, raw)) => print_raw(
|
|
&format!("RAW /storage/{}/status on {}", s.storage, node),
|
|
status,
|
|
&raw,
|
|
),
|
|
Err(e2) => eprintln!(" RAW /storage/{}/status fetch failed: {}", s.storage, e2),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- raw /cluster/resources (type=storage) field shapes --------------------
|
|
label("raw /cluster/resources (storage entries)");
|
|
match raw_fetch(&url, &ticket, "/cluster/resources", &[("type", "storage")]).await {
|
|
Ok((status, body)) => {
|
|
let entries = data_array(&body);
|
|
eprintln!(" HTTP {} -> {} storage entries", status, entries.len());
|
|
if let Some(first) = entries.first() {
|
|
if let Some(obj) = first.as_object() {
|
|
let mut keys: Vec<_> = obj.keys().cloned().collect();
|
|
keys.sort();
|
|
eprintln!(" field keys: {:?}", keys);
|
|
}
|
|
}
|
|
for entry in entries.iter().take(3) {
|
|
eprintln!(" {}", serde_json::to_string(entry).unwrap_or_default());
|
|
}
|
|
}
|
|
Err(e) => eprintln!(" ERROR: raw /cluster/resources fetch failed: {}", e),
|
|
}
|
|
|
|
// ---- get_tasks -------------------------------------------------------------
|
|
label("get_tasks");
|
|
match manager.get_tasks("conn").await {
|
|
Ok(tasks) => {
|
|
eprintln!(" task count: {}", tasks.len());
|
|
for t in tasks.iter().take(5) {
|
|
eprintln!(
|
|
" upid={} node={} type={} id={} user={} status={:?} exitstatus={:?}",
|
|
t.upid, t.node, t.r#type, t.id, t.user, t.status, t.exitstatus
|
|
);
|
|
}
|
|
if tasks.len() > 5 {
|
|
eprintln!(" ... ({} more)", tasks.len() - 5);
|
|
}
|
|
}
|
|
Err(e) => {
|
|
eprintln!(" ERROR: get_tasks failed: {}", e);
|
|
failures.push(format!("get_tasks: {}", e));
|
|
}
|
|
}
|
|
|
|
// ---- raw task probes ----------------------------------------------------------
|
|
// The app now sends plain `/cluster/tasks` (the server rejects any
|
|
// `limit=` query with HTTP 400); probe variants to confirm the rejection
|
|
// and the node-scoped `/nodes/{node}/tasks` alternative.
|
|
label("raw task probes");
|
|
let probe_node = first_online_node.as_deref().unwrap_or("conn");
|
|
let probes: Vec<(String, Vec<(&str, &str)>)> = vec![
|
|
("/cluster/tasks".to_string(), vec![]),
|
|
("/cluster/tasks".to_string(), vec![("limit", "50")]),
|
|
("/cluster/tasks".to_string(), vec![("limit", "10")]),
|
|
(format!("/nodes/{}/tasks", probe_node), vec![("limit", "50")]),
|
|
];
|
|
for (path, query) in &probes {
|
|
match raw_fetch(&url, &ticket, path, query).await {
|
|
Ok((status, body)) => print_raw(&format!("GET {}", path), status, &body),
|
|
Err(e) => eprintln!(" GET {} failed: {}", path, e),
|
|
}
|
|
}
|
|
|
|
// ---- get_backup_jobs (parsed + full raw JSON) --------------------------------
|
|
label("get_backup_jobs");
|
|
match manager.get_backup_jobs("conn").await {
|
|
Ok(jobs) => {
|
|
eprintln!(" parsed job count: {}", jobs.len());
|
|
for j in &jobs {
|
|
eprintln!(
|
|
" id={} store={} schedule={:?} all={} enabled={} node={:?} vmid={:?} compress={:?} mode={:?} quiet={:?}",
|
|
j.id, j.store, j.schedule, j.all, j.enabled, j.node, j.vmid, j.compress, j.mode, j.quiet
|
|
);
|
|
}
|
|
}
|
|
Err(e) => {
|
|
eprintln!(" ERROR: get_backup_jobs failed: {}", e);
|
|
failures.push(format!("get_backup_jobs: {}", e));
|
|
}
|
|
}
|
|
// Raw JSON of every job entry (captures the full field shape the parsed
|
|
// struct drops).
|
|
let client = match live_client() {
|
|
Ok(c) => Some(c),
|
|
Err(e) => {
|
|
failures.push(format!("raw backup jobs client: {}", e));
|
|
None
|
|
}
|
|
};
|
|
if let Some(client) = client {
|
|
let auth = ticket_auth(&ticket);
|
|
match api_request(&client, &url, Method::GET, "/cluster/backup", &auth, &[], None).await {
|
|
Ok(raw) => {
|
|
let entries = raw.as_array().cloned().unwrap_or_default();
|
|
eprintln!(" RAW entries ({}):", entries.len());
|
|
for entry in &entries {
|
|
eprintln!(" {}", serde_json::to_string(entry).unwrap_or_default());
|
|
}
|
|
}
|
|
Err(e) => {
|
|
eprintln!(" ERROR: raw /cluster/backup fetch failed: {}", e);
|
|
failures.push(format!("raw /cluster/backup: {}", e));
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- get_backups(conn, None) aggregation --------------------------------------
|
|
label("get_backups(conn, None)");
|
|
match manager.get_backups("conn", None).await {
|
|
Ok(backups) => {
|
|
eprintln!(" aggregated backup count: {}", backups.len());
|
|
let mut storages = BTreeSet::new();
|
|
for b in &backups {
|
|
storages.insert(b.storage.clone());
|
|
if let Some(prefix) = b.volid.split(':').next() {
|
|
storages.insert(prefix.to_string());
|
|
}
|
|
}
|
|
eprintln!(" storages aggregated (from storage field / volid prefix): {:?}", storages);
|
|
for b in backups.iter().take(8) {
|
|
eprintln!(
|
|
" volid={} backupid={} type={} id={} time={} size={} storage={}",
|
|
b.volid, b.backupid, b.backup_type, b.backup_id, b.backup_time, b.size, b.storage
|
|
);
|
|
}
|
|
if backups.len() > 8 {
|
|
eprintln!(" ... ({} more)", backups.len() - 8);
|
|
}
|
|
}
|
|
Err(e) => failures.push(format!("get_backups(conn, None): {}", e)),
|
|
}
|
|
|
|
// ---- raw backup-storage content ------------------------------------------------
|
|
// The backup jobs target kashyyk / bb-pve-b2 / ots-pve-b2; dump the raw
|
|
// node-scoped content so the actual backup volumeids are visible (the
|
|
// parsed content(None) path returned empty).
|
|
label("raw backup storage content");
|
|
let backup_storages = ["kashyyk", "bb-pve-b2", "ots-pve-b2"];
|
|
let content_node = first_online_node.as_deref().unwrap_or("conn");
|
|
for storage in backup_storages {
|
|
let path = format!("/nodes/{}/storage/{}/content", content_node, storage);
|
|
match raw_fetch(&url, &ticket, &path, &[("content", "backup")]).await {
|
|
Ok((status, body)) => print_raw(
|
|
&format!("GET {}/content?content=backup", path),
|
|
status,
|
|
&body,
|
|
),
|
|
Err(e) => eprintln!(" GET {} failed: {}", path, e),
|
|
}
|
|
}
|
|
|
|
// ---- per-guest disk / network / snapshot raw data --------------------------------
|
|
label("guest disks / nics / snapshots");
|
|
let first_qemu = vms.iter().find(|v| v.r#type == "qemu");
|
|
let first_lxc = vms.iter().find(|v| v.r#type == "lxc");
|
|
let guests: Vec<(&str, u32, &str)> = vec![
|
|
("qemu", first_qemu.map(|v| v.vmid).unwrap_or(0), first_qemu.map(|v| v.node.as_str()).unwrap_or("")),
|
|
("lxc", first_lxc.map(|v| v.vmid).unwrap_or(0), first_lxc.map(|v| v.node.as_str()).unwrap_or("")),
|
|
];
|
|
for (vm_type, vmid, node) in guests {
|
|
if vmid == 0 || node.is_empty() {
|
|
eprintln!(" no {} guest found — skipping", vm_type);
|
|
continue;
|
|
}
|
|
eprintln!(" --- {} guest vmid={} on node={} ---", vm_type, vmid, node);
|
|
|
|
match manager.get_disks("conn", node, vmid, vm_type).await {
|
|
Ok(disks) => {
|
|
eprintln!(" parsed disks ({}):", disks.len());
|
|
for d in &disks {
|
|
eprintln!(
|
|
" {}: storage={} size={} format={}",
|
|
d.device, d.storage, d.size, d.format
|
|
);
|
|
}
|
|
}
|
|
Err(e) => failures.push(format!("get_disks({}, {}, {}): {}", node, vmid, vm_type, e)),
|
|
}
|
|
// Raw disk value strings from the VM config.
|
|
match raw_config(&url, &ticket, node, vmid, vm_type).await {
|
|
Ok(cfg) => {
|
|
eprintln!(" RAW disk config value strings:");
|
|
if let Some(obj) = cfg.as_object() {
|
|
let mut keys: Vec<_> = obj.keys().cloned().collect();
|
|
keys.sort();
|
|
for k in keys {
|
|
if is_disk_key(&k) {
|
|
if let Some(v) = obj.get(&k).and_then(|v| v.as_str()) {
|
|
eprintln!(" {} = {}", k, v);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Err(e) => failures.push(format!("raw_config({}, {}, {}): {}", node, vmid, vm_type, e)),
|
|
}
|
|
|
|
match manager.get_network_interfaces("conn", node, vmid, vm_type).await {
|
|
Ok(nics) => {
|
|
eprintln!(" parsed nics ({}):", nics.len());
|
|
for nic in &nics {
|
|
eprintln!(
|
|
" {}: model={} mac={} bridge={:?} tag={:?} firewall={:?} link_down={:?}",
|
|
nic.name, nic.model, nic.macaddr, nic.bridge, nic.tag, nic.firewall, nic.link_down
|
|
);
|
|
}
|
|
}
|
|
Err(e) => {
|
|
failures.push(format!("get_network_interfaces({}, {}, {}): {}", node, vmid, vm_type, e))
|
|
}
|
|
}
|
|
match raw_config(&url, &ticket, node, vmid, vm_type).await {
|
|
Ok(cfg) => {
|
|
eprintln!(" RAW net config value strings:");
|
|
if let Some(obj) = cfg.as_object() {
|
|
let mut keys: Vec<_> = obj.keys().cloned().collect();
|
|
keys.sort();
|
|
for k in keys {
|
|
if is_net_key(&k) {
|
|
if let Some(v) = obj.get(&k).and_then(|v| v.as_str()) {
|
|
eprintln!(" {} = {}", k, v);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Err(e) => failures.push(format!("raw_config net({}, {}, {}): {}", node, vmid, vm_type, e)),
|
|
}
|
|
|
|
match manager.get_snapshots("conn", node, vmid, vm_type).await {
|
|
Ok(snaps) => {
|
|
eprintln!(" snapshots ({}):", snaps.len());
|
|
for s in &snaps {
|
|
eprintln!(
|
|
" name={} vmstate={} snaptime={} parent={:?}",
|
|
s.name, s.vmstate, s.snaptime, s.parent
|
|
);
|
|
}
|
|
}
|
|
Err(e) => failures.push(format!("get_snapshots({}, {}, {}): {}", node, vmid, vm_type, e)),
|
|
}
|
|
}
|
|
|
|
// ---- get_websocket_url ---------------------------------------------------------
|
|
label("get_websocket_url");
|
|
let ws_node = first_online_node.as_deref().unwrap_or("conn");
|
|
match manager.get_websocket_url("conn", ws_node).await {
|
|
Ok(ws_url) => eprintln!(" websocket base origin: {}", ws_url),
|
|
Err(e) => failures.push(format!("get_websocket_url: {}", e)),
|
|
}
|
|
|
|
// ---- summary --------------------------------------------------------------------
|
|
label("SUMMARY");
|
|
if failures.is_empty() {
|
|
eprintln!(" ALL READ ENDPOINTS PASSED");
|
|
} else {
|
|
eprintln!(" {} endpoint failures:", failures.len());
|
|
for f in &failures {
|
|
eprintln!(" - {}", f);
|
|
}
|
|
panic!("{} endpoint failure(s):\n - {}", failures.len(), failures.join("\n - "));
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 2: node discovery + per-node reachability
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn live_discover_nodes_and_reachability() {
|
|
let Some((url, user, pass)) = live_env() else {
|
|
eprintln!("{}", skip_reason());
|
|
return;
|
|
};
|
|
eprintln!("Live target: {} (user {})", url, user);
|
|
let (mut manager, _dir, ticket) = match manager_with_session(&url, &user, &pass).await {
|
|
Ok(v) => v,
|
|
Err(e) => panic!("setup failed: {}", e),
|
|
};
|
|
|
|
label("discover_nodes");
|
|
let primary_url = url.trim_end_matches('/').to_string();
|
|
let discovered = match manager.discover_nodes("conn").await {
|
|
Ok(d) => Some(d),
|
|
Err(e) => {
|
|
eprintln!(" discover_nodes FAILED: {}", e);
|
|
eprintln!(
|
|
" NOTE: this is a credentials/permission limitation — /cluster/status needs Sys.Audit on /,\n which this user lacks, so cluster node IPs are unavailable for URL derivation.\n Falling back to name-derived URLs via derive_node_url."
|
|
);
|
|
None
|
|
}
|
|
};
|
|
|
|
// Candidate node URLs: real discovered URLs when discovery worked,
|
|
// otherwise name-derived URLs (best effort).
|
|
let mut candidates: Vec<(String, bool)> = Vec::new(); // (url, is_primary)
|
|
match &discovered {
|
|
Some(nodes) => {
|
|
for n in nodes {
|
|
eprintln!(
|
|
" node={:<10} status={:<8} is_primary={:<5} local={} url={}",
|
|
n.name, n.status, n.is_primary, n.local, n.url
|
|
);
|
|
candidates.push((n.url.clone(), n.is_primary));
|
|
}
|
|
}
|
|
None => {
|
|
let nodes = manager
|
|
.get_nodes("conn")
|
|
.await
|
|
.expect("get_nodes must still work even without /cluster/status");
|
|
for n in &nodes {
|
|
let cand_url = derive_node_url(&primary_url, None, &n.node);
|
|
let is_primary = cand_url.eq_ignore_ascii_case(&primary_url);
|
|
eprintln!(
|
|
" node={:<10} url={}{}",
|
|
n.node,
|
|
cand_url,
|
|
if is_primary { " [PRIMARY]" } else { " (name-derived; cluster IP unknown)" }
|
|
);
|
|
candidates.push((cand_url, is_primary));
|
|
}
|
|
if !candidates.iter().any(|(_, is_primary)| *is_primary) {
|
|
eprintln!(
|
|
" (no candidate matched the primary URL — treating the connection's primary URL itself as the primary node)"
|
|
);
|
|
candidates.push((primary_url.clone(), true));
|
|
}
|
|
}
|
|
}
|
|
|
|
label("per-node reachability (GET /api2/json/version)");
|
|
let client = match live_client() {
|
|
Ok(c) => c,
|
|
Err(e) => panic!("client build failed: {}", e),
|
|
};
|
|
let auth = ticket_auth(&ticket);
|
|
let mut reachable = 0usize;
|
|
let mut unreachable: Vec<String> = Vec::new();
|
|
|
|
// Direct reachability against each candidate node URL.
|
|
for (cand_url, is_primary) in &candidates {
|
|
match api_request(&client, cand_url, Method::GET, "/version", &auth, &[], None).await {
|
|
Ok(value) => {
|
|
reachable += 1;
|
|
eprintln!(
|
|
" {} REACHABLE -> version={} release={} (primary={})",
|
|
cand_url,
|
|
value["version"].as_str().unwrap_or("?"),
|
|
value["release"].as_str().unwrap_or("?"),
|
|
is_primary
|
|
);
|
|
}
|
|
Err(e) => {
|
|
unreachable.push(format!("{}: {}", cand_url, e));
|
|
eprintln!(" {} UNREACHABLE -> {} (primary={})", cand_url, e, is_primary);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Cluster-proxy reachability: every node queried through the primary
|
|
// endpoint. This works even when the node IPs are unknown (pveproxy
|
|
// forwards node-scoped requests to the target node) and proves each node
|
|
// is up and servable by the cluster.
|
|
let node_names: Vec<String> = match &discovered {
|
|
Some(nodes) => nodes.iter().map(|n| n.name.clone()).collect(),
|
|
None => manager
|
|
.get_nodes("conn")
|
|
.await
|
|
.expect("get_nodes must work")
|
|
.into_iter()
|
|
.map(|n| n.node)
|
|
.collect(),
|
|
};
|
|
let mut proxied_reachable = 0usize;
|
|
for name in &node_names {
|
|
let path = format!("/nodes/{}/version", name);
|
|
match api_request(&client, &primary_url, Method::GET, &path, &auth, &[], None).await {
|
|
Ok(value) => {
|
|
proxied_reachable += 1;
|
|
eprintln!(
|
|
" primary-proxy /nodes/{}/version -> REACHABLE version={}",
|
|
name,
|
|
value["version"].as_str().unwrap_or("?")
|
|
);
|
|
}
|
|
Err(e) => eprintln!(" primary-proxy /nodes/{}/version -> FAILED: {}", name, e),
|
|
}
|
|
}
|
|
|
|
// Assert the primary node/endpoint is reachable.
|
|
assert!(
|
|
!candidates.is_empty(),
|
|
"no node candidates at all — cluster unreachable?"
|
|
);
|
|
let primary_target = candidates
|
|
.iter()
|
|
.find(|(_, is_primary)| *is_primary)
|
|
.map(|(u, _)| u.clone())
|
|
.unwrap_or_else(|| primary_url.clone());
|
|
let primary_result = api_request(&client, &primary_target, Method::GET, "/version", &auth, &[], None).await;
|
|
assert!(
|
|
primary_result.is_ok(),
|
|
"primary node {} must be reachable, got: {:?}",
|
|
primary_target,
|
|
primary_result.err()
|
|
);
|
|
eprintln!(
|
|
" primary node {} REACHABLE (asserted) — version={}",
|
|
primary_target,
|
|
primary_result.unwrap()["version"].as_str().unwrap_or("?")
|
|
);
|
|
eprintln!(
|
|
" direct reachability: {} of {} candidate URLs reachable; cluster-proxy: {} of {} nodes reachable",
|
|
reachable,
|
|
candidates.len(),
|
|
proxied_reachable,
|
|
node_names.len()
|
|
);
|
|
if !unreachable.is_empty() {
|
|
eprintln!(" NOTE (not asserted): unreachable candidates:");
|
|
for u in &unreachable {
|
|
eprintln!(" - {}", u);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 3: failover to a configured fallback endpoint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn live_failover_to_fallback() {
|
|
let Some((url, user, pass)) = live_env() else {
|
|
eprintln!("{}", skip_reason());
|
|
return;
|
|
};
|
|
eprintln!("Live target: {} (user {})", url, user);
|
|
let (ticket, csrf) = match login(&url, &user, &pass).await {
|
|
Ok(v) => v,
|
|
Err(e) => panic!("login failed: {}", e),
|
|
};
|
|
let dir = match tempfile::tempdir() {
|
|
Ok(d) => d,
|
|
Err(e) => panic!("tempdir failed: {}", e),
|
|
};
|
|
let path = dir.path().join("connections.json");
|
|
let mut manager = ConnectionManager::new();
|
|
let config = ConnectionConfig {
|
|
id: "bad".to_string(),
|
|
name: "bad".to_string(),
|
|
// 127.0.0.1:1 is always connection-refused (nothing listens on port 1).
|
|
primary: EndpointConfig {
|
|
url: "https://127.0.0.1:1".to_string(),
|
|
node: None,
|
|
token: None,
|
|
},
|
|
fallbacks: vec![EndpointConfig {
|
|
url: url.trim_end_matches('/').to_string(),
|
|
node: None,
|
|
token: None,
|
|
}],
|
|
cert_fingerprint: None,
|
|
trusted: false,
|
|
accept_untrusted: true,
|
|
status: "disconnected".to_string(),
|
|
cluster_name: None,
|
|
is_cluster: false,
|
|
auth_mode: "password".to_string(),
|
|
username: Some(user.to_string()),
|
|
nodes: vec![],
|
|
cluster_id: None,
|
|
server_type: "pve".to_string(),
|
|
};
|
|
manager
|
|
.add_connection(config, &path)
|
|
.await
|
|
.expect("connection 'bad' should be added");
|
|
manager
|
|
.set_session_ticket("bad", &ticket, &csrf)
|
|
.await
|
|
.expect("session ticket should be injected");
|
|
|
|
label("failover: get_nodes via fallback");
|
|
let nodes = match manager.get_nodes("bad").await {
|
|
Ok(n) => n,
|
|
Err(e) => panic!("get_nodes('bad') failed — fallback did not serve the request: {}", e),
|
|
};
|
|
eprintln!(" get_nodes succeeded via the real fallback endpoint; node count = {}", nodes.len());
|
|
for n in &nodes {
|
|
eprintln!(" node={} status={}", n.node, n.status);
|
|
}
|
|
|
|
let status = manager
|
|
.runtime_status("bad")
|
|
.expect("runtime_status should be readable");
|
|
eprintln!(" runtime_status('bad') = {:?}", status);
|
|
assert_eq!(status, "failover", "expected runtime_status 'failover' after the primary was refused, got {:?}", status);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 4: cluster identity is consistent across every discovered node
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn live_cluster_identity_consistent_across_nodes() {
|
|
let Some((url, user, pass)) = live_env() else {
|
|
eprintln!("{}", skip_reason());
|
|
return;
|
|
};
|
|
eprintln!("Live target: {} (user {})", url, user);
|
|
let (mut manager, dir, ticket) = match manager_with_session(&url, &user, &pass).await {
|
|
Ok(v) => v,
|
|
Err(e) => panic!("setup failed: {}", e),
|
|
};
|
|
let path = dir.path().join("connections.json");
|
|
|
|
let discovered = match manager.discover_nodes("conn").await {
|
|
Ok(d) => d,
|
|
Err(e) => {
|
|
eprintln!(" discover_nodes FAILED: {}", e);
|
|
eprintln!(
|
|
" Cannot verify cluster identity: /cluster/status is permission-denied for this user\n (needs Sys.Audit on /), so per-node URLs cannot be derived and every node would report the\n same 403. There is no identity data to compare."
|
|
);
|
|
panic!(
|
|
"live_cluster_identity_consistent_across_nodes: cannot derive per-node cluster identity because /cluster/status is denied: {}",
|
|
e
|
|
);
|
|
}
|
|
};
|
|
eprintln!("discovered {} nodes", discovered.len());
|
|
|
|
label("per-node cluster identity");
|
|
let mut identities: Vec<(String, String, String)> = Vec::new(); // (name, id, name)
|
|
let mut direct_failures: Vec<String> = Vec::new();
|
|
for (i, node) in discovered.iter().enumerate() {
|
|
let id = format!("node-{}", i);
|
|
let config = ConnectionConfig {
|
|
id: id.clone(),
|
|
name: id.clone(),
|
|
primary: EndpointConfig {
|
|
url: node.url.clone(),
|
|
node: Some(node.name.clone()),
|
|
token: None,
|
|
},
|
|
fallbacks: vec![],
|
|
cert_fingerprint: None,
|
|
trusted: false,
|
|
accept_untrusted: true,
|
|
status: "disconnected".to_string(),
|
|
cluster_name: None,
|
|
is_cluster: false,
|
|
auth_mode: "password".to_string(),
|
|
username: Some(user.to_string()),
|
|
nodes: vec![],
|
|
cluster_id: None,
|
|
server_type: "pve".to_string(),
|
|
};
|
|
manager
|
|
.add_connection(config, &path)
|
|
.await
|
|
.expect("node connection should be added");
|
|
manager
|
|
.set_session_ticket(&id, &ticket, "")
|
|
.await
|
|
.expect("node session ticket should be injected");
|
|
|
|
match manager.get_cluster_status(&id).await {
|
|
Ok(status) => {
|
|
let node_count = status.nodes.as_ref().map(|n| n.len()).unwrap_or(0);
|
|
eprintln!(
|
|
" node={:<10} url={} cluster_id={:?} cluster_name={:?} node_count={}",
|
|
node.name, node.url, status.id, status.name, node_count
|
|
);
|
|
identities.push((node.name.clone(), status.id.clone(), status.name.clone()));
|
|
}
|
|
Err(e) => {
|
|
eprintln!(" node={:<10} url={} FAILED: {}", node.name, node.url, e);
|
|
direct_failures.push(e.to_string());
|
|
}
|
|
}
|
|
}
|
|
|
|
if identities.is_empty() {
|
|
// When every per-node attempt failed at the transport layer, the
|
|
// derived node URLs are not routable from this host (cluster-internal
|
|
// IPs). That is an environment limitation, not an identity mismatch:
|
|
// verify the cluster identity through the primary endpoint (every
|
|
// node-scoped request is forwarded by pveproxy to the same cluster)
|
|
// and skip the direct comparison.
|
|
let all_transport =
|
|
!direct_failures.is_empty() && direct_failures.iter().all(|f| f.contains("Cannot connect"));
|
|
if all_transport {
|
|
eprintln!(
|
|
" [skip] all {} per-node direct connections failed with transport errors — derived\n\
|
|
node URLs are not routable from this host. Verifying cluster identity through\n\
|
|
the primary endpoint instead.",
|
|
direct_failures.len()
|
|
);
|
|
let proxy_nodes = manager
|
|
.get_nodes("conn")
|
|
.await
|
|
.expect("get_nodes must still work through the primary");
|
|
match manager.get_cluster_status("conn").await {
|
|
Ok(status) => {
|
|
let node_count = status.nodes.as_ref().map(|n| n.len()).unwrap_or(0);
|
|
eprintln!(
|
|
" primary-proxy /cluster/status -> cluster_id={:?} cluster_name={:?} node_count={}",
|
|
status.id, status.name, node_count
|
|
);
|
|
assert!(
|
|
!status.id.is_empty(),
|
|
"cluster id from primary /cluster/status is empty — not a real cluster?"
|
|
);
|
|
eprintln!(
|
|
" primary-proxy per-node reachability: {} nodes served through the primary",
|
|
proxy_nodes.len()
|
|
);
|
|
eprintln!(
|
|
" CONSISTENT (via primary proxy): cluster id {:?} serves all {} nodes",
|
|
status.id,
|
|
proxy_nodes.len()
|
|
);
|
|
}
|
|
Err(e) => panic!("cluster identity via primary proxy failed: {}", e),
|
|
}
|
|
return;
|
|
}
|
|
assert!(
|
|
!identities.is_empty(),
|
|
"no node reported a cluster status — cannot compare cluster identity (failures: {:?})",
|
|
direct_failures
|
|
);
|
|
}
|
|
let (first_name, first_id, _) = &identities[0];
|
|
assert!(
|
|
!first_id.is_empty(),
|
|
"cluster id reported by node {} is empty — not a real cluster?",
|
|
first_name
|
|
);
|
|
for (name, cid, _) in identities.iter().skip(1) {
|
|
assert_eq!(
|
|
cid, first_id,
|
|
"cluster id mismatch: node {} reports {:?} but node {} reports {:?} — same-cluster merging would break",
|
|
name, cid, first_name, first_id
|
|
);
|
|
}
|
|
eprintln!(
|
|
" CONSISTENT: all {} nodes report cluster id {:?}",
|
|
identities.len(),
|
|
first_id
|
|
);
|
|
}
|
|
|