clustri/src-tauri/tests/failover.rs

//! Integration tests for automatic failover across cluster endpoints.
//!
//! Transport-level failures (connection refused, timeouts, DNS resolution)
//! must be distinguishable from real authentication failures so the request
//! core can rotate to the next configured endpoint. These tests cover the
//! error mapping in `api_request` and the rotation logic in
//! `Connection::request`.

use httpmock::prelude::*;
use httpmock::Mock;
use clustri::{
    api_request, AuthContext, AuthMode, ConnectionConfig, ConnectionManager, EndpointConfig, Error,
    ServerType,
};
use reqwest::Client;
use reqwest::Method as RMethod;

const TOKEN: &str = "root@pam!failover-token";

/// Returns a URL pointing at a closed TCP port: a listener is bound to an
/// ephemeral port, its address is captured, and the listener is dropped so
/// any subsequent connection attempt is refused.
fn closed_port_url() -> String {
    let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind");
    let addr = listener.local_addr().expect("address should be available");
    drop(listener);
    format!("http://{}", addr)
}

fn token_auth() -> AuthContext {
    AuthContext {
        mode: AuthMode::Token,
        token: Some(TOKEN.to_string()),
        ticket: None,
        csrf_token: None,
        server_type: ServerType::Pve,
    }
}

/// Builds a `ConnectionManager` with a single token-mode connection whose
/// primary endpoint is `primary_url` and whose fallbacks are `fallback_urls`.
async fn setup_manager_with_fallbacks(
    primary_url: &str,
    fallback_urls: &[String],
) -> (ConnectionManager, tempfile::TempDir) {
    let dir = tempfile::tempdir().expect("temp dir should be created");
    let path = dir.path().join("connections.json");
    let mut manager = ConnectionManager::new();
    let config = ConnectionConfig {
        id: "conn".to_string(),
        name: "conn".to_string(),
        primary: EndpointConfig {
            url: primary_url.to_string(),
            node: None,
            token: Some(TOKEN.to_string()),
        },
        fallbacks: fallback_urls
            .iter()
            .map(|url| EndpointConfig {
                url: url.clone(),
                node: None,
                token: Some(TOKEN.to_string()),
            })
            .collect(),
        cert_fingerprint: None,
        trusted: false,
        accept_untrusted: true,
        status: "disconnected".to_string(),
        cluster_name: None,
        is_cluster: false,
        auth_mode: "token".to_string(),
        username: None,
        nodes: vec![],
        cluster_id: None,
        server_type: "pve".to_string(),
    };
    manager
        .add_connection(config, &path)
        .await
        .expect("connection should be added");
    (manager, dir)
}

/// Stubs `GET /api2/json/cluster/resources?type=vm` on `server`.
fn stub_vm_resources<'a>(server: &'a MockServer, status: u16, body: &str) -> Mock<'a> {
    server.mock(|when, then| {
        when.method(GET)
            .path("/api2/json/cluster/resources")
            .query_param("type", "vm");
        then.status(status)
            .header("content-type", "application/json")
            .body(body);
    })
}

#[tokio::test]
async fn transport_error_maps_to_connection_failed() {
    let url = closed_port_url();
    let error = api_request(
        &Client::new(),
        &url,
        RMethod::GET,
        "/version",
        &token_auth(),
        &[],
        None,
    )
    .await
    .expect_err("connection refused must fail");

    assert!(
        matches!(error, Error::ConnectionFailed(_)),
        "expected ConnectionFailed, got: {}",
        error
    );
}

#[tokio::test]
async fn request_rotates_to_fallback_when_primary_down() {
    let fallback = MockServer::start();
    let fallback_mock = stub_vm_resources(
        &fallback,
        200,
        &serde_json::json!({
            "data": [
                {"vmid": 100, "name": "web01", "type": "qemu", "status": "running",
                 "node": "pve1"}
            ]
        })
        .to_string(),
    );

    let primary_url = closed_port_url();
    let (manager, _dir) = setup_manager_with_fallbacks(&primary_url, &[fallback.base_url()]).await;

    let vms = manager
        .get_vms("conn")
        .await
        .expect("request should succeed via the fallback endpoint");

    assert_eq!(vms.len(), 1);
    assert_eq!(vms[0].vmid, 100);
    assert_eq!(vms[0].name.as_deref(), Some("web01"));
    assert_eq!(
        manager
            .runtime_status("conn")
            .expect("status should be readable"),
        "failover"
    );
    fallback_mock.assert();
}

#[tokio::test]
async fn non_transport_error_does_not_rotate() {
    let primary = MockServer::start();
    let fallback = MockServer::start();
    let primary_mock = stub_vm_resources(&primary, 500, r#"{"message":"boom"}"#);
    let fallback_mock = stub_vm_resources(&fallback, 200, r#"{"data":[]}"#);

    let (manager, _dir) =
        setup_manager_with_fallbacks(&primary.base_url(), &[fallback.base_url()]).await;

    let error = manager
        .get_vms("conn")
        .await
        .expect_err("a 500 response must surface as an ApiError");

    assert!(
        matches!(error, Error::ApiError(ref message) if message == "boom"),
        "expected ApiError with message 'boom', got: {}",
        error
    );
    primary_mock.assert();
    assert_eq!(
        fallback_mock.calls(),
        0,
        "a non-transport error must not trigger failover"
    );
}

#[tokio::test]
async fn request_succeeds_on_primary_again_when_it_returns() {
    let primary = MockServer::start();
    let fallback = MockServer::start();
    let primary_mock = stub_vm_resources(&primary, 200, r#"{"data":[]}"#);
    let fallback_mock = stub_vm_resources(&fallback, 200, r#"{"data":[]}"#);

    let (manager, _dir) =
        setup_manager_with_fallbacks(&primary.base_url(), &[fallback.base_url()]).await;

    let vms = manager
        .get_vms("conn")
        .await
        .expect("request should succeed on the primary endpoint");

    assert!(vms.is_empty());
    primary_mock.assert();
    assert_eq!(fallback_mock.calls(), 0);
    assert_eq!(
        manager
            .runtime_status("conn")
            .expect("status should be readable"),
        "connected"
    );
}