diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3496da..661e305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,6 +263,10 @@ jobs: - name: "EXEC: {Check cluster status}, independent" run: ./foc-devnet status + # Configure /etc/hosts for inter-SP communication via host.docker.internal + - name: "EXEC: {Configure host.docker.internal for SP-to-SP comms}, independent" + run: echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts + # Start the full Filecoin localnet cluster - name: "EXEC: {Start cluster}, independent" id: start_cluster diff --git a/README.md b/README.md index ff98b66..c2c981d 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,27 @@ A developer-friendly tool for spinning up complete Filecoin test networks with s Get up and running in three simple steps: -### Step 0: Ensure non-root user -`foc-devnet` requires itself to be run by a non-root user. Please ensure that you are running as a non-root user which is part of `docker` group. +### Prerequisites -Run the following to see your User ID and groups you are a part of: -``` +**Non-root user with Docker access**: `foc-devnet` must be run by a non-root user in the `docker` group. + +```bash echo $(id -u); groups | grep 'docker' ``` +**Configure host.docker.internal**: Add this entry to `/etc/hosts` so SP URLs work from both host and containers: + +```bash +echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts +``` + +This is required for SP-to-SP fetch. `foc-devnet start` will check for this and fail with instructions if not configured. + +For GitHub Actions, add this step before running foc-devnet: +```yaml +- run: echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts +``` + ### Step 1: Initialize ```bash diff --git a/README_ADVANCED.md b/README_ADVANCED.md index 4c3d75b..1754476 100644 --- a/README_ADVANCED.md +++ b/README_ADVANCED.md @@ -135,6 +135,76 @@ foc-devnet version --- +## Network Architecture & Inter-SP Communication + +### The Challenge + +Running multiple Service Providers (SPs) locally requires them to communicate with each other while remaining accessible from the host (for testing/synapse code). This creates a networking puzzle: + +- **Container-to-container communication:** Docker DNS names (e.g., `foc-curio-1`) work within the docker network but don't resolve from the host +- **Host-to-container communication:** `localhost` works from the host but means different things inside containers +- **The conflict:** Using `localhost:` breaks inter-SP comms because each container sees `localhost` as itself + +### How It Works: `host.docker.internal` + +foc-devnet solves this by using `host.docker.internal` as a unified endpoint that works consistently from both the host and all containers. + +**Setup (mostly automatic):** +1. Verify `/etc/hosts` has: `127.0.0.1 host.docker.internal` + - On macOS with Docker Desktop: automatic + - On Linux: foc-devnet checks for this and provides setup instructions +2. Each Curio container launches with: `--add-host=host.docker.internal:host-gateway` +3. SPs register in the service provider registry as: `http://host.docker.internal:` +4. Curio runs with `CURIO_PULL_ALLOW_INSECURE=1` to allow HTTP/internal connections + +**Why it works:** +- From the host: `host.docker.internal` → `/etc/hosts` → `127.0.0.1` +- From containers: `--add-host` mapping → routes back to the host's IP +- Same hostname everywhere = SPs can call each other + host can access SPs + +### Setup Requirements + +**macOS with Docker Desktop:** +- Works automatically, no setup needed + +**Linux:** +Add this line to `/etc/hosts`: +```bash +echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts +``` + +**CI/CD (GitHub Actions, etc.):** +Add before running foc-devnet: +```yaml +- run: echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts +``` + +### Validation + +foc-devnet validates DNS resolution before startup: +```bash +foc-devnet start +# ✓ host.docker.internal resolves to localhost +# Proceeding with startup... +``` + +If resolution fails, you'll get clear error messages with exact fix instructions for your platform. + +### Tradeoffs + +**Benefits:** +- Single endpoint works from everywhere (host + all containers) +- Enables inter-SP communication out of the box +- Minimal setup overhead +- No architectural changes to core components + +**Considerations:** +- Requires `/etc/hosts` modification (one-liner, done once per machine) +- `CURIO_PULL_ALLOW_INSECURE=1` bypasses TLS validation (acceptable for devnet) +- Docker must support `host-gateway` (standard in modern Docker versions) + +--- + ## Configuration System ### Config File Location @@ -1027,7 +1097,7 @@ docker logs foc--curio-2 # Query provider IDs cat ~/.foc-devnet/state/latest/pdp_sps/*.provider_id.json -# Access Yugabyte (one per SP) +# Access Yugabyte (one per SP, see below for more detail) docker exec -it foc--yugabyte-1 ysqlsh -h localhost -p 5433 # Query Lotus for miner info @@ -1040,6 +1110,16 @@ docker exec foc--builder cast call \ ``` +### Querying Yugabyte Database + +Each Curio has its own Yugabyte (curio-N → yugabyte-N). Tables are in `curio` schema. Credentials: `yugabyte`/`yugabyte`/`yugabyte` (user/pass/db). + +```bash +docker exec foc--yugabyte-1 bash -c "PGPASSWORD=yugabyte /yugabyte/bin/ysqlsh -h 127.0.0.1 -U yugabyte -d yugabyte -c \"\"" +``` + +Key tables: `curio.harmony_machines`, `curio.harmony_task`, `curio.harmony_task_history`, `curio.parked_pieces`. + --- ## Troubleshooting diff --git a/src/commands/start/curio/daemon.rs b/src/commands/start/curio/daemon.rs index 18ef72a..d723271 100644 --- a/src/commands/start/curio/daemon.rs +++ b/src/commands/start/curio/daemon.rs @@ -153,6 +153,8 @@ fn build_docker_create_args( container_name.to_string(), "--network".to_string(), pdp_miner_network_name(run_id, sp_index), + // Enable host.docker.internal for SP-to-SP fetch (resolves to host gateway) + "--add-host=host.docker.internal:host-gateway".to_string(), ]; // Port mappings - get dynamically allocated ports from context diff --git a/src/commands/start/curio/db_setup.rs b/src/commands/start/curio/db_setup.rs index ea6014b..b89fc50 100644 --- a/src/commands/start/curio/db_setup.rs +++ b/src/commands/start/curio/db_setup.rs @@ -68,6 +68,9 @@ pub fn build_foc_contract_env_vars(context: &SetupContext) -> Result .to_string(), ); + // Allow insecure sources (HTTP, localhost, private IPs) for SP-to-SP pull in devnet + env_vars.push("CURIO_PULL_ALLOW_INSECURE=1".to_string()); + Ok(env_vars) } diff --git a/src/commands/start/mod.rs b/src/commands/start/mod.rs index fb09d89..592ec7a 100644 --- a/src/commands/start/mod.rs +++ b/src/commands/start/mod.rs @@ -36,8 +36,71 @@ use crate::paths::{foc_devnet_config, foc_devnet_run_dir}; use crate::run_id::{create_latest_symlink, save_current_run_id}; use crate::version_info::write_version_file; pub use eth_acc_funding::constants::FEVM_ACCOUNTS_PREFUNDED; +use std::net::ToSocketAddrs; use std::path::{Path, PathBuf}; -use tracing::{info, warn}; +use tracing::{error, info, warn}; + +/// Check that host.docker.internal resolves to 127.0.0.1. +/// +/// This is required for SP-to-SP fetch to work. The hostname must resolve to localhost +/// so that URLs registered in the SP registry work from both the host and inside containers. +/// +/// On macOS with Docker Desktop, this works automatically. +/// On Linux, users must add `127.0.0.1 host.docker.internal` to /etc/hosts. +fn check_host_docker_internal() -> Result<(), Box> { + info!("Checking host.docker.internal resolution..."); + + // Try to resolve host.docker.internal:80 (port doesn't matter, just need DNS resolution) + match "host.docker.internal:80".to_socket_addrs() { + Ok(mut addrs) => { + // Check if any resolved address is 127.0.0.1 + let is_localhost = addrs.any(|addr| addr.ip().is_loopback()); + + if is_localhost { + info!("✓ host.docker.internal resolves to localhost"); + Ok(()) + } else { + error!("════════════════════════════════════════════════════════════════════"); + error!("ERROR: host.docker.internal does not resolve to localhost (127.0.0.1)"); + error!("════════════════════════════════════════════════════════════════════"); + error!(""); + error!("SP-to-SP fetch requires host.docker.internal to resolve to 127.0.0.1"); + error!("so that registered SP URLs work from both host and containers."); + error!(""); + error!("To fix this, add the following line to /etc/hosts:"); + error!(""); + error!(" 127.0.0.1 host.docker.internal"); + error!(""); + error!("You can do this with:"); + error!(" echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts"); + error!(""); + error!("════════════════════════════════════════════════════════════════════"); + Err("host.docker.internal must resolve to 127.0.0.1".into()) + } + } + Err(_) => { + error!("════════════════════════════════════════════════════════════════════"); + error!("ERROR: host.docker.internal does not resolve"); + error!("════════════════════════════════════════════════════════════════════"); + error!(""); + error!("SP-to-SP fetch requires host.docker.internal to resolve to 127.0.0.1"); + error!("so that registered SP URLs work from both host and containers."); + error!(""); + error!("Add the following line to /etc/hosts:"); + error!(""); + error!(" 127.0.0.1 host.docker.internal"); + error!(""); + error!("You can do this with:"); + error!(" echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts"); + error!(""); + error!("For GitHub Actions, add this step before running foc-devnet:"); + error!(" - run: echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts"); + error!(""); + error!("════════════════════════════════════════════════════════════════════"); + Err("host.docker.internal must be resolvable".into()) + } + } +} /// Stop any existing cluster before starting a new one. fn stop_existing_cluster() -> Result<(), Box> { @@ -401,6 +464,9 @@ pub fn start_cluster( run_id: String, notest: bool, ) -> Result<(), Box> { + // Check host.docker.internal resolution first (required for SP-to-SP fetch) + check_host_docker_internal()?; + stop_existing_cluster()?; let (volumes_dir, run_dir, run_id) = setup_directories_and_run_id(run_id)?; diff --git a/src/commands/start/pdp_service_provider/pdp_service_provider_step.rs b/src/commands/start/pdp_service_provider/pdp_service_provider_step.rs index beb8694..16c8f8c 100644 --- a/src/commands/start/pdp_service_provider/pdp_service_provider_step.rs +++ b/src/commands/start/pdp_service_provider/pdp_service_provider_step.rs @@ -201,7 +201,8 @@ impl Step for PdpSpRegistrationStep { let mut provider_ids = Vec::new(); for (sp_index, sp_address, sp_eth_address, pdp_port, should_approve) in sp_data { - let service_url = format!("http://localhost:{}", pdp_port); + // Use host.docker.internal so the URL works from both host and containers + let service_url = format!("http://host.docker.internal:{}", pdp_port); match registration::register_single_provider( ®istration::ProviderRegistrationParams { diff --git a/src/config.rs b/src/config.rs index 30fb4e7..a0a8bc1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -230,8 +230,8 @@ impl Default for Config { commit: "3bc6a7fd2d0b66119163c6759241a6ff74ac03e1".to_string(), }, yugabyte_download_url: Self::get_default_yugabyte_url(), - approved_pdp_sp_count: 1, - active_pdp_sp_count: 1, + approved_pdp_sp_count: 2, + active_pdp_sp_count: 2, } } }