diff --git a/.vitepress/config.mts b/.vitepress/config.mts index 195a8ce..0cd17bb 100644 --- a/.vitepress/config.mts +++ b/.vitepress/config.mts @@ -119,10 +119,13 @@ export default defineConfig({ function nav() { return [ + { + text: "Optimum Gateway", + link: "https://getoptimum.github.io/optimum-gateway/versions/latest/", + }, { text: "Menu", items: [ - // { text: "Get Started", link: "/docs/how-to-guides/overview" }, { text: "Learn", link: "/docs/learn/overview/intro" }, { text: "Resources", @@ -160,50 +163,12 @@ function sidebarHome() { text: "mump2p Protocol", link: "/docs/learn/overview/p2p.md", }, - { - text: "Optimum DeRAM", - link: "/docs/learn/overview/deram.md", - }, ], }, { text: "Getting Started", collapsed: false, items: [ - { - text: "Quick Start", - link: "/docs/guides/overview", - }, - { - text: "Getting Started with CLI", - link: "/docs/guides/01-getting-started-cli", - }, - { - text: "Getting Started with Docker", - link: "/docs/guides/02-getting-started-docker", - }, - { - text: "Understanding Key Parameters", - link: "/docs/guides/03-parameters", - }, - { - text: "Common Experiments", - link: "/docs/guides/04-experiments", - }, - { - text: "FAQ", - link: "/docs/guides/05-faq-glossary", - }, - ], - }, - { - text: "Ethereum Integration", - collapsed: false, - items: [ - { - text: "HOP Testing Suite", - link: "https://getoptimum.github.io/optimum-hop/", - }, { text: "Optimum Gateway", link: "https://getoptimum.github.io/optimum-gateway/versions/latest/", @@ -215,37 +180,17 @@ function sidebarHome() { collapsed: false, items: [ { - text: "mump2p Protocol", - collapsed: true, - items: [ - { - text: "Gossip", - link: "/docs/research/gossip/gossip", - }, - { - text: "Transport", - link: "/docs/research/gossip/transport", - }, - { - text: "Decentralized Access", - link: "/docs/research/gossip/decentralized-access", - } - ] + text: "Gossip", + link: "/docs/research/gossip/gossip", }, { - text: "Optimum DeRAM", - collapsed: true, - items: [ - { - text: "Atomicity and Consistency", - link: "/docs/research/deram/atomicity-consistency", - }, - { - text: "Decentralized Data Storage", - link: "/docs/research/deram/decentralized-storage", - } - ] - } + text: "Transport", + link: "/docs/research/gossip/transport", + }, + { + text: "Decentralized Access", + link: "/docs/research/gossip/decentralized-access", + }, ], }, ] diff --git a/docs/guides/01-getting-started-cli.md b/docs/guides/01-getting-started-cli.md deleted file mode 100644 index 3ea8b35..0000000 --- a/docs/guides/01-getting-started-cli.md +++ /dev/null @@ -1,531 +0,0 @@ -# Getting Started with mump2p CLI - -The `mump2p` CLI is the quickest way to interact with Optimum Network without running your own infrastructure. - -In the next 5 minutes, you'll have: - -* A working CLI -* Your first published message -* A subscription feeding you live data - -The `mump2p` CLI is your shortcut into `Optimum Network` — a high-performance, RLNC-enhanced peer-to-peer network. - -Instead of: - -* Manually locating and connecting to active Optimum Network nodes -* Handling low-level peer discovery and connection logic -* Managing complex network and encoding configurations - -The `mump2p` CLI connects you directly to our hosted `optimum-proxy` (available in multiple regions) and start sending or receiving messages instantly. -It connects to an `optimum-proxy` and lets you publish and subscribe to real-time topics — with authentication, usage tracking, and advanced delivery options. - -## Why Optimum Proxy? - -Optimum Network is a **peer-to-peer network** where nodes exchange messages over a `RLNC-enhanced` pubsub mesh. -If you connect directly to a P2P node, you need to: - -* Know node IP/port. -* Handle peer discovery. -* Many more complex configuration operations. - -The **Optimum Proxies** removes that complexity: - -* Acts as **points of entry**. -* Maintains connections to multiple Optimum Network nodes. -* Enforces thresholds and applies filters. -* Tracks usage and applies fair rate limits. - -With `mump2p`, you connect only to the proxy — it does the rest. - -## Why Authentication? - -Authentication in `mump2p-cli` is not just about logging in, it enables: - -* **Access Control**: Only authorized users can publish/subscribe to protected topics. -* **Rate Limits**: Prevents spam and ensures fair use. -* **Usage Tracking**: Monitor your publish/subscription stats. -* **Account Linking**: Associate activity with your user or team. - -## How It Fits into the Network - -![mump2p CLI Architecture](../../static/img/mump2p.png) - -* CLI talks to the Proxy via HTTP/WebSocket or gRPC. -* Proxy connects to the P2P Mesh (multiple nodes across regions). -* Mesh uses RLNC for efficient message delivery and reconstruction. -* Your client receives fully decoded messages in real-time. - -## 1. Install mump2p CLI - -### Quick Install (One Command) - -```bash -curl -sSL https://raw.githubusercontent.com/getoptimum/mump2p-cli/main/install.sh | bash -``` - -This automatically: - -* Detects your OS (Linux/macOS) -* Downloads the latest release -* Sets correct permissions -* Verifies installation works - -### Manual Install - -If you prefer manual installation: - -| OS | Command | -|---|---| -| **Linux** | `curl -L -o mump2p https://github.com/getoptimum/mump2p-cli/releases/latest/download/mump2p-linux && chmod +x mump2p` | -| **macOS** | `curl -L -o mump2p https://github.com/getoptimum/mump2p-cli/releases/latest/download/mump2p-mac && chmod +x mump2p` | - -### Verify Installation - -```bash -./mump2p version -``` - -> Always use the latest version from [mump2p-cli releases](https://github.com/getoptimum/mump2p-cli/releases) - -### 2. Authenticate - -Login via device authorization flow: - -```sh -./mump2p login -``` - -**Output:** - -```bash -Initiating authentication... - -To complete authentication: -1. Visit: https://your-auth-domain.auth0.com/activate?user_code=XXXX-XXXX -2. Or go to https://your-auth-domain.auth0.com/activate and enter code: XXXX-XXXX -3. This code expires in 15 minutes - -Waiting for you to complete authentication in the browser... - -✅ Successfully authenticated -Token expires at: 18 Aug 25 13:15 IST -``` - -1. CLI shows a URL and a code. -2. Open the URL in your browser. -3. Enter the code to complete authentication. -4. CLI stores a JWT for future requests. - -#### Check status - -```sh -./mump2p whoami -``` - -**Output:** - -```bash -Authentication Status: ----------------------- -Client ID: google-oauth2|100677750055416883405 -Expires: 18 Aug 25 13:15 IST -Valid for: 24h0m0s -Is Active: true - -Rate Limits: ------------- -Publish Rate: 1000 per hour -Publish Rate: 8 per second -Max Message Size: 4.00 MB -Daily Quota: 5120.00 MB -``` - -**Important: By default `Is Active` is `false`. Contact us to activate your account.** - -#### Other auth commands - -```sh -./mump2p refresh # Refresh token -./mump2p logout # Logout -``` - -#### Development Mode (No Authentication) - -For testing without authentication: - -```sh -./mump2p --disable-auth --client-id="test-client" publish --topic=demo --message="Hello" --service-url="http://proxy_1:8081" # localhost -./mump2p --disable-auth --client-id="test-client" subscribe --topic=demo --service-url="http://proxy_1:8081" # localhost -``` - -> **Complete Guide:** [Development mode documentation](https://github.com/getoptimum/mump2p-cli/blob/main/docs/guide.md#developmenttesting-mode) - covers all flags, usage, and examples - -#### Custom Authentication Path - -For production deployments or custom storage locations: - -```sh -./mump2p --auth-path /opt/mump2p/auth/token.yml login -# Or use environment variable -export MUMP2P_AUTH_PATH="/opt/mump2p/auth/token.yml" -``` - -> **Complete Guide:** [Custom auth path documentation](https://github.com/getoptimum/mump2p-cli/blob/main/docs/guide.md#custom-authentication-file-location) - use cases, security, deployment - -**Refresh Output:** - -```bash -Current token status: -Expires at: 18 Aug 25 13:15 IST -Valid for: 23h56m0s -Refreshing token... -✅ Token refreshed successfully -New expiration: 18 Aug 25 13:19 IST -Valid for: 24h0m0s -``` - -**Logout Output:** - -```bash -✅ Successfully logged out -``` - -### 3. Choose a Proxy Location - -**Available Service URLs:** - -| Location | URL | -| ------------------- | ------------------- | -| **Tokyo (Default)** | 34.146.222.111:8080 | -| **Tokyo** | 35.221.118.95:8080 | -| **Singapore** | 34.142.205.26:8080 | - -Use a custom location - -```sh ---service-url="http://34.142.205.26:8080" -``` - - -### 4. Subscribe to a Topic - -#### Basic subscription - -```sh -./mump2p subscribe --topic=demo -``` - -**Output:** - -```bash -claims is &{google-oauth2|100677750055416883405 2025-08-17 13:15:07 +0530 IST 2025-08-18 13:15:07 +0530 IST true 4194304 1000 8 5368709120 google-oauth2|100677750055416883405 1755416706719} -claims is google-oauth2|100677750055416883405 -Sending HTTP POST subscription request... -HTTP POST subscription successful: {"status":"subscribed","topic":"demo"} -Opening WebSocket connection... -Listening for messages on topic 'demo'... Press Ctrl+C to exit -``` - -#### Save messages locally - -```sh -./mump2p subscribe --topic=demo --persist=/path/to/ -``` - -**Output:** - -```bash -Persisting data to: /path/to/messages.log -claims is &{google-oauth2|100677750055416883405 2025-08-17 13:15:07 +0530 IST 2025-08-18 13:15:07 +0530 IST true 4194304 1000 8 5368709120 google-oauth2|100677750055416883405 1755416706719} -claims is google-oauth2|100677750055416883405 -Sending HTTP POST subscription request... -HTTP POST subscription successful: {"status":"subscribed","topic":"demo"} -Opening WebSocket connection... -Listening for messages on topic 'demo'... Press Ctrl+C to exit -``` - -**Persisted message format:** - -```bash -[2025-08-17T13:19:08+05:30] Testing persistence! -``` - -#### Forward to webhook - -Basic webhook forwarding: - -```sh -./mump2p subscribe --topic=demo --webhook=https://your-server.com/webhook -``` - -**Webhook with custom schema templates:** - -```sh -# Discord -./mump2p subscribe --topic=alerts --webhook="https://discord.com/api/webhooks/..." --webhook-schema='{"content":"{{.Message}}"}' - -# Slack -./mump2p subscribe --topic=notifications --webhook="https://hooks.slack.com/services/..." --webhook-schema='{"text":"{{.Message}}"}' - -# Custom JSON with metadata -./mump2p subscribe --topic=logs --webhook="https://your-server.com/webhook" --webhook-schema='{"message":"{{.Message}}","timestamp":"{{.Timestamp}}","topic":"{{.Topic}}"}' -``` - -> **Complete Guide:** [Webhook formatting documentation](https://github.com/getoptimum/mump2p-cli/blob/main/docs/guide.md#webhook-formatting) - Discord, Slack, Telegram templates, variables, queue options - -#### gRPC Subscription - -For high-performance streaming, use gRPC mode: - -```sh -./mump2p subscribe --topic=demo --grpc -``` - -**Output:** - -```bash -claims is &{google-oauth2|100677750055416883405 2025-08-21 16:01:29 +0530 IST 2025-08-22 16:01:29 +0530 IST true 4194304 1000 8 5368709120 google-oauth2|100677750055416883405 1755772288994} -claims is google-oauth2|100677750055416883405 -Sending HTTP POST subscription request... -HTTP POST subscription successful: {"client":"google-oauth2|100677750055416883405","status":"subscribed"} -Listening for messages on topic 'demo' via gRPC... Press Ctrl+C to exit -``` - - -### 5. Publish a Message - -#### Text - -```sh -./mump2p publish --topic=demo --message="Hello from CLI!" -``` - -**Output:** - -```bash -✅ Published inline message -{"status":"published","topic":"demo"} -``` - -#### File - -```sh -./mump2p publish --topic=demo --file=/path/to/file.json -``` - -**Output:** - -```bash -✅ Published sample-data.json -{"status":"published","topic":"demo"} -``` - -#### gRPC Publishing - -For high-performance publishing, use gRPC mode: - -```sh -./mump2p publish --topic=demo --message="Hello via gRPC!" --grpc -``` - -**Output:** - -```bash -✅ Published via gRPC inline message -``` - -#### With threshold - -```sh -./mump2p publish --topic=demo --message="High reliability" --threshold=0.9 -``` - - -### 6. Check Usage & Limits - -```sh -./mump2p usage -``` - -**Output:** - -```bash - Publish (hour): 0 / 1000 - Publish (second): 0 / 8 - Data Used: 0.0000 MB / 5120.0000 MB - Next Reset: 18 Aug 25 13:15 IST (24h0m0s from now) - Last Publish: 07 Aug 25 06:33 -0700 -``` - -Shows: - -* Publish count (per hour/day) -* Quota usage -* Time until reset -* Token expiry - - -### 7. List Active Topics - -Check which topics you're currently subscribed to: - -```sh -./mump2p list-topics -``` - -> **Complete Guide:** [List topics documentation](https://github.com/getoptimum/mump2p-cli/blob/main/docs/guide.md#list-your-active-topics) - multi-proxy support, output format - - -### 8. Check Proxy Health - -Monitor the health and system metrics of the proxy server: - -```sh -./mump2p health -``` - -**Output:** - -```bash -Proxy Health Status: -------------------- -Status: ok -Memory Used: 7.06% -CPU Used: 0.30% -Disk Used: 44.91% -``` - -#### Check specific proxy - -```sh -./mump2p health --service-url="http://35.221.118.95:8080" -``` - - -### 9. Debug Mode - -The `--debug` flag provides detailed timing and proxy information for troubleshooting and performance analysis. - -**What it shows:** - -* Message timestamps (send/receive times in nanoseconds) -* Proxy IP addresses (source and destination) -* Message metadata (size, hash, protocol) -* Sequential message numbering - -**Basic usage:** - -```sh -# Debug publish operations -./mump2p --debug publish --topic=test-topic --message='Hello World' - -# Debug subscribe operations -./mump2p --debug subscribe --topic=test-topic - -# Debug with gRPC -./mump2p --debug publish --topic=test-topic --message='Hello World' --grpc -./mump2p --debug subscribe --topic=test-topic --grpc -``` - -**Example output:** - -Publish: - -```text -Publish: sender_info:34.146.222.111, [send_time, size]:[1757606701424811000, 2010] topic:test-topic msg_hash:4bbac12f protocol:HTTP -``` - -Subscribe: - -```text -Recv: [1] receiver_addr:34.146.222.111 [recv_time, size]:[1757606701424811000, 2082] sender_addr:34.146.222.111 [send_time, size]:[1757606700160514000, 2009] topic:test-topic hash:8da69366 protocol:WebSocket -``` - -**Understanding the output:** - -* `[1]` - Message sequence number -* `[send_time, size]` - Unix timestamp (nanoseconds) and message size -* `msg_hash/hash` - First 8 characters of SHA256 hash for tracking -* Calculate latency: `recv_time - send_time` - -> **Complete Guide:** [Debug mode documentation](https://github.com/getoptimum/mump2p-cli/blob/main/docs/guide.md#debug-mode) - blast testing, load testing, performance analysis - - -### 10. Common Issues - -#### Unauthorized - -```sh -Error: your account is inactive -``` - -→ Contact admin to activate account. - -#### Rate limit exceeded - -```sh -Error: per-hour limit reached -``` - -→ Wait until reset or request higher tier. - -#### Connection refused - -```sh -Error: HTTP publish failed: dial tcp ... -``` - -→ Proxy not reachable. Check --service-url. - -#### Topic not assigned - -```sh -Error: publish error: topic not assigned -``` - -→ Topic needs to be subscribed to first or doesn't exist. - -#### Missing message or file - -```sh -Error: either --message or --file must be provided -``` - -→ Provide either --message or --file parameter. - -#### Conflicting parameters - -```sh -Error: only one of --message or --file should be used at a time -``` - -→ Use only one of --message or --file, not both. - -#### Authentication required - -```sh -Error: authentication required: token has expired, please login again -``` - -→ Run `./mump2p login` to authenticate. - - -### 11. Important Tips - -* Use descriptive topic names per team. -* Keep `whoami` and `usage` handy. -* For high-volume topics, increase webhook queue size. -* Start with hosted proxy, then try local deployment for full control. -* Subscribe to a topic before publishing to it. -* Use the `--service-url` flag to connect to different gateways for better performance. -* Use `--grpc` flag for high-performance streaming and publishing. -* Monitor proxy health with `./mump2p health` for troubleshooting. -* Use `--debug` flag for performance monitoring and troubleshooting. -* Check active topics with `./mump2p list-topics` to manage subscriptions. - -## Complete Documentation - -* **[Complete User Guide](https://github.com/getoptimum/mump2p-cli/blob/main/docs/guide.md)** - Advanced features, authentication, webhooks, debug mode -* **[CLI Repository](https://github.com/getoptimum/mump2p-cli)** - Source code and documentation -* **[FAQ & Troubleshooting](https://github.com/getoptimum/mump2p-cli#faq---common-issues--troubleshooting)** - Common issues and solutions -* **[Latest Releases](https://github.com/getoptimum/mump2p-cli/releases)** - Download latest version diff --git a/docs/guides/02-getting-started-docker.md b/docs/guides/02-getting-started-docker.md deleted file mode 100644 index 4b308ad..0000000 --- a/docs/guides/02-getting-started-docker.md +++ /dev/null @@ -1,331 +0,0 @@ -# Getting Started with Docker (Local Deployment) - -Running **Optimum Network** locally with Docker gives you **full control** over configuration, topology, and experiments. -You can run the network in two primary ways: - -**1. OptimumProxy + mump2p** — Clients connect to an **Optimum Proxy**, which manages P2P connections for them. - -![OptimumProxy + mump2p Architecture](../../static/img/docker_1.png) - -* Simplifies client configuration — only the Proxy address is needed. -* Proxy handles shard reassembly, threshold logic, and node selection automatically. -* Easier scaling and centralized policy control. - -**2. Direct mump2p** — Clients connect directly to **mump2p nodes** (each node must run the gRPC API). - -![Direct mump2p Architecture](../../static/img/docker_2.png) - -* Fewer network hops = potentially lower latency. -* Clients must know node addresses and manage failover logic. -* Best for specialized or performance-critical workloads. - - -While the [mump2p-cli (Hosted Proxy)](./01-getting-started-cli.md) lets you get started instantly, -local deployment offers: - -* **Custom Configuration** — Tune thresholds, shard factors, and mesh sizes. -* **Full Control** — Decide how many nodes, their topology, and resource allocation. -* **Private Testing** — Run in isolated networks without using public proxies. -* **Advanced Experiments** — Simulate network conditions, failure scenarios, and scaling. - -## Which mode should I use? - -Choose the deployment mode that best fits your use case: - -| **Mode A: Proxy + mump2p** | **Mode B: Direct mump2p** | -|---|---| -| **One endpoint** — simpler client config | **Lowest latency** — fewer network hops | -| **Policy control** — rate limiting, auth | **Direct control** — no proxy overhead | -| **Auto failover** — proxy handles node selection | **Manual failover** — clients manage addresses | - -**Quick Decision:** - -* **Want simpler setup and client code?** → **[Start with Mode A](#5-mode-a--optimumproxy--mump2p-recommended)** -* **Need maximum performance and control?** → **[Jump to Mode B](#6-mode-b--direct-mump2p-advanced--lower-latency)** - -## 1. Before You Start - -### Requirements - -* **[Docker](https://docs.docker.com/engine/install/)** — Container runtime for running Optimum Network components -* **[Docker Compose](https://docs.docker.com/compose/install/)** — Tool for defining multi-container applications -* **[Go v1.24+](https://golang.org/dl/)** — Required for building custom gRPC clients -* At least **2 GB free RAM** for running multiple nodes locally - -> **Quick Docker Install:** -> -> * **Linux**: `curl -fsSL https://get.docker.com | sh` -> * **macOS**: [Docker Desktop for Mac](https://docs.docker.com/desktop/install/mac-install/) -> * **Windows**: [Docker Desktop for Windows](https://docs.docker.com/desktop/install/windows-install/) - -### Components - -| Component | Purpose | Docker Images | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | -| **mump2p Node** | RLNC-enabled mesh peer, encodes/decodes message shards, handles peer discovery and subscriptions. Optional gRPC API for direct clients. | `getoptimum/p2pnode:${P2P_NODE_VERSION-latest}` | -| **Optimum Proxy** | Bridges clients and the mesh, manages subscriptions, shard reassembly, threshold logic, and node selection. | `getoptimum/proxy:${PROXY_VERSION-latest}` | - - - -### Directory layout - -Create a clean working folder: - -```sh -mkdir -p ~/optimum-local/{proxy-p2p,direct-p2p,identity} -cd ~/optimum-local -``` - -We’ll keep identity in `./identity` folder so you can reuse keys across restarts. - - - -## 2. Pick Your Mode - -| Recommended mode | Why | -| ----------------------------- | ----------------------------------------------------------------------------------------- | -| **OptimumProxy + mump2p** | One endpoint for clients, proxy handles matching, decoding thresholds, fanout, and policy | -| **Direct mump2p** | Fewer hops, you control connection/retry logic and node selection | - - -## 3. Environment Configuration - -Before starting, create your `.env` file: - -```bash -cp .env.example .env -``` - -**Important:** After copying, you need to replace the `BOOTSTRAP_PEER_ID` in your `.env` file with the peer ID generated by `make generate-identity`. - -**Workflow:** - -1. Run `make generate-identity` - this creates a unique peer ID -2. Copy the generated peer ID from the output -3. Edit your `.env` file and replace the example `BOOTSTRAP_PEER_ID` with your generated one - -Edit with your values: - -```bash -BOOTSTRAP_PEER_ID= -CLUSTER_ID=my-cluster -PROXY_VERSION=v0.0.1-rc16 -P2P_NODE_VERSION=v0.0.1-rc16 -``` - -> **Complete Guide:** [Environment configuration](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#environment-variables-env) - -## 4. Generate a Bootstrap Identity - -Generate P2P identity for node discovery: - -```bash -make generate-identity -``` - -This creates `./identity/p2p.key` with your unique Peer ID. - -> **Complete Guide:** [Identity generation and Makefile commands](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#getting-started) - all make commands, direct binary usage - -## 5. Mode A — OptimumProxy + mump2p (Recommended) - -### Docker Compose Setup - -**Key points:** - -* Use `.env` variables for versions and cluster ID -* Network uses static IPs for deterministic bootstrap addresses -* Bootstrap node (p2pnode-1) needs identity volume mount -* Production setup uses 2 proxies and 4 P2P nodes - -**Simplified example:** - -```yaml -services: - proxy-1: - image: 'getoptimum/proxy:${PROXY_VERSION-latest}' - environment: - - CLUSTER_ID=${CLUSTER_ID} - - P2P_NODES=p2pnode-1:33212,p2pnode-2:33212 - ports: - - "8081:8080" - - "50051:50051" - - p2pnode-1: - image: 'getoptimum/p2pnode:${P2P_NODE_VERSION-latest}' - volumes: - - ./identity:/identity - environment: - - CLUSTER_ID=${CLUSTER_ID} - - NODE_MODE=optimum - - IDENTITY_DIR=/identity -``` - -> **Complete Docker Compose:** -> -> * [Full configuration with 2 proxies + 4 nodes](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docker-compose-optimum.yml) -> * [All environment variables](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#environment-variables-env) - -### Start the Network - -```sh -export BOOTSTRAP_PEER_ID= -docker-compose -f docker-compose-optimum.yml up --build -d -``` - -### Verify Health - -```sh -# Check containers -docker-compose -f docker-compose-optimum.yml ps - -# Test endpoints -curl http://localhost:8081/api/v1/health # Proxy -curl http://localhost:9091/api/v1/health # P2P node -``` - -> **Complete Testing Guide:** [Health checks and validation](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#3-verification) - -### Send & receive (Proxy mode) using mump2p-cli - -If you haven't already installed `mump2p-cli`, see the [**Getting Started with mump2p-cli**](./01-getting-started-cli.md) chapter. - -**Subscribe:** - -```sh -./mump2p subscribe --topic=demo --service-url=http://localhost:8081 -``` - -**Publish (in a new terminal):** - -```sh -./mump2p publish --topic=demo --message="Hello via Proxy" --service-url=http://localhost:8081 -``` - -You should see your subscriber print the message immediately. - - -### Use Proxy via REST API (Optional) - -**Basic commands:** - -```sh -# Publish -curl -X POST http://localhost:8081/api/v1/publish \ - -H "Content-Type: application/json" \ - -d '{"client_id":"test","topic":"demo","message":"Hello"}' - -# Subscribe -curl -X POST http://localhost:8081/api/v1/subscribe \ - -H "Content-Type: application/json" \ - -d '{"client_id":"test","topic":"demo","threshold":0.1}' - -# WebSocket -wscat -c "ws://localhost:8081/api/v1/ws?client_id=test" -``` - -> **Complete API Reference:** [Proxy REST and WebSocket API](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#proxy-rest-api) - parameters, rate limits, authentication - -### Use Proxy via gRPC (Optional) - -For gRPC bidirectional streaming (higher performance than WebSocket): - -> **Complete Implementation:** -> -> * [Proxy gRPC Client Source](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_proxy_client/proxy_client.go) -> * [Setup and Usage Guide](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#grpc-proxy-client-implementation) -> * REST subscription + gRPC streaming + flow control settings - -## 6. Mode B — Direct mump2p (Advanced / Lower Latency) - -In this mode, clients connect directly to node sidecar gRPC (no proxy). - -### Docker Compose Setup - -**Simplified example:** - -```yaml -services: - p2pnode-1: - image: 'getoptimum/p2pnode:${P2P_NODE_VERSION-latest}' - volumes: - - ./identity:/identity - environment: - - CLUSTER_ID=${CLUSTER_ID} - - NODE_MODE=optimum - - IDENTITY_DIR=/identity - ports: - - "33221:33212" -``` - -> **Complete Docker Compose:** [Full configuration for direct P2P mode](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docker-compose-optimum.yml) -> -> **Note:** The docker-compose file uses environment variables from `.env` for versions. Ensure your `.env` file has `PROXY_VERSION` and `P2P_NODE_VERSION` set. - -### Start and Verify - -```sh -export BOOTSTRAP_PEER_ID= -docker-compose -f docker-compose-optimum.yml up --build -d -curl http://localhost:9091/api/v1/health -``` - -### Use P2P Client Directly - -Connect using the P2P client with trace handling and metrics: - -```bash -# Subscribe -./grpc_p2p_client/p2p-client -mode=subscribe -topic=testtopic --addr=127.0.0.1:33221 - -# Publish -./grpc_p2p_client/p2p-client -mode=publish -topic=testtopic -msg="Hello" --addr=127.0.0.1:33222 -``` - -> **Complete Implementation:** -> -> * [P2P Client Source](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_p2p_client/cmd/single/main.go) -> * [Usage Guide with Makefile commands](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#using-p2p-nodes-directly-optional--no-proxy) -> * [Message format explanation](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#understanding-message-output-format) - -For all configuration variables, see the [Parameters Section](./03-parameters.md). - -## Troubleshooting - -### "Connection refused" from client - -* Ensure you're pointing to the host-mapped ports (e.g., 33221, 8081). -* Run `docker-compose -f docker-compose-optimum.yml ps` and confirm port bindings. -* Firewalls: allow inbound localhost traffic. - -### Proxy can’t reach nodes - -* Inside the proxy container, resolve and ping node hosts: - -```sh -docker-compose -f docker-compose-optimum.yml exec proxy-1 sh -lc 'nc -zv p2pnode-1 33212' -``` - -* Make sure `P2P_NODES` hostnames match the `service names` in compose. - -### Port conflicts - -* Change host mappings in ports: (e.g., 33223:33212, 9093:9090, 7073:7070). - -### Protocol mismatch - -* All nodes in a mesh must use the same NODE_MODE (optimum or gossipsub). - -### Stop and Clean - -Stop: - -```sh -docker-compose -f docker-compose-optimum.yml down -``` - -Full reset (containers, volumes, images created by this compose file): - -```sh -docker-compose -f docker-compose-optimum.yml down -v --rmi local -``` diff --git a/docs/guides/03-parameters.md b/docs/guides/03-parameters.md deleted file mode 100644 index e1b2098..0000000 --- a/docs/guides/03-parameters.md +++ /dev/null @@ -1,202 +0,0 @@ -# Understanding Key Parameters in mump2p - -> **Complete Reference:** [Full parameter guide and defaults](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#configuration) - all variables, `.env` workflow, production configs - -mump2p nodes can operate in **two distinct protocol modes**, configured via: - -```sh -NODE_MODE=optimum # RLNC-enhanced gossip -NODE_MODE=gossipsub # Standard libp2p gossip -``` - -Each mode has its own parameter set, with some shared configurations. - -We support **two protocol modes** to let developers and researchers compare **performance, reliability, and bandwidth trade-offs** in real-world scenarios — without changing the rest of their infrastructure. - -## mump2p (RLNC) Mode — `NODE_MODE=optimum` - -mump2p extends the gossip protocol with **Random Linear Network Coding**: - -* Messages are split into **shards** -* Shards can be forwarded early once a threshold is reached -* Improves delivery resilience in lossy or high-latency environments - -Example Docker service: - -```yaml -p2pnode-1: - image: 'getoptimum/p2pnode:${P2P_NODE_VERSION-latest}' - environment: - - NODE_MODE=optimum - - CLUSTER_ID=${CLUSTER_ID} - - OPTIMUM_PORT=7070 - - OPTIMUM_MESH_TARGET=6 - - OPTIMUM_SHARD_FACTOR=4 - - OPTIMUM_THRESHOLD=0.75 -``` - -### Parameters - -| Parameter | Default | Purpose | -| ---------------------- | ------------- | ---------------------------------------------- | -| `OPTIMUM_PORT ` | 7070 | TCP port used for RLNC gossip. | -| `OPTIMUM_MAX_MSG_SIZE` | 1048576 (1MB) | Max allowed message size (full payload). | -| `OPTIMUM_MESH_TARGET` | 6 | Desired peers in mesh. | -| `OPTIMUM_MESH_MIN` | 3 | Minimum peers before adding more. | -| `OPTIMUM_MESH_MAX` | 12 | Max peers before pruning. | -| `OPTIMUM_SHARD_FACTOR` | 4 | Number of shards per message. | -| `OPTIMUM_SHARD_MULT` | 1.5 | Redundancy multiplier (extra shards). | -| `OPTIMUM_THRESHOLD` | 0.75 | Fraction of shards required to forward/decode. | -| `BOOTSTRAP_PEERS` | (none) | List of peer multiaddrs to connect at startup. | - - -## GossipSub Mode — `NODE_MODE=gossipsub` - -GossipSub is the **standard libp2p pub/sub** protocol: - -* Maintains topic-specific peer meshes -* Exchanges message availability metadata with non-mesh peers -* Widely used for blockchain gossip - -Example Docker service: - -```yaml -p2pnode-1: - image: 'getoptimum/p2pnode:${P2P_NODE_VERSION-latest}' - environment: - - NODE_MODE=gossipsub - - LOG_LEVEL=debug - - CLUSTER_ID=p2pnode-1 - - GOSSIPSUB_PORT=6060 - - GOSSIPSUB_MAX_MSG_SIZE=1048576 - - GOSSIPSUB_MESH_TARGET=6 - - GOSSIPSUB_MESH_MIN=4 - - GOSSIPSUB_MESH_MAX=12 - - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/6060/p2p/${BOOTSTRAP_PEER_ID} -``` - - -### Parameters - -| Parameter | Default | Purpose | -| ------------------------ | ------------- | ---------------------------------------------- | -| `GOSSIPSUB_PORT` | 6060 | TCP port for gossip pub/sub. | -| `GOSSIPSUB_MAX_MSG_SIZE` | 1048576 (1MB) | Max allowed message size. | -| `GOSSIPSUB_MESH_TARGET` | 6 | Desired peers in mesh. | -| `GOSSIPSUB_MESH_MIN` | 4 | Minimum peers before adding more. | -| `GOSSIPSUB_MESH_MAX` | 12 | Max peers before pruning. | -| `BOOTSTRAP_PEERS` | (none) | List of peer multiaddrs to connect at startup. | - - -You can refer to the [GossipSub parameter specification](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md#parameters) for a detailed explanation of the standard pub/sub settings. - - -### Shared Parameters (Both `optimum` and `gossipsub` mode) - -| Parameter | Default | Purpose | -| -------------- | --------- | -------------------------------------------------- | -| `LOG_LEVEL` | debug | Log verbosity (debug, info, warn, error). | -| `CLUSTER_ID` | (none) | Logical group name for metrics and identification. | -| `SIDECAR_PORT` | 33212 | gRPC sidecar port for proxy ↔ P2P communication. | -| `API_PORT` | 9090 | HTTP API port for node management. | -| `IDENTITY_DIR` | /identity | Directory containing node’s private key. | - -## OptimumProxy Configuration Parameters - -When using **Proxy + P2P deployment**, the proxy service connects clients to P2P nodes and optionally enforces authentication. - -Example Docker service: - -```yaml -proxy-1: - image: 'getoptimum/proxy:${PROXY_VERSION-latest}' - environment: - - PROXY_PORT=:8080 - - PROXY_GRPC_PORT=:50051 - - CLUSTER_ID=${CLUSTER_ID} - - ENABLE_AUTH=false - - LOG_LEVEL=debug - - P2P_NODES=p2pnode-1:33212,p2pnode-2:33212 -``` - -| Parameter | Default | Purpose | -| --------------------------------- | ------- | ---------------------------------------------------------- | -| `PROXY_PORT` | :8080 | HTTP API port for clients. | -| `PROXY_GRPC_PORT` | :50051 | gRPC API port for clients. | -| `ENABLE_AUTH` | false | Enable Auth0 authentication. | -| `AUTH0_DOMAIN` / `AUTH0_AUDIENCE` | (none) | Auth0 settings (required if `ENABLE_AUTH`=true). | -| `P2P_NODES` | (none) | Comma-separated list of host:port for sidecar connections. | -| `SUBSCRIBER_THRESHOLD` | 0.1 | % of connected subscribers needed to forward a message. | - -## Recommended Default Configuration - -The following table shows the **production defaults**, which are optimized for typical deployment scenarios: - -### For Quick Start (Copy-Paste Ready) - -**mump2p Mode (`NODE_MODE=optimum`):** - -```yaml -environment: - - NODE_MODE=optimum - - LOG_LEVEL=debug - - CLUSTER_ID=my-cluster - - SIDECAR_PORT=33212 - - API_PORT=9090 - - IDENTITY_DIR=/identity - - OPTIMUM_PORT=7070 - - OPTIMUM_MAX_MSG_SIZE=1048576 - - OPTIMUM_MESH_TARGET=6 - - OPTIMUM_MESH_MIN=3 - - OPTIMUM_MESH_MAX=12 - - OPTIMUM_SHARD_FACTOR=4 - - OPTIMUM_SHARD_MULT=1.5 - - OPTIMUM_THRESHOLD=0.75 - - BOOTSTRAP_PEERS="" -``` - -**GossipSub Mode (`NODE_MODE=gossipsub`):** - -```yaml -environment: - - NODE_MODE=gossipsub - - LOG_LEVEL=debug - - CLUSTER_ID=my-cluster - - SIDECAR_PORT=33212 - - API_PORT=9090 - - IDENTITY_DIR=/identity - - GOSSIPSUB_PORT=6060 - - GOSSIPSUB_MAX_MSG_SIZE=1048576 - - GOSSIPSUB_MESH_TARGET=6 - - GOSSIPSUB_MESH_MIN=4 - - GOSSIPSUB_MESH_MAX=12 - - BOOTSTRAP_PEERS="" -``` - -### Complete Defaults Reference - -| Parameter | Default Value | Used In Mode | Description | -| ---------------------------- | ------------- | ------------ | ---------------------------------------------- | -| `LOG_LEVEL` | debug | Both | Log verbosity (debug/info/warn/error) | -| `CLUSTER_ID` | "" | Both | Logical group name for metrics | -| `NODE_MODE` | "" | Both | Protocol mode (optimum/gossipsub) | -| `SIDECAR_PORT` | 33212 | Both | gRPC sidecar port | -| `API_PORT` | 9090 | Both | HTTP API port | -| `IDENTITY_DIR` | /identity | Both | Directory for node private key | -| `OPTIMUM_PORT` | 7070 | optimum | TCP port for RLNC gossip | -| `OPTIMUM_MAX_MSG_SIZE` | 1048576 | optimum | Max message size (1MB) | -| `OPTIMUM_MESH_TARGET` | 6 | optimum | Desired peers in mesh | -| `OPTIMUM_MESH_MIN` | 3 | optimum | Minimum peers before adding more | -| `OPTIMUM_MESH_MAX` | 12 | optimum | Max peers before pruning | -| `OPTIMUM_SHARD_FACTOR` | 4 | optimum | Number of shards per message | -| `OPTIMUM_SHARD_MULT` | 1.5 | optimum | Redundancy multiplier (extra shards) | -| `OPTIMUM_THRESHOLD` | 0.75 | optimum | Forward/decode threshold (75%) | -| `GOSSIPSUB_PORT` | 6060 | gossipsub | TCP port for standard gossip | -| `GOSSIPSUB_MAX_MSG_SIZE` | 1048576 | gossipsub | Max message size (1MB) | -| `GOSSIPSUB_MESH_TARGET` | 6 | gossipsub | Desired peers in mesh | -| `GOSSIPSUB_MESH_MIN` | 4 | gossipsub | Minimum peers before adding more | -| `GOSSIPSUB_MESH_MAX` | 12 | gossipsub | Max peers before pruning | -| `BOOTSTRAP_PEERS` | (none) | Both | List of peer multiaddrs for bootstrap | - -> **Note:** These are the production defaults used by mump2p nodes. For experimental tuning, see [Common Experiments](./04-experiments.md). - diff --git a/docs/guides/04-experiments.md b/docs/guides/04-experiments.md deleted file mode 100644 index f43a9a5..0000000 --- a/docs/guides/04-experiments.md +++ /dev/null @@ -1,146 +0,0 @@ -# Common Experiments - -Once your Optimum Network is running (see [Parameters](./03-parameters.md)), you can try different experiments to understand **performance, reliability, and scaling behavior**. - -> **Before You Begin:** -> Make sure you’ve read: -> -> * [Node Modes & Config Parameters](./03-parameters.md) — to understand what each env var controls. -> * [mump2p CLI](./01-getting-started-cli.md) and [gRPC Client Setup](./02-getting-started-docker.md) — for sending and receiving test messages. - -Each experiment below lists the **goal**, a quick **how-to**, and **what to observe**. - -You can run them using: - -* **mump2p CLI** (see [CLI Guide](./01-getting-started-cli.md)) -* **gRPC P2P client** (trace collection is automatic when subscribing - see [Trace Collection](#metrics-collection)) - - - -## 1. GossipSub vs mump2p - -**Goal:** Compare standard libp2p gossip to RLNC-enhanced gossip to confirm mump2p is faster. - -**How:** - -* Run one cluster with `NODE_MODE=gossipsub`. -* Run another with `NODE_MODE=optimum`. -* Publish the same workload to both networks. - -**Observe:** - -* **Delivery latency** (primary metric - mump2p should be faster) -* **Bandwidth usage** (mump2p should use less) -* **Success rate** (both should deliver messages successfully) - -**Expected Result:** mump2p should show lower latency and bandwidth usage. - - -## 2. Shard Factor Sweep - -**Goal:** Find the optimal number of coded shards for your message size and network. - -**How:** - -* Test range: `OPTIMUM_SHARD_FACTOR` = 2, 4, 8, 16, 32, 64 (powers of 2 recommended). -* Keep all other parameters the same. -* Test with 1MB messages. - -**Observe:** - -* **Delivery latency** (primary metric) -* **Success rate** (messages should still deliver) - -**Expected Result:** For 1MB messages, shard factors 4-8 typically provide the best balance of performance and reliability. Too few shards (≤2) cause delivery failures, while too many shards (≥32) increase latency due to overhead. Start with shard factor 4-8 and tune based on your network conditions. - - -## 3. Forward Threshold Tuning - -**Goal:** Find the optimal threshold for forwarding coded shards early. - -**How:** - -* Fix `OPTIMUM_SHARD_FACTOR=8`. -* Test `OPTIMUM_THRESHOLD` at 0.5, 0.7, 0.9, 1.0. -* Publish small messages in a 20-30 node network. - -**Observe:** - -* **Delivery latency** (primary metric) -* **Bandwidth usage** (threshold=1.0 should use most bandwidth) - -**Expected Result:** Threshold 0.7 provides optimal performance for small networks - delivering 100% success rate with lowest latency. Thresholds below 0.6 may cause delivery failures, while values above 0.8 can reduce success rate and increase latency. - - -## 4. Mesh Density Impact - -**Goal:** Compare mump2p vs GossipSub with different mesh sizes. - -**How:** - -* Test both `NODE_MODE=gossipsub` and `NODE_MODE=optimum`. -* For GossipSub: try `GOSSIPSUB_MESH_TARGET` = 4, 6, 8. -* For mump2p: try `OPTIMUM_MESH_TARGET` = 6, 12, 18. -* Run the same publish/subscribe test. - -**Observe:** - -* **Delivery latency** (primary metric) -* **Bandwidth usage** - -**Expected Result:** mump2p should perform better with higher mesh targets (around 12) while GossipSub optimal around 6. - - - - - -## 5. Load Test - -**Goal:** Find when mump2p vs GossipSub starts to fail under stress. - -**How:** - -* Test both `NODE_MODE=gossipsub` and `NODE_MODE=optimum`. -* Vary **message size** (1KB → 10MB) and **message frequency** (1 → 1000 msg/s). -* Use multiple publishers if needed. - -**Observe:** - -* **When delivery starts to fail** (primary metric) -* **Delivery rate degradation** - -**Expected Result:** mump2p should handle higher stress levels before failing compared to GossipSub. - - -> **Tip:** Trace collection is automatic when using the gRPC P2P client. Simply subscribe to a topic and trace events will be automatically parsed and displayed. See [Trace Collection](#metrics-collection) for details. - -## Metrics Collection - -For comprehensive metrics collection during experiments, use the gRPC P2P client with trace handling: - -**[P2P Client with Metrics Collection](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#collecting-trace-data-for-experiments)** - -**[Complete Code](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_p2p_client/cmd/single/main.go)** - -The client automatically captures and displays trace events when you subscribe: - -* **GossipSub traces**: Peer routing, message delivery status, hop counts, latency -* **mump2p traces**: Shard encoding/decoding, reconstruction efficiency, redundancy metrics -* **Message-level data**: Delivery success rates, end-to-end latency, bandwidth usage - -**Usage:** - -```bash -# Subscribe to a topic - trace events are automatically collected and displayed -./grpc_p2p_client/p2p-client -mode=subscribe -topic=your-topic --addr=127.0.0.1:33221 -``` - -Trace parsing is implemented in `grpc_p2p_client/shared/utils.go` and handles both `ResponseType_MessageTraceGossipSub` and `ResponseType_MessageTraceMumP2P` automatically. - -For multi-node experiments with trace data collection, use the multi-subscribe client: - -```bash -./grpc_p2p_client/p2p-multi-subscribe -topic=test-topic -ipfile=ips.txt -output-data=data.tsv -output-trace=trace.tsv -``` - -See the [dev setup guide](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#multi-node-client-tools) for more details on multi-node client tools. diff --git a/docs/guides/05-faq-glossary.md b/docs/guides/05-faq-glossary.md deleted file mode 100644 index 06814b7..0000000 --- a/docs/guides/05-faq-glossary.md +++ /dev/null @@ -1,119 +0,0 @@ -# Frequently Asked Questions (FAQs) - -## CLI Issues - -For all CLI-related problems (authentication, installation, rate limits, connection issues), please refer to the comprehensive FAQ in the CLI repository: - -**👉 [mump2p CLI FAQ & Troubleshooting](https://github.com/getoptimum/mump2p-cli#faq---common-issues--troubleshooting)** - -It includes detailed troubleshooting for: - -* Authentication and login problems -* Installation issues across different operating systems -* Rate limiting and usage issues -* Service URL and connectivity problems -* Common syntax and usage errors - -## Docker Setup Issues - -### Identity Generation Problems - -### Q: `docker run ... generate-key` command doesn't work - -**A:** Use the identity generation script or Makefile command: - -```bash -# Using Makefile (recommended) -make generate-identity - -# Or using the script directly -./script/generate-identity.sh -``` - -This generates `./identity/p2p.key` and displays the `BOOTSTRAP_PEER_ID` that you need to add to your `.env` file. - -### Container Startup Issues - -### Q: Containers fail to start or can't connect to each other - -**A:** Common fixes: - -1. **Check Docker images**: Use correct versions from your `.env` file (`PROXY_VERSION=v0.0.1-rc16`, `P2P_NODE_VERSION=v0.0.1-rc16` by default) -2. **Network conflicts**: Change subnet in docker-compose if `172.28.0.0/16` conflicts -3. **Port conflicts**: Ensure ports 8081, 8082, 50051, 50052, 33221-33224, 9091-9094, 7071-7074 are available -4. **Platform issues**: Add `platform: linux/amd64` for M1 Macs (already included in docker-compose-optimum.yml) - -### Q: "Connection refused" when clients try to connect - -**A:** Verify: - -* Containers are running: `docker ps` -* Ports are properly mapped in docker-compose -* No firewall blocking connections -* Using correct service URLs (localhost:8080 for proxy, localhost:33221 for direct P2P) - - -## gRPC Client Issues - -### Q: gRPC client gets "connection refused" or timeout errors - -**A:** Check: - -1. **Containers are running**: `docker ps` to verify proxy and p2pnode containers are up -2. **Correct ports**: Proxy gRPC on `localhost:50051`, P2P sidecar on `localhost:33221` -3. **Use latest client examples**: Reference [`optimum-dev-setup-guide/docs/guide.md#grpc-proxy-client-implementation`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#grpc-proxy-client-implementation) - - **[Complete Code](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_proxy_client/proxy_client.go)** - - For P2P direct client, see [`grpc_p2p_client/cmd/single/main.go`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_p2p_client/cmd/single/main.go) - -### Q: Getting "method not found" or protobuf errors - -**A:** Use the correct protobuf definitions from [`optimum-dev-setup-guide/docs/guide.md`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#api-reference): - -* See the [API Reference section](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#api-reference) for complete protobuf definitions -* All proto files are available in the repository's `grpc_*_client/proto/` directories: - * [`grpc_proxy_client/proto/proxy_stream.proto`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_proxy_client/proto/proxy_stream.proto) - * [`grpc_p2p_client/proto/p2p_stream.proto`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_p2p_client/proto/p2p_stream.proto) - - - -## Development Issues - -### Q: Go client code compilation errors - -**A:** Use the exact Go versions and dependencies from [`optimum-dev-setup-guide/docs/guide.md`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#client-tools): - -* See the [Client Tools section](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#client-tools) for complete examples -* All go.mod files and dependencies are available in the repository's `grpc_*_client/` directories: - * [`grpc_proxy_client/go.mod`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_proxy_client/go.mod) - * [`grpc_p2p_client/go.mod`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/grpc_p2p_client/go.mod) - -### Q: Code examples don't work as expected - -**A:** All examples are tested against [`optimum-dev-setup-guide/docs/guide.md`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md). Check: - -1. Environment variables are set correctly -2. Docker containers are running -3. Using the latest example code from the repository - -## General Troubleshooting - -### First Steps - -When something doesn't work: - -1. **Check container logs**: `docker-compose -f docker-compose-optimum.yml logs ` or `docker logs ` -2. **Verify network connectivity**: `docker network ls` and `docker network inspect optimum-dev-setup-guide_optimum-network` -3. **Test basic connectivity**: - * Proxy: `curl http://localhost:8081/api/v1/health` - * P2P Node: `curl http://localhost:9091/api/v1/health` -4. **Check authentication**: `mump2p whoami` (if using CLI) -5. **Verify versions**: Check your `.env` file for `PROXY_VERSION` and `P2P_NODE_VERSION` (default: v0.0.1-rc16) -6. **Check service status**: `docker-compose -f docker-compose-optimum.yml ps` - -### Getting Help - -* **CLI Issues**: [mump2p-cli FAQ](https://github.com/getoptimum/mump2p-cli#faq---common-issues--troubleshooting) -* **Setup Issues**: Check [`optimum-dev-setup-guide/docs/guide.md`](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md) -* **Protocol Questions**: See [mump2p Documentation](../learn/overview/p2p.md) diff --git a/docs/guides/overview.md b/docs/guides/overview.md deleted file mode 100644 index 93b3498..0000000 --- a/docs/guides/overview.md +++ /dev/null @@ -1,85 +0,0 @@ -# Quick Start - -This developer guide is your complete reference for installing, configuring, and operating **Optimum Network**. - -This guide walks you through everything you need to **understand**, **deploy**, **run**, and **experiment** with Optimum Network. It's designed for beginners but complete enough for advanced developers to jump in and integrate. - -Whether you are an application developer, systems engineer, or hackathon participant, this guide will help you: - -1. **Install and run** Optimum Network in different environments -2. **Connect clients** via gRPC, REST, or CLI -3. **Tune parameters** for performance or reliability -4. **Run experiments** to validate system behavior -5. **Monitor and debug** your deployment - -## Start Building - -Choose your path to get started with Optimum Network: - -### **Try mump2p (≤5 min)** - -**[Using mump2p-cli](01-getting-started-cli.md)** — Connect to hosted proxy, no setup required - -* One-command installation -* Instant messaging with hosted infrastructure -* Perfect for testing and prototyping - -### **Run Locally** - -**[Local Setup with Docker](02-getting-started-docker.md)** — Full control over configuration - -* Complete local network deployment -* Experiment with different parameters -* Compare mump2p vs GossipSub performance - -### **Understanding & Experimenting** - -Essential for hackathon success: - -* **[Understanding Key Parameters](03-parameters.md)** — Tune thresholds, shards, and mesh settings -* **[Common Experiments](04-experiments.md)** — Test performance under different conditions -* **[FAQ](05-faq-glossary.md)** — Quick answers and debugging help - - -## What This Guide Covers - -This guide is organized into self-contained sections: - -1. **[Using mump2p-cli](01-getting-started-cli.md)** — Connect to hosted proxy with no setup required. -2. **[Local Setup with Docker](02-getting-started-docker.md)** — Run your own local proxy and P2P nodes for full control. -3. **[Understanding Key Parameters](03-parameters.md)** — Tune thresholds, shards, and mesh settings for your workload. -4. **[Common Experiments](04-experiments.md)** — Test performance and reliability under different conditions. -5. **[FAQ](05-faq-glossary.md)** — Quick answers and debugging help. - - -## Who This Guide Is For - -* **Application developers** who want to integrate fast, resilient messaging. -* **Infrastructure engineers** deploying distributed systems. -* **Hackathon teams** experimenting with real-time messaging. -* **Researchers** testing RLNC in real-world scenarios. - - -## Key Concepts - -Before diving deep, familiarize yourself with these terms and how they relate to Optimum Network's architecture: - -| Term | Description | -|--------------|-------------| -| **Topic** | A named channel for publishing and subscribing to messages. Peers in the same topic exchange messages relevant only to that topic. | -| **Message** | The original data you publish to a topic (e.g., text, JSON, binary). | -| **Shard** | A coded fragment of a message produced by RLNC (Random Linear Network Coding). Shards allow messages to be reconstructed even if some are lost. | -| **Threshold**| The fraction of shards required to successfully decode a message. For example, a threshold of 0.7 means only 70% of shards are needed. | -| **RLNC** | Random Linear Network Coding — a technique that mixes message fragments mathematically for redundancy and efficiency. | -| **Mesh** | The set of directly connected peers in a topic. The mesh topology affects redundancy, latency, and fault tolerance. | -| **Bootstrap Node** | A known peer address used to join the P2P network and discover other peers. | -| **Optimum Proxy** | Bridge nodes between clients and the P2P mesh. It manages subscriptions, publications, and threshold logic on behalf of clients. | -| **Direct P2P Mode** | Clients connect directly to P2P nodes via gRPC without going through a proxy. This can reduce latency but requires more configuration. | -| **Mesh Parameters** | Settings such as `MESH_TARGET` that define how many peers a node tries to keep in its topic mesh. | -| **Fanout** | A temporary set of peers a publisher sends messages to when it is not part of the topic mesh. | -| **Control Messages** | Special messages like GRAFT, PRUNE, IHAVE, IWANT used for GossipSub mesh management. mump2p integrates similar control flows for RLNC. | -| **Node Discovery** | The process by which proxies or nodes automatically learn about other nodes to connect to. | -| **gRPC API** | The RPC interface provided by either a P2P node or a proxy for client communication. | - -For more technical details, see the **[mump2p Technical Overview](../learn/overview/p2p.md)**. - diff --git a/docs/learn/overview/deram.md b/docs/learn/overview/deram.md deleted file mode 100644 index 7465300..0000000 --- a/docs/learn/overview/deram.md +++ /dev/null @@ -1,26 +0,0 @@ -# Introduction to Optimum DeRAM - -**Optimum DeRAM (Decentralized Memory)** is a highly effective, low-latency, high-throughput, -and scalable memory abstraction built within the Optimum protocol. -In essence, it introduces an atomic read/write memory designed for the Web3 environment, addressing the unique -challenges posed by asynchronous communication, high node churn, decentralized -decision-making, and the presence of potentially malicious nodes. - -Achieving a reliable shared memory object in this setting requires careful -attention to reducing latency, enhancing fault tolerance, minimizing bandwidth -and storage costs, ensuring high throughput, and maintaining non-blocking, -high-availability performance. - -Our approach leverages Random Linear Network Codes (RLNC), selected for their -flexible structure, which helps avoid costly distributed synchronization -primitives. This flexibility contributes to improved durability, reduced -bandwidth usage, and enhanced fault tolerance within a distributed storage -framework. - -Our system is implemented as a network of functionally heterogeneous nodes, termed -Flexnodes, which collectively provide decentralized storage and communication -services. External clients can interact with any Flexnode to perform read/write -operations, using this system both for data storage and as a communication -socket within the network. - -**More coming soon.** diff --git a/docs/learn/overview/intro.md b/docs/learn/overview/intro.md index fdd0e53..f52414d 100644 --- a/docs/learn/overview/intro.md +++ b/docs/learn/overview/intro.md @@ -12,22 +12,12 @@ Powered by [Random Linear Network Coding (RLNC)](./p2p.md#random-linear-network- ## Products -Builders and operators can adopt **mump2p** now for measurable latency gains. DeRAM and DeROM are next to unlock low-latency reads/writes. +Builders and operators can adopt **mump2p** now for measurable latency gains via the **Optimum Gateway**. ### mump2p RLNC-accelerated, libp2p/gossipsub-compatible pub/sub for fast, resilient propagation of blocks, blobs, and transactions. -**Get started:** [Overview](../../learn/overview/p2p.md) · [Quickstart](../../guides/01-getting-started-cli.md) - -### Optimum DeRAM (Decentralized Random Access Memory) - -Decentralized **read-write** memory exposing low-latency shared-state semantics across nodes. -**Get started:** [Introduction](./deram.md) - -### Optimum DeROM (Decentralized Read-Only Memory) - -Decentralized **read-only/append-oriented** memory optimized for broadcast and caching. -**Get started:** [Coming next] +**Get started:** [Overview](../../learn/overview/p2p.md) · [Optimum Gateway](https://getoptimum.github.io/optimum-gateway/versions/latest/) ## Flexnodes @@ -35,14 +25,11 @@ A **flexnode** is an operator-run node that participates in Optimum's coded goss * encode, decode, and forward RLNC-coded gossip frames; * maintain bounded coded buffers to recover loss and smooth tail latency; -* serve **DeRAM/DeROM** reads/writes per policy and quotas via stable APIs; * interoperate with existing clients and libp2p/gossipsub where applicable. ## Start here -* **Try mump2p (≤5 min):** [Using mump2p-cli](../../guides/01-getting-started-cli.md) -* **Run locally:** [Local Setup with Docker](../../guides/02-getting-started-docker.md) -* **Integrate:** [Publish/Subscribe via Optimum Proxy endpoints](https://github.com/getoptimum/optimum-dev-setup-guide/blob/main/docs/guide.md#proxy-api) +* **Run the Optimum Gateway:** [Gateway docs](https://getoptimum.github.io/optimum-gateway/versions/latest/) ### Intended users diff --git a/docs/research/deram/atomicity-consistency.md b/docs/research/deram/atomicity-consistency.md deleted file mode 100644 index 198adf7..0000000 --- a/docs/research/deram/atomicity-consistency.md +++ /dev/null @@ -1,35 +0,0 @@ -# Network Coding for Atomicity and Consistency - -## [2022] - -**N. Nicolaou, V. Cadambe, N. Prakash, A. Trigeorgi, K. M. Konwar, M. Médard, N. Lynch**, -*[Adaptive, Reconfigurable, Erasure coded, Atomic Storage](https://groups.csail.mit.edu/tds/papers/Konwar/TOS-2021-04-0018.R2_Proof_fl.pdf)*, -**ACM Transactions on Storage**, vol. 18, iss. 4, pp. 11-39, Sept. 2022. - -## [2019] - -**N. Nicolaou, V. Cadambe, N. Prakash, K. M. Konwar, M. Médard, N. Lynch**, -*[ARES: Adaptive, Reconfigurable, Erasure Coded, Atomic Storage](https://drive.google.com/file/d/19O8NFK9VdufSVBB0IH-BzDm8IW_sofqP/view?usp=drive_link)*, -**IEEE 39th International Conference on Distributed Computing Systems (ICDCS)**, 2019. - -**K. Konwar, P. Narayana Moorthy, M. Médard, N. Lynch**, -*[Fast Lean Erasure-Coded Atomic Memory Object](https://groups.csail.mit.edu/tds/papers/Lynch/OPODIS19.pdf)*, -**23rd International Conference on Principles of Distributed Systems (OPODIS)**, 2019. - -## [2017] - -**K. Konwar, P. Narayana Moorthy, N. Lynch, M. Médard**, -*[A Layered Architecture for Erasure-Coded Consistent Distributed Storage](https://dspace.mit.edu/bitstream/handle/1721.1/137772/PODC_final_v7_pnm.pdf?sequence=2&isAllowed=y)*, -**PODC**, 2017. - -## [2016] - -**K. M. Konwar, N. Prakash, E. Kantor, N. Lynch, M. Médard, A. A. Schwarzmann**, -*[Storage-Optimized Data-Atomic Algorithms for Handling Erasures and Errors in Distributed Storage Systems](https://drive.google.com/file/d/18UvTd71LVDDwkl2zHt_p1cnlL161xHHv/view?usp=drive_link)*, -**IEEE International Parallel and Distributed Processing Symposium**, 2016. - -## [2014] - -**V. R. Cadambe, N. Lynch, M. Médard, P. Musial**, -*[A Coded Shared Atomic Memory Algorithm for Message Passing Architectures](https://drive.google.com/file/d/1-Pbdn4O78_1qBkR5ag61NGuOxLn_C1yg/view?usp=drive_link)*, -**IEEE 13th International Symposium on Network Computing and Applications**, 2014. diff --git a/docs/research/deram/decentralized-storage.md b/docs/research/deram/decentralized-storage.md deleted file mode 100644 index 704166a..0000000 --- a/docs/research/deram/decentralized-storage.md +++ /dev/null @@ -1,85 +0,0 @@ -# Network Coding for Decentralized Data Storage - -## [2019] - -**J. Neu and Médard, M.**, -*[Babel Storage: Uncoordinated Content Delivery from Multiple Coded Storage Systems](https://drive.google.com/file/d/11upgvFvkfm7Rbohs_NdAQUbAu-R4pj9d/view?usp=drive_link)*, -**IEEE Global Communications Conference (GLOBECOM)**, December 2019. - -## [2018] - -**N. Prakash, Abdrashitov, V., and Médard, M.**, -*[The Storage Versus Repair-Bandwidth Trade-off for Clustered Storage Systems](https://drive.google.com/file/d/1c6qtEi5ubf9Mca0Go96H8CPGLRqHs_ui/view?usp=drive_link)*, -**IEEE Transactions on Information Theory**, vol. 64, no. 8, August 2018. - -**M. Mahdian, Prakash, N., Médard, M., and Yeh E.**, -*[Updating Content in Cache-Aided Coded Multicast](https://dspace.mit.edu/handle/1721.1/134942.2?show=full)*, -**IEEE Journal on Selected Areas in Communications: Special Issue on Caching for Communication Systems and Networks**, Vol. 36, No.6, June 2018, pp. 1203-1216. - -## [2017] - -**V. Abdrashitov, Prakash, N., and Médard, M.**, -*[The Storage vs Repair Bandwidth Trade-off for Multiple Failures in Clustered Storage Networks](https://drive.google.com/file/d/13IV688G3BJ8iWfaUQiLQGfw7X_yfv6D5/view?usp=drive_link)*, -**IEEE Information Theory Workshop (ITW)**, 2017. - -**A. Abdrashitov, Prakash, N., and Médard, M.**, -*[The Storage vs Repair Bandwidth Trade-off for Multiple Failures in Clustered Storage Networks](https://dspace.mit.edu/bitstream/handle/1721.1/121591/The_Storage_vs_Repair_Bandwidth_Trade-off_for_Mult.pdf?sequence=2)*, -**ITW**, 2017. - -## [2016] - -**V. Abdrashitov and Médard, M.**, -*[Staying Alive — network coding for data persistence in volatile networks](https://drive.google.com/file/d/17egnB2eNEDJruqIqG779CeHkaz3n9Gix/view?usp=drive_link)*, -**50th Asilomar Conference on Signals, Systems and Computers**, 2016. - -**C. Hellge and Médard, M.**, -*[Multi-code distributed storage](https://drive.google.com/file/d/1AaYp9iN-D17Cy7b8xW6lrDknwb_TNXz7/view?usp=drive_link)*, -**IEEE 9th International Conference on Cloud Computing**, 2016. - -## [2015] - -**V. Abdrashitov and Médard, M.**, -*[Durable Network Coded Distributed Storage](https://drive.google.com/file/d/1_Zn7N_90lZO2A_idTj8WLrPZfR5xr8jP/view?usp=drive_link)*, -**Allerton Conference**, October 2015. - -## [2013] - -**U. J. Ferner, Long, Q., Pedroso, M., Voloch, L., and Médard, M.**, -*[Building a Network Coded Storage Testbed for Data Center Energy Reduction](https://drive.google.com/file/d/1Ino0DBfTQwqznpupemBVizlWSh3NRsgK/view?usp=drive_link)*, -**Sustainable Internet and ICT for Sustainability (SustainIT)**, October 2013. - -## [2012] - -**U. J. Ferner, Médard, M., and Soljanin E.**, -*[Toward Sustainable Networking: Storage Area Networks with Network Coding](https://drive.google.com/file/d/1bL-FUhFe5sGtZKYErNZ7V4RlrlV98soj/view?usp=drive_link)*, -**Allerton Conference**, October 2012. - -**P. F. Oliveira, Lima, L., Vinhoza, T. T. V., Barros, J., and Médard, M.**, -*[Coding for Trusted Storage in Untrusted Networks](https://drive.google.com/file/d/1SozemEXp6CR1UFyMAiZrZaXtFYLW8wHH/view?usp=drive_link)*, -**IEEE Transactions on Information Forensics and Security**, vol. 7, no. 6, December 2012. - -## [2010] - -**P. F. Oliveira, Lima, L., Vinhoza, T. T. V., Barros, J., and Médard, M.**, -*[Trusted Storage over Untrusted Networks](https://drive.google.com/file/d/1XIj3ItL1T0KUGspI9qjobHm6NzqtcFdh/view?usp=drive_link)*, -**IEEE GLOBECOM**, December 2010. - -## [2008] - -**S. Jaggi, Langberg, M., Katti, S., Ho, T., Katabi, D., Médard, M., and Effros, M.**, -*[Resilient Network Coding in the Presence of Byzantine Adversaries](https://drive.google.com/file/d/1Jd-ylYZaDSN94EEQfzA9CWFIJ15AAfIO/view?usp=drive_link)*, -**IEEE Transactions on Information Theory**, vol. 52, no. 6, June 2008. - -**S. Jaggi, Langberg, M., Katti, S., Ho, T., Katabi, D., Médard, M., and Effros, M.**, -*[Resilient Network Coding in the Presence of Byzantine Adversaries](https://drive.google.com/file/d/1Jd-ylYZaDSN94EEQfzA9CWFIJ15AAfIO/view?usp=drive_link)*, -**IEEE Transactions on Information Theory**, vol. 54, no. 6, June 2008. - -**T. Ho, Leong, B., Koetter, R., Médard, M., Effros, M., and Karger, D. R.**, -*[Byzantine Modification Detection in Multicast Networks With Random Network Coding](https://drive.google.com/file/d/1AwO2_QjZVg4TpRf9608MnD7iavNN_QMC/view?usp=drive_link)*, -**IEEE Transactions on Information Theory**, vol. 54, no. 6, June 2008. - -## [2007] - -**F. Zhao, Kalkart, T., Médard, M. and Han, K. J.**, -*[Signatures for Content Distribution with Network Coding](https://drive.google.com/file/d/1J3kVfuarE7qkEJPQAT5NbXUG4vVOot1K/view?usp=drive_link)*, -**ISIT**, Nice, FR., June 2007.