Merge branch 'main' into arqu/dns_transfer_tests

.github/workflows/beta.yaml

@@ -38,6 +38,6 @@ jobs:
           severity: error
           details: |
             Rustc beta tests failed
-            See https://github.com/n0-computer/iroh/actions/workflows/beta.yaml
+            See https://github.com/${{ github.repository }}/actions/workflows/beta.yaml
           webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }}
 

.github/workflows/ci.yml

@@ -228,9 +228,6 @@ jobs:
       - name: Docs
         run: cargo doc --workspace --all-features --no-deps --document-private-items
 
-      - name: Docs (default features)
-        run: cargo doc --workspace --no-deps
-
   clippy_check:
     timeout-minutes: 30
     runs-on: ubuntu-latest

.github/workflows/flaky.yaml

@@ -87,7 +87,7 @@ jobs:
             echo "$failure" >> $GITHUB_OUTPUT
           done
           echo "" >> $GITHUB_OUTPUT
-          echo "See https://github.com/n0-computer/iroh/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT
+          echo "See https://github.com/${{ github.repository }}/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT
           echo "$EOF" >> $GITHUB_OUTPUT
       - name: Notify discord on failure
         uses: n0-computer/discord-webhook-notify@v1

iroh/README.md

@@ -73,6 +73,20 @@ event!(
 );
 ```
 
+## Building documentation
+
+Building the documentation is only supported when using
+`--all-features`.
+
+Additionally you might want to enable documenting the cargo features
+required for certain APIs, which is done by also passing the `--cfg
+iroh_docsrs` flag to rustdoc when building the documentation.  This
+also requires using nightly rust, e.g.:
+
+```sh
+RUSTDOCFLAGS="--cfg iroh_docsrs" cargo +nightly doc --workspace --no-deps --all-features
+```
+
 # License
 
 This project is licensed under either of

iroh/src/discovery.rs

@@ -23,18 +23,19 @@
 //!
 //! Some generally useful discovery implementations are provided:
 //!
+//! - [`StaticProvider`] which allows application to add and remove out-of-band addressing
+//!   information.
+//!
 //! - The [`DnsDiscovery`] which performs lookups via the standard DNS systems.  To publish
 //!   to this DNS server a [`PkarrPublisher`] is needed.  [Number 0] runs a public instance
 //!   of a [`PkarrPublisher`] with attached DNS server which is globally available and a
 //!   reliable default choice.
 //!
 //! - The [`PkarrResolver`] which can perform lookups from designated [pkarr relay servers]
 //!   using HTTP.
-#![cfg_attr(
-    feature = "discovery-local-network",
-    doc = "- [`LocalSwarmDiscovery`]: local_swarm_discovery::LocalSwarmDiscovery
-             very similar to mDNS."
-)]
+//!
+//! - [`LocalSwarmDiscovery`]: local_swarm_discovery::LocalSwarmDiscovery which is an mDNS
+//!   implementation.
 //!
 //! - The [`DhtDiscovery`] also uses the [`pkarr`] system but can also publish and lookup
 //!   records to/from the Mainline DHT.
@@ -68,13 +69,7 @@
 //! # }
 //! ```
 //!
-//! To also enable
-#![cfg_attr(feature = "discovery-local-network", doc = "[`LocalSwarmDiscovery`]")]
-#![cfg_attr(
-    not(feature = "discovery-local-network"),
-    doc = "`LocalSwarmDiscovery`"
-)]
-//! it can be added as another service in the
+//! To also enable [`LocalSwarmDiscovery`] it can be added as another service in the
 //! [`ConcurrentDiscovery`]:
 //!
 //! ```no_run
@@ -106,12 +101,10 @@
 //! [`PkarrPublisher`]: pkarr::PkarrPublisher
 //! [`DhtDiscovery`]: pkarr::dht::DhtDiscovery
 //! [pkarr relay servers]: https://pkarr.org/#servers
-#![cfg_attr(
-    feature = "discovery-local-network",
-    doc = "[`LocalSwarmDiscovery`]: local_swarm_discovery::LocalSwarmDiscovery"
-)]
+//! [`LocalSwarmDiscovery`]: local_swarm_discovery::LocalSwarmDiscovery
+//! [`StaticProvider`]: static_provider::StaticProvider
 
-use std::{collections::BTreeSet, net::SocketAddr, time::Duration};
+use std::{collections::BTreeSet, net::SocketAddr, sync::Arc, time::Duration};
 
 use anyhow::{anyhow, ensure, Result};
 use futures_lite::stream::{Boxed as BoxStream, StreamExt};
@@ -194,6 +187,8 @@ pub trait Discovery: std::fmt::Debug + Send + Sync {
     }
 }
 
+impl<T: Discovery> Discovery for Arc<T> {}
+
 /// The results returned from [`Discovery::resolve`].
 #[derive(Debug, Clone)]
 pub struct DiscoveryItem {
@@ -446,6 +441,7 @@ mod tests {
     use anyhow::Context;
     use iroh_base::SecretKey;
     use rand::Rng;
+    use testresult::TestResult;
     use tokio_util::task::AbortOnDropHandle;
 
     use super::*;
@@ -732,6 +728,21 @@ mod tests {
             .expect("time drift")
             .as_micros() as u64
     }
+
+    #[tokio::test]
+    async fn test_arc_discovery() -> TestResult {
+        let discovery = Arc::new(EmptyDiscovery);
+
+        let _ep = Endpoint::builder()
+            .add_discovery({
+                let discovery = discovery.clone();
+                move |_| Some(discovery)
+            })
+            .bind()
+            .await?;
+
+        Ok(())
+    }
 }
 
 /// This module contains end-to-end tests for DNS node discovery.

iroh/src/discovery/static_provider.rs

@@ -1,4 +1,15 @@
-//! A static discovery implementation that allows adding info for nodes manually.
+//! A static node discovery to manually add node addressing information.
+//!
+//! Often an application might get node addressing information out-of-band in an
+//! application-specific way.  [`NodeTicket`]'s are one common way used to achieve this.
+//! This "static" addressing information is often only usable for a limited time so needs to
+//! be able to be removed again once know it is no longer useful.
+//!
+//! This is where the [`StaticProvider`] is useful: it allows applications to add and
+//! retract node addressing information that is otherwise out-of-band to iroh.
+//!
+//! [`NodeTicket`]: https://docs.rs/iroh-base/latest/iroh_base/ticket/struct.NodeTicket
+
 use std::{
     collections::{btree_map::Entry, BTreeMap, BTreeSet},
     net::SocketAddr,
@@ -11,8 +22,49 @@ use iroh_base::{NodeAddr, NodeId, RelayUrl};
 
 use super::{Discovery, DiscoveryItem};
 
-/// A static discovery implementation that allows providing info for nodes manually.
-#[derive(Debug, Default)]
+/// A static node discovery to manually add node addressing information.
+///
+/// Often an application might get node addressing information out-of-band in an
+/// application-specific way.  [`NodeTicket`]'s are one common way used to achieve this.
+/// This "static" addressing information is often only usable for a limited time so needs to
+/// be able to be removed again once know it is no longer useful.
+///
+/// This is where the [`StaticProvider`] is useful: it allows applications to add and
+/// retract node addressing information that is otherwise out-of-band to iroh.
+///
+/// # Examples
+///
+/// ```rust
+/// use iroh::{discovery::static_provider::StaticProvider, Endpoint, NodeAddr};
+/// use iroh_base::SecretKey;
+///
+/// # #[tokio::main]
+/// # async fn main() -> anyhow::Result<()> {
+/// // Create the discovery service and endpoint.
+/// let discovery = StaticProvider::new();
+///
+/// let _ep = Endpoint::builder()
+///     .add_discovery({
+///         let discovery = discovery.clone();
+///         move |_| Some(discovery)
+///     })
+///     .bind()
+///     .await?;
+///
+/// /// Sometime later add a RelayUrl for a fake NodeId.
+/// let key = SecretKey::from_bytes(&[0u8; 32]); // Do not use fake secret keys!
+/// discovery.add_node_addr(NodeAddr {
+///     node_id: key.public(),
+///     relay_url: Some("https://example.com".parse()?),
+///     direct_addresses: Default::default(),
+/// });
+///
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`NodeTicket`]: https://docs.rs/iroh-base/latest/iroh_base/ticket/struct.NodeTicket
+#[derive(Debug, Default, Clone)]
 #[repr(transparent)]
 pub struct StaticProvider {
     nodes: Arc<RwLock<BTreeMap<NodeId, NodeInfo>>>,
@@ -27,16 +79,22 @@ struct NodeInfo {
 
 impl StaticProvider {
     /// The provenance string for this discovery implementation.
+    ///
+    /// This is mostly used for debugging information and allows understanding the origin of
+    /// addressing information used by an iroh [`Endpoint`].
+    ///
+    /// [`Endpoint`]: crate::Endpoint
     pub const PROVENANCE: &'static str = "static_discovery";
 
-    /// Create a new static discovery instance.
+    /// Creates a new static discovery instance.
     pub fn new() -> Self {
         Self::default()
     }
 
-    /// Creates a static discovery instance from something that can be converted into node addresses.
+    /// Creates a static discovery instance from node addresses.
+    ///
+    /// # Examples
     ///
-    /// Example:
     /// ```rust
     /// use std::{net::SocketAddr, str::FromStr};
     ///
@@ -68,7 +126,7 @@ impl StaticProvider {
         res
     }
 
-    /// Add node info for the given node id.
+    /// Sets node addressing information for the given node ID.
     ///
     /// This will completely overwrite any existing info for the node.
     pub fn set_node_addr(&self, info: impl Into<NodeAddr>) -> Option<NodeAddr> {
@@ -90,9 +148,11 @@ impl StaticProvider {
         })
     }
 
-    /// Add node info for the given node id, combining it with any existing info.
+    /// Augments node addressing information for the given node ID.
     ///
-    /// This will add any new direct addresses and overwrite the relay url.
+    /// The provided addressing information is combined with the existing info in the static
+    /// provider.  Any new direct addresses are added to those already present while the
+    /// relay URL is overwritten.
     pub fn add_node_addr(&self, info: impl Into<NodeAddr>) {
         let info: NodeAddr = info.into();
         let last_updated = SystemTime::now();
@@ -114,7 +174,7 @@ impl StaticProvider {
         }
     }
 
-    /// Get node info for the given node id.
+    /// Returns node addressing information for the given node ID.
     pub fn get_node_addr(&self, node_id: NodeId) -> Option<NodeAddr> {
         let guard = self.nodes.read().expect("poisoned");
         let info = guard.get(&node_id)?;
@@ -125,7 +185,9 @@ impl StaticProvider {
         })
     }
 
-    /// Remove node info for the given node id.
+    /// Removes all node addressing information for the given node ID.
+    ///
+    /// Any removed information is returned.
     pub fn remove_node_addr(&self, node_id: NodeId) -> Option<NodeAddr> {
         let mut guard = self.nodes.write().expect("poisoned");
         let info = guard.remove(&node_id)?;
@@ -170,3 +232,47 @@ impl Discovery for StaticProvider {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use anyhow::Context;
+    use iroh_base::SecretKey;
+    use testresult::TestResult;
+
+    use super::*;
+    use crate::Endpoint;
+
+    #[tokio::test]
+    async fn test_basic() -> TestResult {
+        let discovery = StaticProvider::new();
+
+        let _ep = Endpoint::builder()
+            .add_discovery({
+                let discovery = discovery.clone();
+                move |_| Some(discovery)
+            })
+            .bind()
+            .await?;
+
+        let key = SecretKey::from_bytes(&[0u8; 32]);
+        let addr = NodeAddr {
+            node_id: key.public(),
+            relay_url: Some("https://example.com".parse()?),
+            direct_addresses: Default::default(),
+        };
+        discovery.add_node_addr(addr.clone());
+
+        let back = discovery.get_node_addr(key.public()).context("no addr")?;
+
+        assert_eq!(back, addr);
+
+        let removed = discovery
+            .remove_node_addr(key.public())
+            .context("nothing removed")?;
+        assert_eq!(removed, addr);
+        let res = discovery.get_node_addr(key.public());
+        assert!(res.is_none());
+
+        Ok(())
+    }
+}

iroh/src/endpoint.rs

@@ -493,11 +493,14 @@ pub fn make_server_config(
 /// while still remaining independent connections.  This will result in more optimal network
 /// behaviour.
 ///
-/// New connections are typically created using the [`Endpoint::connect`] and
-/// [`Endpoint::accept`] methods.  Once established, the [`Connection`] gives access to most
-/// [QUIC] features.  Individual streams to send data to the peer are created using the
-/// [`Connection::open_bi`], [`Connection::accept_bi`], [`Connection::open_uni`] and
-/// [`Connection::open_bi`] functions.
+/// The endpoint is created using the [`Builder`], which can be created using
+/// [`Endpoint::builder`].
+///
+/// Once an endpoint exists, new connections are typically created using the
+/// [`Endpoint::connect`] and [`Endpoint::accept`] methods.  Once established, the
+/// [`Connection`] gives access to most [QUIC] features.  Individual streams to send data to
+/// the peer are created using the [`Connection::open_bi`], [`Connection::accept_bi`],
+/// [`Connection::open_uni`] and [`Connection::open_bi`] functions.
 ///
 /// Note that due to the light-weight properties of streams a stream will only be accepted
 /// once the initiating peer has sent some data on it.
@@ -713,10 +716,17 @@ impl Endpoint {
     ///
     /// See also [`Endpoint::add_node_addr_with_source`].
     ///
+    /// # Using node discovery instead
+    ///
+    /// It is strongly advised to use node discovery using the [`StaticProvider`] instead.
+    /// This provides more flexibility and future proofing.
+    ///
     /// # Errors
     ///
     /// Will return an error if we attempt to add our own [`PublicKey`] to the node map or if the
     /// direct addresses are a subset of ours.
+    ///
+    /// [`StaticProvider`]: crate::discovery::static_provider::StaticProvider
     pub fn add_node_addr(&self, node_addr: NodeAddr) -> Result<()> {
         self.add_node_addr_inner(node_addr, magicsock::Source::App)
     }
@@ -729,10 +739,17 @@ impl Endpoint {
     /// address that matches this node's direct addresses will be silently ignored. The *source* is
     /// used for logging exclusively and will not be stored.
     ///
+    /// # Using node discovery instead
+    ///
+    /// It is strongly advised to use node discovery using the [`StaticProvider`] instead.
+    /// This provides more flexibility and future proofing.
+    ///
     /// # Errors
     ///
     /// Will return an error if we attempt to add our own [`PublicKey`] to the node map or if the
     /// direct addresses are a subset of ours.
+    ///
+    /// [`StaticProvider`]: crate::discovery::static_provider::StaticProvider
     pub fn add_node_addr_with_source(
         &self,
         node_addr: NodeAddr,