[NUT-15] LND Support for MPP Payments (#536)

---------

Co-authored-by: thesimplekid <tsk@thesimplekid.com>

crates/cdk-cln/src/lib.rs

@@ -241,9 +241,14 @@ impl MintLightning for Cln {
             }
         }
 
-        let amount_msat = melt_quote
-            .msat_to_pay
-            .map(|a| CLN_Amount::from_msat(a.into()));
+        let amount_msat = partial_amount
+            .is_none()
+            .then(|| {
+                melt_quote
+                    .msat_to_pay
+                    .map(|a| CLN_Amount::from_msat(a.into()))
+            })
+            .flatten();
 
         let mut cln_client = self.cln_client.lock().await;
         let cln_response = cln_client

crates/cdk-integration-tests/Cargo.toml

@@ -19,6 +19,7 @@ axum = "0.6.20"
 rand = "0.8.5"
 bip39 = { version = "2.0", features = ["rand"] }
 anyhow = "1"
+cashu = { path = "../cashu", features = ["mint", "wallet"] }
 cdk = { path = "../cdk", features = ["mint", "wallet"] }
 cdk-cln = { path = "../cdk-cln" }
 cdk-lnd = { path = "../cdk-lnd" }

crates/cdk-integration-tests/src/init_regtest.rs

@@ -36,17 +36,17 @@ pub fn get_mint_addr() -> String {
     env::var("cdk_itests_mint_addr").expect("Temp dir set")
 }
 
-pub fn get_mint_port() -> u16 {
-    let dir = env::var("cdk_itests_mint_port").expect("Temp dir set");
+pub fn get_mint_port(which: &str) -> u16 {
+    let dir = env::var(format!("cdk_itests_mint_port_{}", which)).expect("Temp dir set");
     dir.parse().unwrap()
 }
 
-pub fn get_mint_url() -> String {
-    format!("http://{}:{}", get_mint_addr(), get_mint_port())
+pub fn get_mint_url(which: &str) -> String {
+    format!("http://{}:{}", get_mint_addr(), get_mint_port(which))
 }
 
-pub fn get_mint_ws_url() -> String {
-    format!("ws://{}:{}/v1/ws", get_mint_addr(), get_mint_port())
+pub fn get_mint_ws_url(which: &str) -> String {
+    format!("ws://{}:{}/v1/ws", get_mint_addr(), get_mint_port(which))
 }
 
 pub fn get_temp_dir() -> PathBuf {

crates/cdk-integration-tests/tests/regtest.rs

@@ -5,6 +5,7 @@ use std::time::Duration;
 
 use anyhow::{bail, Result};
 use bip39::Mnemonic;
+use cashu::{MeltOptions, Mpp};
 use cdk::amount::{Amount, SplitTarget};
 use cdk::cdk_database::WalletMemoryDatabase;
 use cdk::nuts::nut00::ProofsMethods;
@@ -19,7 +20,7 @@ use cdk_integration_tests::init_regtest::{
     get_mint_url, get_mint_ws_url, LND_RPC_ADDR, LND_TWO_RPC_ADDR,
 };
 use cdk_integration_tests::wait_for_mint_to_be_paid;
-use futures::{SinkExt, StreamExt};
+use futures::{join, SinkExt, StreamExt};
 use lightning_invoice::Bolt11Invoice;
 use ln_regtest_rs::ln_client::{ClnClient, LightningClient, LndClient};
 use ln_regtest_rs::InvoiceStatus;
@@ -79,14 +80,14 @@ async fn test_regtest_mint_melt_round_trip() -> Result<()> {
     let lnd_client = init_lnd_client().await;
 
     let wallet = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &Mnemonic::generate(12)?.to_seed_normalized(""),
         None,
     )?;
 
-    let (ws_stream, _) = connect_async(get_mint_ws_url())
+    let (ws_stream, _) = connect_async(get_mint_ws_url("0"))
         .await
         .expect("Failed to connect");
     let (mut write, mut reader) = ws_stream.split();
@@ -164,7 +165,7 @@ async fn test_regtest_mint_melt() -> Result<()> {
     let lnd_client = init_lnd_client().await;
 
     let wallet = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &Mnemonic::generate(12)?.to_seed_normalized(""),
@@ -198,7 +199,7 @@ async fn test_restore() -> Result<()> {
 
     let seed = Mnemonic::generate(12)?.to_seed_normalized("");
     let wallet = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &seed,
@@ -218,7 +219,7 @@ async fn test_restore() -> Result<()> {
     assert!(wallet.total_balance().await? == 100.into());
 
     let wallet_2 = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &seed,
@@ -257,7 +258,7 @@ async fn test_pay_invoice_twice() -> Result<()> {
 
     let seed = Mnemonic::generate(12)?.to_seed_normalized("");
     let wallet = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &seed,
@@ -316,7 +317,7 @@ async fn test_internal_payment() -> Result<()> {
 
     let seed = Mnemonic::generate(12)?.to_seed_normalized("");
     let wallet = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &seed,
@@ -338,7 +339,7 @@ async fn test_internal_payment() -> Result<()> {
     let seed = Mnemonic::generate(12)?.to_seed_normalized("");
 
     let wallet_2 = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &seed,
@@ -360,7 +361,7 @@ async fn test_internal_payment() -> Result<()> {
         .await
         .unwrap();
 
-    let check_paid = match get_mint_port() {
+    let check_paid = match get_mint_port("0") {
         8085 => {
             let cln_one_dir = get_cln_dir("one");
             let cln_client = ClnClient::new(cln_one_dir.clone(), None).await?;
@@ -411,7 +412,7 @@ async fn test_cached_mint() -> Result<()> {
     let lnd_client = init_lnd_client().await;
 
     let wallet = Wallet::new(
-        &get_mint_url(),
+        &get_mint_url("0"),
         CurrencyUnit::Sat,
         Arc::new(WalletMemoryDatabase::default()),
         &Mnemonic::generate(12)?.to_seed_normalized(""),
@@ -438,7 +439,7 @@ async fn test_cached_mint() -> Result<()> {
     }
 
     let active_keyset_id = wallet.get_active_mint_keyset().await?.id;
-    let http_client = HttpClient::new(get_mint_url().as_str().parse()?);
+    let http_client = HttpClient::new(get_mint_url("0").as_str().parse()?);
     let premint_secrets =
         PreMintSecrets::random(active_keyset_id, 100.into(), &SplitTarget::default()).unwrap();
 
@@ -458,3 +459,84 @@ async fn test_cached_mint() -> Result<()> {
     assert!(response == response1);
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+async fn test_multimint_melt() -> Result<()> {
+    let lnd_client = init_lnd_client().await;
+
+    let wallet1 = Wallet::new(
+        &get_mint_url("0"),
+        CurrencyUnit::Sat,
+        Arc::new(WalletMemoryDatabase::default()),
+        &Mnemonic::generate(12)?.to_seed_normalized(""),
+        None,
+    )?;
+    let wallet2 = Wallet::new(
+        &get_mint_url("1"),
+        CurrencyUnit::Sat,
+        Arc::new(WalletMemoryDatabase::default()),
+        &Mnemonic::generate(12)?.to_seed_normalized(""),
+        None,
+    )?;
+
+    let mint_amount = Amount::from(100);
+
+    // Fund the wallets
+    let quote = wallet1.mint_quote(mint_amount, None).await?;
+    lnd_client.pay_invoice(quote.request.clone()).await?;
+    loop {
+        let quote_status = wallet1.mint_quote_state(&quote.id).await?;
+        if quote_status.state == MintQuoteState::Paid {
+            break;
+        }
+        tracing::debug!("Quote not yet paid");
+    }
+    wallet1
+        .mint(&quote.id, SplitTarget::default(), None)
+        .await?;
+
+    let quote = wallet2.mint_quote(mint_amount, None).await?;
+    lnd_client.pay_invoice(quote.request.clone()).await?;
+    loop {
+        let quote_status = wallet2.mint_quote_state(&quote.id).await?;
+        if quote_status.state == MintQuoteState::Paid {
+            break;
+        }
+        tracing::debug!("Quote not yet paid");
+    }
+    wallet2
+        .mint(&quote.id, SplitTarget::default(), None)
+        .await?;
+
+    // Get an invoice
+    let invoice = lnd_client.create_invoice(Some(50)).await?;
+
+    // Get multi-part melt quotes
+    let melt_options = MeltOptions::Mpp {
+        mpp: Mpp {
+            amount: Amount::from(25000),
+        },
+    };
+    let quote_1 = wallet1
+        .melt_quote(invoice.clone(), Some(melt_options))
+        .await
+        .expect("Could not get melt quote");
+    let quote_2 = wallet2
+        .melt_quote(invoice.clone(), Some(melt_options))
+        .await
+        .expect("Could not get melt quote");
+
+    // Multimint pay invoice
+    let result1 = wallet1.melt(&quote_1.id);
+    let result2 = wallet2.melt(&quote_2.id);
+    let result = join!(result1, result2);
+
+    // Unpack results
+    let result1 = result.0.unwrap();
+    let result2 = result.1.unwrap();
+
+    // Check
+    assert!(result1.state == result2.state);
+    assert!(result1.state == MeltQuoteState::Paid);
+    Ok(())
+}

crates/cdk-lnd/src/error.rs

@@ -1,5 +1,6 @@
 //! LND Errors
 
+use fedimint_tonic_lnd::tonic::Status;
 use thiserror::Error;
 
 /// LND Error
@@ -23,6 +24,12 @@ pub enum Error {
     /// Unknown payment status
     #[error("LND unknown payment status")]
     UnknownPaymentStatus,
+    /// Missing last hop in route
+    #[error("LND missing last hop in route")]
+    MissingLastHop,
+    /// Errors coming from the backend
+    #[error("LND error: `{0}`")]
+    LndError(Status),
 }
 
 impl From<Error> for cdk::cdk_lightning::Error {

crates/cdk-lnd/src/lib.rs

@@ -19,12 +19,13 @@ use cdk::cdk_lightning::{
 };
 use cdk::mint::FeeReserve;
 use cdk::nuts::{CurrencyUnit, MeltQuoteBolt11Request, MeltQuoteState, MintQuoteState};
+use cdk::secp256k1::hashes::Hash;
 use cdk::util::{hex, unix_time};
 use cdk::{mint, Bolt11Invoice};
 use error::Error;
 use fedimint_tonic_lnd::lnrpc::fee_limit::Limit;
 use fedimint_tonic_lnd::lnrpc::payment::PaymentStatus;
-use fedimint_tonic_lnd::lnrpc::FeeLimit;
+use fedimint_tonic_lnd::lnrpc::{FeeLimit, Hop, HtlcAttempt, MppRecord};
 use fedimint_tonic_lnd::tonic::Code;
 use fedimint_tonic_lnd::Client;
 use futures::{Stream, StreamExt};
@@ -80,7 +81,7 @@ impl MintLightning for Lnd {
     #[instrument(skip_all)]
     fn get_settings(&self) -> Settings {
         Settings {
-            mpp: false,
+            mpp: true,
             unit: CurrencyUnit::Msat,
             invoice_description: true,
         }
@@ -200,7 +201,7 @@ impl MintLightning for Lnd {
     async fn pay_invoice(
         &self,
         melt_quote: mint::MeltQuote,
-        _partial_amount: Option<Amount>,
+        partial_amount: Option<Amount>,
         max_fee: Option<Amount>,
     ) -> Result<PayInvoiceResponse, Self::Err> {
         let payment_request = melt_quote.request;
@@ -222,60 +223,159 @@ impl MintLightning for Lnd {
             }
         }
 
-        let amount_msat: u64 = match melt_quote.msat_to_pay {
-            Some(amount_msat) => amount_msat.into(),
-            None => {
-                let bolt11 = Bolt11Invoice::from_str(&payment_request)?;
-                bolt11
-                    .amount_milli_satoshis()
-                    .ok_or(Error::UnknownInvoiceAmount)?
-            }
-        };
-
-        let pay_req = fedimint_tonic_lnd::lnrpc::SendRequest {
-            payment_request,
-            fee_limit: max_fee.map(|f| {
-                let limit = Limit::Fixed(u64::from(f) as i64);
-
-                FeeLimit { limit: Some(limit) }
-            }),
-            amt_msat: amount_msat as i64,
-            ..Default::default()
+        let bolt11 = Bolt11Invoice::from_str(&payment_request)?;
+        let amount_msat: u64 = match bolt11.amount_milli_satoshis() {
+            Some(amount_msat) => amount_msat,
+            None => melt_quote
+                .msat_to_pay
+                .ok_or(Error::UnknownInvoiceAmount)?
+                .into(),
         };
 
-        let payment_response = self
-            .client
-            .lock()
-            .await
-            .lightning()
-            .send_payment_sync(fedimint_tonic_lnd::tonic::Request::new(pay_req))
-            .await
-            .map_err(|err| {
-                tracing::warn!("Lightning payment failed: {}", err);
-                Error::PaymentFailed
-            })?
-            .into_inner();
-
-        let total_amount = payment_response
-            .payment_route
-            .map_or(0, |route| route.total_amt_msat / MSAT_IN_SAT as i64)
-            as u64;
+        // Detect partial payments
+        match partial_amount {
+            Some(part_amt) => {
+                let partial_amount_msat = to_unit(part_amt, &melt_quote.unit, &CurrencyUnit::Msat)?;
+                let invoice = Bolt11Invoice::from_str(&payment_request)?;
+
+                // Extract information from invoice
+                let pub_key = invoice.get_payee_pub_key();
+                let payer_addr = invoice.payment_secret().0.to_vec();
+                let payment_hash = invoice.payment_hash();
+
+                // Create a request for the routes
+                let route_req = fedimint_tonic_lnd::lnrpc::QueryRoutesRequest {
+                    pub_key: hex::encode(pub_key.serialize()),
+                    amt_msat: u64::from(partial_amount_msat) as i64,
+                    fee_limit: max_fee.map(|f| {
+                        let limit = Limit::Fixed(u64::from(f) as i64);
+                        FeeLimit { limit: Some(limit) }
+                    }),
+                    ..Default::default()
+                };
+
+                // Query the routes
+                let routes_response: fedimint_tonic_lnd::lnrpc::QueryRoutesResponse = self
+                    .client
+                    .lock()
+                    .await
+                    .lightning()
+                    .query_routes(route_req)
+                    .await
+                    .map_err(Error::LndError)?
+                    .into_inner();
+
+                let mut payment_response: HtlcAttempt = HtlcAttempt {
+                    ..Default::default()
+                };
+
+                // For each route:
+                // update its MPP record,
+                // attempt it and check the result
+                for mut route in routes_response.routes.into_iter() {
+                    let last_hop: &mut Hop = route.hops.last_mut().ok_or(Error::MissingLastHop)?;
+                    let mpp_record = MppRecord {
+                        payment_addr: payer_addr.clone(),
+                        total_amt_msat: amount_msat as i64,
+                    };
+                    last_hop.mpp_record = Some(mpp_record);
+                    tracing::debug!("sendToRouteV2 needle");
+                    payment_response = self
+                        .client
+                        .lock()
+                        .await
+                        .router()
+                        .send_to_route_v2(fedimint_tonic_lnd::routerrpc::SendToRouteRequest {
+                            payment_hash: payment_hash.to_byte_array().to_vec(),
+                            route: Some(route),
+                            ..Default::default()
+                        })
+                        .await
+                        .map_err(Error::LndError)?
+                        .into_inner();
+
+                    if let Some(failure) = payment_response.failure {
+                        if failure.code == 15 {
+                            // Try a different route
+                            continue;
+                        }
+                    } else {
+                        break;
+                    }
+                }
 
-        let (status, payment_preimage) = match total_amount == 0 {
-            true => (MeltQuoteState::Unpaid, None),
-            false => (
-                MeltQuoteState::Paid,
-                Some(hex::encode(payment_response.payment_preimage)),
-            ),
-        };
+                // Get status and maybe the preimage
+                let (status, payment_preimage) = match payment_response.status {
+                    0 => (MeltQuoteState::Pending, None),
+                    1 => (
+                        MeltQuoteState::Paid,
+                        Some(hex::encode(payment_response.preimage)),
+                    ),
+                    2 => (MeltQuoteState::Unpaid, None),
+                    _ => (MeltQuoteState::Unknown, None),
+                };
+
+                // Get the actual amount paid in sats
+                let mut total_amt: u64 = 0;
+                if let Some(route) = payment_response.route {
+                    total_amt = (route.total_amt_msat / 1000) as u64;
+                }
 
-        Ok(PayInvoiceResponse {
-            payment_lookup_id: hex::encode(payment_response.payment_hash),
-            payment_preimage,
-            status,
-            total_spent: total_amount.into(),
-            unit: CurrencyUnit::Sat,
-        })
+                Ok(PayInvoiceResponse {
+                    payment_lookup_id: hex::encode(payment_hash),
+                    payment_preimage,
+                    status,
+                    total_spent: total_amt.into(),
+                    unit: CurrencyUnit::Sat,
+                })
+            }
+            None => {
+                let pay_req = fedimint_tonic_lnd::lnrpc::SendRequest {
+                    payment_request,
+                    fee_limit: max_fee.map(|f| {
+                        let limit = Limit::Fixed(u64::from(f) as i64);
+
+                        FeeLimit { limit: Some(limit) }
+                    }),
+                    amt_msat: amount_msat as i64,
+                    ..Default::default()
+                };
+
+                let payment_response = self
+                    .client
+                    .lock()
+                    .await
+                    .lightning()
+                    .send_payment_sync(fedimint_tonic_lnd::tonic::Request::new(pay_req))
+                    .await
+                    .map_err(|err| {
+                        tracing::warn!("Lightning payment failed: {}", err);
+                        Error::PaymentFailed
+                    })?
+                    .into_inner();
+
+                let total_amount = payment_response
+                    .payment_route
+                    .map_or(0, |route| route.total_amt_msat / MSAT_IN_SAT as i64)
+                    as u64;
+
+                let (status, payment_preimage) = match total_amount == 0 {
+                    true => (MeltQuoteState::Unpaid, None),
+                    false => (
+                        MeltQuoteState::Paid,
+                        Some(hex::encode(payment_response.payment_preimage)),
+                    ),
+                };
+
+                Ok(PayInvoiceResponse {
+                    payment_lookup_id: hex::encode(payment_response.payment_hash),
+                    payment_preimage,
+                    status,
+                    total_spent: total_amount.into(),
+                    unit: CurrencyUnit::Sat,
+                })
+            }
+        }
     }
 
     #[instrument(skip(self, description))]

crates/cdk/src/mint/melt.rs

@@ -43,30 +43,45 @@ impl Mint {
             .get_settings(&unit, &method)
             .ok_or(Error::UnsupportedUnit)?;
 
-        if matches!(options, Some(MeltOptions::Mpp { mpp: _ })) {
-            // Verify there is no corresponding mint quote.
-            // Otherwise a wallet is trying to pay someone internally, but
-            // with a multi-part quote. And that's just not possible.
-            if (self.localstore.get_mint_quote_by_request(&request).await?).is_some() {
-                return Err(Error::InternalMultiPartMeltQuote);
-            }
-            // Verify MPP is enabled for unit and method
-            if !nut15
-                .methods
-                .into_iter()
-                .any(|m| m.method == method && m.unit == unit)
-            {
-                return Err(Error::MppUnitMethodNotSupported(unit, method));
+        let amount = match options {
+            Some(MeltOptions::Mpp { mpp: _ }) => {
+                // Verify there is no corresponding mint quote.
+                // Otherwise a wallet is trying to pay someone internally, but
+                // with a multi-part quote. And that's just not possible.
+                if (self.localstore.get_mint_quote_by_request(&request).await?).is_some() {
+                    return Err(Error::InternalMultiPartMeltQuote);
+                }
+                // Verify MPP is enabled for unit and method
+                if !nut15
+                    .methods
+                    .into_iter()
+                    .any(|m| m.method == method && m.unit == unit)
+                {
+                    return Err(Error::MppUnitMethodNotSupported(unit, method));
+                }
+                // Assign `amount`
+                // because should have already been converted to the partial amount
+                amount
             }
-        }
+            None => amount,
+        };
+
         let is_above_max = matches!(settings.max_amount, Some(max) if amount > max);
         let is_below_min = matches!(settings.min_amount, Some(min) if amount < min);
         match is_above_max || is_below_min {
-            true => Err(Error::AmountOutofLimitRange(
-                settings.min_amount.unwrap_or_default(),
-                settings.max_amount.unwrap_or_default(),
-                amount,
-            )),
+            true => {
+                tracing::error!(
+                    "Melt amount out of range: {} is not within {} and {}",
+                    amount,
+                    settings.min_amount.unwrap_or_default(),
+                    settings.max_amount.unwrap_or_default(),
+                );
+                Err(Error::AmountOutofLimitRange(
+                    settings.min_amount.unwrap_or_default(),
+                    settings.max_amount.unwrap_or_default(),
+                    amount,
+                ))
+            }
             false => Ok(()),
         }
     }
@@ -210,6 +225,13 @@ impl Mint {
         let quote_msats = to_unit(melt_quote.amount, &melt_quote.unit, &CurrencyUnit::Msat)
             .expect("Quote unit is checked above that it can convert to msat");
 
+        let invoice_amount_msats: Amount = match invoice.amount_milli_satoshis() {
+            Some(amt) => amt.into(),
+            None => melt_quote
+                .msat_to_pay
+                .ok_or(Error::InvoiceAmountUndefined)?,
+        };
+        /*
         let invoice_amount_msats: Amount = match melt_quote.msat_to_pay {
             Some(amount) => amount,
             None => invoice
@@ -217,11 +239,11 @@ impl Mint {
                 .ok_or(Error::InvoiceAmountUndefined)?
                 .into(),
         };
+        */
 
         let partial_amount = match invoice_amount_msats > quote_msats {
             true => {
                 let partial_msats = invoice_amount_msats - quote_msats;
-
                 Some(
                     to_unit(partial_msats, &CurrencyUnit::Msat, &melt_quote.unit)
                         .map_err(|_| Error::UnsupportedUnit)?,
@@ -491,6 +513,7 @@ impl Mint {
                     }
                     _ => None,
                 };
+                tracing::debug!("partial_amount: {:?}", partial_amount);
                 let ln = match self
                     .ln
                     .get(&LnKey::new(quote.unit.clone(), PaymentMethod::Bolt11))

misc/itests.sh

@@ -33,9 +33,10 @@ trap cleanup EXIT
 # Create a temporary directory
 export cdk_itests=$(mktemp -d)
 export cdk_itests_mint_addr="127.0.0.1";
-export cdk_itests_mint_port=8085;
+export cdk_itests_mint_port_0=8085;
+export cdk_itests_mint_port_1=8087;
 
-URL="http://$cdk_itests_mint_addr:$cdk_itests_mint_port/v1/info"
+URL="http://$cdk_itests_mint_addr:$cdk_itests_mint_port_0/v1/info"
 # Check if the temporary directory was created successfully
 if [[ ! -d "$cdk_itests" ]]; then
     echo "Failed to create temp directory"
@@ -90,8 +91,9 @@ cargo test -p cdk-integration-tests --test regtest
 # # Run cargo test with the http_subscription feature
 cargo test -p cdk-integration-tests --test regtest --features http_subscription
 
-# Run tests with lnd mint
-export cdk_itests_mint_port=8087;
+# Switch Mints: Run tests with LND mint
+export cdk_itests_mint_port_0=8087;
+export cdk_itests_mint_port_1=8085;
 cargo test -p cdk-integration-tests --test regtest
 
 # Capture the exit status of cargo test