wallet: update wallet address generation logic to use unified BIP86/BIP32 approach

diff --git a/lightningd/hsm_control.c b/lightningd/hsm_control.c
index f8be6cc..182cbe0 100644
--- a/lightningd/hsm_control.c
+++ b/lightningd/hsm_control.c
@@ -189,21 +189,36 @@ struct ext_key *hsm_init(struct lightningd *ld)
 		fatal("--experimental-splicing needs HSM capable of signing splices!");
 	}
 
-	/* Try to get BIP86 base key from HSM (works only for mnemonic secrets) */
-	ld->bip86_base = tal(ld, struct ext_key);
-	msg = towire_hsmd_derive_bip86_key(NULL, 0, false);
-	const u8 *reply = hsm_sync_req(tmpctx, ld, take(msg));
-	if (fromwire_hsmd_derive_bip86_key_reply(reply, ld->bip86_base)) {
-		/* BIP86 derivation succeeded - we have a mnemonic-based secret */
-		log_info(ld->log, "Using BIP86 for new addresses, BIP32 for channels (mnemonic HSM secret)");
-		/* Keep bip32_base for channel operations, database, etc. */
+	/* Check if we have a mnemonic-based HSM secret from TLV */
+	bool is_mnemonic_secret = false;
+	if (tlvs && tlvs->hsm_secret_type) {
+		u8 secret_type = *tlvs->hsm_secret_type;
+		is_mnemonic_secret = (secret_type == 2 || secret_type == 3); /* HSM_SECRET_MNEMONIC_NO_PASS or HSM_SECRET_MNEMONIC_WITH_PASS */
+	}
+
+	if (is_mnemonic_secret) {
+		/* Try to get BIP86 base key from HSM (works only for mnemonic secrets) */
+		ld->bip86_base = tal(ld, struct ext_key);
+		msg = towire_hsmd_derive_bip86_key(NULL, 0, false);
+		const u8 *reply = hsm_sync_req(tmpctx, ld, take(msg));
+		if (fromwire_hsmd_derive_bip86_key_reply(reply, ld->bip86_base)) {
+			/* BIP86 derivation succeeded */
+			log_info(ld->log, "Using BIP86 for new addresses, BIP32 for channels (mnemonic HSM secret)");
+			/* Keep bip32_base for channel operations, database, etc. */
+		} else {
+			/* BIP86 derivation failed unexpectedly */
+			ld->bip86_base = tal_free(ld->bip86_base);
+		}
 	} else {
-		/* BIP86 derivation failed - we have a legacy secret */
+		/* Legacy HSM secret - don't attempt BIP86 derivation */
 		log_info(ld->log, "Using BIP32 derivation for all operations (legacy HSM secret)");
-		ld->bip86_base = tal_free(ld->bip86_base);
-		/* bip32_base was already set by the HSM init reply */
+		ld->bip86_base = NULL;
 	}
 
+	/* Free the TLV structure to prevent memory leak */
+	if (tlvs)
+		tal_free(tlvs);
+
 	/* This is equivalent to makesecret("bolt12-invoice-base") */
 	msg = towire_hsmd_derive_secret(NULL, tal_dup_arr(tmpctx, u8,
 							  (const u8 *)BOLT12_ID_BASE_STRING,
diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c
index 9a5494b..59c06c4 100644
--- a/lightningd/peer_control.c
+++ b/lightningd/peer_control.c
@@ -210,7 +210,12 @@ u8 *p2tr_for_keyidx(const tal_t *ctx, struct lightningd *ld, u64 keyidx)
 {
 	struct pubkey shutdownkey;
 
-	bip32_pubkey(ld, &shutdownkey, keyidx);
+	/* Use BIP86 derivation if wallet has BIP86 base, otherwise use BIP32 */
+	if (ld->bip86_base) {
+		bip86_pubkey(ld, &shutdownkey, keyidx);
+	} else {
+		bip32_pubkey(ld, &shutdownkey, keyidx);
+	}
 
 	return scriptpubkey_p2tr(ctx, &shutdownkey);
 }
diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c
index fe895a4..96dc1d6 100644
--- a/lightningd/test/run-invoice-select-inchan.c
+++ b/lightningd/test/run-invoice-select-inchan.c
@@ -17,6 +17,9 @@ struct channel *any_channel_by_scid(struct lightningd *ld UNNEEDED,
 /* Generated stub for bip32_pubkey */
 void bip32_pubkey(struct lightningd *ld UNNEEDED, struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
 { fprintf(stderr, "bip32_pubkey called!\n"); abort(); }
+/* Generated stub for bip86_pubkey */
+void bip86_pubkey(struct lightningd *ld UNNEEDED, struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
+{ fprintf(stderr, "bip86_pubkey called!\n"); abort(); }
 /* Generated stub for bitcoind_getutxout_ */
 void bitcoind_getutxout_(const tal_t *ctx UNNEEDED,
 			 struct bitcoind *bitcoind UNNEEDED,
diff --git a/wallet/wallet.c b/wallet/wallet.c
index 5061798..b325875 100644
--- a/wallet/wallet.c
+++ b/wallet/wallet.c
@@ -112,98 +112,53 @@ static void our_addresses_add(struct wallet_address_htable *our_addresses,
 	wallet_address_htable_add(our_addresses, waddr);
 }
 
-/* Add BIP86 address for a given index */
-static bool our_addresses_add_bip86_for_index(struct wallet *w, u32 i)
-{
-	struct pubkey pubkey;
-	const u8 *scriptpubkey;
-
-	/* Use BIP86 derivation from the base key */
-	bip86_pubkey(w->ld, &pubkey, i);
-
-	/* Create P2TR scriptpubkey from the BIP86 public key */
-	scriptpubkey = scriptpubkey_p2tr(tmpctx, &pubkey);
-	our_addresses_add(w->our_addresses,
-			  i,
-			  take(scriptpubkey),
-			  tal_bytelen(scriptpubkey),
-			  ADDR_P2TR_MNEMONIC);
-	return true;
-}
-
 static void our_addresses_add_for_index(struct wallet *w, u32 i)
 {
-	struct ext_key ext;
+	struct pubkey pubkey;
 	enum addrtype addrtype;
 	const u8 *scriptpubkey;
-	const u32 flags = BIP32_FLAG_KEY_PUBLIC | BIP32_FLAG_SKIP_HASH;
+	bool legacy = (w->ld->bip86_base == NULL);
 
-	if (bip32_key_from_parent(w->ld->bip32_base, i,
-				  flags, &ext) != WALLY_OK) {
-		abort();
+	/* Choose derivation method based on wallet type */
+	if (w->ld->bip86_base) {
+		bip86_pubkey(w->ld, &pubkey, i);
+	} else {
+		bip32_pubkey(w->ld, &pubkey, i);
 	}
 
-	/* If we don't know (prior to 24.11), just add all
-	 * possibilities. */
-	/* FIXME: We could deprecate P2SH once we don't see
-	 * any, since we stopped publishing them in 24.02 */
+	/* Determine which address types to generate */
 	if (!wallet_get_addrtype(w, i, &addrtype)) {
-		if (w->ld->bip86_base) {
-			/* Derive and add BIP86 script for this index */
-			our_addresses_add_bip86_for_index(w, i);
-			return;
-		}
-
-		const u8 *addr;
-		scriptpubkey = scriptpubkey_p2wpkh_derkey(NULL, ext.pub_key);
-		addr = scriptpubkey_p2sh(NULL, scriptpubkey);
-		our_addresses_add(w->our_addresses,
-				  i, take(addr), tal_bytelen(addr),
-				  ADDR_P2SH_SEGWIT);
-		our_addresses_add(w->our_addresses,
-				  i,
-				  take(scriptpubkey), tal_bytelen(scriptpubkey),
-				  ADDR_BECH32);
-		scriptpubkey = scriptpubkey_p2tr_derkey(NULL, ext.pub_key);
-		our_addresses_add(w->our_addresses,
-				  i,
-				  take(scriptpubkey), tal_bytelen(scriptpubkey),
-				  ADDR_P2TR);
-		return;
+		/* Unknown (prior to 24.11): add all possibilities */
+		//assert(legacy);
+		addrtype = ADDR_ALL;
 	}
 
-	switch (addrtype) {
+	/* Generate scripts based on address type */
+	if (addrtype == ADDR_P2SH_SEGWIT) {
 		/* This doesn't happen */
-	case ADDR_P2SH_SEGWIT:
 		abort();
-	case ADDR_BECH32:
-	case ADDR_ALL:
-		scriptpubkey = scriptpubkey_p2wpkh_derkey(NULL, ext.pub_key);
-		our_addresses_add(w->our_addresses,
-				  i,
-				  take(scriptpubkey),
-				  tal_bytelen(scriptpubkey),
-				  ADDR_BECH32);
-		if (addrtype != ADDR_ALL)
-			return;
-	/* Fall thru */
-	case ADDR_P2TR:
-		scriptpubkey = scriptpubkey_p2tr_derkey(NULL, ext.pub_key);
-		our_addresses_add(w->our_addresses,
-				  i,
-				  take(scriptpubkey),
-				  tal_bytelen(scriptpubkey),
-				  ADDR_P2TR);
-		return;
-	case ADDR_P2TR_MNEMONIC:
-		/* BIP86 addresses require HSM derivation */
-		if (our_addresses_add_bip86_for_index(w, i)) {
-			return;
+	}
+
+	if (addrtype & ADDR_BECH32) {
+		scriptpubkey = scriptpubkey_p2wpkh(NULL, &pubkey);
+
+		/* Add P2SH-wrapped version for legacy compatibility */
+		if (addrtype == ADDR_ALL && legacy) {
+			const u8 *addr = scriptpubkey_p2sh(NULL, scriptpubkey);
+			our_addresses_add(w->our_addresses, i, take(addr),
+					  tal_bytelen(addr), ADDR_P2SH_SEGWIT);
 		}
-		/* If BIP86 derivation fails, skip this address */
-		return;
+
+		/* Add native BECH32 */
+		our_addresses_add(w->our_addresses, i, take(scriptpubkey),
+				  tal_bytelen(scriptpubkey), ADDR_BECH32);
+	}
+
+	if (addrtype & ADDR_P2TR) {
+		scriptpubkey = scriptpubkey_p2tr(NULL, &pubkey);
+		our_addresses_add(w->our_addresses, i, take(scriptpubkey),
+				  tal_bytelen(scriptpubkey), ADDR_P2TR);
 	}
-	abort();
 }
 
 static void our_addresses_init(struct wallet *w)
@@ -211,17 +166,12 @@ static void our_addresses_init(struct wallet *w)
 	w->our_addresses_maxindex = 0;
 	w->our_addresses = new_htable(w, wallet_address_htable);
 
-	/* If BIP86 is enabled, prefill the address table up to keyscan_gap so
-	 * rescans immediately include BIP86 scripts without needing prior
-	 * address allocations. */
-	if (w->ld->bip86_base) {
-		for (u32 i = 0; i <= w->keyscan_gap; i++) {
-			our_addresses_add_for_index(w, i);
-		}
-		w->our_addresses_maxindex = w->keyscan_gap;
-	} else {
-		our_addresses_add_for_index(w, w->our_addresses_maxindex);
+	/* Prefill the address table up to keyscan_gap so rescans immediately
+	 * include scripts without needing prior address allocations. */
+	for (u32 i = 0; i <= w->keyscan_gap; i++) {
+		our_addresses_add_for_index(w, i);
 	}
+	w->our_addresses_maxindex = w->keyscan_gap;
 }
 
 static void outpointfilters_init(struct wallet *w)
@@ -1046,10 +996,12 @@ bool wallet_can_spend(struct wallet *w, const u8 *script, size_t script_len,
 
 	/* If we found a used key in the keyscan_gap we should
 	 * remember that. */
-	if (waddr->addrtype == ADDR_P2TR_MNEMONIC) {
+	if (w->ld->bip86_base) {
+		/* BIP86-based wallet: all addresses use BIP86 derivation */
 		if (waddr->index > bip86_max_index)
 			db_set_intvar(w->db, "bip86_max_index", waddr->index);
 	} else {
+		/* Legacy wallet: all addresses use BIP32 derivation */
 		if (waddr->index > bip32_max_index)
 			db_set_intvar(w->db, "bip32_max_index", waddr->index);
 	}
@@ -1063,12 +1015,22 @@ bool wallet_can_spend(struct wallet *w, const u8 *script, size_t script_len,
 s64 wallet_get_newindex(struct lightningd *ld, enum addrtype addrtype)
 {
 	struct db_stmt *stmt;
-	u64 newidx = db_get_intvar(ld->wallet->db, "bip32_max_index", 0) + 1;
+	u64 newidx;
+	const char *index_var;
+
+	/* Choose index variable based on wallet type */
+	if (ld->bip86_base) {
+		index_var = "bip86_max_index";
+	} else {
+		index_var = "bip32_max_index";
+	}
+
+	newidx = db_get_intvar(ld->wallet->db, index_var, 0) + 1;
 
 	if (newidx == BIP32_INITIAL_HARDENED_CHILD)
 		return -1;
 
-	db_set_intvar(ld->wallet->db, "bip32_max_index", newidx);
+	db_set_intvar(ld->wallet->db, index_var, newidx);
 	stmt = db_prepare_v2(ld->wallet->db,
 			     SQL("INSERT INTO addresses ("
 				 "  keyidx"
@@ -1081,26 +1043,6 @@ s64 wallet_get_newindex(struct lightningd *ld, enum addrtype addrtype)
 	return newidx;
 }
 
-s64 wallet_get_new_bip86_index(struct lightningd *ld)
-{
-	struct db_stmt *db_stmt;
-	u64 newidx = db_get_intvar(ld->wallet->db, "bip86_max_index", 0) + 1;
-
-	if (newidx == BIP32_INITIAL_HARDENED_CHILD)
-		return -1;
-
-	db_set_intvar(ld->wallet->db, "bip86_max_index", newidx);
-	db_stmt = db_prepare_v2(ld->wallet->db,
-			     SQL("INSERT INTO addresses ("
-				 "  keyidx"
-				 ", addrtype"
-				 ") VALUES (?, ?);"));
-	db_bind_u64(db_stmt, newidx);
-	db_bind_int(db_stmt, wallet_addrtype_in_db(ADDR_P2TR_MNEMONIC));
-	db_exec_prepared_v2(take(db_stmt));
-
-	return newidx;
-}
 
 bool wallet_get_addrtype(struct wallet *wallet, u64 idx,
 			 enum addrtype *addrtype)
@@ -3259,9 +3201,6 @@ static void got_utxo(struct wallet *w,
 	case ADDR_P2TR:
 		utxo->utxotype = UTXO_P2TR;
 		goto type_ok;
-	case ADDR_P2TR_MNEMONIC:
-		utxo->utxotype = UTXO_P2TR;
-		goto type_ok;
 	case ADDR_ALL:
 		break;
 	}
diff --git a/wallet/wallet.h b/wallet/wallet.h
index b4b0ff3..cacd363 100644
--- a/wallet/wallet.h
+++ b/wallet/wallet.h
@@ -283,7 +283,6 @@ enum addrtype {
 	ADDR_P2SH_SEGWIT = 1,
 	ADDR_BECH32 = 2,
 	ADDR_P2TR = 4,
-	ADDR_P2TR_MNEMONIC = 8,
 	ADDR_ALL = (ADDR_BECH32 + ADDR_P2TR)
 };
 
@@ -296,9 +295,6 @@ static inline enum addrtype wallet_addrtype_in_db(enum addrtype t)
 	case ADDR_P2TR:
 		BUILD_ASSERT(ADDR_P2TR == 4);
 		return t;
-	case ADDR_P2TR_MNEMONIC:
-		BUILD_ASSERT(ADDR_P2TR_MNEMONIC == 8);
-		return t;
 	case ADDR_ALL:
 		BUILD_ASSERT(ADDR_ALL == 6);
 		return t;
@@ -618,13 +614,6 @@ bool wallet_can_spend(struct wallet *w,
  */
 s64 wallet_get_newindex(struct lightningd *ld, enum addrtype addrtype);
 
-/**
- * wallet_get_new_bip86_index - get a new BIP86 index from the wallet.
- * @ld: (in) lightning daemon
- *
- * Returns -1 on error (key exhaustion).
- */
-s64 wallet_get_new_bip86_index(struct lightningd *ld);
 
 /**
  * wallet_get_addrtype - get the address types for this key.
diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c
index c6229e8..a91b2aa 100644
--- a/wallet/walletrpc.c
+++ b/wallet/walletrpc.c
@@ -50,7 +50,6 @@ encode_pubkey_to_addr(const tal_t *ctx,
 		ok = segwit_addr_encode(out, hrp, 0, h160.u.u8, sizeof(h160));
 		goto done;
 
-	case ADDR_P2TR_MNEMONIC:
 	case ADDR_P2TR: {
 		u8 *p2tr_spk = scriptpubkey_p2tr(ctx, pubkey);
 		u8 *x_key = p2tr_spk + 2;
@@ -96,13 +95,11 @@ static struct command_result *param_newaddr(struct command *cmd,
 		**addrtype = ADDR_BECH32;
 	else if (!chainparams->is_elements && json_tok_streq(buffer, tok, "p2tr"))
 		**addrtype = ADDR_P2TR;
-	else if (!chainparams->is_elements && json_tok_streq(buffer, tok, "bip86"))
-		**addrtype = ADDR_P2TR_MNEMONIC;
 	else if (json_tok_streq(buffer, tok, "all"))
 		**addrtype = ADDR_ALL;
 	else
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
-				    "'%s' should be 'p2tr', 'bip86', 'bech32', or 'all', not '%.*s'",
+				    "'%s' should be 'p2tr', 'bech32', or 'all', not '%.*s'",
 				    name, tok->end - tok->start, buffer + tok->start);
 	return NULL;
 }
@@ -112,34 +109,31 @@ bool WARN_UNUSED_RESULT newaddr_inner(struct command *cmd, struct pubkey *pubkey
 	s64 keyidx;
 	u8 *b32script;
 	u8 *p2tr_script;
+	bool use_bip86_base = (cmd->ld->bip86_base != NULL);
 
-	/* Handle BIP86 separately since it only supports P2TR */
-	if (addrtype == ADDR_P2TR_MNEMONIC) {
-		keyidx = wallet_get_new_bip86_index(cmd->ld);
-		if (keyidx < 0) return false;
+	/* Get new index - wallet_get_newindex now handles both BIP32 and BIP86 */
+	keyidx = wallet_get_newindex(cmd->ld, addrtype);
+	if (keyidx < 0) return false;
 
-		/* Use HSM for BIP86 derivation */
+	/* Choose derivation method based on wallet type */
+	if (use_bip86_base) {
+		/* Wallet has mnemonic - use BIP86 derivation */
 		bip86_pubkey(cmd->ld, pubkey, keyidx);
-
-		u8 *script = scriptpubkey_p2tr(tmpctx, pubkey);
-		txfilter_add_scriptpubkey(cmd->ld->owned_txfilter, script);
-		return true;
-	}
-
-	keyidx = wallet_get_newindex(cmd->ld, addrtype);
-	if (keyidx < 0) {
-		// return command_fail(cmd, LIGHTNINGD, "Keys exhausted ");
-		return false;
+	} else {
+		/* Legacy wallet - use BIP32 derivation */
+		bip32_pubkey(cmd->ld, pubkey, keyidx);
 	}
 
-	bip32_pubkey(cmd->ld, pubkey, keyidx);
-
+	/* Generate scripts from pubkey (same logic for both wallet types) */
 	b32script = scriptpubkey_p2wpkh(tmpctx, pubkey);
 	p2tr_script = scriptpubkey_p2tr(tmpctx, pubkey);
+
+	/* Add scripts to filter based on requested address type */
 	if (addrtype & ADDR_BECH32)
 		txfilter_add_scriptpubkey(cmd->ld->owned_txfilter, b32script);
 	if (addrtype & ADDR_P2TR)
 		txfilter_add_scriptpubkey(cmd->ld->owned_txfilter, p2tr_script);
+
 	return true;
 }
 
@@ -165,28 +159,18 @@ static struct command_result *json_newaddr(struct command *cmd,
 
 	response = json_stream_success(cmd);
 
-	/* For BIP86, only return P2TR address */
-	if (*addrtype == ADDR_P2TR_MNEMONIC) {
-		p2tr = encode_pubkey_to_addr(cmd, &pubkey, ADDR_P2TR, NULL);
-		if (!p2tr) {
-			return command_fail(cmd, LIGHTNINGD,
-					    "BIP86 P2TR address encoding failure.");
-		}
-		json_add_string(response, "p2tr", p2tr);
-	} else {
-		/* For other address types, generate both bech32 and p2tr */
-		bech32 = encode_pubkey_to_addr(cmd, &pubkey, ADDR_BECH32, NULL);
-		p2tr = encode_pubkey_to_addr(cmd, &pubkey, ADDR_P2TR, NULL);
-		if (!bech32 || !p2tr) {
-			return command_fail(cmd, LIGHTNINGD,
-					    "p2wpkh address encoding failure.");
-		}
-
-		if (*addrtype & ADDR_BECH32)
-			json_add_string(response, "bech32", bech32);
-		if (*addrtype & ADDR_P2TR)
-			json_add_string(response, "p2tr", p2tr);
+	/* Generate addresses based on requested type */
+	bech32 = encode_pubkey_to_addr(cmd, &pubkey, ADDR_BECH32, NULL);
+	p2tr = encode_pubkey_to_addr(cmd, &pubkey, ADDR_P2TR, NULL);
+	if (!bech32 || !p2tr) {
+		return command_fail(cmd, LIGHTNINGD,
+				    "p2wpkh address encoding failure.");
 	}
+
+	if (*addrtype & ADDR_BECH32)
+		json_add_string(response, "bech32", bech32);
+	if (*addrtype & ADDR_P2TR)
+		json_add_string(response, "p2tr", p2tr);
 	return command_success(cmd, response);
 }
 
@@ -204,7 +188,7 @@ static void json_add_address_details(struct json_stream *response,
 {
 	json_object_start(response, NULL);
 	json_add_u64(response, "keyidx", keyidx);
-	if (!streq(out_p2wpkh, "") && addrtype != ADDR_P2TR_MNEMONIC) {
+	if (!streq(out_p2wpkh, "")) {
 		json_add_string(response, "bech32", out_p2wpkh);
 	}
 	if (!streq(out_p2tr,"")) {
@@ -245,12 +229,12 @@ static struct command_result *json_listaddresses(struct command *cmd,
 		if (listaddrtypes[i].keyidx == BIP32_INITIAL_HARDENED_CHILD){
 			break;
 		}
-		/* Use appropriate derivation based on address type */
-		if (listaddrtypes[i].addrtype == ADDR_P2TR_MNEMONIC) {
-			/* For BIP86 addresses, use BIP86 derivation */
+		/* Use appropriate derivation based on wallet type */
+		if (cmd->ld->bip86_base) {
+			/* Mnemonic wallet - use BIP86 derivation */
 			bip86_pubkey(cmd->ld, &pubkey, listaddrtypes[i].keyidx);
 		} else {
-			/* For regular addresses, use standard BIP32 derivation */
+			/* Legacy wallet - use BIP32 derivation */
 			bip32_pubkey(cmd->ld, &pubkey, listaddrtypes[i].keyidx);
 		}
 		char *out_p2wpkh = "";
@@ -265,7 +249,7 @@ static struct command_result *json_listaddresses(struct command *cmd,
 				abort();
 			}
 		}
-		if (listaddrtypes[i].addrtype == ADDR_P2TR || listaddrtypes[i].addrtype == ADDR_ALL || listaddrtypes[i].addrtype == ADDR_P2TR_MNEMONIC) {
+		if (listaddrtypes[i].addrtype == ADDR_P2TR || listaddrtypes[i].addrtype == ADDR_ALL) {
 			out_p2tr = encode_pubkey_to_addr(cmd,
 								&pubkey,
 								ADDR_P2TR,