From f8789907674755f544ebafea5f47964119b0d0e6 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 15 Nov 2024 14:48:43 +0100 Subject: [PATCH 1/6] tapdb: fix comments and args --- tapdb/asset_minting.go | 2 +- tapdb/assets_common.go | 2 +- tapdb/assets_store.go | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tapdb/asset_minting.go b/tapdb/asset_minting.go index 779f72137..012855d76 100644 --- a/tapdb/asset_minting.go +++ b/tapdb/asset_minting.go @@ -253,7 +253,7 @@ var ( // fails. ErrBindBatchTx = errors.New("unable to bind batch tx") - // ErrEcodePsbt is returned when serializing a PSBT fails. + // ErrEncodePsbt is returned when serializing a PSBT fails. ErrEncodePsbt = errors.New("unable to encode psbt") ) diff --git a/tapdb/assets_common.go b/tapdb/assets_common.go index 921716700..8d2e29c69 100644 --- a/tapdb/assets_common.go +++ b/tapdb/assets_common.go @@ -118,7 +118,7 @@ var ( // exactly 32 bytes. ErrTapscriptRootSize = errors.New("tapscript root invalid: wrong size") - // ErrFetchGenesisAsset is returned when fetching the database ID for an + // ErrFetchGenesisID is returned when fetching the database ID for an // asset genesis fails. ErrFetchGenesisID = errors.New("unable to fetch genesis asset") ) diff --git a/tapdb/assets_store.go b/tapdb/assets_store.go index d5ee469ef..948046e28 100644 --- a/tapdb/assets_store.go +++ b/tapdb/assets_store.go @@ -212,7 +212,7 @@ type ActiveAssetsStore interface { // disk. FetchAssetProofs(ctx context.Context) ([]AssetProof, error) - // FetchAssetsProofsSizes fetches all the asset proofs lengths that are + // FetchAssetProofsSizes fetches all the asset proofs lengths that are // stored on disk. FetchAssetProofsSizes(ctx context.Context) ([]AssetProofSize, error) @@ -363,12 +363,12 @@ type ActiveAssetsStore interface { // MetaStore is a sub-set of the main sqlc.Querier interface that contains // methods related to metadata of the daemon. type MetaStore interface { - // AssetsDBSize returns the total size of the taproot assets sqlite - // database. + // AssetsDBSizeSqlite returns the total size of the taproot assets + // sqlite database. AssetsDBSizeSqlite(ctx context.Context) (int32, error) - // AssetsDBSize returns the total size of the taproot assets postgres - // database. + // AssetsDBSizePostgres returns the total size of the taproot assets + // postgres database. AssetsDBSizePostgres(ctx context.Context) (int64, error) } @@ -1471,8 +1471,8 @@ func locatorToProofQuery(locator proof.Locator) (FetchAssetProof, error) { // the FileArchiver. // // NOTE: This implements the proof.Archiver interface. -func (a *AssetStore) FetchIssuanceProof(ctx context.Context, id asset.ID, - anchorOutpoint wire.OutPoint) (proof.Blob, error) { +func (a *AssetStore) FetchIssuanceProof(_ context.Context, _ asset.ID, + _ wire.OutPoint) (proof.Blob, error) { return nil, proof.ErrProofNotFound } From 116bdcd9a286cdf640dc6d3daaa221a050d6ff63 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 15 Nov 2024 19:40:01 +0100 Subject: [PATCH 2/6] tapdb: use embeddings for script key --- tapdb/addrs.go | 64 +++------------- tapdb/asset_minting.go | 31 +------- tapdb/assets_common.go | 64 ++++++++++++---- tapdb/assets_store.go | 85 +++++--------------- tapdb/sqlc/addrs.sql.go | 118 +++++++++++++--------------- tapdb/sqlc/assets.sql.go | 128 +++++++++++++++---------------- tapdb/sqlc/queries/addrs.sql | 16 +--- tapdb/sqlc/queries/assets.sql | 21 +++-- tapdb/sqlc/queries/transfers.sql | 9 +-- tapdb/sqlc/transfers.sql.go | 34 ++++---- 10 files changed, 229 insertions(+), 341 deletions(-) diff --git a/tapdb/addrs.go b/tapdb/addrs.go index 538514b49..8f4c544d0 100644 --- a/tapdb/addrs.go +++ b/tapdb/addrs.go @@ -393,22 +393,13 @@ func (t *TapAddressBook) QueryAddrs(ctx context.Context, } } - rawScriptKey, err := btcec.ParsePubKey( - addr.RawScriptKey, + scriptKey, err := parseScriptKey( + addr.InternalKey, addr.ScriptKey, ) if err != nil { return fmt.Errorf("unable to decode "+ "script key: %w", err) } - rawScriptKeyDesc := keychain.KeyDescriptor{ - KeyLocator: keychain.KeyLocator{ - Family: keychain.KeyFamily( - addr.ScriptKeyFamily, - ), - Index: uint32(addr.ScriptKeyIndex), - }, - PubKey: rawScriptKey, - } internalKey, err := btcec.ParsePubKey(addr.RawTaprootKey) if err != nil { @@ -425,11 +416,6 @@ func (t *TapAddressBook) QueryAddrs(ctx context.Context, PubKey: internalKey, } - scriptKey, err := btcec.ParsePubKey(addr.TweakedScriptKey) - if err != nil { - return err - } - taprootOutputKey, err := schnorr.ParsePubKey( addr.TaprootOutputKey, ) @@ -456,8 +442,8 @@ func (t *TapAddressBook) QueryAddrs(ctx context.Context, tapAddr, err := address.New( address.Version(addr.Version), assetGenesis, - groupKey, groupWitness, - *scriptKey, *internalKey, uint64(addr.Amount), + groupKey, groupWitness, *scriptKey.PubKey, + *internalKey, uint64(addr.Amount), tapscriptSibling, t.params, *proofCourierAddr, address.WithAssetVersion( asset.Version(addr.AssetVersion), @@ -467,16 +453,9 @@ func (t *TapAddressBook) QueryAddrs(ctx context.Context, return fmt.Errorf("unable to make addr: %w", err) } - declaredKnown := extractBool( - addr.ScriptKeyDeclaredKnown, - ) addrs = append(addrs, address.AddrWithKeyInfo{ - Tap: tapAddr, - ScriptKeyTweak: asset.TweakedScriptKey{ - RawKey: rawScriptKeyDesc, - Tweak: addr.ScriptKeyTweak, - DeclaredKnown: declaredKnown, - }, + Tap: tapAddr, + ScriptKeyTweak: *scriptKey.TweakedScriptKey, InternalKeyDesc: internalKeyDesc, TaprootOutputKey: *taprootOutputKey, CreationTime: addr.CreationTime.UTC(), @@ -560,21 +539,7 @@ func fetchAddr(ctx context.Context, db AddrBook, params *address.ChainParams, } } - rawScriptKey, err := btcec.ParsePubKey(dbAddr.RawScriptKey) - if err != nil { - return nil, fmt.Errorf("unable to decode script key: %w", err) - } - scriptKeyDesc := keychain.KeyDescriptor{ - KeyLocator: keychain.KeyLocator{ - Family: keychain.KeyFamily( - dbAddr.ScriptKeyFamily, - ), - Index: uint32(dbAddr.ScriptKeyIndex), - }, - PubKey: rawScriptKey, - } - - scriptKey, err := btcec.ParsePubKey(dbAddr.TweakedScriptKey) + scriptKey, err := parseScriptKey(dbAddr.InternalKey, dbAddr.ScriptKey) if err != nil { return nil, fmt.Errorf("unable to decode script key: %w", err) } @@ -611,8 +576,9 @@ func fetchAddr(ctx context.Context, db AddrBook, params *address.ChainParams, tapAddr, err := address.New( address.Version(dbAddr.Version), genesis, groupKey, - groupWitness, *scriptKey, *internalKey, uint64(dbAddr.Amount), - tapscriptSibling, params, *proofCourierAddr, + groupWitness, *scriptKey.PubKey, *internalKey, + uint64(dbAddr.Amount), tapscriptSibling, params, + *proofCourierAddr, address.WithAssetVersion(asset.Version(dbAddr.AssetVersion)), ) if err != nil { @@ -620,14 +586,8 @@ func fetchAddr(ctx context.Context, db AddrBook, params *address.ChainParams, } return &address.AddrWithKeyInfo{ - Tap: tapAddr, - ScriptKeyTweak: asset.TweakedScriptKey{ - RawKey: scriptKeyDesc, - Tweak: dbAddr.ScriptKeyTweak, - DeclaredKnown: extractBool( - dbAddr.ScriptKeyDeclaredKnown, - ), - }, + Tap: tapAddr, + ScriptKeyTweak: *scriptKey.TweakedScriptKey, InternalKeyDesc: internalKeyDesc, TaprootOutputKey: *taprootOutputKey, CreationTime: dbAddr.CreationTime.UTC(), diff --git a/tapdb/asset_minting.go b/tapdb/asset_minting.go index 012855d76..9faa15d21 100644 --- a/tapdb/asset_minting.go +++ b/tapdb/asset_minting.go @@ -781,35 +781,12 @@ func fetchAssetSprouts(ctx context.Context, q PendingAssetStore, for i, sprout := range dbSprout { // First, we'll decode the script key which very asset must // specify, and populate the key locator information - tweakedScriptKey, err := btcec.ParsePubKey( - sprout.TweakedScriptKey, + scriptKey, err := parseScriptKey( + sprout.InternalKey, sprout.ScriptKey, ) if err != nil { - return nil, err - } - - internalScriptKey, err := btcec.ParsePubKey( - sprout.ScriptKeyRaw, - ) - if err != nil { - return nil, err - } - - scriptKeyDesc := keychain.KeyDescriptor{ - PubKey: internalScriptKey, - KeyLocator: keychain.KeyLocator{ - Index: uint32(sprout.ScriptKeyIndex), - Family: keychain.KeyFamily(sprout.ScriptKeyFam), - }, - } - declaredKnown := extractBool(sprout.ScriptKeyDeclaredKnown) - scriptKey := asset.ScriptKey{ - PubKey: tweakedScriptKey, - TweakedScriptKey: &asset.TweakedScriptKey{ - RawKey: scriptKeyDesc, - Tweak: sprout.Tweak, - DeclaredKnown: declaredKnown, - }, + return nil, fmt.Errorf("unable to decode script key: "+ + "%w", err) } // Not all assets have a key group, so we only need to diff --git a/tapdb/assets_common.go b/tapdb/assets_common.go index 8d2e29c69..260ee7a6c 100644 --- a/tapdb/assets_common.go +++ b/tapdb/assets_common.go @@ -443,26 +443,58 @@ func fetchScriptKey(ctx context.Context, q FetchScriptKeyStore, return nil, err } - rawKey, err := btcec.ParsePubKey(dbKey.RawKey) + scriptKey, err := parseScriptKey(dbKey.InternalKey, dbKey.ScriptKey) if err != nil { - return nil, fmt.Errorf("unable to parse raw key: %w", err) - } - - scriptKey := &asset.TweakedScriptKey{ - Tweak: dbKey.Tweak, - RawKey: keychain.KeyDescriptor{ - PubKey: rawKey, - KeyLocator: keychain.KeyLocator{ - Family: keychain.KeyFamily( - dbKey.KeyFamily, - ), - Index: uint32(dbKey.KeyIndex), + return nil, err + } + + return scriptKey.TweakedScriptKey, nil +} + +// parseScriptKey maps a script key and internal key from the database into a +// ScriptKey struct. Both the internal raw public key and the tweaked public key +// must be set and valid. +func parseScriptKey(ik sqlc.InternalKey, sk sqlc.ScriptKey) (asset.ScriptKey, + error) { + + var ( + locator = keychain.KeyLocator{ + Index: uint32(ik.KeyIndex), + Family: keychain.KeyFamily(ik.KeyFamily), + } + result = asset.ScriptKey{ + TweakedScriptKey: &asset.TweakedScriptKey{ + RawKey: keychain.KeyDescriptor{ + KeyLocator: locator, + }, + Tweak: sk.Tweak, + DeclaredKnown: extractBool(sk.DeclaredKnown), }, - }, - DeclaredKnown: extractBool(dbKey.DeclaredKnown), + } + err error + ) + + if len(sk.TweakedScriptKey) == 0 { + return result, fmt.Errorf("tweaked script key is empty") + } + + if len(ik.RawKey) == 0 { + return result, fmt.Errorf("internal raw key is empty") + } + + result.PubKey, err = btcec.ParsePubKey(sk.TweakedScriptKey) + if err != nil { + return result, fmt.Errorf("error parsing tweaked "+ + "script key: %w", err) + } + + result.RawKey.PubKey, err = btcec.ParsePubKey(ik.RawKey) + if err != nil { + return result, fmt.Errorf("error parsing internal "+ + "raw key: %w", err) } - return scriptKey, nil + return result, nil } // FetchGenesisStore houses the methods related to fetching genesis assets. diff --git a/tapdb/assets_store.go b/tapdb/assets_store.go index 948046e28..31f2529d8 100644 --- a/tapdb/assets_store.go +++ b/tapdb/assets_store.go @@ -630,16 +630,12 @@ func (a *AssetStore) dbAssetsToChainAssets(dbAssets []ConfirmedAsset, // First, we'll decode the script key which every asset must // specify, and populate the key locator information. - rawScriptKeyPub, err := btcec.ParsePubKey(sprout.ScriptKeyRaw) + scriptKey, err := parseScriptKey( + sprout.InternalKey, sprout.ScriptKey, + ) if err != nil { - return nil, err - } - rawScriptKeyDesc := keychain.KeyDescriptor{ - PubKey: rawScriptKeyPub, - KeyLocator: keychain.KeyLocator{ - Index: uint32(sprout.ScriptKeyIndex), - Family: keychain.KeyFamily(sprout.ScriptKeyFam), - }, + return nil, fmt.Errorf("unable to decode script key: "+ + "%w", err) } // Not all assets have a key group, so we only need to @@ -723,20 +719,6 @@ func (a *AssetStore) dbAssetsToChainAssets(dbAssets []ConfirmedAsset, amount = 1 } - scriptKeyPub, err := btcec.ParsePubKey(sprout.TweakedScriptKey) - if err != nil { - return nil, err - } - declaredKnown := extractBool(sprout.ScriptKeyDeclaredKnown) - scriptKey := asset.ScriptKey{ - PubKey: scriptKeyPub, - TweakedScriptKey: &asset.TweakedScriptKey{ - RawKey: rawScriptKeyDesc, - Tweak: sprout.ScriptKeyTweak, - DeclaredKnown: declaredKnown, - }, - } - assetSprout, err := asset.New( assetGenesis, amount, lockTime, relativeLocktime, scriptKey, groupKey, @@ -750,7 +732,9 @@ func (a *AssetStore) dbAssetsToChainAssets(dbAssets []ConfirmedAsset, // We cannot use 0 as the amount when creating a new asset with // the New function above. But if this is a tombstone asset, we // actually have to set the amount to 0. - if scriptKeyPub.IsEqual(asset.NUMSPubKey) && sprout.Amount == 0 { + if scriptKey.PubKey.IsEqual(asset.NUMSPubKey) && + sprout.Amount == 0 { + assetSprout.Amount = 0 } @@ -2649,29 +2633,14 @@ func fetchAssetTransferOutputs(ctx context.Context, q ActiveAssetsStore, "key: %w", err) } - scriptKey, err := btcec.ParsePubKey(dbOut.ScriptKeyBytes) + scriptKey, err := parseScriptKey( + dbOut.InternalKey, dbOut.ScriptKey, + ) if err != nil { return nil, fmt.Errorf("unable to decode script key: "+ "%w", err) } - rawScriptKey, err := btcec.ParsePubKey( - dbOut.ScriptKeyRawKeyBytes, - ) - if err != nil { - return nil, fmt.Errorf("unable to decode raw script "+ - "key: %w", err) - } - - scriptKeyLocator := keychain.KeyLocator{ - Family: keychain.KeyFamily( - dbOut.ScriptKeyFamily, - ), - Index: uint32( - dbOut.ScriptKeyIndex, - ), - } - var splitRootHash mssmt.NodeHash copy(splitRootHash[:], dbOut.SplitCommitmentRootHash) @@ -2686,7 +2655,6 @@ func fetchAssetTransferOutputs(ctx context.Context, q ActiveAssetsStore, err) } - declaredKnown := extractBool(dbOut.ScriptKeyDeclaredKnown) outputAnchor := tapfreighter.Anchor{ Value: btcutil.Amount( dbOut.AnchorValue, @@ -2738,19 +2706,9 @@ func fetchAssetTransferOutputs(ctx context.Context, q ActiveAssetsStore, LockTime: uint64(dbOut.LockTime.Int32), RelativeLockTime: uint64(dbOut.RelativeLockTime.Int32), AssetVersion: asset.Version(dbOut.AssetVersion), - ScriptKey: asset.ScriptKey{ - PubKey: scriptKey, - TweakedScriptKey: &asset.TweakedScriptKey{ - RawKey: keychain.KeyDescriptor{ - PubKey: rawScriptKey, - KeyLocator: scriptKeyLocator, - }, - Tweak: dbOut.ScriptKeyTweak, - DeclaredKnown: declaredKnown, - }, - }, - ScriptKeyLocal: dbOut.ScriptKeyLocal, - WitnessData: witnessData, + ScriptKey: scriptKey, + ScriptKeyLocal: dbOut.ScriptKeyLocal, + WitnessData: witnessData, SplitCommitmentRoot: mssmt.NewComputedNode( splitRootHash, uint64(dbOut.SplitCommitmentRootValue.Int64), @@ -3021,13 +2979,14 @@ func (a *AssetStore) LogAnchorTxConfirm(ctx context.Context, "witness: %w", err) } - scriptPubKey, err := btcec.ParsePubKey( - out.ScriptKeyBytes, + fullScriptKey, err := parseScriptKey( + out.InternalKey, out.ScriptKey, ) if err != nil { return fmt.Errorf("unable to decode script "+ "key: %w", err) } + scriptPubKey := fullScriptKey.PubKey isNumsKey := scriptPubKey.IsEqual(asset.NUMSPubKey) isTombstone := isNumsKey && @@ -3035,7 +2994,7 @@ func (a *AssetStore) LogAnchorTxConfirm(ctx context.Context, out.OutputType == int16(tappsbt.TypeSplitRoot) isBurn := !isNumsKey && len(witnessData) > 0 && asset.IsBurnKey(scriptPubKey, witnessData[0]) - isKnown := extractBool(out.ScriptKeyDeclaredKnown) + isKnown := fullScriptKey.DeclaredKnown skipAssetCreation := !isTombstone && !isBurn && !out.ScriptKeyLocal && !isKnown @@ -3066,7 +3025,7 @@ func (a *AssetStore) LogAnchorTxConfirm(ctx context.Context, // overwrite all other fields. templateID := spentAssetIDs[0] params := ApplyPendingOutput{ - ScriptKeyID: out.ScriptKeyID, + ScriptKeyID: out.ScriptKey.ScriptKeyID, AnchorUtxoID: sqlInt64( out.AnchorUtxoID, ), @@ -3099,13 +3058,11 @@ func (a *AssetStore) LogAnchorTxConfirm(ctx context.Context, "witnesses: %w", err) } - var scriptKey asset.SerializedKey - copy(scriptKey[:], out.ScriptKeyBytes) + scriptKey := asset.ToSerialized(scriptPubKey) receiverProof, ok := conf.FinalProofs[scriptKey] if !ok { return fmt.Errorf("no proof found for output "+ - "with script key %x", - out.ScriptKeyBytes) + "with script key %x", scriptKey[:]) } localProofKeys = append(localProofKeys, scriptKey) diff --git a/tapdb/sqlc/addrs.sql.go b/tapdb/sqlc/addrs.sql.go index 820d10593..ade3d9a75 100644 --- a/tapdb/sqlc/addrs.sql.go +++ b/tapdb/sqlc/addrs.sql.go @@ -16,12 +16,8 @@ SELECT version, asset_version, genesis_asset_id, group_key, tapscript_sibling, taproot_output_key, amount, asset_type, creation_time, managed_from, proof_courier_addr, - script_keys.tweaked_script_key, - script_keys.tweak AS script_key_tweak, - script_keys.declared_known AS script_key_declared_known, - raw_script_keys.raw_key as raw_script_key, - raw_script_keys.key_family AS script_key_family, - raw_script_keys.key_index AS script_key_index, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + raw_script_keys.key_id, raw_script_keys.raw_key, raw_script_keys.key_family, raw_script_keys.key_index, taproot_keys.raw_key AS raw_taproot_key, taproot_keys.key_family AS taproot_key_family, taproot_keys.key_index AS taproot_key_index @@ -36,26 +32,22 @@ WHERE taproot_output_key = $1 ` type FetchAddrByTaprootOutputKeyRow struct { - Version int16 - AssetVersion int16 - GenesisAssetID int64 - GroupKey []byte - TapscriptSibling []byte - TaprootOutputKey []byte - Amount int64 - AssetType int16 - CreationTime time.Time - ManagedFrom sql.NullTime - ProofCourierAddr []byte - TweakedScriptKey []byte - ScriptKeyTweak []byte - ScriptKeyDeclaredKnown sql.NullBool - RawScriptKey []byte - ScriptKeyFamily int32 - ScriptKeyIndex int32 - RawTaprootKey []byte - TaprootKeyFamily int32 - TaprootKeyIndex int32 + Version int16 + AssetVersion int16 + GenesisAssetID int64 + GroupKey []byte + TapscriptSibling []byte + TaprootOutputKey []byte + Amount int64 + AssetType int16 + CreationTime time.Time + ManagedFrom sql.NullTime + ProofCourierAddr []byte + ScriptKey ScriptKey + InternalKey InternalKey + RawTaprootKey []byte + TaprootKeyFamily int32 + TaprootKeyIndex int32 } func (q *Queries) FetchAddrByTaprootOutputKey(ctx context.Context, taprootOutputKey []byte) (FetchAddrByTaprootOutputKeyRow, error) { @@ -73,12 +65,15 @@ func (q *Queries) FetchAddrByTaprootOutputKey(ctx context.Context, taprootOutput &i.CreationTime, &i.ManagedFrom, &i.ProofCourierAddr, - &i.TweakedScriptKey, - &i.ScriptKeyTweak, - &i.ScriptKeyDeclaredKnown, - &i.RawScriptKey, - &i.ScriptKeyFamily, - &i.ScriptKeyIndex, + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, &i.RawTaprootKey, &i.TaprootKeyFamily, &i.TaprootKeyIndex, @@ -207,12 +202,8 @@ SELECT version, asset_version, genesis_asset_id, group_key, tapscript_sibling, taproot_output_key, amount, asset_type, creation_time, managed_from, proof_courier_addr, - script_keys.tweaked_script_key, - script_keys.tweak AS script_key_tweak, - script_keys.declared_known AS script_key_declared_known, - raw_script_keys.raw_key AS raw_script_key, - raw_script_keys.key_family AS script_key_family, - raw_script_keys.key_index AS script_key_index, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + raw_script_keys.key_id, raw_script_keys.raw_key, raw_script_keys.key_family, raw_script_keys.key_index, taproot_keys.raw_key AS raw_taproot_key, taproot_keys.key_family AS taproot_key_family, taproot_keys.key_index AS taproot_key_index @@ -240,26 +231,22 @@ type FetchAddrsParams struct { } type FetchAddrsRow struct { - Version int16 - AssetVersion int16 - GenesisAssetID int64 - GroupKey []byte - TapscriptSibling []byte - TaprootOutputKey []byte - Amount int64 - AssetType int16 - CreationTime time.Time - ManagedFrom sql.NullTime - ProofCourierAddr []byte - TweakedScriptKey []byte - ScriptKeyTweak []byte - ScriptKeyDeclaredKnown sql.NullBool - RawScriptKey []byte - ScriptKeyFamily int32 - ScriptKeyIndex int32 - RawTaprootKey []byte - TaprootKeyFamily int32 - TaprootKeyIndex int32 + Version int16 + AssetVersion int16 + GenesisAssetID int64 + GroupKey []byte + TapscriptSibling []byte + TaprootOutputKey []byte + Amount int64 + AssetType int16 + CreationTime time.Time + ManagedFrom sql.NullTime + ProofCourierAddr []byte + ScriptKey ScriptKey + InternalKey InternalKey + RawTaprootKey []byte + TaprootKeyFamily int32 + TaprootKeyIndex int32 } func (q *Queries) FetchAddrs(ctx context.Context, arg FetchAddrsParams) ([]FetchAddrsRow, error) { @@ -289,12 +276,15 @@ func (q *Queries) FetchAddrs(ctx context.Context, arg FetchAddrsParams) ([]Fetch &i.CreationTime, &i.ManagedFrom, &i.ProofCourierAddr, - &i.TweakedScriptKey, - &i.ScriptKeyTweak, - &i.ScriptKeyDeclaredKnown, - &i.RawScriptKey, - &i.ScriptKeyFamily, - &i.ScriptKeyIndex, + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, &i.RawTaprootKey, &i.TaprootKeyFamily, &i.TaprootKeyIndex, diff --git a/tapdb/sqlc/assets.sql.go b/tapdb/sqlc/assets.sql.go index fa1ca8f87..5318584b7 100644 --- a/tapdb/sqlc/assets.sql.go +++ b/tapdb/sqlc/assets.sql.go @@ -936,11 +936,9 @@ WITH genesis_info AS ( WHERE wit.gen_asset_id IN (SELECT gen_asset_id FROM genesis_info) ) SELECT - version, script_keys.tweak, script_keys.tweaked_script_key, - script_keys.declared_known AS script_key_declared_known, - internal_keys.raw_key AS script_key_raw, - internal_keys.key_family AS script_key_fam, - internal_keys.key_index AS script_key_index, + version, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index, key_group_info.tapscript_root, key_group_info.witness_stack, key_group_info.tweaked_group_key, @@ -964,32 +962,28 @@ JOIN internal_keys ` type FetchAssetsForBatchRow struct { - Version int32 - Tweak []byte - TweakedScriptKey []byte - ScriptKeyDeclaredKnown sql.NullBool - ScriptKeyRaw []byte - ScriptKeyFam int32 - ScriptKeyIndex int32 - TapscriptRoot []byte - WitnessStack []byte - TweakedGroupKey []byte - GroupKeyRaw []byte - GroupKeyFamily sql.NullInt32 - GroupKeyIndex sql.NullInt32 - ScriptVersion int32 - Amount int64 - LockTime sql.NullInt32 - RelativeLockTime sql.NullInt32 - Spent bool - AssetID []byte - AssetTag string - MetaHash []byte - MetaType sql.NullInt16 - MetaBlob []byte - GenesisOutputIndex int32 - AssetType int16 - GenesisPrevOut []byte + Version int32 + ScriptKey ScriptKey + InternalKey InternalKey + TapscriptRoot []byte + WitnessStack []byte + TweakedGroupKey []byte + GroupKeyRaw []byte + GroupKeyFamily sql.NullInt32 + GroupKeyIndex sql.NullInt32 + ScriptVersion int32 + Amount int64 + LockTime sql.NullInt32 + RelativeLockTime sql.NullInt32 + Spent bool + AssetID []byte + AssetTag string + MetaHash []byte + MetaType sql.NullInt16 + MetaBlob []byte + GenesisOutputIndex int32 + AssetType int16 + GenesisPrevOut []byte } // We use a LEFT JOIN here as not every asset has a group key, so this'll @@ -1007,12 +1001,15 @@ func (q *Queries) FetchAssetsForBatch(ctx context.Context, rawKey []byte) ([]Fet var i FetchAssetsForBatchRow if err := rows.Scan( &i.Version, - &i.Tweak, - &i.TweakedScriptKey, - &i.ScriptKeyDeclaredKnown, - &i.ScriptKeyRaw, - &i.ScriptKeyFam, - &i.ScriptKeyIndex, + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, &i.TapscriptRoot, &i.WitnessStack, &i.TweakedGroupKey, @@ -1578,7 +1575,7 @@ func (q *Queries) FetchMintingBatchesByInverseState(ctx context.Context, batchSt } const fetchScriptKeyByTweakedKey = `-- name: FetchScriptKeyByTweakedKey :one -SELECT tweak, raw_key, key_family, key_index, declared_known +SELECT script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index FROM script_keys JOIN internal_keys ON script_keys.internal_key_id = internal_keys.key_id @@ -1586,22 +1583,23 @@ WHERE script_keys.tweaked_script_key = $1 ` type FetchScriptKeyByTweakedKeyRow struct { - Tweak []byte - RawKey []byte - KeyFamily int32 - KeyIndex int32 - DeclaredKnown sql.NullBool + ScriptKey ScriptKey + InternalKey InternalKey } func (q *Queries) FetchScriptKeyByTweakedKey(ctx context.Context, tweakedScriptKey []byte) (FetchScriptKeyByTweakedKeyRow, error) { row := q.db.QueryRowContext(ctx, fetchScriptKeyByTweakedKey, tweakedScriptKey) var i FetchScriptKeyByTweakedKeyRow err := row.Scan( - &i.Tweak, - &i.RawKey, - &i.KeyFamily, - &i.KeyIndex, - &i.DeclaredKnown, + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, ) return i, err } @@ -1688,6 +1686,9 @@ SELECT seedling_id, asset_name, asset_type, asset_version, asset_supply, assets_meta.meta_data_hash, assets_meta.meta_data_type, assets_meta.meta_data_blob, emission_enabled, batch_id, group_genesis_id, group_anchor_id, group_tapscript_root, + -- TODO(guggero): We should use sqlc.embed() for the script key and internal + -- key fields, but we can't because it's a LEFT JOIN. We should check if the + -- LEFT JOIN is actually necessary or if we always have keys for seedlings. script_keys.tweak AS script_key_tweak, script_keys.tweaked_script_key, script_keys.declared_known AS script_key_declared_known, @@ -2197,12 +2198,8 @@ func (q *Queries) QueryAssetBalancesByGroup(ctx context.Context, arg QueryAssetB const queryAssets = `-- name: QueryAssets :many SELECT assets.asset_id AS asset_primary_key, assets.genesis_id, version, spent, - script_keys.tweak AS script_key_tweak, - script_keys.tweaked_script_key, - script_keys.declared_known AS script_key_declared_known, - internal_keys.raw_key AS script_key_raw, - internal_keys.key_family AS script_key_fam, - internal_keys.key_index AS script_key_index, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index, key_group_info_view.tapscript_root, key_group_info_view.witness_stack, key_group_info_view.tweaked_group_key, @@ -2297,12 +2294,8 @@ type QueryAssetsRow struct { GenesisID int64 Version int32 Spent bool - ScriptKeyTweak []byte - TweakedScriptKey []byte - ScriptKeyDeclaredKnown sql.NullBool - ScriptKeyRaw []byte - ScriptKeyFam int32 - ScriptKeyIndex int32 + ScriptKey ScriptKey + InternalKey InternalKey TapscriptRoot []byte WitnessStack []byte TweakedGroupKey []byte @@ -2371,12 +2364,15 @@ func (q *Queries) QueryAssets(ctx context.Context, arg QueryAssetsParams) ([]Que &i.GenesisID, &i.Version, &i.Spent, - &i.ScriptKeyTweak, - &i.TweakedScriptKey, - &i.ScriptKeyDeclaredKnown, - &i.ScriptKeyRaw, - &i.ScriptKeyFam, - &i.ScriptKeyIndex, + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, &i.TapscriptRoot, &i.WitnessStack, &i.TweakedGroupKey, diff --git a/tapdb/sqlc/queries/addrs.sql b/tapdb/sqlc/queries/addrs.sql index ece4b3cbb..a7764347a 100644 --- a/tapdb/sqlc/queries/addrs.sql +++ b/tapdb/sqlc/queries/addrs.sql @@ -10,12 +10,8 @@ SELECT version, asset_version, genesis_asset_id, group_key, tapscript_sibling, taproot_output_key, amount, asset_type, creation_time, managed_from, proof_courier_addr, - script_keys.tweaked_script_key, - script_keys.tweak AS script_key_tweak, - script_keys.declared_known AS script_key_declared_known, - raw_script_keys.raw_key AS raw_script_key, - raw_script_keys.key_family AS script_key_family, - raw_script_keys.key_index AS script_key_index, + sqlc.embed(script_keys), + sqlc.embed(raw_script_keys), taproot_keys.raw_key AS raw_taproot_key, taproot_keys.key_family AS taproot_key_family, taproot_keys.key_index AS taproot_key_index @@ -38,12 +34,8 @@ SELECT version, asset_version, genesis_asset_id, group_key, tapscript_sibling, taproot_output_key, amount, asset_type, creation_time, managed_from, proof_courier_addr, - script_keys.tweaked_script_key, - script_keys.tweak AS script_key_tweak, - script_keys.declared_known AS script_key_declared_known, - raw_script_keys.raw_key as raw_script_key, - raw_script_keys.key_family AS script_key_family, - raw_script_keys.key_index AS script_key_index, + sqlc.embed(script_keys), + sqlc.embed(raw_script_keys), taproot_keys.raw_key AS raw_taproot_key, taproot_keys.key_family AS taproot_key_family, taproot_keys.key_index AS taproot_key_index diff --git a/tapdb/sqlc/queries/assets.sql b/tapdb/sqlc/queries/assets.sql index a16302d27..27b63e2f6 100644 --- a/tapdb/sqlc/queries/assets.sql +++ b/tapdb/sqlc/queries/assets.sql @@ -135,6 +135,9 @@ SELECT seedling_id, asset_name, asset_type, asset_version, asset_supply, assets_meta.meta_data_hash, assets_meta.meta_data_type, assets_meta.meta_data_blob, emission_enabled, batch_id, group_genesis_id, group_anchor_id, group_tapscript_root, + -- TODO(guggero): We should use sqlc.embed() for the script key and internal + -- key fields, but we can't because it's a LEFT JOIN. We should check if the + -- LEFT JOIN is actually necessary or if we always have keys for seedlings. script_keys.tweak AS script_key_tweak, script_keys.tweaked_script_key, script_keys.declared_known AS script_key_declared_known, @@ -254,11 +257,9 @@ WITH genesis_info AS ( WHERE wit.gen_asset_id IN (SELECT gen_asset_id FROM genesis_info) ) SELECT - version, script_keys.tweak, script_keys.tweaked_script_key, - script_keys.declared_known AS script_key_declared_known, - internal_keys.raw_key AS script_key_raw, - internal_keys.key_family AS script_key_fam, - internal_keys.key_index AS script_key_index, + version, + sqlc.embed(script_keys), + sqlc.embed(internal_keys), key_group_info.tapscript_root, key_group_info.witness_stack, key_group_info.tweaked_group_key, @@ -417,12 +418,8 @@ WHERE ( -- name: QueryAssets :many SELECT assets.asset_id AS asset_primary_key, assets.genesis_id, version, spent, - script_keys.tweak AS script_key_tweak, - script_keys.tweaked_script_key, - script_keys.declared_known AS script_key_declared_known, - internal_keys.raw_key AS script_key_raw, - internal_keys.key_family AS script_key_fam, - internal_keys.key_index AS script_key_index, + sqlc.embed(script_keys), + sqlc.embed(internal_keys), key_group_info_view.tapscript_root, key_group_info_view.witness_stack, key_group_info_view.tweaked_group_key, @@ -871,7 +868,7 @@ FROM script_keys WHERE tweaked_script_key = $1; -- name: FetchScriptKeyByTweakedKey :one -SELECT tweak, raw_key, key_family, key_index, declared_known +SELECT sqlc.embed(script_keys), sqlc.embed(internal_keys) FROM script_keys JOIN internal_keys ON script_keys.internal_key_id = internal_keys.key_id diff --git a/tapdb/sqlc/queries/transfers.sql b/tapdb/sqlc/queries/transfers.sql index c88671a45..d7240047e 100644 --- a/tapdb/sqlc/queries/transfers.sql +++ b/tapdb/sqlc/queries/transfers.sql @@ -91,13 +91,8 @@ SELECT utxo_internal_keys.raw_key AS internal_key_raw_key_bytes, utxo_internal_keys.key_family AS internal_key_family, utxo_internal_keys.key_index AS internal_key_index, - script_keys.tweaked_script_key AS script_key_bytes, - script_keys.tweak AS script_key_tweak, - script_keys.declared_known AS script_key_declared_known, - script_key AS script_key_id, - script_internal_keys.raw_key AS script_key_raw_key_bytes, - script_internal_keys.key_family AS script_key_family, - script_internal_keys.key_index AS script_key_index + sqlc.embed(script_keys), + sqlc.embed(script_internal_keys) FROM asset_transfer_outputs outputs JOIN managed_utxos utxos ON outputs.anchor_utxo = utxos.utxo_id diff --git a/tapdb/sqlc/transfers.sql.go b/tapdb/sqlc/transfers.sql.go index a5ed3aede..7fc74c63f 100644 --- a/tapdb/sqlc/transfers.sql.go +++ b/tapdb/sqlc/transfers.sql.go @@ -137,13 +137,8 @@ SELECT utxo_internal_keys.raw_key AS internal_key_raw_key_bytes, utxo_internal_keys.key_family AS internal_key_family, utxo_internal_keys.key_index AS internal_key_index, - script_keys.tweaked_script_key AS script_key_bytes, - script_keys.tweak AS script_key_tweak, - script_keys.declared_known AS script_key_declared_known, - script_key AS script_key_id, - script_internal_keys.raw_key AS script_key_raw_key_bytes, - script_internal_keys.key_family AS script_key_family, - script_internal_keys.key_index AS script_key_index + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + script_internal_keys.key_id, script_internal_keys.raw_key, script_internal_keys.key_family, script_internal_keys.key_index FROM asset_transfer_outputs outputs JOIN managed_utxos utxos ON outputs.anchor_utxo = utxos.utxo_id @@ -183,13 +178,8 @@ type FetchTransferOutputsRow struct { InternalKeyRawKeyBytes []byte InternalKeyFamily int32 InternalKeyIndex int32 - ScriptKeyBytes []byte - ScriptKeyTweak []byte - ScriptKeyDeclaredKnown sql.NullBool - ScriptKeyID int64 - ScriptKeyRawKeyBytes []byte - ScriptKeyFamily int32 - ScriptKeyIndex int32 + ScriptKey ScriptKey + InternalKey InternalKey } func (q *Queries) FetchTransferOutputs(ctx context.Context, transferID int64) ([]FetchTransferOutputsRow, error) { @@ -227,13 +217,15 @@ func (q *Queries) FetchTransferOutputs(ctx context.Context, transferID int64) ([ &i.InternalKeyRawKeyBytes, &i.InternalKeyFamily, &i.InternalKeyIndex, - &i.ScriptKeyBytes, - &i.ScriptKeyTweak, - &i.ScriptKeyDeclaredKnown, - &i.ScriptKeyID, - &i.ScriptKeyRawKeyBytes, - &i.ScriptKeyFamily, - &i.ScriptKeyIndex, + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, ); err != nil { return nil, err } From 0790564d4022fd55001468f16ce275f4c710f515 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 15 Nov 2024 15:03:40 +0100 Subject: [PATCH 3/6] tapdb: add script_keys.key_type column With this commit we add a new numeric type for the type of a script key. This will be a Golang enum/const that's going to be assigned manually when declaring a key as known. For existing keys, we'll add a new mechanism in the following commits that runs on startup after we detect a SQL migration was applied that will attempt the detection of the type of those keys. --- tapdb/migrations.go | 2 +- tapdb/sqlc/addrs.sql.go | 6 +- tapdb/sqlc/assets.sql.go | 89 ++++++++++++++++--- .../000024_script_key_type.down.sql | 1 + .../migrations/000024_script_key_type.up.sql | 7 ++ tapdb/sqlc/models.go | 1 + tapdb/sqlc/querier.go | 1 + tapdb/sqlc/queries/assets.sql | 36 +++++--- tapdb/sqlc/transfers.sql.go | 3 +- 9 files changed, 119 insertions(+), 27 deletions(-) create mode 100644 tapdb/sqlc/migrations/000024_script_key_type.down.sql create mode 100644 tapdb/sqlc/migrations/000024_script_key_type.up.sql diff --git a/tapdb/migrations.go b/tapdb/migrations.go index 7b3f1a6fb..e49bc6e1d 100644 --- a/tapdb/migrations.go +++ b/tapdb/migrations.go @@ -22,7 +22,7 @@ const ( // daemon. // // NOTE: This MUST be updated when a new migration is added. - LatestMigrationVersion = 23 + LatestMigrationVersion = 24 ) // MigrationTarget is a functional option that can be passed to applyMigrations diff --git a/tapdb/sqlc/addrs.sql.go b/tapdb/sqlc/addrs.sql.go index ade3d9a75..c566adf94 100644 --- a/tapdb/sqlc/addrs.sql.go +++ b/tapdb/sqlc/addrs.sql.go @@ -16,7 +16,7 @@ SELECT version, asset_version, genesis_asset_id, group_key, tapscript_sibling, taproot_output_key, amount, asset_type, creation_time, managed_from, proof_courier_addr, - script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, raw_script_keys.key_id, raw_script_keys.raw_key, raw_script_keys.key_family, raw_script_keys.key_index, taproot_keys.raw_key AS raw_taproot_key, taproot_keys.key_family AS taproot_key_family, @@ -70,6 +70,7 @@ func (q *Queries) FetchAddrByTaprootOutputKey(ctx context.Context, taprootOutput &i.ScriptKey.TweakedScriptKey, &i.ScriptKey.Tweak, &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, &i.InternalKey.KeyID, &i.InternalKey.RawKey, &i.InternalKey.KeyFamily, @@ -202,7 +203,7 @@ SELECT version, asset_version, genesis_asset_id, group_key, tapscript_sibling, taproot_output_key, amount, asset_type, creation_time, managed_from, proof_courier_addr, - script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, raw_script_keys.key_id, raw_script_keys.raw_key, raw_script_keys.key_family, raw_script_keys.key_index, taproot_keys.raw_key AS raw_taproot_key, taproot_keys.key_family AS taproot_key_family, @@ -281,6 +282,7 @@ func (q *Queries) FetchAddrs(ctx context.Context, arg FetchAddrsParams) ([]Fetch &i.ScriptKey.TweakedScriptKey, &i.ScriptKey.Tweak, &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, &i.InternalKey.KeyID, &i.InternalKey.RawKey, &i.InternalKey.KeyFamily, diff --git a/tapdb/sqlc/assets.sql.go b/tapdb/sqlc/assets.sql.go index 5318584b7..e24b486d2 100644 --- a/tapdb/sqlc/assets.sql.go +++ b/tapdb/sqlc/assets.sql.go @@ -937,7 +937,7 @@ WITH genesis_info AS ( ) SELECT version, - script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index, key_group_info.tapscript_root, key_group_info.witness_stack, @@ -1006,6 +1006,7 @@ func (q *Queries) FetchAssetsForBatch(ctx context.Context, rawKey []byte) ([]Fet &i.ScriptKey.TweakedScriptKey, &i.ScriptKey.Tweak, &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, &i.InternalKey.KeyID, &i.InternalKey.RawKey, &i.InternalKey.KeyFamily, @@ -1575,7 +1576,7 @@ func (q *Queries) FetchMintingBatchesByInverseState(ctx context.Context, batchSt } const fetchScriptKeyByTweakedKey = `-- name: FetchScriptKeyByTweakedKey :one -SELECT script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index +SELECT script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index FROM script_keys JOIN internal_keys ON script_keys.internal_key_id = internal_keys.key_id @@ -1596,6 +1597,7 @@ func (q *Queries) FetchScriptKeyByTweakedKey(ctx context.Context, tweakedScriptK &i.ScriptKey.TweakedScriptKey, &i.ScriptKey.Tweak, &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, &i.InternalKey.KeyID, &i.InternalKey.RawKey, &i.InternalKey.KeyFamily, @@ -1692,6 +1694,7 @@ SELECT seedling_id, asset_name, asset_type, asset_version, asset_supply, script_keys.tweak AS script_key_tweak, script_keys.tweaked_script_key, script_keys.declared_known AS script_key_declared_known, + script_keys.key_type AS script_key_type, internal_keys.raw_key AS script_key_raw, internal_keys.key_family AS script_key_fam, internal_keys.key_index AS script_key_index, @@ -1727,6 +1730,7 @@ type FetchSeedlingsForBatchRow struct { ScriptKeyTweak []byte TweakedScriptKey []byte ScriptKeyDeclaredKnown sql.NullBool + ScriptKeyType sql.NullInt16 ScriptKeyRaw []byte ScriptKeyFam sql.NullInt32 ScriptKeyIndex sql.NullInt32 @@ -1761,6 +1765,7 @@ func (q *Queries) FetchSeedlingsForBatch(ctx context.Context, rawKey []byte) ([] &i.ScriptKeyTweak, &i.TweakedScriptKey, &i.ScriptKeyDeclaredKnown, + &i.ScriptKeyType, &i.ScriptKeyRaw, &i.ScriptKeyFam, &i.ScriptKeyIndex, @@ -1829,6 +1834,53 @@ func (q *Queries) FetchTapscriptTree(ctx context.Context, rootHash []byte) ([]Fe return items, nil } +const fetchUnknownTypeScriptKeys = `-- name: FetchUnknownTypeScriptKeys :many +SELECT script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index +FROM script_keys +JOIN internal_keys + ON script_keys.internal_key_id = internal_keys.key_id +WHERE script_keys.key_type IS NULL +` + +type FetchUnknownTypeScriptKeysRow struct { + ScriptKey ScriptKey + InternalKey InternalKey +} + +func (q *Queries) FetchUnknownTypeScriptKeys(ctx context.Context) ([]FetchUnknownTypeScriptKeysRow, error) { + rows, err := q.db.QueryContext(ctx, fetchUnknownTypeScriptKeys) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchUnknownTypeScriptKeysRow + for rows.Next() { + var i FetchUnknownTypeScriptKeysRow + if err := rows.Scan( + &i.ScriptKey.ScriptKeyID, + &i.ScriptKey.InternalKeyID, + &i.ScriptKey.TweakedScriptKey, + &i.ScriptKey.Tweak, + &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, + &i.InternalKey.KeyID, + &i.InternalKey.RawKey, + &i.InternalKey.KeyFamily, + &i.InternalKey.KeyIndex, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const genesisAssets = `-- name: GenesisAssets :many SELECT gen_asset_id, asset_id, asset_tag, meta_data_id, output_index, asset_type, genesis_point_id FROM genesis_assets @@ -2198,7 +2250,7 @@ func (q *Queries) QueryAssetBalancesByGroup(ctx context.Context, arg QueryAssetB const queryAssets = `-- name: QueryAssets :many SELECT assets.asset_id AS asset_primary_key, assets.genesis_id, version, spent, - script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, internal_keys.key_id, internal_keys.raw_key, internal_keys.key_family, internal_keys.key_index, key_group_info_view.tapscript_root, key_group_info_view.witness_stack, @@ -2369,6 +2421,7 @@ func (q *Queries) QueryAssets(ctx context.Context, arg QueryAssetsParams) ([]Que &i.ScriptKey.TweakedScriptKey, &i.ScriptKey.Tweak, &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, &i.InternalKey.KeyID, &i.InternalKey.RawKey, &i.InternalKey.KeyFamily, @@ -2870,21 +2923,29 @@ func (q *Queries) UpsertManagedUTXO(ctx context.Context, arg UpsertManagedUTXOPa const upsertScriptKey = `-- name: UpsertScriptKey :one INSERT INTO script_keys ( - internal_key_id, tweaked_script_key, tweak, declared_known + internal_key_id, tweaked_script_key, tweak, declared_known, key_type ) VALUES ( - $1, $2, $3, $4 + $1, $2, $3, $4, $5 ) ON CONFLICT (tweaked_script_key) - -- As a NOP, we just set the script key to the one that triggered the - -- conflict. + -- This is not a NOP, we overwrite the declared_known and key_type value. DO UPDATE SET tweaked_script_key = EXCLUDED.tweaked_script_key, -- If the script key was previously unknown, we'll update to the new - -- value. - declared_known = CASE - WHEN script_keys.declared_known IS NULL OR script_keys.declared_known = FALSE - THEN COALESCE(EXCLUDED.declared_known, script_keys.declared_known) - ELSE script_keys.declared_known - END + -- value, if that is non-NULL. + declared_known = + CASE + WHEN COALESCE(script_keys.declared_known, FALSE) = FALSE + THEN COALESCE(EXCLUDED.declared_known, script_keys.declared_known) + ELSE script_keys.declared_known + END, + -- We only overwrite the key type with a value that does not mean + -- "unknown" (0 or NULL). + key_type = + CASE + WHEN COALESCE(EXCLUDED.key_type, 0) != 0 + THEN EXCLUDED.key_type + ELSE script_keys.key_type + END RETURNING script_key_id ` @@ -2893,6 +2954,7 @@ type UpsertScriptKeyParams struct { TweakedScriptKey []byte Tweak []byte DeclaredKnown sql.NullBool + KeyType sql.NullInt16 } func (q *Queries) UpsertScriptKey(ctx context.Context, arg UpsertScriptKeyParams) (int64, error) { @@ -2901,6 +2963,7 @@ func (q *Queries) UpsertScriptKey(ctx context.Context, arg UpsertScriptKeyParams arg.TweakedScriptKey, arg.Tweak, arg.DeclaredKnown, + arg.KeyType, ) var script_key_id int64 err := row.Scan(&script_key_id) diff --git a/tapdb/sqlc/migrations/000024_script_key_type.down.sql b/tapdb/sqlc/migrations/000024_script_key_type.down.sql new file mode 100644 index 000000000..1414c9cad --- /dev/null +++ b/tapdb/sqlc/migrations/000024_script_key_type.down.sql @@ -0,0 +1 @@ +ALTER TABLE script_keys DROP COLUMN key_type; diff --git a/tapdb/sqlc/migrations/000024_script_key_type.up.sql b/tapdb/sqlc/migrations/000024_script_key_type.up.sql new file mode 100644 index 000000000..6166205c5 --- /dev/null +++ b/tapdb/sqlc/migrations/000024_script_key_type.up.sql @@ -0,0 +1,7 @@ +-- The key_type column is used to store the type of key that is stored in the +-- script_keys table. The type is a Golang numeric type that will have values +-- such as BIP-0086, script path with custom (externally defined) script, script +-- path with Taproot Asset Channel related script, etc. The NULL value +-- will mean the type is not known. Existing script keys at the time of this +-- migration will be updated at startup after the migration is applied. +ALTER TABLE script_keys ADD COLUMN key_type SMALLINT; diff --git a/tapdb/sqlc/models.go b/tapdb/sqlc/models.go index 4a7aba095..759c07f55 100644 --- a/tapdb/sqlc/models.go +++ b/tapdb/sqlc/models.go @@ -312,6 +312,7 @@ type ScriptKey struct { TweakedScriptKey []byte Tweak []byte DeclaredKnown sql.NullBool + KeyType sql.NullInt16 } type TapscriptEdge struct { diff --git a/tapdb/sqlc/querier.go b/tapdb/sqlc/querier.go index 62c213c34..693146eaa 100644 --- a/tapdb/sqlc/querier.go +++ b/tapdb/sqlc/querier.go @@ -90,6 +90,7 @@ type Querier interface { FetchTransferOutputs(ctx context.Context, transferID int64) ([]FetchTransferOutputsRow, error) FetchUniverseKeys(ctx context.Context, arg FetchUniverseKeysParams) ([]FetchUniverseKeysRow, error) FetchUniverseRoot(ctx context.Context, namespace string) (FetchUniverseRootRow, error) + FetchUnknownTypeScriptKeys(ctx context.Context) ([]FetchUnknownTypeScriptKeysRow, error) GenesisAssets(ctx context.Context) ([]GenesisAsset, error) GenesisPoints(ctx context.Context) ([]GenesisPoint, error) GetRootKey(ctx context.Context, id []byte) (Macaroon, error) diff --git a/tapdb/sqlc/queries/assets.sql b/tapdb/sqlc/queries/assets.sql index 27b63e2f6..e6838f6c3 100644 --- a/tapdb/sqlc/queries/assets.sql +++ b/tapdb/sqlc/queries/assets.sql @@ -141,6 +141,7 @@ SELECT seedling_id, asset_name, asset_type, asset_version, asset_supply, script_keys.tweak AS script_key_tweak, script_keys.tweaked_script_key, script_keys.declared_known AS script_key_declared_known, + script_keys.key_type AS script_key_type, internal_keys.raw_key AS script_key_raw, internal_keys.key_family AS script_key_fam, internal_keys.key_index AS script_key_index, @@ -845,21 +846,29 @@ WHERE txid = $1; -- name: UpsertScriptKey :one INSERT INTO script_keys ( - internal_key_id, tweaked_script_key, tweak, declared_known + internal_key_id, tweaked_script_key, tweak, declared_known, key_type ) VALUES ( - $1, $2, $3, $4 + $1, $2, $3, $4, $5 ) ON CONFLICT (tweaked_script_key) - -- As a NOP, we just set the script key to the one that triggered the - -- conflict. + -- This is not a NOP, we overwrite the declared_known and key_type value. DO UPDATE SET tweaked_script_key = EXCLUDED.tweaked_script_key, -- If the script key was previously unknown, we'll update to the new - -- value. - declared_known = CASE - WHEN script_keys.declared_known IS NULL OR script_keys.declared_known = FALSE - THEN COALESCE(EXCLUDED.declared_known, script_keys.declared_known) - ELSE script_keys.declared_known - END + -- value, if that is non-NULL. + declared_known = + CASE + WHEN COALESCE(script_keys.declared_known, FALSE) = FALSE + THEN COALESCE(EXCLUDED.declared_known, script_keys.declared_known) + ELSE script_keys.declared_known + END, + -- We only overwrite the key type with a value that does not mean + -- "unknown" (0 or NULL). + key_type = + CASE + WHEN COALESCE(EXCLUDED.key_type, 0) != 0 + THEN EXCLUDED.key_type + ELSE script_keys.key_type + END RETURNING script_key_id; -- name: FetchScriptKeyIDByTweakedKey :one @@ -874,6 +883,13 @@ JOIN internal_keys ON script_keys.internal_key_id = internal_keys.key_id WHERE script_keys.tweaked_script_key = $1; +-- name: FetchUnknownTypeScriptKeys :many +SELECT sqlc.embed(script_keys), sqlc.embed(internal_keys) +FROM script_keys +JOIN internal_keys + ON script_keys.internal_key_id = internal_keys.key_id +WHERE script_keys.key_type IS NULL; + -- name: FetchInternalKeyLocator :one SELECT key_family, key_index FROM internal_keys diff --git a/tapdb/sqlc/transfers.sql.go b/tapdb/sqlc/transfers.sql.go index 7fc74c63f..ba1312882 100644 --- a/tapdb/sqlc/transfers.sql.go +++ b/tapdb/sqlc/transfers.sql.go @@ -137,7 +137,7 @@ SELECT utxo_internal_keys.raw_key AS internal_key_raw_key_bytes, utxo_internal_keys.key_family AS internal_key_family, utxo_internal_keys.key_index AS internal_key_index, - script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, + script_keys.script_key_id, script_keys.internal_key_id, script_keys.tweaked_script_key, script_keys.tweak, script_keys.declared_known, script_keys.key_type, script_internal_keys.key_id, script_internal_keys.raw_key, script_internal_keys.key_family, script_internal_keys.key_index FROM asset_transfer_outputs outputs JOIN managed_utxos utxos @@ -222,6 +222,7 @@ func (q *Queries) FetchTransferOutputs(ctx context.Context, transferID int64) ([ &i.ScriptKey.TweakedScriptKey, &i.ScriptKey.Tweak, &i.ScriptKey.DeclaredKnown, + &i.ScriptKey.KeyType, &i.InternalKey.KeyID, &i.InternalKey.RawKey, &i.InternalKey.KeyFamily, From 7ac3c110c74335d029e21b98c1369655e38efec2 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 15 Nov 2024 15:42:00 +0100 Subject: [PATCH 4/6] multi: add type to script key With this commit we add a type enum to the script key. This will mostly be used on the database level to filter assets when listing or calculating balances. We can only be 100% certain about the type of a key in the case of BIP-0086 or where we explicitly create them in the tapchannel package. Anything else is a bit of a guess, since the keys are coming over the RPC interface. So we just assume any key that has a non-empty script path is an externally-defined (external app) script spend. This heuristic should be good enough to filter out channel related assets or assets that can't be spent without external instructions in balance RPC calls. --- address/book.go | 19 +++++-- asset/asset.go | 80 +++++++++++++++++++++++++--- rpcserver.go | 7 ++- tapchannel/aux_funding_controller.go | 4 +- tapchannel/aux_sweeper.go | 8 ++- tapdb/addrs.go | 4 +- tapdb/addrs_test.go | 4 +- tapdb/asset_minting.go | 4 ++ tapdb/assets_common.go | 5 ++ tapdb/assets_store.go | 7 ++- 10 files changed, 123 insertions(+), 19 deletions(-) diff --git a/address/book.go b/address/book.go index 00ae6cf4d..7cde14cec 100644 --- a/address/book.go +++ b/address/book.go @@ -125,7 +125,7 @@ type Storage interface { // being relevant for the local wallet (e.g. show assets received on // this key in the asset list and balances). InsertScriptKey(ctx context.Context, scriptKey asset.ScriptKey, - declareAsKnown bool) error + declareAsKnown bool, keyType asset.ScriptKeyType) error } // KeyRing is used to create script and internal keys for Taproot Asset @@ -337,7 +337,12 @@ func (b *Book) NewAddressWithKeys(ctx context.Context, addrVersion Version, if err != nil { return nil, fmt.Errorf("unable to insert internal key: %w", err) } - err = b.cfg.Store.InsertScriptKey(ctx, scriptKey, true) + + // We might not know the type of script key, if it was given to us + // through an RPC call. So we make a guess here. + keyType := scriptKey.GuessType() + + err = b.cfg.Store.InsertScriptKey(ctx, scriptKey, true, keyType) if err != nil { return nil, fmt.Errorf("unable to insert script key: %w", err) } @@ -374,9 +379,11 @@ func (b *Book) IsLocalKey(ctx context.Context, // InsertScriptKey inserts an address related script key into the database. func (b *Book) InsertScriptKey(ctx context.Context, scriptKey asset.ScriptKey, - declareAsKnown bool) error { + declareAsKnown bool, keyType asset.ScriptKeyType) error { - return b.cfg.Store.InsertScriptKey(ctx, scriptKey, declareAsKnown) + return b.cfg.Store.InsertScriptKey( + ctx, scriptKey, declareAsKnown, keyType, + ) } // NextInternalKey derives then inserts an internal key into the database to @@ -411,7 +418,9 @@ func (b *Book) NextScriptKey(ctx context.Context, } scriptKey := asset.NewScriptKeyBip86(keyDesc) - err = b.cfg.Store.InsertScriptKey(ctx, scriptKey, true) + err = b.cfg.Store.InsertScriptKey( + ctx, scriptKey, true, asset.ScriptKeyBip86, + ) if err != nil { return asset.ScriptKey{}, err } diff --git a/asset/asset.go b/asset/asset.go index ad5ba6839..68f840bf2 100644 --- a/asset/asset.go +++ b/asset/asset.go @@ -101,6 +101,43 @@ const ( EncodeSegwit ) +// ScriptKeyType denotes the type of script key used for an asset. This type is +// serialized to the database, so we don't use iota for the values to ensure +// they don't change by accident. +type ScriptKeyType uint8 + +const ( + // ScriptKeyUnknown is the default script key type used for assets that + // we don't know the type of. This should only be stored for assets + // where we don't know the internal key of the script key (e.g. for + // imported proofs). + ScriptKeyUnknown ScriptKeyType = 0 + + // ScriptKeyBip86 is the script key type used for assets that use the + // BIP86 style tweak (e.g. an empty tweak). + ScriptKeyBip86 ScriptKeyType = 1 + + // ScriptKeyScriptPathExternal is the script key type used for assets + // that use a script path that is defined by an external application. + // Keys with script paths are normally not shown in asset balances and + // by default aren't used for coin selection unless specifically + // requested. + ScriptKeyScriptPathExternal ScriptKeyType = 2 + + // ScriptKeyBurn is the script key type used for assets that are burned + // and not spendable. + ScriptKeyBurn ScriptKeyType = 3 + + // ScriptKeyScriptPathChannel is the script key type used for assets + // that use a script path that is somehow related to Taproot Asset + // Channels. That means the script key is either a funding key + // (OP_TRUE), a commitment output key (to_local, to_remote, htlc), or a + // HTLC second-level transaction output key. + // Keys related to channels are not shown in asset balances (unless + // specifically requested) and _never_ used for coin selection. + ScriptKeyScriptPathChannel ScriptKeyType = 4 +) + var ( // ZeroPrevID is the blank prev ID used for genesis assets and also // asset split leaves. @@ -1176,6 +1213,9 @@ type TweakedScriptKey struct { // flag has is that assets with a declared key are shown in the asset // list/balance. DeclaredKnown bool + + // Type is the type of script key that is being used. + Type ScriptKeyType } // IsEqual returns true is this tweaked script key is exactly equivalent to the @@ -1261,15 +1301,44 @@ func (s *ScriptKey) HasScriptPath() bool { return s.TweakedScriptKey != nil && len(s.TweakedScriptKey.Tweak) > 0 } +// GuessType tries to guess the type of the script key based on the information +// available. +func (s *ScriptKey) GuessType() ScriptKeyType { + // If we have an explicit script key type set, we can return that. + if s.TweakedScriptKey != nil && + s.TweakedScriptKey.Type != ScriptKeyUnknown { + + return s.TweakedScriptKey.Type + } + + // If there is a known tweak, then we know that this is a script path + // key. We never return the channel type, since those keys should always + // be declared properly, and we never should need to guess their type. + if s.HasScriptPath() { + return ScriptKeyScriptPathExternal + } + + // Do we know the internal key? Then we can check whether it is a + // BIP-0086 key. + if s.PubKey != nil && s.TweakedScriptKey != nil && + s.TweakedScriptKey.RawKey.PubKey != nil { + + bip86 := NewScriptKeyBip86(s.TweakedScriptKey.RawKey) + if bip86.PubKey.IsEqual(s.PubKey) { + return ScriptKeyBip86 + } + } + + return ScriptKeyUnknown +} + // NewScriptKey constructs a ScriptKey with only the publicly available // information. This resulting key may or may not have a tweak applied to it. func NewScriptKey(key *btcec.PublicKey) ScriptKey { // Since we'll never query lnd for a tweaked key, it doesn't matter if // we lose the parity information here. And this will only ever be // serialized on chain in a 32-bit representation as well. - key, _ = schnorr.ParsePubKey( - schnorr.SerializePubKey(key), - ) + key, _ = schnorr.ParsePubKey(schnorr.SerializePubKey(key)) return ScriptKey{ PubKey: key, } @@ -1281,9 +1350,7 @@ func NewScriptKey(key *btcec.PublicKey) ScriptKey { func NewScriptKeyBip86(rawKey keychain.KeyDescriptor) ScriptKey { // Tweak the script key BIP-0086 style (such that we only commit to the // internal key when signing). - tweakedPubKey := txscript.ComputeTaprootKeyNoScript( - rawKey.PubKey, - ) + tweakedPubKey := txscript.ComputeTaprootKeyNoScript(rawKey.PubKey) // Since we'll never query lnd for a tweaked key, it doesn't matter if // we lose the parity information here. And this will only ever be @@ -1296,6 +1363,7 @@ func NewScriptKeyBip86(rawKey keychain.KeyDescriptor) ScriptKey { PubKey: tweakedPubKey, TweakedScriptKey: &TweakedScriptKey{ RawKey: rawKey, + Type: ScriptKeyBip86, }, } } diff --git a/rpcserver.go b/rpcserver.go index 092b48bf9..bd2ec3d0e 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -7322,7 +7322,12 @@ func (r *rpcServer) DeclareScriptKey(ctx context.Context, err) } - err = r.cfg.TapAddrBook.InsertScriptKey(ctx, *scriptKey, true) + // Because we've been given the key over the RPC interface, we can't be + // 100% sure of the type. But we can make a best effort guess based on + // the fields the user has set. + keyType := scriptKey.GuessType() + + err = r.cfg.TapAddrBook.InsertScriptKey(ctx, *scriptKey, true, keyType) if err != nil { return nil, fmt.Errorf("error inserting script key: %w", err) } diff --git a/tapchannel/aux_funding_controller.go b/tapchannel/aux_funding_controller.go index e0c5c9e37..9d09a5da3 100644 --- a/tapchannel/aux_funding_controller.go +++ b/tapchannel/aux_funding_controller.go @@ -663,7 +663,9 @@ func (f *FundingController) fundVirtualPacket(ctx context.Context, // We'll also need to import the funding script key into the wallet so // the asset will be materialized in the asset table and show up in the // balance correctly. - err := f.cfg.AddrBook.InsertScriptKey(ctx, fundingScriptKey, true) + err := f.cfg.AddrBook.InsertScriptKey( + ctx, fundingScriptKey, true, asset.ScriptKeyScriptPathChannel, + ) if err != nil { return nil, fmt.Errorf("unable to insert script key: %w", err) } diff --git a/tapchannel/aux_sweeper.go b/tapchannel/aux_sweeper.go index 40522e846..249ffa681 100644 --- a/tapchannel/aux_sweeper.go +++ b/tapchannel/aux_sweeper.go @@ -755,7 +755,9 @@ func (a *AuxSweeper) importCommitScriptKeys(req lnwallet.ResolutionReq) error { ctxb := context.Background() for _, key := range keysToImport { - err := a.cfg.AddrBook.InsertScriptKey(ctxb, key, true) + err := a.cfg.AddrBook.InsertScriptKey( + ctxb, key, true, asset.ScriptKeyScriptPathChannel, + ) if err != nil { return fmt.Errorf("unable to insert script "+ "key: %w", err) @@ -938,7 +940,9 @@ func (a *AuxSweeper) importCommitTx(req lnwallet.ResolutionReq, // the asset will be materialized in the asset table and show up in the // balance correctly. ctxb := context.Background() - err := a.cfg.AddrBook.InsertScriptKey(ctxb, fundingScriptKey, true) + err := a.cfg.AddrBook.InsertScriptKey( + ctxb, fundingScriptKey, true, asset.ScriptKeyScriptPathChannel, + ) if err != nil { return fmt.Errorf("unable to insert script key: %w", err) } diff --git a/tapdb/addrs.go b/tapdb/addrs.go index 8f4c544d0..388514dc5 100644 --- a/tapdb/addrs.go +++ b/tapdb/addrs.go @@ -638,7 +638,8 @@ func (t *TapAddressBook) InsertInternalKey(ctx context.Context, // it can be recognized as belonging to the wallet when a transfer comes in // later on. func (t *TapAddressBook) InsertScriptKey(ctx context.Context, - scriptKey asset.ScriptKey, declaredKnown bool) error { + scriptKey asset.ScriptKey, declaredKnown bool, + keyType asset.ScriptKeyType) error { var writeTxOpts AddrBookTxOptions return t.db.ExecTx(ctx, &writeTxOpts, func(q AddrBook) error { @@ -655,6 +656,7 @@ func (t *TapAddressBook) InsertScriptKey(ctx context.Context, TweakedScriptKey: scriptKey.PubKey.SerializeCompressed(), Tweak: scriptKey.Tweak, DeclaredKnown: sqlBool(declaredKnown), + KeyType: sqlInt16(keyType), }) return err }) diff --git a/tapdb/addrs_test.go b/tapdb/addrs_test.go index 443197fe4..a5bd59b32 100644 --- a/tapdb/addrs_test.go +++ b/tapdb/addrs_test.go @@ -663,8 +663,8 @@ func randScriptKey(t *testing.T) asset.ScriptKey { // insertScriptKeyWithNull is a helper function that inserts a script key with a // a NULL value for declared known. We use this so we can insert a NULL vs an // actual value. It is identical to the InsertScriptKey. -func insertScriptKeyWithNull(ctx context.Context, key asset.ScriptKey, -) func(AddrBook) error { +func insertScriptKeyWithNull(ctx context.Context, + key asset.ScriptKey) func(AddrBook) error { return func(q AddrBook) error { internalKeyID, err := insertInternalKey( diff --git a/tapdb/asset_minting.go b/tapdb/asset_minting.go index 9faa15d21..166cfd911 100644 --- a/tapdb/asset_minting.go +++ b/tapdb/asset_minting.go @@ -669,12 +669,16 @@ func fetchAssetSeedlings(ctx context.Context, q PendingAssetStore, declaredKnown := extractBool( dbSeedling.ScriptKeyDeclaredKnown, ) + scriptType := extractSqlInt16[asset.ScriptKeyType]( + dbSeedling.ScriptKeyType, + ) seedling.ScriptKey = asset.ScriptKey{ PubKey: tweakedScriptKey, TweakedScriptKey: &asset.TweakedScriptKey{ RawKey: scriptKeyRawKey, Tweak: scriptKeyTweak, DeclaredKnown: declaredKnown, + Type: scriptType, }, } } diff --git a/tapdb/assets_common.go b/tapdb/assets_common.go index 260ee7a6c..81ccdb66d 100644 --- a/tapdb/assets_common.go +++ b/tapdb/assets_common.go @@ -382,6 +382,7 @@ func upsertScriptKey(ctx context.Context, scriptKey asset.ScriptKey, InternalKeyID: rawScriptKeyID, TweakedScriptKey: scriptKey.PubKey.SerializeCompressed(), Tweak: scriptKey.Tweak, + KeyType: sqlInt16(scriptKey.Type), }) if err != nil { return 0, fmt.Errorf("%w: %w", ErrUpsertScriptKey, err) @@ -413,6 +414,7 @@ func upsertScriptKey(ctx context.Context, scriptKey asset.ScriptKey, scriptKeyID, err = q.UpsertScriptKey(ctx, NewScriptKey{ InternalKeyID: rawScriptKeyID, TweakedScriptKey: scriptKey.PubKey.SerializeCompressed(), + KeyType: sqlInt16(asset.ScriptKeyUnknown), }) if err != nil { return 0, fmt.Errorf("%w: %w", ErrUpsertScriptKey, err) @@ -469,6 +471,9 @@ func parseScriptKey(ik sqlc.InternalKey, sk sqlc.ScriptKey) (asset.ScriptKey, }, Tweak: sk.Tweak, DeclaredKnown: extractBool(sk.DeclaredKnown), + Type: extractSqlInt16[asset.ScriptKeyType]( + sk.KeyType, + ), }, } err error diff --git a/tapdb/assets_store.go b/tapdb/assets_store.go index 31f2529d8..ef6244d8c 100644 --- a/tapdb/assets_store.go +++ b/tapdb/assets_store.go @@ -2533,10 +2533,14 @@ func insertAssetTransferOutput(ctx context.Context, q ActiveAssetsStore, scriptInternalKey := keychain.KeyDescriptor{ PubKey: output.ScriptKey.PubKey, } - var tweak []byte + var ( + tweak []byte + scriptKeyType asset.ScriptKeyType + ) if output.ScriptKey.TweakedScriptKey != nil { scriptInternalKey = output.ScriptKey.RawKey tweak = output.ScriptKey.Tweak + scriptKeyType = output.ScriptKey.Type } scriptInternalKeyID, err := q.UpsertInternalKey(ctx, InternalKey{ RawKey: scriptInternalKey.PubKey.SerializeCompressed(), @@ -2551,6 +2555,7 @@ func insertAssetTransferOutput(ctx context.Context, q ActiveAssetsStore, InternalKeyID: scriptInternalKeyID, TweakedScriptKey: output.ScriptKey.PubKey.SerializeCompressed(), Tweak: tweak, + KeyType: sqlInt16(scriptKeyType), }) if err != nil { return fmt.Errorf("unable to insert script key: %w", err) From 1d27ee23af7578b6650f8be7ca3b5ddf319c5dee Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 15 Nov 2024 18:04:07 +0100 Subject: [PATCH 5/6] tapfreighter: set type for burn keys correctly In order for burn keys to be stored with the correct type, we need to detect them before storing the transfer outputs to the database. We can't set the script key type before, because we're carrying around the script key in a virtual packet, where that information isn't available. --- tapfreighter/chain_porter.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tapfreighter/chain_porter.go b/tapfreighter/chain_porter.go index 3fbb6790d..4fb4e2894 100644 --- a/tapfreighter/chain_porter.go +++ b/tapfreighter/chain_porter.go @@ -1219,6 +1219,12 @@ func (p *ChainPorter) stateStep(currentPkg sendPackage) (*sendPackage, error) { "%w", err) } + // Burn keys are the only keys that we don't explicitly store + // in the DB before this point. But we'll want them to have the + // correct type when creating the transfer, so we'll set that + // now. + detectBurnKeys(currentPkg.VirtualPackets) + // We now need to find out if this is a transfer to ourselves // (e.g. a change output) or an outbound transfer. A key being // local means the lnd node connected to this daemon knows how @@ -1456,6 +1462,26 @@ func (p *ChainPorter) publishSubscriberEvent(event fn.Event) { } } +// detectBurnKeys checks if any of the outputs in the virtual packets are burn +// keys and sets the appropriate type on the output script key. +func detectBurnKeys(activeTransfers []*tappsbt.VPacket) { + for _, vPkt := range activeTransfers { + for _, vOut := range vPkt.Outputs { + if vOut.Asset == nil { + continue + } + + witness := vOut.Asset.PrevWitnesses + if len(witness) > 0 && asset.IsBurnKey( + vOut.ScriptKey.PubKey, witness[0], + ) { + vOut.Asset.ScriptKey.Type = asset.ScriptKeyBurn + vOut.ScriptKey.Type = asset.ScriptKeyBurn + } + } + } +} + // A compile-time assertion to make sure ChainPorter satisfies the // fn.EventPublisher interface. var _ fn.EventPublisher[fn.Event, bool] = (*ChainPorter)(nil) From bd9d22ddb99973ef684c00cbaab4a913a034b18e Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 15 Nov 2024 18:59:59 +0100 Subject: [PATCH 6/6] tapcfg+tapdb: run post migration checks --- tapcfg/server.go | 9 +- tapdb/assets_store.go | 12 +-- tapdb/migrations.go | 46 ++++++---- tapdb/post_migration_checks.go | 150 +++++++++++++++++++++++++++++++++ tapdb/postgres.go | 12 ++- tapdb/sqlite.go | 12 ++- 6 files changed, 211 insertions(+), 30 deletions(-) create mode 100644 tapdb/post_migration_checks.go diff --git a/tapcfg/server.go b/tapcfg/server.go index 31364185d..b260e6d64 100644 --- a/tapcfg/server.go +++ b/tapcfg/server.go @@ -26,13 +26,6 @@ import ( "github.com/lightningnetwork/lnd/signal" ) -// databaseBackend is an interface that contains all methods our different -// database backends implement. -type databaseBackend interface { - tapdb.BatchedQuerier - WithTx(tx *sql.Tx) *sqlc.Queries -} - // genServerConfig generates a server config from the given tapd config. // // NOTE: The RPCConfig and SignalInterceptor fields must be set by the caller @@ -43,7 +36,7 @@ func genServerConfig(cfg *Config, cfgLogger btclog.Logger, var ( err error - db databaseBackend + db tapdb.DatabaseBackend dbType sqlc.BackendType ) diff --git a/tapdb/assets_store.go b/tapdb/assets_store.go index ef6244d8c..362b6d73c 100644 --- a/tapdb/assets_store.go +++ b/tapdb/assets_store.go @@ -621,8 +621,8 @@ func parseAssetWitness(input AssetWitness) (asset.Witness, error) { // dbAssetsToChainAssets maps a set of confirmed assets in the database, and // the witnesses of those assets to a set of normal ChainAsset structs needed // by a higher level application. -func (a *AssetStore) dbAssetsToChainAssets(dbAssets []ConfirmedAsset, - witnesses assetWitnesses) ([]*asset.ChainAsset, error) { +func dbAssetsToChainAssets(dbAssets []ConfirmedAsset, witnesses assetWitnesses, + clock clock.Clock) ([]*asset.ChainAsset, error) { chainAssets := make([]*asset.ChainAsset, len(dbAssets)) for i := range dbAssets { @@ -826,7 +826,7 @@ func (a *AssetStore) dbAssetsToChainAssets(dbAssets []ConfirmedAsset, owner := sprout.AnchorLeaseOwner expiry := sprout.AnchorLeaseExpiry if len(owner) > 0 && expiry.Valid && - expiry.Time.UTC().After(a.clock.Now().UTC()) { + expiry.Time.UTC().After(clock.Now().UTC()) { copy(chainAssets[i].AnchorLeaseOwner[:], owner) chainAssets[i].AnchorLeaseExpiry = &expiry.Time @@ -1198,7 +1198,7 @@ func (a *AssetStore) FetchAllAssets(ctx context.Context, includeSpent, return nil, dbErr } - return a.dbAssetsToChainAssets(dbAssets, assetWitnesses) + return dbAssetsToChainAssets(dbAssets, assetWitnesses, a.clock) } // FetchManagedUTXOs fetches all UTXOs we manage. @@ -1901,7 +1901,9 @@ func (a *AssetStore) queryChainAssets(ctx context.Context, q ActiveAssetsStore, if err != nil { return nil, err } - matchingAssets, err := a.dbAssetsToChainAssets(dbAssets, assetWitnesses) + matchingAssets, err := dbAssetsToChainAssets( + dbAssets, assetWitnesses, a.clock, + ) if err != nil { return nil, err } diff --git a/tapdb/migrations.go b/tapdb/migrations.go index e49bc6e1d..2377cfd29 100644 --- a/tapdb/migrations.go +++ b/tapdb/migrations.go @@ -2,6 +2,7 @@ package tapdb import ( "bytes" + "database/sql" "errors" "fmt" "io" @@ -14,6 +15,7 @@ import ( "github.com/golang-migrate/migrate/v4/database" "github.com/golang-migrate/migrate/v4/source/httpfs" "github.com/lightninglabs/taproot-assets/fn" + "github.com/lightninglabs/taproot-assets/tapdb/sqlc" ) const ( @@ -25,6 +27,13 @@ const ( LatestMigrationVersion = 24 ) +// DatabaseBackend is an interface that contains all methods our different +// Database backends implement. +type DatabaseBackend interface { + BatchedQuerier + WithTx(tx *sql.Tx) *sqlc.Queries +} + // MigrationTarget is a functional option that can be passed to applyMigrations // to specify a target version to migrate to. `currentDbVersion` is the current // (migration) version of the database, or None if unknown. @@ -115,9 +124,10 @@ func (m *migrationLogger) Verbose() bool { // applyMigrations executes database migration files found in the given file // system under the given path, using the passed database driver and database -// name, up to or down to the given target version. +// name, up to or down to the given target version. The boolean return value +// indicates whether any migrations were applied. func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string, - targetVersion MigrationTarget, opts *migrateOptions) error { + targetVersion MigrationTarget, opts *migrateOptions) (bool, error) { // With the migrate instance open, we'll create a new migration source // using the embedded file system stored in sqlSchemas. The library @@ -125,7 +135,7 @@ func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string, // in this intermediate layer. migrateFileServer, err := httpfs.New(http.FS(fs), path) if err != nil { - return err + return false, err } // Finally, we'll run the migration with our driver above based on the @@ -135,7 +145,7 @@ func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string, "migrations", migrateFileServer, dbName, driver, ) if err != nil { - return err + return false, err } migrationVersion, _, _ := sqlMigrate.Version() @@ -144,38 +154,44 @@ func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string, // prevent that without explicit accounting. latestVersion := opts.latestVersion.UnwrapOr(LatestMigrationVersion) if migrationVersion > latestVersion { - return fmt.Errorf("%w: database version is newer than the "+ - "latest migration version, preventing downgrade: "+ + return false, fmt.Errorf("%w: database version is newer than "+ + "the latest migration version, preventing downgrade: "+ "db_version=%v, latest_migration_version=%v", ErrMigrationDowngrade, migrationVersion, latestVersion) } // Report the current version of the database before the migration. - currentDbVersion, _, err := driver.Version() + versionBeforeMigration, _, err := driver.Version() if err != nil { - return fmt.Errorf("unable to get current db version: %w", err) + return false, fmt.Errorf("unable to get current db version: %w", + err) } log.Infof("Attempting to apply migration(s) "+ "(current_db_version=%v, latest_migration_version=%v)", - currentDbVersion, latestVersion) + versionBeforeMigration, latestVersion) // Apply our local logger to the migration instance. sqlMigrate.Log = &migrationLogger{log} // Execute the migration based on the target given. - err = targetVersion(sqlMigrate, currentDbVersion, latestVersion) + err = targetVersion(sqlMigrate, versionBeforeMigration, latestVersion) if err != nil && !errors.Is(err, migrate.ErrNoChange) { - return err + return false, err } + // If we actually did migrate, we'll now run the Golang based + // post-migration checks that ensure the database is in a consistent + // state, based on properties not fully expressible in SQL. + // Report the current version of the database after the migration. - currentDbVersion, _, err = driver.Version() + versionAfterMigration, _, err := driver.Version() if err != nil { - return fmt.Errorf("unable to get current db version: %w", err) + return true, fmt.Errorf("unable to get current db version: %w", + err) } - log.Infof("Database version after migration: %v", currentDbVersion) + log.Infof("Database version after migration: %v", versionAfterMigration) - return nil + return true, nil } // replacerFS is an implementation of a fs.FS virtual file system that wraps an diff --git a/tapdb/post_migration_checks.go b/tapdb/post_migration_checks.go new file mode 100644 index 000000000..f4f382d4c --- /dev/null +++ b/tapdb/post_migration_checks.go @@ -0,0 +1,150 @@ +package tapdb + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/lightninglabs/taproot-assets/asset" + "github.com/lightninglabs/taproot-assets/fn" + "github.com/lightninglabs/taproot-assets/tapdb/sqlc" + "github.com/lightninglabs/taproot-assets/tapscript" + "github.com/lightningnetwork/lnd/clock" +) + +// postMigrationCheck is a function type for a function that performs a +// post-migration check on the database. +type postMigrationCheck func(context.Context, sqlc.Querier) error + +var ( + // postMigrationChecks is a list of functions that are run after the + // database migrations have been applied. These functions are used to + // perform additional checks on the database state that are not fully + // expressible in SQL. + postMigrationChecks = []postMigrationCheck{ + detectScriptKeyType, + } +) + +// runPostMigrationChecks runs a set of post-migration checks on the database +// using the given database backend. +func runPostMigrationChecks(db DatabaseBackend) error { + var ( + ctx = context.Background() + txDb = NewTransactionExecutor( + db, func(tx *sql.Tx) sqlc.Querier { + return db.WithTx(tx) + }, + ) + writeTxOpts AssetStoreTxOptions + ) + + return txDb.ExecTx(ctx, &writeTxOpts, func(q sqlc.Querier) error { + log.Infof("Running %d post-migration checks", + len(postMigrationChecks)) + start := time.Now() + + for _, check := range postMigrationChecks { + err := check(ctx, q) + if err != nil { + return err + } + } + + log.Infof("Post-migration checks completed in %v", + time.Since(start)) + + return nil + }) +} + +// detectScriptKeyType attempts to detect the type of the script keys that don't +// have a type set yet. +func detectScriptKeyType(ctx context.Context, q sqlc.Querier) error { + defaultClock := clock.NewDefaultClock() + + // We start by fetching all assets, even the spent ones. We then collect + // a list of the burn keys from the assets (because burn keys can only + // be calculated from the asset's witness). + assetFilter := QueryAssetFilters{ + Now: sql.NullTime{ + Time: defaultClock.Now().UTC(), + Valid: true, + }, + } + dbAssets, assetWitnesses, err := fetchAssetsWithWitness( + ctx, q, assetFilter, + ) + if err != nil { + return fmt.Errorf("error fetching assets: %w", err) + } + + chainAssets, err := dbAssetsToChainAssets( + dbAssets, assetWitnesses, defaultClock, + ) + if err != nil { + return fmt.Errorf("error converting assets: %w", err) + } + + burnAssets := fn.Filter(chainAssets, func(a *asset.ChainAsset) bool { + return a.IsBurn() + }) + burnKeys := make(map[asset.SerializedKey]struct{}) + for _, a := range burnAssets { + serializedKey := asset.ToSerialized(a.ScriptKey.PubKey) + burnKeys[serializedKey] = struct{}{} + } + + untypedKeys, err := q.FetchUnknownTypeScriptKeys(ctx) + if err != nil { + return fmt.Errorf("error fetching script keys: %w", err) + } + + channelFundingKey := asset.NewScriptKey( + tapscript.NewChannelFundingScriptTree().TaprootKey, + ).PubKey + + for _, k := range untypedKeys { + scriptKey, err := parseScriptKey(k.InternalKey, k.ScriptKey) + if err != nil { + return fmt.Errorf("error parsing script key: %w", err) + } + + serializedKey := asset.ToSerialized(scriptKey.PubKey) + newType := asset.ScriptKeyUnknown + + if _, ok := burnKeys[serializedKey]; ok { + newType = asset.ScriptKeyBurn + } else { + guessedType := scriptKey.GuessType() + if guessedType == asset.ScriptKeyBip86 { + newType = asset.ScriptKeyBip86 + } + + if guessedType == asset.ScriptKeyScriptPathExternal && + scriptKey.PubKey.IsEqual(channelFundingKey) { + + newType = asset.ScriptKeyScriptPathChannel + } + } + + // If we were able to identify the key type, we update the key + // in the database. + if newType != asset.ScriptKeyUnknown { + _, err := q.UpsertScriptKey(ctx, NewScriptKey{ + InternalKeyID: k.InternalKey.KeyID, + TweakedScriptKey: k.ScriptKey.TweakedScriptKey, + Tweak: k.ScriptKey.Tweak, + DeclaredKnown: k.ScriptKey.DeclaredKnown, + KeyType: sqlInt16(newType), + }) + if err != nil { + return fmt.Errorf("error updating script key "+ + "type: %w", err) + } + } + } + + return nil +} diff --git a/tapdb/postgres.go b/tapdb/postgres.go index 12f91ad8f..61b17aeeb 100644 --- a/tapdb/postgres.go +++ b/tapdb/postgres.go @@ -158,10 +158,20 @@ func (s *PostgresStore) ExecuteMigrations(target MigrationTarget, } postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements) - return applyMigrations( + didMigrate, err := applyMigrations( postgresFS, driver, "sqlc/migrations", s.cfg.DBName, target, opts, ) + if err != nil { + return fmt.Errorf("error applying migrations: %w", err) + } + + // Run post-migration checks if we actually did migrate. + if didMigrate { + return runPostMigrationChecks(s) + } + + return nil } // NewTestPostgresDB is a helper function that creates a Postgres database for diff --git a/tapdb/sqlite.go b/tapdb/sqlite.go index 9e3cb4ed8..9c378c799 100644 --- a/tapdb/sqlite.go +++ b/tapdb/sqlite.go @@ -244,9 +244,19 @@ func (s *SqliteStore) ExecuteMigrations(target MigrationTarget, } sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements) - return applyMigrations( + didMigrate, err := applyMigrations( sqliteFS, driver, "sqlc/migrations", "sqlite", target, opts, ) + if err != nil { + return fmt.Errorf("error applying migrations: %w", err) + } + + // Run post-migration checks if we actually did migrate. + if didMigrate { + return runPostMigrationChecks(s) + } + + return nil } // NewTestSqliteDB is a helper function that creates an SQLite database for