kvdb: migrate between different backends

.github/workflows/main.yml

@@ -203,7 +203,7 @@ jobs:
       matrix:
         unit_type:
           - btcd unit-cover
-          - unit tags=kvdb_etcd
+          - unit tags="kvdb_etcd kvdb_postgres"
           - travis-race
     steps:
       - name: git checkout

.gitignore

@@ -30,6 +30,8 @@ _testmain.go
 /lncli-debug
 /lnd-itest
 /lncli-itest
+/lndinit
+/lndinit-debug
 
 # Integration test log files
 lntest/itest/*.log

.travis.yml

@@ -61,6 +61,11 @@ jobs:
         - bash ./scripts/install_bitcoind.sh
         - make itest-parallel backend=bitcoind dbbackend=etcd
 
+    - name: Bitcoind Integration with postgres (txindex enabled)
+      script:
+        - bash ./scripts/install_bitcoind.sh
+        - make itest-parallel backend=bitcoind dbbackend=postgres
+
     - name: Bitcoind Integration (txindex disabled)
       script:
         - bash ./scripts/install_bitcoind.sh

Dockerfile

@@ -43,6 +43,7 @@ RUN apk --no-cache add \
 # Copy the binaries from the builder image.
 COPY --from=builder /go/bin/lncli /bin/
 COPY --from=builder /go/bin/lnd /bin/
+COPY --from=builder /go/bin/lndinit /bin/
 COPY --from=builder /go/src/github.com/lightningnetwork/lnd/scripts/verify-install.sh /
 
 # Store the SHA256 hash of the binaries that were just produced for later

Makefile

@@ -128,9 +128,10 @@ $(GOFUZZ_BUILD_BIN):
 # ============
 
 build:
-	@$(call print, "Building debug lnd and lncli.")
+	@$(call print, "Building debug binaries.")
 	$(GOBUILD) -tags="$(DEV_TAGS)" -o lnd-debug $(DEV_LDFLAGS) $(PKG)/cmd/lnd
 	$(GOBUILD) -tags="$(DEV_TAGS)" -o lncli-debug $(DEV_LDFLAGS) $(PKG)/cmd/lncli
+	cd cmd/lndinit; $(GOBUILD) -tags="$(DEV_TAGS)" -o ../../lndinit-debug $(DEV_LDFLAGS) .
 
 build-itest:
 	@$(call print, "Building itest btcd and lnd.")
@@ -141,19 +142,21 @@ build-itest:
 	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -c -o lntest/itest/itest.test$(EXEC_SUFFIX)
 
 install:
-	@$(call print, "Installing lnd and lncli.")
+	@$(call print, "Installing binaries.")
 	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/lnd
 	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/lncli
+	cd cmd/lndinit; $(GOINSTALL) -tags="${tags}" $(LDFLAGS) .
 
 release-install:
-	@$(call print, "Installing release lnd and lncli.")
+	@$(call print, "Installing release binaries.")
 	env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd
 	env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lncli
+	cd cmd/lndinit; env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" .
 
 # Make sure the generated mobile RPC stubs don't influence our vendor package
 # by removing them first in the clean-mobile target.
 release: clean-mobile
-	@$(call print, "Releasing lnd and lncli binaries.")
+	@$(call print, "Releasing binaries.")
 	$(VERSION_CHECK)
 	./scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)"
 
@@ -176,18 +179,27 @@ scratch: build
 
 check: unit itest
 
-itest-only:
+db-instance:
+ifeq ($(dbbackend),postgres)
+	docker rm lnd-postgres --force
+	docker run --name lnd-postgres -e POSTGRES_PASSWORD=postgres -p 6432:5432 -d postgres -N 500
+
+	# Wait for it to be started
+	sleep 3
+endif
+
+itest-only: db-instance
 	@$(call print, "Running integration tests with ${backend} backend.")
 	rm -rf lntest/itest/*.log lntest/itest/.logs-*; date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS)
+	EXEC_SUFFIX=$(EXEC_SUFFIX) DBBACKEND=$(dbbackend) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS)
 	lntest/itest/log_check_errors.sh
 
 itest: build-itest itest-only
 
-itest-parallel: build-itest
+itest-parallel: build-itest db-instance
 	@$(call print, "Running tests")
 	rm -rf lntest/itest/*.log lntest/itest/.logs-*; date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) echo "$$(seq 0 $$(expr $(ITEST_PARALLELISM) - 1))" | xargs -P $(ITEST_PARALLELISM) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
+	EXEC_SUFFIX=$(EXEC_SUFFIX) DBBACKEND=$(dbbackend) echo "$$(seq 0 $$(expr $(ITEST_PARALLELISM) - 1))" | xargs -P $(ITEST_PARALLELISM) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
 	lntest/itest/log_check_errors.sh
 
 unit: btcd
@@ -303,7 +315,7 @@ mobile: ios android
 
 clean:
 	@$(call print, "Cleaning source.$(NC)")
-	$(RM) ./lnd-debug ./lncli-debug
+	$(RM) ./lnd-debug ./lncli-debug ./lndinit-debug
 	$(RM) ./lnd-itest ./lncli-itest
 	$(RM) -r ./vendor .vendor-new
 

chainntnfs/height_hint_cache.go

@@ -84,7 +84,7 @@ type ConfirmHintCache interface {
 // will be stored.
 type HeightHintCache struct {
 	cfg CacheConfig
-	db  *channeldb.DB
+	db  kvdb.Backend
 }
 
 // Compile-time checks to ensure HeightHintCache satisfies the SpendHintCache
@@ -93,7 +93,9 @@ var _ SpendHintCache = (*HeightHintCache)(nil)
 var _ ConfirmHintCache = (*HeightHintCache)(nil)
 
 // NewHeightHintCache returns a new height hint cache backed by a database.
-func NewHeightHintCache(cfg CacheConfig, db *channeldb.DB) (*HeightHintCache, error) {
+func NewHeightHintCache(cfg CacheConfig, db kvdb.Backend) (*HeightHintCache,
+	error) {
+
 	cache := &HeightHintCache{cfg, db}
 	if err := cache.initBuckets(); err != nil {
 		return nil, err
@@ -105,7 +107,7 @@ func NewHeightHintCache(cfg CacheConfig, db *channeldb.DB) (*HeightHintCache, er
 // initBuckets ensures that the primary buckets used by the circuit are
 // initialized so that we can assume their existence after startup.
 func (c *HeightHintCache) initBuckets() error {
-	return kvdb.Batch(c.db.Backend, func(tx kvdb.RwTx) error {
+	return kvdb.Batch(c.db, func(tx kvdb.RwTx) error {
 		_, err := tx.CreateTopLevelBucket(spendHintBucket)
 		if err != nil {
 			return err
@@ -127,7 +129,7 @@ func (c *HeightHintCache) CommitSpendHint(height uint32,
 	Log.Tracef("Updating spend hint to height %d for %v", height,
 		spendRequests)
 
-	return kvdb.Batch(c.db.Backend, func(tx kvdb.RwTx) error {
+	return kvdb.Batch(c.db, func(tx kvdb.RwTx) error {
 		spendHints := tx.ReadWriteBucket(spendHintBucket)
 		if spendHints == nil {
 			return ErrCorruptedHeightHintCache
@@ -197,7 +199,7 @@ func (c *HeightHintCache) PurgeSpendHint(spendRequests ...SpendRequest) error {
 
 	Log.Tracef("Removing spend hints for %v", spendRequests)
 
-	return kvdb.Batch(c.db.Backend, func(tx kvdb.RwTx) error {
+	return kvdb.Batch(c.db, func(tx kvdb.RwTx) error {
 		spendHints := tx.ReadWriteBucket(spendHintBucket)
 		if spendHints == nil {
 			return ErrCorruptedHeightHintCache
@@ -228,7 +230,7 @@ func (c *HeightHintCache) CommitConfirmHint(height uint32,
 	Log.Tracef("Updating confirm hints to height %d for %v", height,
 		confRequests)
 
-	return kvdb.Batch(c.db.Backend, func(tx kvdb.RwTx) error {
+	return kvdb.Batch(c.db, func(tx kvdb.RwTx) error {
 		confirmHints := tx.ReadWriteBucket(confirmHintBucket)
 		if confirmHints == nil {
 			return ErrCorruptedHeightHintCache
@@ -299,7 +301,7 @@ func (c *HeightHintCache) PurgeConfirmHint(confRequests ...ConfRequest) error {
 
 	Log.Tracef("Removing confirm hints for %v", confRequests)
 
-	return kvdb.Batch(c.db.Backend, func(tx kvdb.RwTx) error {
+	return kvdb.Batch(c.db, func(tx kvdb.RwTx) error {
 		confirmHints := tx.ReadWriteBucket(confirmHintBucket)
 		if confirmHints == nil {
 			return ErrCorruptedHeightHintCache

chainreg/chainregistry.go

@@ -28,6 +28,7 @@ import (
 	"github.com/lightningnetwork/lnd/htlcswitch"
 	"github.com/lightningnetwork/lnd/input"
 	"github.com/lightningnetwork/lnd/keychain"
+	"github.com/lightningnetwork/lnd/kvdb"
 	"github.com/lightningnetwork/lnd/lncfg"
 	"github.com/lightningnetwork/lnd/lnwallet"
 	"github.com/lightningnetwork/lnd/lnwallet/btcwallet"
@@ -68,11 +69,13 @@ type Config struct {
 	// LtcdMode defines settings for connecting to an ltcd node.
 	LtcdMode *lncfg.Btcd
 
-	// LocalChanDB is a pointer to the local backing channel database.
-	LocalChanDB *channeldb.DB
+	// HeightHintDB is a pointer to the database that stores the height
+	// hints.
+	HeightHintDB kvdb.Backend
 
-	// RemoteChanDB is a pointer to the remote backing channel database.
-	RemoteChanDB *channeldb.DB
+	// ChanStateDB is a pointer to the database that stores the channel
+	// state.
+	ChanStateDB *channeldb.DB
 
 	// BlockCacheSize is the size (in bytes) of blocks kept in memory.
 	BlockCacheSize uint64
@@ -304,7 +307,7 @@ func NewChainControl(cfg *Config, blockCache *blockcache.BlockCache) (
 
 	// Initialize the height hint cache within the chain directory.
 	hintCache, err := chainntnfs.NewHeightHintCache(
-		heightHintCacheConfig, cfg.LocalChanDB,
+		heightHintCacheConfig, cfg.HeightHintDB,
 	)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to initialize height hint "+
@@ -684,7 +687,7 @@ func NewChainControl(cfg *Config, blockCache *blockcache.BlockCache) (
 	// Create, and start the lnwallet, which handles the core payment
 	// channel logic, and exposes control via proxy state machines.
 	walletCfg := lnwallet.Config{
-		Database:           cfg.RemoteChanDB,
+		Database:           cfg.ChanStateDB,
 		Notifier:           cc.ChainNotifier,
 		WalletController:   wc,
 		Signer:             cc.Signer,