challenger: paginage ListInvoices to avoid resource exhaustion

challenger/lnd.go

@@ -74,7 +74,7 @@ func NewLndChallenger(client InvoiceClient,
 
 // Start starts the challenger's main work which is to keep track of all
 // invoices and their states. For that the backing lnd node is queried for all
-// invoices on startup and the a subscription to all subsequent invoice updates
+// invoices on startup and a subscription to all subsequent invoice updates
 // is created.
 func (l *LndChallenger) Start() error {
 	// These are the default values for the subscription. In case there are
@@ -84,49 +84,61 @@ func (l *LndChallenger) Start() error {
 	addIndex := uint64(0)
 	settleIndex := uint64(0)
 
-	// Get a list of all existing invoices on startup and add them to our
-	// cache. We need to keep track of all invoices, even quite old ones to
-	// make sure tokens are valid. But to save space we only keep track of
-	// an invoice's state.
+	// Paginate through all existing invoices on startup and add them to our
+	// cache. We need to keep track of all invoices to ensure tokens are
+	// valid.
 	ctx := l.clientCtx()
-	invoiceResp, err := l.client.ListInvoices(
-		ctx, &lnrpc.ListInvoiceRequest{
-			NumMaxInvoices: math.MaxUint64,
-		},
-	)
-	if err != nil {
-		return err
-	}
-
-	// Advance our indices to the latest known one so we'll only receive
-	// updates for new invoices and/or newly settled invoices.
-	l.invoicesMtx.Lock()
-	for _, invoice := range invoiceResp.Invoices {
-		// Some invoices like AMP invoices may not have a payment hash
-		// populated.
-		if invoice.RHash == nil {
-			continue
+	indexOffset := uint64(0)
+	for {
+		invoiceResp, err := l.client.ListInvoices(
+			ctx, &lnrpc.ListInvoiceRequest{
+				IndexOffset:    indexOffset,
+				NumMaxInvoices: 1000,
+			},
+		)
+		if err != nil {
+			return err
 		}
 
-		if invoice.AddIndex > addIndex {
-			addIndex = invoice.AddIndex
-		}
-		if invoice.SettleIndex > settleIndex {
-			settleIndex = invoice.SettleIndex
-		}
-		hash, err := lntypes.MakeHash(invoice.RHash)
-		if err != nil {
-			l.invoicesMtx.Unlock()
-			return fmt.Errorf("error parsing invoice hash: %v", err)
+		// Lock the mutex to safely update the invoice states.
+		l.invoicesMtx.Lock()
+		for _, invoice := range invoiceResp.Invoices {
+			// Skip invoices that do not have a payment hash
+			// populated.
+			if invoice.RHash == nil {
+				continue
+			}
+
+			if invoice.AddIndex > addIndex {
+				addIndex = invoice.AddIndex
+			}
+			if invoice.SettleIndex > settleIndex {
+				settleIndex = invoice.SettleIndex
+			}
+			hash, err := lntypes.MakeHash(invoice.RHash)
+			if err != nil {
+				l.invoicesMtx.Unlock()
+				return fmt.Errorf("error parsing invoice "+
+					"hash: %v", err)
+			}
+
+			// Skip tracking the state of canceled or expired
+			// invoices.
+			if invoiceIrrelevant(invoice) {
+				continue
+			}
+			l.invoiceStates[hash] = invoice.State
 		}
+		l.invoicesMtx.Unlock()
 
-		// Don't track the state of canceled or expired invoices.
-		if invoiceIrrelevant(invoice) {
-			continue
+		// If there are no more invoices, stop pagination.
+		if len(invoiceResp.Invoices) < 1000 {
+			break
 		}
-		l.invoiceStates[hash] = invoice.State
+
+		// Update the index offset for the next batch.
+		indexOffset = invoiceResp.LastIndexOffset
 	}
-	l.invoicesMtx.Unlock()
 
 	// We need to be able to cancel any subscription we make.
 	ctxc, cancel := context.WithCancel(l.clientCtx())