fix: market add-url, online/offline filters (#381)

* fix add-url command

* start deal after import

* mark deal failed for removed pipelines

* deal type filter

cmd/curio/market.go

@@ -224,9 +224,10 @@ var marketAddOfflineURLCmd = &cli.Command{
 			_, err = dep.DB.Exec(ctx, `INSERT INTO market_offline_urls (
 								uuid,
 								url,
+                                headers,
 								raw_size
 							) VALUES ($1, $2, $3, $4);`,
-				uuid, url, size)
+				uuid, url, []byte("{}"), size)
 			if err != nil {
 				return xerrors.Errorf("adding details to DB: %w", err)
 			}

deps/config/types.go

@@ -687,6 +687,12 @@ type MK12Config struct {
 	// DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list
 	// If True then all deals coming from unknown clients will be rejected.
 	DenyUnknownClients bool
+
+	// DenyOnlineDeals determines if the storage provider will accept online deals
+	DenyOnlineDeals bool
+
+	// DenyOfflineDeals determines if the storage provider will accept offline deals
+	DenyOfflineDeals bool
 }
 
 type PieceLocatorConfig struct {

documentation/en/configuration/default-curio-configuration.md

@@ -472,6 +472,16 @@ description: The default curio configuration
       # type: bool
       #DenyUnknownClients = false
 
+      # DenyOnlineDeals determines if the storage provider will accept online deals
+      #
+      # type: bool
+      #DenyOnlineDeals = false
+
+      # DenyOfflineDeals determines if the storage provider will accept offline deals
+      #
+      # type: bool
+      #DenyOfflineDeals = false
+
     [Market.StorageMarketConfig.IPNI]
       # Disable set whether to disable indexing announcement to the network and expose endpoints that
       # allow indexer nodes to process announcements. Default: False

market/mk12/mk12.go

@@ -96,6 +96,22 @@ func NewMK12Handler(miners []address.Address, db *harmonydb.DB, sc *ffi.SealCall
 // from the network
 func (m *MK12) ExecuteDeal(ctx context.Context, dp *DealParams, clientPeer peer.ID) (*ProviderDealRejectionInfo, error) {
 
+	if m.cfg.Market.StorageMarketConfig.MK12.DenyOfflineDeals {
+		if dp.IsOffline {
+			return &ProviderDealRejectionInfo{
+				Reason: "offline deals are not allowed on this provider",
+			}, nil
+		}
+	}
+
+	if m.cfg.Market.StorageMarketConfig.MK12.DenyOnlineDeals {
+		if !dp.IsOffline {
+			return &ProviderDealRejectionInfo{
+				Reason: "online deals are not allowed on this provider",
+			}, nil
+		}
+	}
+
 	ds := &ProviderDealState{
 		DealUuid:           dp.DealUUID,
 		ClientDealProposal: dp.ClientDealProposal,

tasks/storage-market/storage_market.go

@@ -453,7 +453,8 @@ func (d *CurioStorageDealMarket) findURLForOfflineDeals(ctx context.Context, dea
 						UPDATE market_mk12_deal_pipeline
 						SET url = selected_data.url,
 							headers = selected_data.headers,
-							raw_size = selected_data.raw_size
+							raw_size = selected_data.raw_size,
+							started = TRUE
 						FROM selected_data
 						WHERE market_mk12_deal_pipeline.uuid = $1
 						RETURNING CASE 

web/api/webrpc/market.go

@@ -952,6 +952,15 @@ func (a *WebRPC) MK12DealPipelineRemove(ctx context.Context, uuid string) error
 			}
 		}
 
+		//Mark failure for deal
+		n, err := tx.Exec(`UPDATE market_mk12_deals SET error = $1 WHERE uuid = $2`, "Deal pipeline removed by SP", uuid)
+		if err != nil {
+			return false, xerrors.Errorf("failed to mark deal %s as failed", uuid)
+		}
+		if n != 1 {
+			return false, xerrors.Errorf("expected 1 row to be updated, got %d", n)
+		}
+
 		// Remove market_mk12_deal_pipeline entry
 		_, err = tx.Exec(`DELETE FROM market_mk12_deal_pipeline WHERE uuid = $1`, uuid)
 		if err != nil {