improve reputation score, ColdInfo enabled flag, fixes (textileio#476)

jsign · web-flow · commit ec0da08add5e · 2020-06-26T17:52:15.000-03:00
* improve reputation score, enabled coldconfig flag, fixes

Signed-off-by: Ignacio Hagopian &lt;jsign.uy@gmail.com&gt;

* fix build

Signed-off-by: Ignacio Hagopian &lt;jsign.uy@gmail.com&gt;

* improve parallelizability of tests

Signed-off-by: Ignacio Hagopian &lt;jsign.uy@gmail.com&gt;

* change to switch statement

Signed-off-by: Ignacio Hagopian &lt;jsign.uy@gmail.com&gt;
diff --git a/Makefile b/Makefile
@@ -21,7 +21,7 @@ docs-pow:
 .PHONY: docs-pow
 
 test:
-	go test -short -parallel 4 -race -timeout 30m ./... 
+	go test -short -parallel 8 -race -timeout 30m ./... 
 .PHONY: test
 
 clean-protos:
diff --git a/deals/deals_test.go b/deals/deals_test.go
@@ -144,14 +144,19 @@ func waitForDealComplete(client *apistruct.FullNodeStruct, deals []cid.Cid) erro
 				finished[d] = struct{}{}
 				continue
 			}
-			if di.State != storagemarket.StorageDealUnknown &&
-				di.State != storagemarket.StorageDealProposalAccepted &&
-				di.State != storagemarket.StorageDealStaged &&
-				di.State != storagemarket.StorageDealValidating &&
-				di.State != storagemarket.StorageDealClientFunding &&
-				di.State != storagemarket.StorageDealPublish &&
-				di.State != storagemarket.StorageDealPublishing &&
-				di.State != storagemarket.StorageDealSealing {
+			switch di.State {
+			case
+				storagemarket.StorageDealUnknown,
+				storagemarket.StorageDealWaitingForDataRequest,
+				storagemarket.StorageDealProposalAccepted,
+				storagemarket.StorageDealStaged,
+				storagemarket.StorageDealValidating,
+				storagemarket.StorageDealClientFunding,
+				storagemarket.StorageDealPublish,
+				storagemarket.StorageDealPublishing,
+				storagemarket.StorageDealSealing:
+			default:
+
 				return fmt.Errorf("unexpected deal state: %s", storagemarket.DealStates[di.State])
 			}
 		}
diff --git a/ffs/rpc/rpc.go b/ffs/rpc/rpc.go
@@ -728,8 +728,10 @@ func toRPCCidInfo(info ffs.CidInfo) *CidInfo {
 			},
 		},
 		Cold: &ColdInfo{
+			Enabled: info.Cold.Enabled,
 			Filecoin: &FilInfo{
 				DataCid:   info.Cold.Filecoin.DataCid.String(),
+				Size:      info.Cold.Filecoin.Size,
 				Proposals: make([]*FilStorage, len(info.Cold.Filecoin.Proposals)),
 			},
 		},
diff --git a/ffs/rpc/rpc.pb.go b/ffs/rpc/rpc.pb.go
diff --git a/ffs/rpc/rpc.proto b/ffs/rpc/rpc.proto
@@ -78,12 +78,13 @@ message FilStorage {
 
 message FilInfo {
    string data_cid = 1;
-   int64 size = 2;
+   uint64 size = 2;
    repeated FilStorage proposals = 3;
 }
 
 message ColdInfo {
-   FilInfo filecoin = 1;
+   bool enabled = 1;
+   FilInfo filecoin = 2;
 }
 
 message CidInfo {
diff --git a/ffs/scheduler/scheduler.go b/ffs/scheduler/scheduler.go
@@ -644,11 +644,14 @@ func (s *Scheduler) executeColdStorage(ctx context.Context, curr ffs.CidInfo, cf
 	}
 
 	// At least 1 of the proposal deals reached a successful final status.
-	return ffs.ColdInfo{Filecoin: ffs.FilInfo{
-		DataCid:   curr.Cid,
-		Size:      size,
-		Proposals: append(okDeals, curr.Cold.Filecoin.Proposals...), // Append to any existing other proposals
-	}}, allErrors, nil
+	return ffs.ColdInfo{
+		Enabled: true,
+		Filecoin: ffs.FilInfo{
+			DataCid:   curr.Cid,
+			Size:      size,
+			Proposals: append(okDeals, curr.Cold.Filecoin.Proposals...), // Append to any existing other proposals
+		},
+	}, allErrors, nil
 }
 
 func (s *Scheduler) waitForDeals(ctx context.Context, c cid.Cid, startedProposals []cid.Cid) ([]ffs.FilStorage, []ffs.DealError) {
@@ -676,6 +679,7 @@ func (s *Scheduler) waitForDeals(ctx context.Context, c cid.Cid, startedProposal
 				lock.Lock()
 				failedDeals = append(failedDeals, dealError)
 				lock.Unlock()
+				return
 			}
 			lock.Lock()
 			okDeals = append(okDeals, res)
diff --git a/ffs/types.go b/ffs/types.go
@@ -374,6 +374,7 @@ type IpfsHotInfo struct {
 // ColdInfo contains information about the current storage state
 // of a Cid in the cold layer.
 type ColdInfo struct {
+	Enabled  bool
 	Filecoin FilInfo
 }
 
diff --git a/go.sum b/go.sum
@@ -1544,7 +1544,9 @@ github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg=
 github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
diff --git a/reputation/reputation.go b/reputation/reputation.go
@@ -252,7 +252,7 @@ func calculateScore(addr string, mi miner.IndexSnapshot, si faults.IndexSnapshot
 		askScore = 1
 	}
 
-	score := 50*faultsScore + 20*powerScore + 20*externalScore + 10*askScore
+	score := 50*faultsScore + 20*powerScore + 20*externalScore + 10*askScore + 1000*float64(miner.ActiveDeals)
 	return MinerScore{
 		Addr:  addr,
 		Score: int(score),

Original file line number	Diff line number	Diff line change
`@@ -78,12 +78,13 @@ message FilStorage {`
`78`	`78`
`79`	`79`	`message FilInfo {`
`80`	`80`	`string data_cid = 1;`
`81`		`- int64 size = 2;`
	`81`	`+ uint64 size = 2;`
`82`	`82`	`repeated FilStorage proposals = 3;`
`83`	`83`	`}`
`84`	`84`
`85`	`85`	`message ColdInfo {`
`86`		`- FilInfo filecoin = 1;`
	`86`	`+ bool enabled = 1;`
	`87`	`+ FilInfo filecoin = 2;`
`87`	`88`	`}`
`88`	`89`
`89`	`90`	`message CidInfo {`
Original file line number	Diff line number	Diff line change
`@@ -374,6 +374,7 @@ type IpfsHotInfo struct {`
`374`	`374`	`// ColdInfo contains information about the current storage state`
`375`	`375`	`// of a Cid in the cold layer.`
`376`	`376`	`type ColdInfo struct {`
	`377`	`+ Enabled bool`
`377`	`378`	`Filecoin FilInfo`
`378`	`379`	`}`
`379`	`380`
Original file line number	Diff line number	Diff line change
`@@ -252,7 +252,7 @@ func calculateScore(addr string, mi miner.IndexSnapshot, si faults.IndexSnapshot`
`252`	`252`	`askScore = 1`
`253`	`253`	`}`
`254`	`254`
`255`		`- score := 50faultsScore + 20powerScore + 20externalScore + 10askScore`
	`255`	`+ score := 50faultsScore + 20powerScore + 20externalScore + 10askScore + 1000*float64(miner.ActiveDeals)`
`256`	`256`	`return MinerScore{`
`257`	`257`	`Addr: addr,`
`258`	`258`	`Score: int(score),`