Skip to content

Commit 217ea04

Browse files
asutulaasutula
authored
Simplification with StorageConfig (textileio#526)
* rename DefaultConfig to StorageConfig Signed-off-by: Aaron Sutula <hi@asutula.com> * remove CidConfig Signed-off-by: Aaron Sutula <hi@asutula.com> * update cli docs Signed-off-by: Aaron Sutula <hi@asutula.com> * addToHot > stage Signed-off-by: Aaron Sutula <hi@asutula.com> * cleaning up wording in various places Signed-off-by: Aaron Sutula <hi@asutula.com> * mod tidy Signed-off-by: Aaron Sutula <hi@asutula.com> * fix readme cli example Signed-off-by: Aaron Sutula <hi@asutula.com> * better DefaultStorageConfig naming for public apis Signed-off-by: Aaron Sutula <hi@asutula.com> * pr feedback Signed-off-by: Aaron Sutula <hi@asutula.com> * pr feedback, some todos for key renaming Signed-off-by: Aaron Sutula <hi@asutula.com> * update ds keys Signed-off-by: Aaron Sutula <hi@asutula.com> * readme update Signed-off-by: Aaron Sutula <hi@asutula.com> * avoid v4 of google protobuf js Signed-off-by: Aaron Sutula <hi@asutula.com>
1 parent 3ee0be9 commit 217ea04

39 files changed

Lines changed: 1534 additions & 2015 deletions

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ buildtools/protoc-gen-go
2424
myfile
2525
myfile2
2626

27-
# CidConfig name that can be used for testing.
27+
# StorageConfig name that can be used for testing.
2828
pconfig.pow
2929

3030
tags

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,9 +191,9 @@ make build
191191
❯ head -c 700 </dev/urandom > myfile
192192
❯ pow ffs create
193193
> Instance created with id 0ac0fb4d-581c-4276-bd90-a9aa30dd4cb4 and token 883f57b1-4e66-47f8-b291-7cf8b10f6370
194-
❯ pow ffs addToHot -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 myfile
195-
> Success! Added file to FFS hot storage with cid: QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm
196-
❯ pow ffs push -w -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm
194+
❯ pow ffs stage -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 myfile
195+
> Success! Cached file in FFS hot storage with cid: QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm
196+
❯ pow ffs config push -w -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm
197197
> Success! Pushed cid config for QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm to FFS with job id: 966dcb44-9ef4-4d2a-9c90-a8103c77c354
198198
JOB ID STATUS
199199
966dcb44-9ef4-4d2a-9c90-a8103c77c354 Success

‎api/client/ffs.go‎

Lines changed: 79 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -44,25 +44,25 @@ func WithAddressType(addressType string) NewAddressOption {
4444
}
4545
}
4646

47-
// PushConfigOption mutates a push request.
48-
type PushConfigOption func(r *rpc.PushConfigRequest)
47+
// PushStorageConfigOption mutates a push request.
48+
type PushStorageConfigOption func(r *rpc.PushStorageConfigRequest)
4949

50-
// WithCidConfig overrides the Api default Cid configuration.
51-
func WithCidConfig(c ffs.CidConfig) PushConfigOption {
52-
return func(r *rpc.PushConfigRequest) {
50+
// WithStorageConfig overrides the Api default Cid configuration.
51+
func WithStorageConfig(c ffs.StorageConfig) PushStorageConfigOption {
52+
return func(r *rpc.PushStorageConfigRequest) {
5353
r.HasConfig = true
54-
r.Config = &rpc.CidConfig{
55-
Cid: util.CidToString(c.Cid),
56-
Hot: toRPCHotConfig(c.Hot),
57-
Cold: toRPCColdConfig(c.Cold),
54+
r.Config = &rpc.StorageConfig{
55+
Repairable: c.Repairable,
56+
Hot: toRPCHotConfig(c.Hot),
57+
Cold: toRPCColdConfig(c.Cold),
5858
}
5959
}
6060
}
6161

6262
// WithOverride allows a new push configuration to override an existing one.
6363
// It's used as an extra security measure to avoid unwanted configuration changes.
64-
func WithOverride(override bool) PushConfigOption {
65-
return func(r *rpc.PushConfigRequest) {
64+
func WithOverride(override bool) PushStorageConfigOption {
65+
return func(r *rpc.PushStorageConfigRequest) {
6666
r.HasOverrideConfig = true
6767
r.OverrideConfig = override
6868
}
@@ -184,38 +184,13 @@ func (f *FFS) Addrs(ctx context.Context) ([]api.AddrInfo, error) {
184184
return addrs, nil
185185
}
186186

187-
// DefaultConfig returns the default storage config.
188-
func (f *FFS) DefaultConfig(ctx context.Context) (ffs.DefaultConfig, error) {
189-
resp, err := f.client.DefaultConfig(ctx, &rpc.DefaultConfigRequest{})
187+
// DefaultStorageConfig returns the default storage config.
188+
func (f *FFS) DefaultStorageConfig(ctx context.Context) (ffs.StorageConfig, error) {
189+
resp, err := f.client.DefaultStorageConfig(ctx, &rpc.DefaultStorageConfigRequest{})
190190
if err != nil {
191-
return ffs.DefaultConfig{}, err
192-
}
193-
return ffs.DefaultConfig{
194-
Hot: ffs.HotConfig{
195-
Enabled: resp.DefaultConfig.Hot.Enabled,
196-
AllowUnfreeze: resp.DefaultConfig.Hot.AllowUnfreeze,
197-
Ipfs: ffs.IpfsConfig{
198-
AddTimeout: int(resp.DefaultConfig.Hot.Ipfs.AddTimeout),
199-
},
200-
},
201-
Cold: ffs.ColdConfig{
202-
Enabled: resp.DefaultConfig.Cold.Enabled,
203-
Filecoin: ffs.FilConfig{
204-
RepFactor: int(resp.DefaultConfig.Cold.Filecoin.RepFactor),
205-
DealMinDuration: resp.DefaultConfig.Cold.Filecoin.DealMinDuration,
206-
ExcludedMiners: resp.DefaultConfig.Cold.Filecoin.ExcludedMiners,
207-
CountryCodes: resp.DefaultConfig.Cold.Filecoin.CountryCodes,
208-
TrustedMiners: resp.DefaultConfig.Cold.Filecoin.TrustedMiners,
209-
Renew: ffs.FilRenew{
210-
Enabled: resp.DefaultConfig.Cold.Filecoin.Renew.Enabled,
211-
Threshold: int(resp.DefaultConfig.Cold.Filecoin.Renew.Threshold),
212-
},
213-
Addr: resp.DefaultConfig.Cold.Filecoin.Addr,
214-
MaxPrice: resp.DefaultConfig.Cold.Filecoin.MaxPrice,
215-
},
216-
},
217-
Repairable: resp.DefaultConfig.Repairable,
218-
}, nil
191+
return ffs.StorageConfig{}, err
192+
}
193+
return fromRPCStorageConfig(resp.DefaultStorageConfig), nil
219194
}
220195

221196
// NewAddr created a new wallet address managed by the FFS instance.
@@ -228,60 +203,21 @@ func (f *FFS) NewAddr(ctx context.Context, name string, options ...NewAddressOpt
228203
return resp.Addr, err
229204
}
230205

231-
// GetDefaultCidConfig returns a CidConfig built from the default storage config and prepped for the provided cid.
232-
func (f *FFS) GetDefaultCidConfig(ctx context.Context, c cid.Cid) (ffs.CidConfig, error) {
233-
res, err := f.client.GetDefaultCidConfig(ctx, &rpc.GetDefaultCidConfigRequest{Cid: util.CidToString(c)})
234-
if err != nil {
235-
return ffs.CidConfig{}, err
236-
}
237-
resCid, err := util.CidFromString(res.Config.Cid)
238-
if err != nil {
239-
return ffs.CidConfig{}, err
240-
}
241-
return ffs.CidConfig{
242-
Cid: resCid,
243-
Repairable: res.Config.Repairable,
244-
Hot: ffs.HotConfig{
245-
AllowUnfreeze: res.Config.Hot.AllowUnfreeze,
246-
Enabled: res.Config.Hot.Enabled,
247-
Ipfs: ffs.IpfsConfig{
248-
AddTimeout: int(res.Config.Hot.Ipfs.AddTimeout),
249-
},
250-
},
251-
Cold: ffs.ColdConfig{
252-
Enabled: res.Config.Cold.Enabled,
253-
Filecoin: ffs.FilConfig{
254-
RepFactor: int(res.Config.Cold.Filecoin.RepFactor),
255-
Addr: res.Config.Cold.Filecoin.Addr,
256-
CountryCodes: res.Config.Cold.Filecoin.CountryCodes,
257-
DealMinDuration: res.Config.Cold.Filecoin.DealMinDuration,
258-
ExcludedMiners: res.Config.Cold.Filecoin.ExcludedMiners,
259-
Renew: ffs.FilRenew{
260-
Enabled: res.Config.Cold.Filecoin.Renew.Enabled,
261-
Threshold: int(res.Config.Cold.Filecoin.Renew.Threshold),
262-
},
263-
TrustedMiners: res.Config.Cold.Filecoin.TrustedMiners,
264-
MaxPrice: res.Config.Cold.Filecoin.MaxPrice,
265-
},
266-
},
267-
}, nil
268-
}
269-
270-
// GetCidConfig gets the current config for a cid.
271-
func (f *FFS) GetCidConfig(ctx context.Context, c cid.Cid) (*rpc.GetCidConfigResponse, error) {
272-
return f.client.GetCidConfig(ctx, &rpc.GetCidConfigRequest{Cid: util.CidToString(c)})
206+
// GetStorageConfig gets the current config for a cid.
207+
func (f *FFS) GetStorageConfig(ctx context.Context, c cid.Cid) (*rpc.GetStorageConfigResponse, error) {
208+
return f.client.GetStorageConfig(ctx, &rpc.GetStorageConfigRequest{Cid: c.String()})
273209
}
274210

275-
// SetDefaultConfig sets the default storage config.
276-
func (f *FFS) SetDefaultConfig(ctx context.Context, config ffs.DefaultConfig) error {
277-
req := &rpc.SetDefaultConfigRequest{
278-
Config: &rpc.DefaultConfig{
211+
// SetDefaultStorageConfig sets the default storage config.
212+
func (f *FFS) SetDefaultStorageConfig(ctx context.Context, config ffs.StorageConfig) error {
213+
req := &rpc.SetDefaultStorageConfigRequest{
214+
Config: &rpc.StorageConfig{
279215
Hot: toRPCHotConfig(config.Hot),
280216
Cold: toRPCColdConfig(config.Cold),
281217
Repairable: config.Repairable,
282218
},
283219
}
284-
_, err := f.client.SetDefaultConfig(ctx, req)
220+
_, err := f.client.SetDefaultStorageConfig(ctx, req)
285221
return err
286222
}
287223

@@ -321,35 +257,10 @@ func (f *FFS) Info(ctx context.Context) (api.InstanceInfo, error) {
321257
}
322258

323259
return api.InstanceInfo{
324-
ID: ffs.APIID(res.Info.Id),
325-
DefaultConfig: ffs.DefaultConfig{
326-
Hot: ffs.HotConfig{
327-
Enabled: res.Info.DefaultConfig.Hot.Enabled,
328-
AllowUnfreeze: res.Info.DefaultConfig.Hot.AllowUnfreeze,
329-
Ipfs: ffs.IpfsConfig{
330-
AddTimeout: int(res.Info.DefaultConfig.Hot.Ipfs.AddTimeout),
331-
},
332-
},
333-
Cold: ffs.ColdConfig{
334-
Enabled: res.Info.DefaultConfig.Cold.Enabled,
335-
Filecoin: ffs.FilConfig{
336-
RepFactor: int(res.Info.DefaultConfig.Cold.Filecoin.RepFactor),
337-
DealMinDuration: res.Info.DefaultConfig.Cold.Filecoin.DealMinDuration,
338-
ExcludedMiners: res.Info.DefaultConfig.Cold.Filecoin.ExcludedMiners,
339-
TrustedMiners: res.Info.DefaultConfig.Cold.Filecoin.TrustedMiners,
340-
CountryCodes: res.Info.DefaultConfig.Cold.Filecoin.CountryCodes,
341-
Renew: ffs.FilRenew{
342-
Enabled: res.Info.DefaultConfig.Cold.Filecoin.Renew.Enabled,
343-
Threshold: int(res.Info.DefaultConfig.Cold.Filecoin.Renew.Threshold),
344-
},
345-
Addr: res.Info.DefaultConfig.Cold.Filecoin.Addr,
346-
MaxPrice: res.Info.DefaultConfig.Cold.Filecoin.MaxPrice,
347-
},
348-
},
349-
Repairable: res.Info.DefaultConfig.Repairable,
350-
},
351-
Balances: balances,
352-
Pins: pins,
260+
ID: ffs.APIID(res.Info.Id),
261+
DefaultStorageConfig: fromRPCStorageConfig(res.Info.DefaultStorageConfig),
262+
Balances: balances,
263+
Pins: pins,
353264
}, nil
354265
}
355266

@@ -428,7 +339,7 @@ func (f *FFS) WatchJobs(ctx context.Context, ch chan<- JobEvent, jids ...ffs.Job
428339
return nil
429340
}
430341

431-
// Replace pushes a CidConfig of c2 equal to c1, and removes c1. This operation
342+
// Replace pushes a StorageConfig for c2 equal to that of c1, and removes c1. This operation
432343
// is more efficient than manually removing and adding in two separate operations.
433344
func (f *FFS) Replace(ctx context.Context, c1 cid.Cid, c2 cid.Cid) (ffs.JobID, error) {
434345
resp, err := f.client.Replace(ctx, &rpc.ReplaceRequest{Cid1: util.CidToString(c1), Cid2: util.CidToString(c2)})
@@ -438,14 +349,14 @@ func (f *FFS) Replace(ctx context.Context, c1 cid.Cid, c2 cid.Cid) (ffs.JobID, e
438349
return ffs.JobID(resp.JobId), nil
439350
}
440351

441-
// PushConfig push a new configuration for the Cid in the Hot and Cold layers.
442-
func (f *FFS) PushConfig(ctx context.Context, c cid.Cid, opts ...PushConfigOption) (ffs.JobID, error) {
443-
req := &rpc.PushConfigRequest{Cid: util.CidToString(c)}
352+
// PushStorageConfig push a new configuration for the Cid in the Hot and Cold layers.
353+
func (f *FFS) PushStorageConfig(ctx context.Context, c cid.Cid, opts ...PushStorageConfigOption) (ffs.JobID, error) {
354+
req := &rpc.PushStorageConfigRequest{Cid: util.CidToString(c)}
444355
for _, opt := range opts {
445356
opt(req)
446357
}
447358

448-
resp, err := f.client.PushConfig(ctx, req)
359+
resp, err := f.client.PushStorageConfig(ctx, req)
449360
if err != nil {
450361
return ffs.EmptyJobID, err
451362
}
@@ -552,9 +463,9 @@ func (f *FFS) Close(ctx context.Context) error {
552463
return err
553464
}
554465

555-
// AddToHot allows you to add data to the Hot layer in preparation for pushing a cid config.
556-
func (f *FFS) AddToHot(ctx context.Context, data io.Reader) (*cid.Cid, error) {
557-
stream, err := f.client.AddToHot(ctx)
466+
// Stage allows you to temporarily cache data in the Hot layer in preparation for pushing a cid storage config.
467+
func (f *FFS) Stage(ctx context.Context, data io.Reader) (*cid.Cid, error) {
468+
stream, err := f.client.Stage(ctx)
558469
if err != nil {
559470
return nil, err
560471
}
@@ -565,7 +476,7 @@ func (f *FFS) AddToHot(ctx context.Context, data io.Reader) (*cid.Cid, error) {
565476
if err != nil && err != io.EOF {
566477
return nil, err
567478
}
568-
sendErr := stream.Send(&rpc.AddToHotRequest{Chunk: buffer[:bytesRead]})
479+
sendErr := stream.Send(&rpc.StageRequest{Chunk: buffer[:bytesRead]})
569480
if sendErr != nil {
570481
if sendErr == io.EOF {
571482
var noOp interface{}
@@ -689,6 +600,47 @@ func toRPCColdConfig(config ffs.ColdConfig) *rpc.ColdConfig {
689600
}
690601
}
691602

603+
func fromRPCStorageConfig(config *rpc.StorageConfig) ffs.StorageConfig {
604+
if config == nil {
605+
return ffs.StorageConfig{}
606+
}
607+
ret := ffs.StorageConfig{Repairable: config.Repairable}
608+
if config.Hot != nil {
609+
ret.Hot = ffs.HotConfig{
610+
Enabled: config.Hot.Enabled,
611+
AllowUnfreeze: config.Hot.AllowUnfreeze,
612+
}
613+
if config.Hot.Ipfs != nil {
614+
ret.Hot.Ipfs = ffs.IpfsConfig{
615+
AddTimeout: int(config.Hot.Ipfs.AddTimeout),
616+
}
617+
}
618+
}
619+
if config.Cold != nil {
620+
ret.Cold = ffs.ColdConfig{
621+
Enabled: config.Cold.Enabled,
622+
}
623+
if config.Cold.Filecoin != nil {
624+
ret.Cold.Filecoin = ffs.FilConfig{
625+
RepFactor: int(config.Cold.Filecoin.RepFactor),
626+
DealMinDuration: config.Cold.Filecoin.DealMinDuration,
627+
ExcludedMiners: config.Cold.Filecoin.ExcludedMiners,
628+
CountryCodes: config.Cold.Filecoin.CountryCodes,
629+
TrustedMiners: config.Cold.Filecoin.TrustedMiners,
630+
Addr: config.Cold.Filecoin.Addr,
631+
MaxPrice: config.Cold.Filecoin.MaxPrice,
632+
}
633+
if config.Cold.Filecoin.Renew != nil {
634+
ret.Cold.Filecoin.Renew = ffs.FilRenew{
635+
Enabled: config.Cold.Filecoin.Renew.Enabled,
636+
Threshold: int(config.Cold.Filecoin.Renew.Threshold),
637+
}
638+
}
639+
}
640+
}
641+
return ret
642+
}
643+
692644
func fromRPCDealErrors(des []*rpc.DealError) ([]ffs.DealError, error) {
693645
res := make([]ffs.DealError, len(des))
694646
for i, de := range des {

‎cli-docs/pow/pow_ffs.md‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ Provides commands to manage ffs
2121
### SEE ALSO
2222

2323
* [pow](pow.md) - A client for storage and retreival of powergate data
24-
* [pow ffs addToHot](pow_ffs_addToHot.md) - Add data to FFS hot storage via file path
2524
* [pow ffs addrs](pow_ffs_addrs.md) - Provides commands to manage wallet addresses
2625
* [pow ffs cancel](pow_ffs_cancel.md) - Cancel an executing job
2726
* [pow ffs close](pow_ffs_close.md) - Close the FFS instance
@@ -32,13 +31,12 @@ Provides commands to manage ffs
3231
* [pow ffs info](pow_ffs_info.md) - Get info from ffs instance
3332
* [pow ffs log](pow_ffs_log.md) - Display logs for specified cid
3433
* [pow ffs paych](pow_ffs_paych.md) - Provides commands to manage payment channels
35-
* [pow ffs pull](pow_ffs_pull.md) - Fetches the config for the provided cid
36-
* [pow ffs push](pow_ffs_push.md) - Add data to FFS via cid
3734
* [pow ffs remove](pow_ffs_remove.md) - Removes a Cid from being tracked as an active storage
38-
* [pow ffs replace](pow_ffs_replace.md) - Pushes a CidConfig of c2 equal to c1, and removes c1
35+
* [pow ffs replace](pow_ffs_replace.md) - Pushes a StorageConfig for c2 equal to that of c1, and removes c1
3936
* [pow ffs retrievals](pow_ffs_retrievals.md) - List retrieval deal records for an FFS instance
4037
* [pow ffs send](pow_ffs_send.md) - Send fil from one managed address to any other address
4138
* [pow ffs show](pow_ffs_show.md) - Show pinned cid data
39+
* [pow ffs stage](pow_ffs_stage.md) - Temporarily cache data in the Hot layer in preparation for pushing a cid storage config
4240
* [pow ffs storage](pow_ffs_storage.md) - List storage deal records for an FFS instance
4341
* [pow ffs watch](pow_ffs_watch.md) - Watch for job status updates
4442

‎cli-docs/pow/pow_ffs_config.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Provides commands to manage storage configuration
2222

2323
* [pow ffs](pow_ffs.md) - Provides commands to manage ffs
2424
* [pow ffs config default](pow_ffs_config_default.md) - Returns the default storage config
25-
* [pow ffs config prep](pow_ffs_config_prep.md) - Returns the default config prepped for the provided cid
26-
* [pow ffs config set](pow_ffs_config_set.md) - Sets the default cid storage config from stdin or a file
25+
* [pow ffs config get](pow_ffs_config_get.md) - Fetches the storage config for the provided cid
26+
* [pow ffs config push](pow_ffs_config_push.md) - Add data to FFS via cid
27+
* [pow ffs config set-default](pow_ffs_config_set-default.md) - Sets the default cid storage config from stdin or a file
2728

cli-docs/pow/pow_ffs_config_prep.md renamed to cli-docs/pow/pow_ffs_config_get.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
1-
## pow ffs config prep
1+
## pow ffs config get
22

3-
Returns the default config prepped for the provided cid
3+
Fetches the storage config for the provided cid
44

55
### Synopsis
66

7-
Returns the default config prepped for the provided cid
7+
Fetches the storage config for the provided cid
88

99
```
10-
pow ffs config prep [cid] [flags]
10+
pow ffs config get [cid] [flags]
1111
```
1212

1313
### Options
1414

1515
```
16-
-h, --help help for prep
16+
-h, --help help for get
1717
-t, --token string FFS auth token
1818
```
1919

0 commit comments

Comments
 (0)