diff --git a/private/pkg/storage/bucket.go b/private/pkg/storage/bucket.go index efe5fbbf97..bb0ea42752 100644 --- a/private/pkg/storage/bucket.go +++ b/private/pkg/storage/bucket.go @@ -17,6 +17,7 @@ package storage import ( "context" "io" + "time" ) // ReadBucket is a simple read-only bucket. @@ -132,6 +133,17 @@ func NopReadWriteBucketCloser(readWriteBucket ReadWriteBucket) ReadWriteBucketCl // ObjectInfo contains object info. // +// Size and last-modified time are not part of this interface because +// ObjectInfo is embedded by types outside this package (e.g. +// bufmodule.FileInfo) that cannot be extended. Instead, implementations +// may optionally implement Sizer and ModTimeProvider. Use +// ObjectInfoSize and ObjectInfoModTime to access these through the +// wrapper chain. +// +// Wrapper implementations (map, strip) should implement +// ObjectInfoUnwrapper so that callers can traverse the chain to reach +// provider-specific ObjectInfo types via ObjectInfoAs. +// // An ObjectInfo will always be the same for a given path within a given Bucket, // that is an ObjectInfo is cacheable for a given Bucket. type ObjectInfo interface { @@ -176,6 +188,81 @@ type ObjectInfo interface { LocalPath() string } +// Sizer is an optional interface implemented by ObjectInfo values that can +// report the size of the object in bytes. +// +// Implementations should return -1 if the size is unknown. +// Use ObjectInfoSize to retrieve the size from an ObjectInfo, which returns +// -1 when the ObjectInfo does not implement Sizer. +type Sizer interface { + Size() int64 +} + +// ModTimeProvider is an optional interface implemented by ObjectInfo values +// that can report when the object was last modified. +// +// Implementations should return a zero time.Time if the modification time is +// unknown. Use ObjectInfoModTime to retrieve the time from an ObjectInfo, +// which returns a zero time.Time when the ObjectInfo does not implement +// ModTimeProvider. +type ModTimeProvider interface { + ModTime() time.Time +} + +// ObjectInfoUnwrapper is an optional interface implemented by ObjectInfo +// wrapper values to expose the ObjectInfo they are wrapping. +// +// This enables callers to traverse the wrapper chain and access +// provider-specific ObjectInfo implementations. Use ObjectInfoAs to find +// a specific type in the chain. +type ObjectInfoUnwrapper interface { + UnwrapObjectInfo() ObjectInfo +} + +// ObjectInfoAs finds the first value in the ObjectInfo unwrap chain that is +// assignable to T, and returns it. Returns the zero value and false if no +// match exists in the chain. +// +// The chain consists of objectInfo itself followed by the sequence of +// ObjectInfos obtained by calling UnwrapObjectInfo repeatedly. +// +// T is unconstrained so that callers can search for both concrete ObjectInfo +// implementations (e.g. a provider-specific type) and optional interfaces +// such as Sizer or ModTimeProvider. +func ObjectInfoAs[T any](objectInfo ObjectInfo) (T, bool) { + for objectInfo != nil { + if typed, ok := objectInfo.(T); ok { + return typed, true + } + unwrapper, ok := objectInfo.(ObjectInfoUnwrapper) + if !ok { + break + } + objectInfo = unwrapper.UnwrapObjectInfo() + } + var zero T + return zero, false +} + +// ObjectInfoSize returns the size of the object in bytes by traversing the +// ObjectInfo unwrap chain for a Sizer. Returns -1 if no Sizer is found. +func ObjectInfoSize(objectInfo ObjectInfo) int64 { + if sizer, ok := ObjectInfoAs[Sizer](objectInfo); ok { + return sizer.Size() + } + return -1 +} + +// ObjectInfoModTime returns the last-modified time by traversing the +// ObjectInfo unwrap chain for a ModTimeProvider. Returns a zero time.Time +// if no ModTimeProvider is found. +func ObjectInfoModTime(objectInfo ObjectInfo) time.Time { + if provider, ok := ObjectInfoAs[ModTimeProvider](objectInfo); ok { + return provider.ModTime() + } + return time.Time{} +} + // ReadObject is an object read from a bucket. type ReadObject interface { ObjectInfo diff --git a/private/pkg/storage/map.go b/private/pkg/storage/map.go index db60c37971..cb4ab4a258 100644 --- a/private/pkg/storage/map.go +++ b/private/pkg/storage/map.go @@ -22,7 +22,6 @@ import ( "io/fs" "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/bufbuild/buf/private/pkg/storage/storageutil" ) // MapReadBucket maps the ReadBucket. @@ -301,10 +300,11 @@ func replaceObjectInfoPath(objectInfo ObjectInfo, path string) ObjectInfo { if objectInfo.Path() == path { return objectInfo } - return storageutil.NewObjectInfo( + return newWrappedObjectInfo( path, objectInfo.ExternalPath(), objectInfo.LocalPath(), + objectInfo, ) } @@ -312,7 +312,7 @@ func replaceReadObjectCloserPath(readObjectCloser ReadObjectCloser, path string) if readObjectCloser.Path() == path { return readObjectCloser } - return compositeReadObjectCloser{replaceObjectInfoPath(readObjectCloser, path), readObjectCloser} + return newCompositeReadObjectCloser(replaceObjectInfoPath(readObjectCloser, path), readObjectCloser) } func replaceWriteObjectCloserExternalAndLocalPathsNotSupported(writeObjectCloser WriteObjectCloser) WriteObjectCloser { diff --git a/private/pkg/storage/storagemem/internal/internal.go b/private/pkg/storage/storagemem/internal/internal.go index b919a09ee8..c9c3051dc9 100644 --- a/private/pkg/storage/storagemem/internal/internal.go +++ b/private/pkg/storage/storagemem/internal/internal.go @@ -56,3 +56,8 @@ func NewImmutableObject( func (i *ImmutableObject) Data() []byte { return i.data } + +// Size returns the size of the data in bytes. +func (i *ImmutableObject) Size() int64 { + return int64(len(i.data)) +} diff --git a/private/pkg/storage/storagemem/read_object_closer.go b/private/pkg/storage/storagemem/read_object_closer.go index 217a8752cc..c0926709bf 100644 --- a/private/pkg/storage/storagemem/read_object_closer.go +++ b/private/pkg/storage/storagemem/read_object_closer.go @@ -16,6 +16,7 @@ package storagemem import ( "bytes" + "io" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storagemem/internal" @@ -26,6 +27,7 @@ type readObjectCloser struct { storageutil.ObjectInfo reader *bytes.Reader + size int64 closed bool } @@ -33,9 +35,14 @@ func newReadObjectCloser(immutableObject *internal.ImmutableObject) *readObjectC return &readObjectCloser{ ObjectInfo: immutableObject.ObjectInfo, reader: bytes.NewReader(immutableObject.Data()), + size: immutableObject.Size(), } } +func (r *readObjectCloser) Size() int64 { + return r.size +} + func (r *readObjectCloser) Read(p []byte) (int, error) { if r.closed { return 0, storage.ErrClosed @@ -43,6 +50,13 @@ func (r *readObjectCloser) Read(p []byte) (int, error) { return r.reader.Read(p) } +func (r *readObjectCloser) WriteTo(w io.Writer) (int64, error) { + if r.closed { + return 0, storage.ErrClosed + } + return r.reader.WriteTo(w) +} + func (r *readObjectCloser) Close() error { if r.closed { return storage.ErrClosed diff --git a/private/pkg/storage/storageos/bucket.go b/private/pkg/storage/storageos/bucket.go index 69a18c6552..71cf82f561 100644 --- a/private/pkg/storage/storageos/bucket.go +++ b/private/pkg/storage/storageos/bucket.go @@ -17,10 +17,12 @@ package storageos import ( "context" "errors" + "io" "io/fs" "os" "path/filepath" "sync/atomic" + "time" "buf.build/go/standard/xpath/xfilepath" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -57,12 +59,12 @@ func newBucket(rootPath string, symlinks bool) (*bucket, error) { }, nil } -func (b *bucket) Get(ctx context.Context, path string) (storage.ReadObjectCloser, error) { +func (b *bucket) Get(ctx context.Context, path string) (_ storage.ReadObjectCloser, retErr error) { externalPath, err := b.getExternalPath(path) if err != nil { return nil, err } - if err := b.validateExternalPath(path, externalPath); err != nil { + if _, err := b.validateExternalPath(path, externalPath); err != nil { return nil, err } resolvedPath := externalPath @@ -76,12 +78,17 @@ func (b *bucket) Get(ctx context.Context, path string) (storage.ReadObjectCloser if err != nil { return nil, err } + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, file.Close()) + } + }() // we could use fileInfo.Name() however we might as well use the externalPath return newReadObjectCloser( path, externalPath, file, - ), nil + ) } func (b *bucket) Stat(ctx context.Context, path string) (storage.ObjectInfo, error) { @@ -89,16 +96,12 @@ func (b *bucket) Stat(ctx context.Context, path string) (storage.ObjectInfo, err if err != nil { return nil, err } - if err := b.validateExternalPath(path, externalPath); err != nil { + fileInfo, err := b.validateExternalPath(path, externalPath) + if err != nil { return nil, err } // we could use fileInfo.Name() however we might as well use the externalPath - return storageutil.NewObjectInfo( - path, - externalPath, - // In storageos, the external path is also the local path. - externalPath, - ), nil + return newOSObjectInfo(path, externalPath, fileInfo), nil } func (b *bucket) Walk( @@ -143,14 +146,7 @@ func (b *bucket) Walk( if err != nil { return err } - if err := f( - storageutil.NewObjectInfo( - path, - externalPath, - // In storageos, the external path is also the local path. - externalPath, - ), - ); err != nil { + if err := f(newOSObjectInfo(path, externalPath, fileInfo)); err != nil { return err } } @@ -255,7 +251,7 @@ func (b *bucket) getExternalPath(path string) (string, error) { return normalpath.Unnormalize(realClean), nil } -func (b *bucket) validateExternalPath(path string, externalPath string) error { +func (b *bucket) validateExternalPath(path string, externalPath string) (os.FileInfo, error) { // this is potentially introducing two calls to a file // instead of one, ie we do both Stat and Open as opposed to just Open // we do this to make sure we are only reading regular files @@ -268,7 +264,7 @@ func (b *bucket) validateExternalPath(path string, externalPath string) error { } if err != nil { if os.IsNotExist(err) { - return &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist} + return nil, &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist} } // The path might have a regular file in one of its // elements (e.g. 'foo/bar/baz.proto' where 'bar' is a @@ -294,17 +290,17 @@ func (b *bucket) validateExternalPath(path string, externalPath string) error { // This error primarily serves as a sentinel error, // but we preserve the original path argument so that // the error still makes sense to the user. - return &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist} + return nil, &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist} } } - return err + return nil, err } if !fileInfo.Mode().IsRegular() { // making this a user error as any access means this was generally requested // by the user, since we only call the function for Walk on regular files - return &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist} + return nil, &fs.PathError{Op: "stat", Path: path, Err: fs.ErrNotExist} } - return nil + return fileInfo, nil } func (b *bucket) getExternalPrefix(prefix string) (string, error) { @@ -319,11 +315,42 @@ func (b *bucket) getExternalPrefix(prefix string) (string, error) { return normalpath.Unnormalize(realClean), nil } -type readObjectCloser struct { - // we use ObjectInfo for Path, ExternalPath, etc to make sure this is static - // we put ObjectInfos in maps in other places so we do not want this to change - // this could be a problem if the underlying file is concurrently moved or resized however +// osObjectInfo is an ObjectInfo that includes file size and modification time. +// +// It implements storage.Sizer and storage.ModTimeProvider. +type osObjectInfo struct { storageutil.ObjectInfo + size int64 + modTime time.Time +} + +func newOSObjectInfo(path string, externalPath string, fileInfo os.FileInfo) *osObjectInfo { + return &osObjectInfo{ + ObjectInfo: storageutil.NewObjectInfo( + path, + externalPath, + // In storageos, the external path is also the local path. + externalPath, + ), + size: fileInfo.Size(), + modTime: fileInfo.ModTime(), + } +} + +func (o *osObjectInfo) Size() int64 { + return o.size +} + +func (o *osObjectInfo) ModTime() time.Time { + return o.modTime +} + +type readObjectCloser struct { + // We embed osObjectInfo for Path, ExternalPath, Size, ModTime, etc. + // to make sure this is static. We put ObjectInfos in maps in other places + // so we do not want this to change. This could be a problem if the + // underlying file is concurrently moved or resized however. + *osObjectInfo file *os.File } @@ -332,16 +359,15 @@ func newReadObjectCloser( path string, externalPath string, file *os.File, -) *readObjectCloser { - return &readObjectCloser{ - ObjectInfo: storageutil.NewObjectInfo( - path, - externalPath, - // In storageos, the external path is the local path - externalPath, - ), - file: file, +) (*readObjectCloser, error) { + fileInfo, err := file.Stat() + if err != nil { + return nil, err } + return &readObjectCloser{ + osObjectInfo: newOSObjectInfo(path, externalPath, fileInfo), + file: file, + }, nil } func (r *readObjectCloser) Read(p []byte) (int, error) { @@ -349,6 +375,11 @@ func (r *readObjectCloser) Read(p []byte) (int, error) { return n, toStorageError(err) } +func (r *readObjectCloser) WriteTo(w io.Writer) (int64, error) { + n, err := r.file.WriteTo(w) + return n, toStorageError(err) +} + func (r *readObjectCloser) Close() error { return toStorageError(r.file.Close()) } diff --git a/private/pkg/storage/strip.go b/private/pkg/storage/strip.go index 4d2bf73ccd..c962ab291a 100644 --- a/private/pkg/storage/strip.go +++ b/private/pkg/storage/strip.go @@ -16,8 +16,6 @@ package storage import ( "context" - - "github.com/bufbuild/buf/private/pkg/storage/storageutil" ) // StripReadBucketExternalPaths strips the differentiated ExternalPaths from objects @@ -70,12 +68,12 @@ func stripObjectInfoExternalPath(objectInfo ObjectInfo) ObjectInfo { if path == objectInfo.ExternalPath() { return objectInfo } - return storageutil.NewObjectInfo(path, path, objectInfo.LocalPath()) + return newWrappedObjectInfo(path, path, objectInfo.LocalPath(), objectInfo) } func stripReadObjectCloserExternalPath(readObjectCloser ReadObjectCloser) ReadObjectCloser { if readObjectCloser.Path() == readObjectCloser.ExternalPath() { return readObjectCloser } - return compositeReadObjectCloser{stripObjectInfoExternalPath(readObjectCloser), readObjectCloser} + return newCompositeReadObjectCloser(stripObjectInfoExternalPath(readObjectCloser), readObjectCloser) } diff --git a/private/pkg/storage/util.go b/private/pkg/storage/util.go index 8d7916c436..88219a9712 100644 --- a/private/pkg/storage/util.go +++ b/private/pkg/storage/util.go @@ -19,6 +19,8 @@ import ( "errors" "io" "sort" + + "github.com/bufbuild/buf/private/pkg/storage/storageutil" ) // errIsNotEmpty is used to break out of the Walk function early in IsEmpty. @@ -205,11 +207,55 @@ func sortObjectInfos(objectInfos []ObjectInfo) { ) } +// wrappedObjectInfo wraps an ObjectInfo with replacement path fields while +// preserving the original for unwrapping and metadata delegation. +type wrappedObjectInfo struct { + storageutil.ObjectInfo + original ObjectInfo +} + +func newWrappedObjectInfo(path string, externalPath string, localPath string, original ObjectInfo) *wrappedObjectInfo { + return &wrappedObjectInfo{ + ObjectInfo: storageutil.NewObjectInfo(path, externalPath, localPath), + original: original, + } +} + +func (w *wrappedObjectInfo) UnwrapObjectInfo() ObjectInfo { + return w.original +} + type compositeReadObjectCloser struct { ObjectInfo io.ReadCloser } +// compositeReadObjectCloserWithWriterTo preserves io.WriterTo from the +// underlying ReadCloser (e.g. *os.File) through the wrapper chain. +type compositeReadObjectCloserWithWriterTo struct { + ObjectInfo + io.ReadCloser + writerTo io.WriterTo +} + +func (c *compositeReadObjectCloserWithWriterTo) WriteTo(w io.Writer) (int64, error) { + return c.writerTo.WriteTo(w) +} + +// newCompositeReadObjectCloser creates a ReadObjectCloser that combines an +// ObjectInfo with an io.ReadCloser. If the ReadCloser implements io.WriterTo, +// the returned value preserves that interface. +func newCompositeReadObjectCloser(objectInfo ObjectInfo, readCloser io.ReadCloser) ReadObjectCloser { + if writerTo, ok := readCloser.(io.WriterTo); ok { + return &compositeReadObjectCloserWithWriterTo{ + ObjectInfo: objectInfo, + ReadCloser: readCloser, + writerTo: writerTo, + } + } + return compositeReadObjectCloser{ObjectInfo: objectInfo, ReadCloser: readCloser} +} + type compositeReadWriteBucketCloser struct { ReadBucket WriteBucket