Mirrors/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2025-12-22 10:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(snapshots): localfs support for passing options (#5044)

Browse Source
This commit is contained in:
Jarek Kowalski
2025-11-27 22:25:06 -08:00
committed by GitHub
parent 39fb62970f
commit c8c4615595
3 changed files with 327 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
fs/localfs/local_fs.go
View File

@@ -13,6 +13,16 @@ import (
const numEntriesToRead = 100 // number of directory entries to read in one shot
// Options contains configuration options for localfs operations.
type Options struct {
// IgnoreUnreadableDirEntries, when true, causes unreadable directory entries
// to be silently skipped during directory iteration instead of causing errors.
IgnoreUnreadableDirEntries bool
}
// DefaultOptions stores the default options used by localfs functions.
var DefaultOptions = &Options{}
type filesystemEntry struct {
name string
size int64
@@ -21,7 +31,8 @@ type filesystemEntry struct {
owner fs.OwnerInfo
device fs.DeviceInfo
prefix string
prefix string
options *Options
}
func (e *filesystemEntry) Name() string {
@@ -92,6 +103,7 @@ func (fsd *filesystemDirectory) Size() int64 {
type fileWithMetadata struct {
*os.File
options *Options
}
func (f *fileWithMetadata) Entry() (fs.Entry, error) {
@@ -102,7 +114,7 @@ func (f *fileWithMetadata) Entry() (fs.Entry, error) {
basename, prefix := splitDirPrefix(f.Name())
return newFilesystemFile(newEntry(basename, fi, prefix)), nil
return newFilesystemFile(newEntry(basename, fi, prefix, f.options)), nil
}
func (fsf *filesystemFile) Open(_ context.Context) (fs.Reader, error) {
@@ -111,7 +123,7 @@ func (fsf *filesystemFile) Open(_ context.Context) (fs.Reader, error) {
return nil, errors.Wrap(err, "unable to open local file")
}
return &fileWithMetadata{f}, nil
return &fileWithMetadata{File: f, options: fsf.options}, nil
}
func (fsl *filesystemSymlink) Readlink(_ context.Context) (string, error) {
@@ -125,7 +137,7 @@ func (fsl *filesystemSymlink) Resolve(_ context.Context) (fs.Entry, error) {
return nil, errors.Wrapf(err, "cannot resolve symlink for '%q'", fsl.fullPath())
}
return NewEntry(target)
return NewEntryWithOptions(target, fsl.options)
}
func (e *filesystemErrorEntry) ErrorInfo() error {
@@ -145,8 +157,15 @@ func splitDirPrefix(s string) (basename, prefix string) {
}
// Directory returns fs.Directory for the specified path.
// It uses DefaultOptions for configuration.
func Directory(path string) (fs.Directory, error) {
e, err := NewEntry(path)
return DirectoryWithOptions(path, DefaultOptions)
}
// DirectoryWithOptions returns fs.Directory for the specified path.
// It uses the provided Options for configuration.
func DirectoryWithOptions(path string, options *Options) (fs.Directory, error) {
e, err := NewEntryWithOptions(path, options)
if err != nil {
return nil, err
}

44
fs/localfs/local_fs_os.go
View File

@@ -18,6 +18,7 @@ const separatorStr = string(filepath.Separator)
type filesystemDirectoryIterator struct {
dirHandle *os.File
childPrefix string
options *Options
currentIndex int
currentBatch []os.DirEntry
@@ -45,7 +46,7 @@ func (it *filesystemDirectoryIterator) Next(_ context.Context) (fs.Entry, error)
n := it.currentIndex
it.currentIndex++
e, err := toDirEntryOrNil(it.currentBatch[n], it.childPrefix)
e, err := toDirEntryOrNil(it.currentBatch[n], it.childPrefix, it.options)
if err != nil {
// stop iteration
return nil, err
@@ -74,7 +75,7 @@ func (fsd *filesystemDirectory) Iterate(_ context.Context) (fs.DirectoryIterator
childPrefix := fullPath + separatorStr
return &filesystemDirectoryIterator{dirHandle: d, childPrefix: childPrefix}, nil
return &filesystemDirectoryIterator{dirHandle: d, childPrefix: childPrefix, options: fsd.options}, nil
}
func (fsd *filesystemDirectory) Child(_ context.Context, name string) (fs.Entry, error) {
@@ -89,10 +90,10 @@ func (fsd *filesystemDirectory) Child(_ context.Context, name string) (fs.Entry,
return nil, errors.Wrap(err, "unable to get child")
}
return entryFromDirEntry(name, st, fullPath+separatorStr), nil
return entryFromDirEntry(name, st, fullPath+separatorStr, fsd.options), nil
}
func toDirEntryOrNil(dirEntry os.DirEntry, prefix string) (fs.Entry, error) {
func toDirEntryOrNil(dirEntry os.DirEntry, prefix string, options *Options) (fs.Entry, error) {
n := dirEntry.Name()
fi, err := os.Lstat(prefix + n)
@@ -101,15 +102,27 @@ func toDirEntryOrNil(dirEntry os.DirEntry, prefix string) (fs.Entry, error) {
return nil, nil
}
if options != nil && options.IgnoreUnreadableDirEntries {
return nil, nil
}
return nil, errors.Wrap(err, "error reading directory")
}
return entryFromDirEntry(n, fi, prefix), nil
return entryFromDirEntry(n, fi, prefix, options), nil
}
// NewEntry returns fs.Entry for the specified path, the result will be one of supported entry types: fs.File, fs.Directory, fs.Symlink
// or fs.UnsupportedEntry.
// It uses DefaultOptions for configuration.
func NewEntry(path string) (fs.Entry, error) {
return NewEntryWithOptions(path, DefaultOptions)
}
// NewEntryWithOptions returns fs.Entry for the specified path, the result will be one of supported entry types: fs.File, fs.Directory, fs.Symlink
// or fs.UnsupportedEntry.
// It uses the provided Options for configuration.
func NewEntryWithOptions(path string, options *Options) (fs.Entry, error) {
path = filepath.Clean(path)
fi, err := os.Lstat(path)
@@ -130,42 +143,42 @@ func NewEntry(path string) (fs.Entry, error) {
}
if path == "/" {
return entryFromDirEntry("/", fi, ""), nil
return entryFromDirEntry("/", fi, "", options), nil
}
basename, prefix := splitDirPrefix(path)
return entryFromDirEntry(basename, fi, prefix), nil
return entryFromDirEntry(basename, fi, prefix, options), nil
}
func entryFromDirEntry(basename string, fi os.FileInfo, prefix string) fs.Entry {
func entryFromDirEntry(basename string, fi os.FileInfo, prefix string, options *Options) fs.Entry {
isplaceholder := strings.HasSuffix(basename, ShallowEntrySuffix)
maskedmode := fi.Mode() & os.ModeType
switch {
case maskedmode == os.ModeDir && !isplaceholder:
return newFilesystemDirectory(newEntry(basename, fi, prefix))
return newFilesystemDirectory(newEntry(basename, fi, prefix, options))
case maskedmode == os.ModeDir && isplaceholder:
return newShallowFilesystemDirectory(newEntry(basename, fi, prefix))
return newShallowFilesystemDirectory(newEntry(basename, fi, prefix, options))
case maskedmode == os.ModeSymlink && !isplaceholder:
return newFilesystemSymlink(newEntry(basename, fi, prefix))
return newFilesystemSymlink(newEntry(basename, fi, prefix, options))
case maskedmode == 0 && !isplaceholder:
return newFilesystemFile(newEntry(basename, fi, prefix))
return newFilesystemFile(newEntry(basename, fi, prefix, options))
case maskedmode == 0 && isplaceholder:
return newShallowFilesystemFile(newEntry(basename, fi, prefix))
return newShallowFilesystemFile(newEntry(basename, fi, prefix, options))
default:
return newFilesystemErrorEntry(newEntry(basename, fi, prefix), fs.ErrUnknown)
return newFilesystemErrorEntry(newEntry(basename, fi, prefix, options), fs.ErrUnknown)
}
}
var _ os.FileInfo = (*filesystemEntry)(nil)
func newEntry(basename string, fi os.FileInfo, prefix string) filesystemEntry {
func newEntry(basename string, fi os.FileInfo, prefix string, options *Options) filesystemEntry {
return filesystemEntry{
TrimShallowSuffix(basename),
fi.Size(),
@@ -174,5 +187,6 @@ func newEntry(basename string, fi os.FileInfo, prefix string) filesystemEntry {
platformSpecificOwnerInfo(fi),
platformSpecificDeviceInfo(fi),
prefix,
options,
}
}

274
fs/localfs/local_fs_test.go
View File

@@ -306,3 +306,277 @@ func TestSplitDirPrefix(t *testing.T) {
require.Equal(t, want.prefix, prefix, input)
}
}
// getOptionsFromEntry extracts the options pointer from an fs.Entry by type assertion.
// Returns nil if the entry doesn't have options or if type assertion fails.
func getOptionsFromEntry(entry fs.Entry) *Options {
switch e := entry.(type) {
case *filesystemDirectory:
return e.options
case *filesystemFile:
return e.options
case *filesystemSymlink:
return e.options
case *filesystemErrorEntry:
return e.options
default:
return nil
}
}
func TestOptionsPassedToChildEntries(t *testing.T) {
ctx := testlogging.Context(t)
tmp := testutil.TempDirectory(t)
// Create a test directory structure
require.NoError(t, os.WriteFile(filepath.Join(tmp, "file1.txt"), []byte{1, 2, 3}, 0o777))
require.NoError(t, os.WriteFile(filepath.Join(tmp, "file2.txt"), []byte{4, 5, 6}, 0o777))
subdir := filepath.Join(tmp, "subdir")
require.NoError(t, os.Mkdir(subdir, 0o777))
require.NoError(t, os.WriteFile(filepath.Join(subdir, "subfile.txt"), []byte{7, 8, 9}, 0o777))
// Create custom options
customOptions := &Options{
IgnoreUnreadableDirEntries: true,
}
// Create directory with custom options
dir, err := DirectoryWithOptions(tmp, customOptions)
require.NoError(t, err)
// Verify the directory itself has the correct options
dirOptions := getOptionsFromEntry(dir)
require.NotNil(t, dirOptions, "directory should have options")
require.Equal(t, customOptions, dirOptions, "directory should have the same options pointer")
require.True(t, dirOptions.IgnoreUnreadableDirEntries, "directory options should match")
// Test that options are passed to children via Child()
childFile, err := dir.Child(ctx, "file1.txt")
require.NoError(t, err)
childOptions := getOptionsFromEntry(childFile)
require.NotNil(t, childOptions, "child file should have options")
require.Equal(t, customOptions, childOptions, "child file should have the same options pointer")
// Test that options are passed to subdirectories
childDir, err := dir.Child(ctx, "subdir")
require.NoError(t, err)
subdirOptions := getOptionsFromEntry(childDir)
require.NotNil(t, subdirOptions, "subdirectory should have options")
require.Equal(t, customOptions, subdirOptions, "subdirectory should have the same options pointer")
// Test that options are passed to nested children
subdirEntry, ok := childDir.(fs.Directory)
require.True(t, ok, "child directory should be a directory")
nestedFile, err := subdirEntry.Child(ctx, "subfile.txt")
require.NoError(t, err)
nestedOptions := getOptionsFromEntry(nestedFile)
require.NotNil(t, nestedOptions, "nested file should have options")
require.Equal(t, customOptions, nestedOptions, "nested file should have the same options pointer")
}
func TestOptionsPassedThroughIteration(t *testing.T) {
ctx := testlogging.Context(t)
tmp := testutil.TempDirectory(t)
// Create a test directory structure
require.NoError(t, os.WriteFile(filepath.Join(tmp, "file1.txt"), []byte{1, 2, 3}, 0o777))
require.NoError(t, os.WriteFile(filepath.Join(tmp, "file2.txt"), []byte{4, 5, 6}, 0o777))
require.NoError(t, os.Mkdir(filepath.Join(tmp, "subdir"), 0o777))
// Create custom options
customOptions := &Options{
IgnoreUnreadableDirEntries: true,
}
// Create directory with custom options
dir, err := DirectoryWithOptions(tmp, customOptions)
require.NoError(t, err)
// Iterate through entries and verify all have the same options pointer
iter, err := dir.Iterate(ctx)
require.NoError(t, err)
defer iter.Close()
entryCount := 0
for {
entry, err := iter.Next(ctx)
if err != nil {
t.Fatalf("iteration error: %v", err)
}
if entry == nil {
break
}
entryCount++
entryOptions := getOptionsFromEntry(entry)
require.NotNil(t, entryOptions, "entry %s should have options", entry.Name())
require.Equal(t, customOptions, entryOptions, "entry %s should have the same options pointer", entry.Name())
}
require.Equal(t, 3, entryCount, "should have found 3 entries")
}
func TestOptionsPassedThroughSymlinkResolution(t *testing.T) {
ctx := testlogging.Context(t)
tmp := testutil.TempDirectory(t)
// Create a target file
targetFile := filepath.Join(tmp, "target.txt")
require.NoError(t, os.WriteFile(targetFile, []byte{1, 2, 3}, 0o777))
// Create a symlink
symlinkPath := filepath.Join(tmp, "link")
require.NoError(t, os.Symlink(targetFile, symlinkPath))
// Create custom options
customOptions := &Options{
IgnoreUnreadableDirEntries: true,
}
// Create symlink entry with custom options
symlinkEntry, err := NewEntryWithOptions(symlinkPath, customOptions)
require.NoError(t, err)
// Verify the symlink has the correct options
symlinkOptions := getOptionsFromEntry(symlinkEntry)
require.NotNil(t, symlinkOptions, "symlink should have options")
require.Equal(t, customOptions, symlinkOptions, "symlink should have the same options pointer")
// Resolve the symlink and verify the resolved entry has the same options
symlink, ok := symlinkEntry.(fs.Symlink)
require.True(t, ok, "entry should be a symlink")
resolved, err := symlink.Resolve(ctx)
require.NoError(t, err)
resolvedOptions := getOptionsFromEntry(resolved)
require.NotNil(t, resolvedOptions, "resolved entry should have options")
require.Equal(t, customOptions, resolvedOptions, "resolved entry should have the same options pointer")
}
func TestOptionsPassedToNewEntry(t *testing.T) {
tmp := testutil.TempDirectory(t)
// Create a file
filePath := filepath.Join(tmp, "testfile.txt")
require.NoError(t, os.WriteFile(filePath, []byte{1, 2, 3}, 0o777))
// Create custom options
customOptions := &Options{
IgnoreUnreadableDirEntries: true,
}
// Create entry with custom options
entry, err := NewEntryWithOptions(filePath, customOptions)
require.NoError(t, err)
// Verify the entry has the correct options
entryOptions := getOptionsFromEntry(entry)
require.NotNil(t, entryOptions, "entry should have options")
require.Equal(t, customOptions, entryOptions, "entry should have the same options pointer")
}
func TestOptionsPassedToNestedDirectories(t *testing.T) {
ctx := testlogging.Context(t)
tmp := testutil.TempDirectory(t)
// Create nested directory structure
level1 := filepath.Join(tmp, "level1")
level2 := filepath.Join(level1, "level2")
level3 := filepath.Join(level2, "level3")
require.NoError(t, os.MkdirAll(level3, 0o777))
require.NoError(t, os.WriteFile(filepath.Join(level3, "file.txt"), []byte{1, 2, 3}, 0o777))
// Create custom options
customOptions := &Options{
IgnoreUnreadableDirEntries: true,
}
// Create root directory with custom options
rootDir, err := DirectoryWithOptions(tmp, customOptions)
require.NoError(t, err)
// Navigate through nested directories and verify options are passed
currentDir := rootDir
levels := []string{"level1", "level2", "level3"}
for _, level := range levels {
child, err := currentDir.Child(ctx, level)
require.NoError(t, err)
childOptions := getOptionsFromEntry(child)
require.NotNil(t, childOptions, "directory %s should have options", level)
require.Equal(t, customOptions, childOptions, "directory %s should have the same options pointer", level)
var ok bool
currentDir, ok = child.(fs.Directory)
require.True(t, ok, "child should be a directory")
}
// Verify the file in the deepest directory has the same options
file, err := currentDir.Child(ctx, "file.txt")
require.NoError(t, err)
fileOptions := getOptionsFromEntry(file)
require.NotNil(t, fileOptions, "file should have options")
require.Equal(t, customOptions, fileOptions, "file should have the same options pointer")
}
func TestDefaultOptionsUsedByDefault(t *testing.T) {
tmp := testutil.TempDirectory(t)
// Create a file
filePath := filepath.Join(tmp, "testfile.txt")
require.NoError(t, os.WriteFile(filePath, []byte{1, 2, 3}, 0o777))
// Use default NewEntry (should use DefaultOptions)
entry, err := NewEntry(filePath)
require.NoError(t, err)
// Verify the entry has DefaultOptions
entryOptions := getOptionsFromEntry(entry)
require.NotNil(t, entryOptions, "entry should have options")
require.Equal(t, DefaultOptions, entryOptions, "entry should have DefaultOptions pointer")
}
func TestDifferentOptionsInstances(t *testing.T) {
tmp := testutil.TempDirectory(t)
// Create two different files
filePath1 := filepath.Join(tmp, "testfile1.txt")
filePath2 := filepath.Join(tmp, "testfile2.txt")
require.NoError(t, os.WriteFile(filePath1, []byte{1, 2, 3}, 0o777))
require.NoError(t, os.WriteFile(filePath2, []byte{4, 5, 6}, 0o777))
// Create two different options instances with same values
options1 := &Options{IgnoreUnreadableDirEntries: true}
options2 := &Options{IgnoreUnreadableDirEntries: false}
// Create entries with different options instances
entry1, err := NewEntryWithOptions(filePath1, options1)
require.NoError(t, err)
entry2, err := NewEntryWithOptions(filePath2, options2)
require.NoError(t, err)
// Verify they have the correct options pointers
entry1Options := getOptionsFromEntry(entry1)
entry2Options := getOptionsFromEntry(entry2)
require.NotNil(t, entry1Options)
require.NotNil(t, entry2Options)
require.Equal(t, options1, entry1Options, "entry1 should have options1 pointer")
require.Equal(t, options2, entry2Options, "entry2 should have options2 pointer")
require.NotEqual(t, entry1Options, entry2Options, "entries should have different options pointers")
require.True(t, entry1Options.IgnoreUnreadableDirEntries, "entry1 options should have IgnoreUnreadableDirEntries=true")
require.False(t, entry2Options.IgnoreUnreadableDirEntries, "entry2 options should have IgnoreUnreadableDirEntries=false")
}