README
¶
S3FS - S3 FileSystem
The s3fs package implements an absfs.Filer for S3-compatible object storage. It provides file operations on S3 buckets using the AWS SDK v2.
Features
- S3-compatible storage: Works with AWS S3 and compatible services
- Standard interface: Implements
absfs.Filerfor seamless integration - Basic operations: Read, write, delete files in S3 buckets
Install
go get github.com/absfs/s3fs
Example Usage
Basic Usage
package main
import (
"log"
"os"
"github.com/absfs/s3fs"
)
func main() {
// Create S3 filesystem
fs, err := s3fs.New(&s3fs.Config{
Bucket: "my-bucket",
Region: "us-east-1",
})
if err != nil {
log.Fatal(err)
}
// Write a file
f, _ := fs.OpenFile("path/to/file.txt", os.O_CREATE|os.O_WRONLY, 0644)
f.Write([]byte("Hello, S3!"))
f.Close()
// Read a file
f, _ = fs.OpenFile("path/to/file.txt", os.O_RDONLY, 0)
defer f.Close()
buf := make([]byte, 1024)
n, _ := f.Read(buf)
log.Printf("Read: %s", buf[:n])
// Delete a file
fs.Remove("path/to/file.txt")
}
With Custom AWS Config
package main
import (
"context"
"log"
"github.com/absfs/s3fs"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
)
func main() {
// Load custom AWS config
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion("us-west-2"),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
"access-key",
"secret-key",
"",
)),
)
if err != nil {
log.Fatal(err)
}
// Create S3 filesystem with custom config
fs, err := s3fs.New(&s3fs.Config{
Bucket: "my-bucket",
Config: &cfg,
})
if err != nil {
log.Fatal(err)
}
// Use filesystem
}
Limitations
- Chmod, Chtimes, Chown: Not supported (S3 doesn't have POSIX permissions)
- Directories: Represented as zero-byte objects with trailing slash
- Seeking: Limited support (S3 is object storage, not a traditional filesystem)
- Atomic operations: Some operations (like Rename) require copy+delete
Authentication
The filesystem uses the AWS SDK's default credential chain, which checks:
- Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
- Shared credentials file (~/.aws/credentials)
- IAM role (when running on EC2, ECS, Lambda, etc.)
absfs
Check out the absfs repo for more information about the abstract filesystem interface and features like filesystem composition.
LICENSE
This project is governed by the MIT License. See LICENSE
Documentation
¶
Overview ¶
Package s3fs implements an absfs.Filer backed by S3-compatible object storage.
Directories are represented as zero-byte objects with a trailing slash. Write operations are buffered in memory and uploaded to S3 on File.Close. The FileSystem is safe for concurrent use; individual File instances are also safe for concurrent use as of v1.1.
S3 does not support POSIX permissions, ownership, or timestamps, so FileSystem.Chmod, FileSystem.Chtimes, and FileSystem.Chown return absfs.ErrNotImplemented.
Index ¶
- Constants
- type Config
- type File
- func (f *File) Close() error
- func (f *File) Name() string
- func (f *File) Read(b []byte) (int, error)
- func (f *File) ReadAt(b []byte, off int64) (int, error)
- func (f *File) ReadDir(n int) ([]iosfs.DirEntry, error)
- func (f *File) Readdir(n int) ([]os.FileInfo, error)
- func (f *File) Readdirnames(n int) ([]string, error)
- func (f *File) Seek(offset int64, whence int) (int64, error)
- func (f *File) Stat() (os.FileInfo, error)
- func (f *File) Sync() error
- func (f *File) Truncate(size int64) error
- func (f *File) Write(b []byte) (int, error)
- func (f *File) WriteAt(b []byte, off int64) (int, error)
- func (f *File) WriteString(s string) (int, error)
- type FileSystem
- func (fs *FileSystem) Chmod(name string, mode os.FileMode) error
- func (fs *FileSystem) Chown(name string, uid, gid int) error
- func (fs *FileSystem) Chtimes(name string, atime time.Time, mtime time.Time) error
- func (fs *FileSystem) Mkdir(name string, perm os.FileMode) error
- func (fs *FileSystem) OpenFile(name string, flag int, perm os.FileMode) (absfs.File, error)
- func (fs *FileSystem) ReadDir(name string) ([]iosfs.DirEntry, error)
- func (fs *FileSystem) ReadFile(name string) ([]byte, error)
- func (fs *FileSystem) Remove(name string) error
- func (fs *FileSystem) Rename(oldpath, newpath string) error
- func (fs *FileSystem) Stat(name string) (os.FileInfo, error)
- func (fs *FileSystem) Sub(dir string) (iosfs.FS, error)
- func (fs *FileSystem) Truncate(name string, size int64) error
- func (fs *FileSystem) WithContext(ctx context.Context) *FileSystem
- type MockS3Client
- func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, ...) (*s3.AbortMultipartUploadOutput, error)
- func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, ...) (*s3.CompleteMultipartUploadOutput, error)
- func (m *MockS3Client) CopyObject(ctx context.Context, params *s3.CopyObjectInput, optFns ...func(*s3.Options)) (*s3.CopyObjectOutput, error)
- func (m *MockS3Client) CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, ...) (*s3.CreateMultipartUploadOutput, error)
- func (m *MockS3Client) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
- func (m *MockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
- func (m *MockS3Client) GetTestObject(key string) ([]byte, bool)
- func (m *MockS3Client) HasObject(key string) bool
- func (m *MockS3Client) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
- func (m *MockS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, ...) (*s3.ListObjectsV2Output, error)
- func (m *MockS3Client) ObjectCount() int
- func (m *MockS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
- func (m *MockS3Client) PutTestObject(key string, data []byte)
- func (m *MockS3Client) Reset()
- func (m *MockS3Client) UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error)
- type S3Client
Constants ¶
const DefaultPartSize = 5 * 1024 * 1024 // 5MB
DefaultPartSize is the buffer threshold for switching to multipart upload. S3 requires a minimum part size of 5MB for all parts except the last.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Config ¶
type Config struct {
Bucket string // S3 bucket name
Region string // AWS region
Config *aws.Config // Optional AWS config (if nil, uses default config loading)
PartSize int64 // Multipart upload part size (0 means DefaultPartSize)
}
Config contains the configuration for connecting to S3.
type File ¶
type File struct {
// contains filtered or unexported fields
}
File represents a file in S3. File instances are safe for concurrent use; all mutable state is protected by an internal mutex.
func (*File) Close ¶
Close closes the file. For files opened for writing, the buffered content is uploaded to S3. If a multipart upload is in progress, the remaining buffer is uploaded as the final part and the upload is completed.
func (*File) Read ¶
Read reads from the S3 object. On first call, the object is fetched from S3 and cached for subsequent reads.
func (*File) ReadAt ¶
ReadAt reads from the S3 object at a specific offset. On first call, the full object is fetched and cached in memory for efficient subsequent random access. For streaming reads of large files, use File.Read instead.
func (*File) ReadDir ¶
ReadDir reads the contents of the directory and returns a slice of up to n DirEntry values. If n > 0, ReadDir returns at most n entries. If n <= 0, it returns all entries.
func (*File) Readdir ¶
Readdir reads directory entries (lists objects with the file's key as prefix). If n > 0, Readdir returns at most n entries. If n <= 0, it returns all entries.
func (*File) Readdirnames ¶
Readdirnames reads directory entry names.
func (*File) Seek ¶
Seek sets the offset for the next read or write. SeekEnd is not supported for S3 files.
func (*File) Truncate ¶
Truncate changes the size of the file buffer. It does not change the I/O offset. If a multipart upload is in progress, it is aborted since already-uploaded parts cannot be truncated.
func (*File) Write ¶
Write writes to the file buffer. The buffer is uploaded to S3 on Close. For large writes, multipart upload is used automatically when the buffer exceeds the configured part size (default 5MB).
type FileSystem ¶
type FileSystem struct {
// contains filtered or unexported fields
}
FileSystem implements absfs.Filer for S3 object storage.
func New ¶
func New(cfg *Config) (*FileSystem, error)
New creates a new S3 filesystem with the given configuration.
func NewWithClient ¶
func NewWithClient(client S3Client, bucket string) *FileSystem
NewWithClient creates a new S3 filesystem with a custom S3 client. This is primarily useful for testing with mock clients.
func NewWithClientAndContext ¶
func NewWithClientAndContext(ctx context.Context, client S3Client, bucket string) *FileSystem
NewWithClientAndContext creates a new S3 filesystem with a custom S3 client and context.
func (*FileSystem) Chmod ¶
func (fs *FileSystem) Chmod(name string, mode os.FileMode) error
Chmod is not supported for S3.
func (*FileSystem) Chown ¶
func (fs *FileSystem) Chown(name string, uid, gid int) error
Chown is not supported for S3.
func (*FileSystem) Mkdir ¶
func (fs *FileSystem) Mkdir(name string, perm os.FileMode) error
Mkdir creates a directory in S3 by writing a zero-byte object with a trailing slash. Returns os.ErrExist if the directory already exists.
func (*FileSystem) OpenFile ¶
OpenFile opens the named file in S3.
Supported flags: O_RDONLY, O_WRONLY, O_RDWR, O_CREATE, O_EXCL, O_TRUNC, O_APPEND. When O_APPEND is set, existing content is loaded and new writes append to it. When O_TRUNC is set (or O_CREATE for a new file), the file starts empty. Paths are sanitized to prevent traversal attacks.
func (*FileSystem) ReadDir ¶
func (fs *FileSystem) ReadDir(name string) ([]iosfs.DirEntry, error)
ReadDir reads the named directory and returns a list of directory entries sorted by filename.
func (*FileSystem) ReadFile ¶
func (fs *FileSystem) ReadFile(name string) ([]byte, error)
ReadFile reads the named file and returns its contents.
func (*FileSystem) Remove ¶
func (fs *FileSystem) Remove(name string) error
Remove removes a file or directory from S3. Returns os.ErrNotExist if neither the file nor a directory marker exists. For directories, it removes the directory marker (key with trailing slash).
func (*FileSystem) Rename ¶
func (fs *FileSystem) Rename(oldpath, newpath string) error
Rename moves a file in S3 by copying to the new key and deleting the old one. This is not atomic. If the delete fails after a successful copy, the copy is rolled back (best-effort).
func (*FileSystem) Stat ¶
func (fs *FileSystem) Stat(name string) (os.FileInfo, error)
Stat returns file info for an S3 object.
func (*FileSystem) Sub ¶
func (fs *FileSystem) Sub(dir string) (iosfs.FS, error)
Sub returns an fs.FS corresponding to the subtree rooted at dir.
func (*FileSystem) Truncate ¶
func (fs *FileSystem) Truncate(name string, size int64) error
Truncate truncates a file to the specified size. For S3, this requires reading the file, truncating the data, and writing it back.
func (*FileSystem) WithContext ¶ added in v1.1.0
func (fs *FileSystem) WithContext(ctx context.Context) *FileSystem
WithContext returns a copy of the FileSystem that uses the given context for all S3 operations.
type MockS3Client ¶
type MockS3Client struct {
// Error injection for testing failure scenarios
GetObjectErr error
PutObjectErr error
DeleteObjectErr error
CopyObjectErr error
HeadObjectErr error
ListObjectsErr error
// Per-key error injection for DeleteObject (checked before global DeleteObjectErr)
DeleteObjectKeyErrs map[string]error
// DefaultMaxKeys overrides the default 1000 max keys for ListObjectsV2.
// Set to a small value in tests to force pagination behavior.
DefaultMaxKeys int32
// Call tracking for assertions
GetObjectCalls []string
PutObjectCalls []string
DeleteObjectCalls []string
CopyObjectCalls []copyObjectCall
HeadObjectCalls []string
ListObjectsCalls []string
// contains filtered or unexported fields
}
MockS3Client is an in-memory mock implementation of S3Client for testing.
func NewMockS3Client ¶
func NewMockS3Client() *MockS3Client
NewMockS3Client creates a new mock S3 client for testing.
func (*MockS3Client) AbortMultipartUpload ¶ added in v1.2.0
func (m *MockS3Client) AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
AbortMultipartUpload implements S3Client.
func (*MockS3Client) CompleteMultipartUpload ¶ added in v1.2.0
func (m *MockS3Client) CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
CompleteMultipartUpload implements S3Client.
func (*MockS3Client) CopyObject ¶
func (m *MockS3Client) CopyObject(ctx context.Context, params *s3.CopyObjectInput, optFns ...func(*s3.Options)) (*s3.CopyObjectOutput, error)
CopyObject implements S3Client.
func (*MockS3Client) CreateMultipartUpload ¶ added in v1.2.0
func (m *MockS3Client) CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
CreateMultipartUpload implements S3Client.
func (*MockS3Client) DeleteObject ¶
func (m *MockS3Client) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
DeleteObject implements S3Client.
func (*MockS3Client) GetObject ¶
func (m *MockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
GetObject implements S3Client.
func (*MockS3Client) GetTestObject ¶
func (m *MockS3Client) GetTestObject(key string) ([]byte, bool)
GetTestObject retrieves a test object from the mock store.
func (*MockS3Client) HasObject ¶
func (m *MockS3Client) HasObject(key string) bool
HasObject checks if an object exists in the mock store.
func (*MockS3Client) HeadObject ¶
func (m *MockS3Client) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
HeadObject implements S3Client.
func (*MockS3Client) ListObjectsV2 ¶
func (m *MockS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
ListObjectsV2 implements S3Client with pagination support via ContinuationToken.
func (*MockS3Client) ObjectCount ¶
func (m *MockS3Client) ObjectCount() int
ObjectCount returns the number of objects in the mock store.
func (*MockS3Client) PutObject ¶
func (m *MockS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
PutObject implements S3Client.
func (*MockS3Client) PutTestObject ¶
func (m *MockS3Client) PutTestObject(key string, data []byte)
PutTestObject adds a test object directly to the mock store.
func (*MockS3Client) Reset ¶
func (m *MockS3Client) Reset()
Reset clears all objects and error injections.
func (*MockS3Client) UploadPart ¶ added in v1.2.0
func (m *MockS3Client) UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error)
UploadPart implements S3Client.
type S3Client ¶
type S3Client interface {
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
CopyObject(ctx context.Context, params *s3.CopyObjectInput, optFns ...func(*s3.Options)) (*s3.CopyObjectOutput, error)
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error)
CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
}
S3Client defines the subset of the S3 API used by FileSystem. Both s3.Client and MockS3Client satisfy this interface.
Source Files