feat: cache revalidation

This commit is contained in:
2026-02-27 23:10:02 +01:00
parent 6f98d730bb
commit abfddca518
12 changed files with 768 additions and 143 deletions
+2
View File
@@ -12,6 +12,7 @@ require (
require ( require (
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e // indirect github.com/a-h/parse v0.0.0-20250122154542-74294addb73e // indirect
github.com/alicebob/miniredis/v2 v2.36.1 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cli/browser v1.3.0 // indirect github.com/cli/browser v1.3.0 // indirect
@@ -20,6 +21,7 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-isatty v0.0.20 // indirect
github.com/natefinch/atomic v1.0.1 // indirect github.com/natefinch/atomic v1.0.1 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
golang.org/x/mod v0.26.0 // indirect golang.org/x/mod v0.26.0 // indirect
golang.org/x/net v0.48.0 // indirect golang.org/x/net v0.48.0 // indirect
golang.org/x/sync v0.16.0 // indirect golang.org/x/sync v0.16.0 // indirect
+4
View File
@@ -2,6 +2,8 @@ github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ6
github.com/a-h/parse v0.0.0-20250122154542-74294addb73e/go.mod h1:3mnrkvGpurZ4ZrTDbYU84xhwXW2TjTKShSwjRi2ihfQ= github.com/a-h/parse v0.0.0-20250122154542-74294addb73e/go.mod h1:3mnrkvGpurZ4ZrTDbYU84xhwXW2TjTKShSwjRi2ihfQ=
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg= github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo= github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/alicebob/miniredis/v2 v2.36.1 h1:Dvc5oAnNOr7BIfPn7tF269U8DvRW1dBG2D5n0WrfYMI=
github.com/alicebob/miniredis/v2 v2.36.1/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -33,6 +35,8 @@ github.com/studio-b12/gowebdav v0.12.0 h1:kFRtQECt8jmVAvA6RHBz3geXUGJHUZA6/IKpOV
github.com/studio-b12/gowebdav v0.12.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/studio-b12/gowebdav v0.12.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
github.com/valkey-io/valkey-go v1.0.72 h1:iRWt1hJyOchcEgbHSkRY3aKkcBudxvMaVMsmxuYxuxE= github.com/valkey-io/valkey-go v1.0.72 h1:iRWt1hJyOchcEgbHSkRY3aKkcBudxvMaVMsmxuYxuxE=
github.com/valkey-io/valkey-go v1.0.72/go.mod h1:VGhZ6fs68Qrn2+OhH+6waZH27bjpgQOiLyUQyXuYK5k= github.com/valkey-io/valkey-go v1.0.72/go.mod h1:VGhZ6fs68Qrn2+OhH+6waZH27bjpgQOiLyUQyXuYK5k=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
-8
View File
@@ -1,8 +1,6 @@
package page package page
import ( import (
"log/slog"
"git.schreifuchs.ch/schreifuchs/schreifuchs.ch/internal/filesystem" "git.schreifuchs.ch/schreifuchs/schreifuchs.ch/internal/filesystem"
) )
@@ -11,12 +9,6 @@ type Service struct {
} }
func New(fs filesystem.FS) *Service { func New(fs filesystem.FS) *Service {
info, err := fs.ReadDir("/")
if err != nil {
slog.Error("could not read dir", "err", err)
}
slog.Info("file info", "info", info)
return &Service{ return &Service{
fs: fs, fs: fs,
} }
+34 -126
View File
@@ -2,21 +2,20 @@ package filesystem
import ( import (
"context" "context"
"encoding/json"
"fmt"
"log/slog" "log/slog"
"os"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/valkey-io/valkey-go" "github.com/valkey-io/valkey-go"
) )
const keyStat = "webdav:stat:"
// CachedClient wraps a FileSystem implementation with Valkey caching. // CachedClient wraps a FileSystem implementation with Valkey caching.
type CachedClient struct { type CachedClient struct {
impl FS impl FS
client valkey.Client client valkey.Client
refreshTime time.Duration ttl time.Duration
hits atomic.Int64 hits atomic.Int64
misses atomic.Int64 misses atomic.Int64
readDirHits atomic.Int64 readDirHits atomic.Int64
@@ -24,139 +23,48 @@ type CachedClient struct {
} }
// NewCachedClient creates a new CachedClient. // NewCachedClient creates a new CachedClient.
func NewCachedClient(impl FS, client valkey.Client, ttl time.Duration) *CachedClient { func NewCachedClient(impl FS, client valkey.Client, refreshtime time.Duration) *CachedClient {
return &CachedClient{ c := &CachedClient{
impl: impl, impl: impl,
client: client, client: client,
refreshTime: ttl, ttl: refreshtime * 5,
} }
go func() {
t := time.NewTicker(refreshtime)
for {
<-t.C
c.revalidate(context.Background())
}
}()
return c
} }
func (c *CachedClient) ReadDir(path string) ([]os.FileInfo, error) { func (c *CachedClient) revalidate(ctx context.Context) {
key := fmt.Sprintf("webdav:readdir:%s", path) go c.revalidateReadDir(ctx)
ctx := context.Background() go c.revalidateRead(ctx)
go c.revalidateStat(ctx)
// Try cache
val, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).ToString()
if err == nil && val != "" {
var cached []File
if err := json.Unmarshal([]byte(val), &cached); err == nil {
c.readDirHits.Add(1)
c.hits.Add(1)
slog.Debug("cache hit", "key", key)
infos := make([]os.FileInfo, len(cached))
for i, f := range cached {
infos[i] = f
}
return infos, nil
}
}
// Cache miss
c.misses.Add(1)
slog.Debug("cache miss", "key", key)
infos, err := c.impl.ReadDir(path)
if err != nil {
return nil, err
}
// Serialize and cache
cached := make([]File, 0, len(infos))
for _, info := range infos {
// Type assert to ensure we capture ContentType and Path
if fi, ok := info.(FileInfo); ok {
cached = append(cached, File{
name: fi.Name(),
size: fi.Size(),
mode: fi.Mode(),
modTime: fi.ModTime(),
isDir: fi.IsDir(),
contentType: fi.ContentType(),
path: fi.Path(),
})
} else {
// Fallback if underlying impl returns generic os.FileInfo
cached = append(cached, File{
name: info.Name(),
size: info.Size(),
mode: info.Mode(),
modTime: info.ModTime(),
isDir: info.IsDir(),
})
}
}
bytes, err := json.Marshal(cached)
if err == nil {
c.client.Do(ctx, c.client.B().Set().Key(key).Value(string(bytes)).Ex(c.refreshTime).Build())
}
return infos, nil
} }
func (c *CachedClient) Read(path string) ([]byte, error) { func (c *CachedClient) scanAndProcess(ctx context.Context, match string, process func(key string) error) {
key := fmt.Sprintf("webdav:read:%s", path) cursor := uint64(0)
ctx := context.Background() for {
res, err := c.client.Do(ctx, c.client.B().Scan().Cursor(cursor).Match(match).Build()).AsScanEntry()
// Try cache
val, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).AsBytes()
if err == nil && len(val) > 0 {
c.readHits.Add(1)
c.hits.Add(1)
slog.Debug("cache hit", "key", key)
return val, nil
}
// Cache miss
c.misses.Add(1)
slog.Debug("cache miss", "key", key)
data, err := c.impl.Read(path)
if err != nil {
return nil, err
}
// Cache
c.client.Do(ctx, c.client.B().Set().Key(key).Value(valkey.BinaryString(data)).Ex(c.refreshTime).Build())
return data, nil
}
func (c *CachedClient) Stat(path string) (info os.FileInfo, err error) {
key := fmt.Sprintf("webdav:stat:%s", path)
ctx := context.Background()
// Try cache
val, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).ToString()
if err == nil && val != "" {
if err := json.Unmarshal([]byte(val), &info); err == nil {
c.hits.Add(1)
slog.Debug("cache hit", "key", key)
return info, nil
}
}
// Cache miss
c.misses.Add(1)
slog.Debug("cache miss", "key", key)
fileInfo, err := c.impl.Stat(path)
if err != nil { if err != nil {
slog.Error("scan failed", "match", match, "err", err)
return return
} }
// Serialize and cache for _, key := range res.Elements {
// Type assert to ensure we capture ContentType and Path if err := process(key); err != nil {
info = ParseToFile(fileInfo) slog.Debug("process failed", "key", key, "err", err)
}
bytes, err := json.Marshal(info)
if err == nil {
c.client.Do(ctx, c.client.B().Set().Key(key).Value(string(bytes)).Ex(c.refreshTime).Build())
} }
return cursor = res.Cursor
} if cursor == 0 {
break
func (c *CachedClient) Invalidate(path string) { }
ctx := context.Background() }
c.client.Do(ctx, c.client.B().Del().Key(fmt.Sprintf("webdav:readdir:%s", path)).Key(fmt.Sprintf("webdav:read:%s", path)).Build())
} }
+464
View File
@@ -0,0 +1,464 @@
package filesystem
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/valkey-io/valkey-go"
)
// MockFS implements filesystem.FS interface for testing.
type MockFS struct {
files map[string]*MockFile
}
// MockFile implements FileInfo and holds content.
type MockFile struct {
name string
content []byte
mode os.FileMode
modTime time.Time
isDir bool
contentType string
path string
}
func (f *MockFile) Name() string { return f.name }
func (f *MockFile) Size() int64 { return int64(len(f.content)) }
func (f *MockFile) Mode() os.FileMode { return f.mode }
func (f *MockFile) ModTime() time.Time { return f.modTime }
func (f *MockFile) IsDir() bool { return f.isDir }
func (f *MockFile) Sys() any { return nil }
func (f *MockFile) ContentType() string { return f.contentType }
func (f *MockFile) Path() string { return f.path }
func NewMockFS() *MockFS {
return &MockFS{
files: make(map[string]*MockFile),
}
}
func (m *MockFS) AddFile(path string, content []byte) {
m.files[path] = &MockFile{
name: path,
content: content,
mode: 0o644,
modTime: time.Now().Truncate(time.Second),
isDir: false,
contentType: "text/plain",
path: path,
}
}
func (m *MockFS) AddDir(path string) {
m.files[path] = &MockFile{
name: path,
mode: 0o755,
modTime: time.Now().Truncate(time.Second),
isDir: true,
path: path,
}
}
func (m *MockFS) Read(path string) ([]byte, error) {
f, ok := m.files[path]
if !ok {
return nil, os.ErrNotExist
}
if f.isDir {
return nil, errors.New("is a directory")
}
return f.content, nil
}
func (m *MockFS) ReadDir(path string) ([]os.FileInfo, error) {
// Simple implementation: return files that start with path + "/"
// For deeper nesting, simulate direct children only.
// Assume flat structure for simplicity or specific test setup.
var infos []os.FileInfo
for p, f := range m.files {
// Just check if it's logically "inside" the directory
// e.g. path="/foo", file="/foo/bar" -> yes
// file="/foo/bar/baz" -> no (if strict direct children)
// file="/other" -> no
if p == path {
continue // self
}
// Check prefix
// if path is root "/", check if p has no other slashes?
// if path is "/foo", check if p starts with "/foo/" and has no further slashes
// This logic is simple but sufficient for tests.
// Let's assume absolute paths.
// If path does not end with /, append it for prefix check
prefix := path
if len(prefix) > 0 && prefix[len(prefix)-1] != '/' {
prefix += "/"
}
if len(p) > len(prefix) && p[:len(prefix)] == prefix {
// Ensure direct child (no more slashes)
rel := p[len(prefix):]
// Count slashes in rel. If 0, it's a direct child.
slashCount := 0
for _, c := range rel {
if c == '/' {
slashCount++
}
}
if slashCount == 0 {
infos = append(infos, f)
}
}
}
if len(infos) == 0 {
// Check if directory exists at all
if _, ok := m.files[path]; !ok {
return nil, os.ErrNotExist
}
}
return infos, nil
}
func (m *MockFS) Stat(path string) (os.FileInfo, error) {
f, ok := m.files[path]
if !ok {
return nil, os.ErrNotExist
}
return f, nil
}
func TestCachedClient_Stat(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("failed to start miniredis: %v", err)
}
defer mr.Close()
client, err := valkey.NewClient(valkey.ClientOption{
InitAddress: []string{mr.Addr()},
DisableCache: true,
})
if err != nil {
t.Fatalf("failed to create valkey client: %v", err)
}
defer client.Close()
mockFS := NewMockFS()
mockFS.AddFile("/test.txt", []byte("hello"))
c := NewCachedClient(mockFS, client, time.Hour)
// First call: Cache miss
info, err := c.Stat("/test.txt")
if err != nil {
t.Fatalf("Stat failed: %v", err)
}
if info.Name() != "/test.txt" {
t.Errorf("expected name /test.txt, got %s", info.Name())
}
if c.misses.Load() != 1 {
t.Errorf("expected 1 miss, got %d", c.misses.Load())
}
if c.hits.Load() != 0 {
t.Errorf("expected 0 hits, got %d", c.hits.Load())
}
// Second call: Cache hit
info2, err := c.Stat("/test.txt")
if err != nil {
t.Fatalf("Stat failed: %v", err)
}
if info2.Size() != info.Size() {
t.Errorf("size mismatch")
}
if c.misses.Load() != 1 {
t.Errorf("expected 1 miss, got %d", c.misses.Load())
}
if c.hits.Load() != 1 {
t.Errorf("expected 1 hit, got %d", c.hits.Load())
}
// Verify valkey content
keys := mr.Keys()
if len(keys) != 1 {
t.Errorf("expected 1 key in redis, got %d", len(keys))
}
}
func TestCachedClient_Read(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("failed to start miniredis: %v", err)
}
defer mr.Close()
client, err := valkey.NewClient(valkey.ClientOption{
InitAddress: []string{mr.Addr()},
DisableCache: true,
})
if err != nil {
t.Fatalf("failed to create valkey client: %v", err)
}
defer client.Close()
mockFS := NewMockFS()
content := []byte("hello world")
mockFS.AddFile("/read.txt", content)
c := NewCachedClient(mockFS, client, time.Hour)
// First call: Miss
data, err := c.Read("/read.txt")
if err != nil {
t.Fatalf("Read failed: %v", err)
}
if string(data) != string(content) {
t.Errorf("content mismatch")
}
if c.misses.Load() != 1 {
t.Errorf("expected 1 miss, got %d", c.misses.Load())
}
// Second call: Hit
data2, err := c.Read("/read.txt")
if err != nil {
t.Fatalf("Read failed: %v", err)
}
if string(data2) != string(content) {
t.Errorf("content mismatch")
}
if c.hits.Load() != 1 {
t.Errorf("expected 1 hit, got %d", c.hits.Load())
}
}
func TestCachedClient_ReadDir(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("failed to start miniredis: %v", err)
}
defer mr.Close()
client, err := valkey.NewClient(valkey.ClientOption{
InitAddress: []string{mr.Addr()},
DisableCache: true,
})
if err != nil {
t.Fatalf("failed to create valkey client: %v", err)
}
defer client.Close()
mockFS := NewMockFS()
mockFS.AddDir("/dir")
mockFS.AddFile("/dir/f1.txt", []byte("1"))
mockFS.AddFile("/dir/f2.txt", []byte("2"))
c := NewCachedClient(mockFS, client, time.Hour)
// First call: Miss
infos, err := c.ReadDir("/dir")
if err != nil {
t.Fatalf("ReadDir failed: %v", err)
}
if len(infos) != 2 {
t.Errorf("expected 2 files, got %d", len(infos))
}
if c.misses.Load() != 1 {
t.Errorf("expected 1 miss, got %d", c.misses.Load())
}
// Second call: Hit
infos2, err := c.ReadDir("/dir")
if err != nil {
t.Fatalf("ReadDir failed: %v", err)
}
if len(infos2) != 2 {
t.Errorf("expected 2 files, got %d", len(infos2))
}
if c.hits.Load() != 1 {
t.Errorf("expected 1 hit, got %d", c.hits.Load())
}
}
func TestCachedClient_RevalidateStat(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("failed to start miniredis: %v", err)
}
defer mr.Close()
client, err := valkey.NewClient(valkey.ClientOption{
InitAddress: []string{mr.Addr()},
DisableCache: true,
})
if err != nil {
t.Fatalf("failed to create valkey client: %v", err)
}
defer client.Close()
mockFS := NewMockFS()
mockFS.AddFile("/reval.txt", []byte("initial"))
c := NewCachedClient(mockFS, client, time.Hour)
// Populate cache
_, _ = c.Stat("/reval.txt")
// Update mock file (simulate external change)
// For simplicity, just update size
f := mockFS.files["/reval.txt"]
f.content = []byte("updated content")
// Important: update modTime to be newer than cached
f.modTime = time.Now().Add(time.Minute)
// Run revalidation manually
// Note: revalidateStat is a private method, so we can test it directly here
// because we are in package filesystem (if using package filesystem_test we couldn't)
// But `revalidateStat` only iterates over keys in redis. So we need to ensure the key is there.
// `Stat` put it there.
// Wait, Stat puts it there. `revalidateStat` iterates keys matching `keyStat*`.
// For each key, it checks `Stat` again.
// Wait, looking at `revalidateStat` implementation:
// It calls `c.impl.Stat(path)`. If successful, it updates cache.
// It doesn't check ModTime explicitly for `Stat` revalidation, it just blindly updates?
// Let's check `revalidateStat` implementation in `cache.go`.
// func (c *CachedClient) revalidateStat(ctx context.Context) { ...
// info, err := c.impl.Stat(path)
// ...
// bytes, err := json.Marshal(f)
// c.client.Do(...Set...)
// }
// Yes, it blindly updates. So modification time doesn't matter for `Stat`, it just fetches fresh info.
c.revalidateStat(context.Background())
// Verify cache is updated by fetching from cache again
// We can inspect redis directly or trust `Stat` returns cached value (which should be updated).
// Let's inspect redis to be sure.
// Or just call Stat again. It should be a cache HIT, but return NEW values.
// Reset stats to check hit
c.hits.Store(0)
c.misses.Store(0)
info, err := c.Stat("/reval.txt")
if err != nil {
t.Fatalf("Stat failed: %v", err)
}
if info.Size() != int64(len("updated content")) {
t.Errorf("expected size %d, got %d", len("updated content"), info.Size())
}
// Should be a hit because revalidate updated the cache entry, so it exists and is valid.
if c.hits.Load() != 1 {
t.Errorf("expected 1 hit, got %d", c.hits.Load())
}
}
func TestCachedClient_RevalidateRead(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("failed to start miniredis: %v", err)
}
defer mr.Close()
client, err := valkey.NewClient(valkey.ClientOption{
InitAddress: []string{mr.Addr()},
DisableCache: true,
})
if err != nil {
t.Fatalf("failed to create valkey client: %v", err)
}
defer client.Close()
mockFS := NewMockFS()
mockFS.AddFile("/read_reval.txt", []byte("initial"))
c := NewCachedClient(mockFS, client, time.Hour)
// Populate cache
_, _ = c.Read("/read_reval.txt")
// Update mock file (simulate external change)
f := mockFS.files["/read_reval.txt"]
f.content = []byte("updated content")
f.modTime = time.Now().Add(time.Minute) // Newer ModTime triggers update
// Run revalidation manually
c.revalidateRead(context.Background())
// Verify cache is updated
// Reset stats
c.hits.Store(0)
c.misses.Store(0)
data, err := c.Read("/read_reval.txt")
if err != nil {
t.Fatalf("Read failed: %v", err)
}
if string(data) != "updated content" {
t.Errorf("expected updated content, got %s", string(data))
}
if c.hits.Load() != 1 {
t.Errorf("expected 1 hit, got %d", c.hits.Load())
}
}
func TestCachedClient_RevalidateReadDir(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("failed to start miniredis: %v", err)
}
defer mr.Close()
client, err := valkey.NewClient(valkey.ClientOption{
InitAddress: []string{mr.Addr()},
DisableCache: true,
})
if err != nil {
t.Fatalf("failed to create valkey client: %v", err)
}
defer client.Close()
mockFS := NewMockFS()
mockFS.AddDir("/dir_reval")
mockFS.AddFile("/dir_reval/f1.txt", []byte("1"))
c := NewCachedClient(mockFS, client, time.Hour)
// Populate cache
_, err = c.ReadDir("/dir_reval")
if err != nil {
t.Fatalf("ReadDir failed: %v", err)
}
// Update mock file (simulate external change)
mockFS.AddFile("/dir_reval/f2.txt", []byte("2"))
// Run revalidation manually
c.revalidateReadDir(context.Background())
// Verify cache is updated
// Reset stats
c.hits.Store(0)
c.misses.Store(0)
infos2, err := c.ReadDir("/dir_reval")
if err != nil {
t.Fatalf("ReadDir failed: %v", err)
}
if len(infos2) != 2 {
t.Errorf("expected 2 files after revalidation, got %d", len(infos2))
}
if c.hits.Load() != 1 {
t.Errorf("expected 1 hit, got %d", c.hits.Load())
}
}
+109
View File
@@ -0,0 +1,109 @@
package filesystem
import (
"context"
"encoding/json"
"log/slog"
"strings"
"time"
"github.com/valkey-io/valkey-go"
)
const keyRead = "webdav:read:"
type readFile struct {
Age time.Time `json:"age"`
Content []byte `json:"content"`
}
func (c *CachedClient) Read(path string) ([]byte, error) {
key := keyRead + path
ctx := context.Background()
// Try cache
cached, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).AsBytes()
if err == nil && len(cached) > 0 {
c.readHits.Add(1)
c.hits.Add(1)
slog.Debug("cache hit", "key", key)
var file readFile
err := json.Unmarshal(cached, &file)
if err != nil {
slog.Error("could not Unmarshal cache", "key", key, "err", err)
} else {
return file.Content, nil
}
}
// Cache miss
c.misses.Add(1)
slog.Debug("cache miss", "key", key)
data, err := c.impl.Read(path)
if err != nil {
return nil, err
}
file := readFile{
Age: time.Now(),
Content: data,
}
cached, err = json.Marshal(file)
if err != nil {
slog.Error("could not marshal file for cache", "key", key, "err", err)
return data, nil
}
// Cache
c.client.Do(ctx, c.client.B().Set().Key(key).Value(valkey.BinaryString(cached)).Ex(c.ttl).Build())
return data, nil
}
func (c *CachedClient) revalidateRead(ctx context.Context) {
startTime := time.Now()
c.scanAndProcess(ctx, keyRead+"*", func(key string) error {
path := strings.TrimPrefix(key, keyRead)
cached, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).AsBytes()
if err != nil {
// Key might be gone or error fetching
return nil
}
var cachedFile readFile
err = json.Unmarshal(cached, &cachedFile)
if err != nil {
c.client.Do(ctx, c.client.B().Del().Key(key).Build())
return err
}
info, err := c.impl.Stat(path)
if err != nil {
c.client.Do(ctx, c.client.B().Del().Key(key).Build())
return nil
}
if info.ModTime().After(cachedFile.Age) {
data, err := c.impl.Read(path)
if err != nil {
return err
}
cachedFile.Age = time.Now()
cachedFile.Content = data
newCached, err := json.Marshal(cachedFile)
if err != nil {
return err
}
c.client.Do(ctx, c.client.B().Set().Key(key).Value(valkey.BinaryString(newCached)).Ex(c.ttl).Build())
} else {
c.client.Do(ctx, c.client.B().Expire().Key(key).Seconds(int64(c.ttl/time.Second)).Build())
}
return nil
})
slog.Debug("revalidation of read complete", "duration", time.Since(startTime))
}
+79
View File
@@ -0,0 +1,79 @@
package filesystem
import (
"context"
"encoding/json"
"log/slog"
"os"
"strings"
"time"
)
const keyReadDir = "webdav:readdir:"
func (c *CachedClient) ReadDir(path string) ([]os.FileInfo, error) {
key := keyReadDir + path
ctx := context.Background()
// Try cache
val, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).ToString()
if err == nil && val != "" {
var cached []File
if err := json.Unmarshal([]byte(val), &cached); err == nil {
c.readDirHits.Add(1)
c.hits.Add(1)
slog.Debug("cache hit", "key", key)
infos := make([]os.FileInfo, len(cached))
for i, f := range cached {
infos[i] = f
}
return infos, nil
}
}
// Cache miss
c.misses.Add(1)
slog.Debug("cache miss", "key", key)
infos, err := c.impl.ReadDir(path)
if err != nil {
return nil, err
}
// Serialize and cache
cached := make([]File, 0, len(infos))
for _, info := range infos {
cached = append(cached, ParseToFile(info))
}
bytes, err := json.Marshal(cached)
if err == nil {
c.client.Do(ctx, c.client.B().Set().Key(key).Value(string(bytes)).Ex(c.ttl).Build())
}
return infos, nil
}
func (c *CachedClient) revalidateReadDir(ctx context.Context) {
startTime := time.Now()
c.scanAndProcess(ctx, keyReadDir+"*", func(key string) error {
path := strings.TrimPrefix(key, keyReadDir)
infos, err := c.impl.ReadDir(path)
if err != nil {
c.client.Do(ctx, c.client.B().Del().Key(key).Build())
return err
}
// Serialize and cache
cached := make([]File, 0, len(infos))
for _, info := range infos {
cached = append(cached, ParseToFile(info))
}
bytes, err := json.Marshal(cached)
if err == nil {
c.client.Do(ctx, c.client.B().Set().Key(key).Value(string(bytes)).Ex(c.ttl).Build())
}
return nil
})
slog.Debug("revalidation of readDir complete", "duration", time.Since(startTime))
}
+67
View File
@@ -0,0 +1,67 @@
package filesystem
import (
"context"
"encoding/json"
"log/slog"
"os"
"strings"
"time"
)
func (c *CachedClient) Stat(path string) (info os.FileInfo, err error) {
key := keyStat + path
ctx := context.Background()
// Try cache
val, err := c.client.Do(ctx, c.client.B().Get().Key(key).Build()).ToString()
if err == nil && val != "" {
var f File
if err := json.Unmarshal([]byte(val), &f); err == nil {
c.hits.Add(1)
slog.Debug("cache hit", "key", key)
return f, nil
}
}
// Cache miss
c.misses.Add(1)
slog.Debug("cache miss", "key", key)
fileInfo, err := c.impl.Stat(path)
if err != nil {
return
}
// Serialize and cache
// Type assert to ensure we capture ContentType and Path
info = ParseToFile(fileInfo)
bytes, err := json.Marshal(info)
if err == nil {
c.client.Do(ctx, c.client.B().Set().Key(key).Value(string(bytes)).Ex(c.ttl).Build())
}
return
}
func (c *CachedClient) revalidateStat(ctx context.Context) {
startTime := time.Now()
c.scanAndProcess(ctx, keyStat+"*", func(key string) error {
path := strings.TrimPrefix(key, keyStat)
info, err := c.impl.Stat(path)
if err != nil {
c.client.Do(ctx, c.client.B().Del().Key(key).Build())
return err
}
// Serialize and cache
f := ParseToFile(info)
bytes, err := json.Marshal(f)
if err == nil {
c.client.Do(ctx, c.client.B().Set().Key(key).Value(string(bytes)).Ex(c.ttl).Build())
}
return nil
})
slog.Debug("revalidation of Stat complete", "duration", time.Since(startTime))
}
+1 -1
View File
@@ -51,7 +51,7 @@ func registerOther() (h http.Handler) {
if err != nil { if err != nil {
slog.Error("failed to create valkey client", "err", err) slog.Error("failed to create valkey client", "err", err)
} else { } else {
fs = filesystem.NewCachedClient(webdavClient, valkeyClient, 5*time.Minute) fs = filesystem.NewCachedClient(webdavClient, valkeyClient, 5*time.Hour)
} }
_ = handlehome.New(mux, page.New(fs)) _ = handlehome.New(mux, page.New(fs))
+1 -1
View File
@@ -1 +1 @@
exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1 exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1
BIN
View File
Binary file not shown.
File diff suppressed because one or more lines are too long