lukegb/depot
1
0
Fork 0
Code Issues 1 Pull requests Projects Packages Activity Actions

bcacheup: add --deep_check_galactic flag for ensuring that all references are properly uploaded

Browse source
This commit is contained in:
Luke Granger-Brown 2022-11-08 15:25:48 +00:00
parent 1be0098156
commit 78d3689dfe
1 changed files with 19 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
go/nix/bcacheup/bcacheup.go
View file

@ -32,6 +32,7 @@ import (
var ( var (
blobURLFlag = flag.String("cache_url", "", "Cache URL") blobURLFlag = flag.String("cache_url", "", "Cache URL")
stateSummaryIntervalFlag = flag.Duration("state_summary_interval", 10*time.Second, "Time between state summary outputs.") stateSummaryIntervalFlag = flag.Duration("state_summary_interval", 10*time.Second, "Time between state summary outputs.")
deepCheckGalacticFlag = flag.Bool("deep_check_galactic", false, "Ensure that all references are available in the cache before skipping, rather than just checking that the path itself is available.")
) )
var ( var (
@ -159,7 +160,8 @@ type uploader struct {
storePath string storePath string
st stateTracker st stateTracker
uploadSF singleflight.Group uploadSF singleflight.Group
deepCheckGalactic bool // if true, don't skip if this item is already present; always check the references to make sure they exist too.
} }
type byteCounterWriter struct{ n uint64 } type byteCounterWriter struct{ n uint64 }
@ -314,15 +316,19 @@ func (u *uploader) uploadRefs(ctx context.Context, current string, refs []string
func (u *uploader) upload(ctx context.Context, path string) error { func (u *uploader) upload(ctx context.Context, path string) error {
u.st.SetState(path, stateCheckingShouldUpload) u.st.SetState(path, stateCheckingShouldUpload)
if ok, err := u.shouldUpload(ctx, path); err != nil { shouldUploadThis, err := u.shouldUpload(ctx, path)
if err != nil {
u.st.SetState(path, stateFailed) u.st.SetState(path, stateFailed)
return fmt.Errorf("determining if we should upload %v: %w", path, err) return fmt.Errorf("determining if we should upload %v: %w", path, err)
} else if !ok { }
if !shouldUploadThis && !u.deepCheckGalactic {
u.st.SetState(path, stateSkipped) u.st.SetState(path, stateSkipped)
return nil return nil
} }
log.Printf("Uploading %v", path) if shouldUploadThis {
log.Printf("Uploading %v", path)
}
ni, err := u.store.NARInfo(path) ni, err := u.store.NARInfo(path)
if err != nil { if err != nil {
@ -336,6 +342,11 @@ func (u *uploader) upload(ctx context.Context, path string) error {
return fmt.Errorf("uploading references for %v: %w", path, err) return fmt.Errorf("uploading references for %v: %w", path, err)
} }
if !shouldUploadThis {
u.st.SetState(path, stateSkipped)
return nil
}
u.st.SetState(path, stateUploadingContent) u.st.SetState(path, stateUploadingContent)
if !ni.NarHash.Valid() { if !ni.NarHash.Valid() {
u.st.SetState(path, stateFailed) u.st.SetState(path, stateFailed)
@ -427,9 +438,10 @@ func main() {
defer store.Close() defer store.Close()
u := &uploader{ u := &uploader{
bucket: bucket, bucket: bucket,
store: store, store: store,
storePath: "/nix/store", storePath: "/nix/store",
deepCheckGalactic: *deepCheckGalacticFlag,
} }
go func() { go func() {