diff --git a/go/nix/bcacheup/bcacheup.go b/go/nix/bcacheup/bcacheup.go index e44a3a6eb5..3f7777504a 100644 --- a/go/nix/bcacheup/bcacheup.go +++ b/go/nix/bcacheup/bcacheup.go @@ -32,6 +32,7 @@ import ( var ( blobURLFlag = flag.String("cache_url", "", "Cache URL") stateSummaryIntervalFlag = flag.Duration("state_summary_interval", 10*time.Second, "Time between state summary outputs.") + deepCheckGalacticFlag = flag.Bool("deep_check_galactic", false, "Ensure that all references are available in the cache before skipping, rather than just checking that the path itself is available.") ) var ( @@ -159,7 +160,8 @@ type uploader struct { storePath string st stateTracker - uploadSF singleflight.Group + uploadSF singleflight.Group + deepCheckGalactic bool // if true, don't skip if this item is already present; always check the references to make sure they exist too. } type byteCounterWriter struct{ n uint64 } @@ -314,15 +316,19 @@ func (u *uploader) uploadRefs(ctx context.Context, current string, refs []string func (u *uploader) upload(ctx context.Context, path string) error { u.st.SetState(path, stateCheckingShouldUpload) - if ok, err := u.shouldUpload(ctx, path); err != nil { + shouldUploadThis, err := u.shouldUpload(ctx, path) + if err != nil { u.st.SetState(path, stateFailed) return fmt.Errorf("determining if we should upload %v: %w", path, err) - } else if !ok { + } + if !shouldUploadThis && !u.deepCheckGalactic { u.st.SetState(path, stateSkipped) return nil } - log.Printf("Uploading %v", path) + if shouldUploadThis { + log.Printf("Uploading %v", path) + } ni, err := u.store.NARInfo(path) if err != nil { @@ -336,6 +342,11 @@ func (u *uploader) upload(ctx context.Context, path string) error { return fmt.Errorf("uploading references for %v: %w", path, err) } + if !shouldUploadThis { + u.st.SetState(path, stateSkipped) + return nil + } + u.st.SetState(path, stateUploadingContent) if !ni.NarHash.Valid() { u.st.SetState(path, stateFailed) @@ -427,9 +438,10 @@ func main() { defer store.Close() u := &uploader{ - bucket: bucket, - store: store, - storePath: "/nix/store", + bucket: bucket, + store: store, + storePath: "/nix/store", + deepCheckGalactic: *deepCheckGalacticFlag, } go func() {