Reuse temporary files (fixes #4)

internal/config/config.go

@@ -145,6 +145,7 @@ type OptionsConfiguration struct {
 	URAccepted           int      `xml:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
 	RestartOnWakeup      bool     `xml:"restartOnWakeup" default:"true"`
 	AutoUpgradeIntervalH int      `xml:"autoUpgradeIntervalH" default:"12"` // 0 for off
+	KeepTemporariesH     int      `xml:"keepTemporariesH" default:"24"`     // 0 for off
 
 	Deprecated_RescanIntervalS int    `xml:"rescanIntervalS,omitempty" json:"-"`
 	Deprecated_UREnabled       bool   `xml:"urEnabled,omitempty" json:"-"`

internal/config/config_test.go

@@ -49,6 +49,7 @@ func TestDefaultValues(t *testing.T) {
 		UPnPRenewal:          30,
 		RestartOnWakeup:      true,
 		AutoUpgradeIntervalH: 12,
+		KeepTemporariesH:     24,
 	}
 
 	cfg := New("test", device1)
@@ -141,6 +142,7 @@ func TestOverriddenValues(t *testing.T) {
 		UPnPRenewal:          15,
 		RestartOnWakeup:      false,
 		AutoUpgradeIntervalH: 24,
+		KeepTemporariesH:     48,
 	}
 
 	cfg, err := Load("testdata/overridenvalues.xml", device1)

internal/config/testdata/overridenvalues.xml

@@ -17,5 +17,6 @@
         <upnpRenewalMinutes>15</upnpRenewalMinutes>
         <restartOnWakeup>false</restartOnWakeup>
         <autoUpgradeIntervalH>24</autoUpgradeIntervalH>
+        <keepTemporariesH>48</keepTemporariesH>
     </options>
 </configuration>

internal/model/puller.go

@@ -412,12 +412,62 @@ func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksSt
 	tempName := filepath.Join(p.dir, defTempNamer.TempName(file.Name))
 	realName := filepath.Join(p.dir, file.Name)
 
+	var reuse bool
+
+	// Check for an old temporary file which might have some blocks we could
+	// reuse.
+	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize)
+	if err == nil {
+		// Check for any reusable blocks in the temp file
+		tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
+
+		// block.String() returns a string unique to the block
+		existingBlocks := make(map[string]bool, len(tempCopyBlocks))
+		for _, block := range tempCopyBlocks {
+			existingBlocks[block.String()] = true
+		}
+
+		// Since the blocks are already there, we don't need to copy them
+		// nor we need to pull them, hence discard blocks which are already
+		// there, if they are exactly the same...
+		var newCopyBlocks []protocol.BlockInfo
+		for _, block := range copyBlocks {
+			_, ok := existingBlocks[block.String()]
+			if !ok {
+				newCopyBlocks = append(newCopyBlocks, block)
+			}
+		}
+
+		var newPullBlocks []protocol.BlockInfo
+		for _, block := range pullBlocks {
+			_, ok := existingBlocks[block.String()]
+			if !ok {
+				newPullBlocks = append(newPullBlocks, block)
+			}
+		}
+
+		// If any blocks could be reused, let the sharedpullerstate know
+		// which flags it is expected to set on the file.
+		// Also update the list of work for the routines.
+		if len(copyBlocks) != len(newCopyBlocks) || len(pullBlocks) != len(newPullBlocks) {
+			reuse = true
+			copyBlocks = newCopyBlocks
+			pullBlocks = newPullBlocks
+		} else {
+			// Otherwise, discard the file ourselves in order for the
+			// sharedpuller not to panic when it fails to exlusively create a
+			// file which already exists
+			os.Remove(tempName)
+		}
+	}
+
 	s := sharedPullerState{
 		file:       file,
 		folder:     p.folder,
 		tempName:   tempName,
 		realName:   realName,
 		pullNeeded: len(pullBlocks),
+		reuse:      reuse,
 	}
 	if len(copyBlocks) > 0 {
 		s.copyNeeded = 1
@@ -628,12 +678,14 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 
 // clean deletes orphaned temporary files
 func (p *Puller) clean() {
+	keep := time.Duration(p.model.cfg.Options.KeepTemporariesH) * time.Hour
+	now := time.Now()
 	filepath.Walk(p.dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) {
+		if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) && info.ModTime().Add(keep).Before(now) {
 			os.Remove(path)
 		}
 

internal/model/sharedpullerstate.go

@@ -31,6 +31,7 @@ type sharedPullerState struct {
 	folder   string
 	tempName string
 	realName string
+	reuse    bool
 
 	// Mutable, must be locked for access
 	err        error      // The first error we hit
@@ -77,7 +78,11 @@ func (s *sharedPullerState) tempFile() (*os.File, error) {
 	}
 
 	// Attempt to create the temp file
-	fd, err := os.OpenFile(s.tempName, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
+	flags := os.O_WRONLY
+	if !s.reuse {
+		flags |= os.O_CREATE | os.O_EXCL
+	}
+	fd, err := os.OpenFile(s.tempName, flags, 0644)
 	if err != nil {
 		s.earlyCloseLocked("dst create", err)
 		return nil, err

internal/scanner/blockqueue.go

@@ -34,7 +34,7 @@ func newParallelHasher(dir string, blockSize, workers int, outbox, inbox chan pr
 
 	for i := 0; i < workers; i++ {
 		go func() {
-			hashFile(dir, blockSize, outbox, inbox)
+			hashFiles(dir, blockSize, outbox, inbox)
 			wg.Done()
 		}()
 	}
@@ -45,32 +45,35 @@ func newParallelHasher(dir string, blockSize, workers int, outbox, inbox chan pr
 	}()
 }
 
-func hashFile(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
+func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
+	fd, err := os.Open(path)
+	if err != nil {
+		if debug {
+			l.Debugln("open:", err)
+		}
+		return []protocol.BlockInfo{}, err
+	}
+
+	fi, err := fd.Stat()
+	if err != nil {
+		fd.Close()
+		if debug {
+			l.Debugln("stat:", err)
+		}
+		return []protocol.BlockInfo{}, err
+	}
+	defer fd.Close()
+	return Blocks(fd, blockSize, fi.Size())
+}
+
+func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
 	for f := range inbox {
 		if protocol.IsDirectory(f.Flags) || protocol.IsDeleted(f.Flags) {
 			outbox <- f
 			continue
 		}
 
-		fd, err := os.Open(filepath.Join(dir, f.Name))
-		if err != nil {
-			if debug {
-				l.Debugln("open:", err)
-			}
-			continue
-		}
-
-		fi, err := fd.Stat()
-		if err != nil {
-			fd.Close()
-			if debug {
-				l.Debugln("stat:", err)
-			}
-			continue
-		}
-		blocks, err := Blocks(fd, blockSize, fi.Size())
-		fd.Close()
-
+		blocks, err := HashFile(filepath.Join(dir, f.Name), blockSize)
 		if err != nil {
 			if debug {
 				l.Debugln("hash error:", f.Name, err)