restic/cmd/restic/cmd_rebuild_index.go

package main

import (
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "Build a new index",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(rebuildIndexOptions, globalOptions)
	},
}

// RebuildIndexOptions collects all options for the rebuild-index command.
type RebuildIndexOptions struct {
	ReadAllPacks bool
}

var rebuildIndexOptions RebuildIndexOptions

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
	f := cmdRebuildIndex.Flags()
	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")

}

func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(gopts.ctx, repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
}

func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
	ctx := gopts.ctx

	var obsoleteIndexes restic.IDs
	packSizeFromList := make(map[restic.ID]int64)
	removePacks := restic.NewIDSet()
	totalPacks := 0

	if opts.ReadAllPacks {
		// get old index files
		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
			obsoleteIndexes = append(obsoleteIndexes, id)
			return nil
		})
		if err != nil {
			return err
		}

		Verbosef("finding pack files in repo...\n")
		err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
			packSizeFromList[id] = size
			removePacks.Insert(id)
			totalPacks++
			return nil
		})
		if err != nil {
			return err
		}
	} else {
		Verbosef("loading indexes...\n")
		err := repo.LoadIndex(gopts.ctx)
		if err != nil {
			return err
		}

		Verbosef("getting pack files to read...\n")
		packSizeFromIndex := repo.Index().PackSize(ctx, false)

		err = repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
			size, ok := packSizeFromIndex[id]
			if !ok || size != packSize {
				// Pack was not referenced in index or size does not match
				packSizeFromList[id] = packSize
				removePacks.Insert(id)
			}
			totalPacks++
			delete(packSizeFromIndex, id)
			return nil
		})
		if err != nil {
			return err
		}
		for id := range packSizeFromIndex {
			// forget pack files that are referenced in the index but do not exist
			// when rebuilding the index
			removePacks.Insert(id)
		}
	}

	if len(packSizeFromList) > 0 {
		Verbosef("reading pack files\n")
		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
		if err != nil {
			return err
		}

		for _, id := range invalidFiles {
			Verboseff("skipped incomplete pack file: %v\n", id)
			totalPacks--
		}
	}

	err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes, uint64(totalPacks))
	if err != nil {
		return err
	}
	Verbosef("done\n")

	return nil
}