Added ContinueOnError flag to allow indexing of repositories with bro…

…ken / malformed files.

Change-Id: Ic155cb854d367d4a0910dad65b3801303816b08d

build/builder.go

@@ -73,6 +73,9 @@ type Options struct {
 
 	// Write memory profiles to this file.
 	MemProfile string
+
+	// continue indexing and skip malformed documents
+	ContinueOnError bool
 }
 
 // Builder manages (parallel) creation of uniformly sized shards.
@@ -435,6 +438,10 @@ func (b *Builder) buildShard(todo []*zoekt.Document, nextShardNum int) (*finishe
 	sortDocuments(todo)
 	for _, t := range todo {
 		if err := shardBuilder.Add(*t); err != nil {
+			if b.opts.ContinueOnError {
+				log.Printf("failed to add document %s : %s", t.Name, err)
+				continue
+			}
 			return nil, err
 		}
 	}

gitindex/index.go

@@ -304,6 +304,12 @@ type Options struct {
 
 	// List of branch names to index, e.g. []string{"HEAD", "stable"}
 	Branches []string
+
+	// Continue indexing on errors / ( ignore malformed documents, missing blobs in repo ....)
+	ContinueOnError bool
+
+	// List all documents which are added
+	Debug bool
 }
 
 func expandBranches(repo *git.Repository, bs []string, prefix string) ([]string, error) {
@@ -473,33 +479,56 @@ func IndexGitRepo(opts Options) error {
 			brs := branchMap[key]
 			blob, err := repos[key].Repo.BlobObject(key.ID)
 			if err != nil {
+				if opts.ContinueOnError {
+					log.Printf("%s failed to load blob %s : %s", opts.RepoDir, key.ID, err)
+					continue
+				}
 				return err
 			}
 
 			if blob.Size > int64(opts.BuildOptions.SizeMax) {
-				if err := builder.Add(zoekt.Document{
+				doc := zoekt.Document{
 					SkipReason:        fmt.Sprintf("file size %d exceeds maximum size %d", blob.Size, opts.BuildOptions.SizeMax),
 					Name:              key.FullPath(),
 					Branches:          brs,
 					SubRepositoryPath: key.SubRepoPath,
-				}); err != nil {
+				}
+				err := builder.Add(doc)
+				if err != nil {
+					if opts.ContinueOnError {
+						log.Printf("%s failed to add document %s : %s", opts.RepoDir, doc.Name, err)
+						continue
+					}
 					return err
 				}
 				continue
 			}
 
 			contents, err := blobContents(blob)
 			if err != nil {
+				if opts.ContinueOnError {
+					log.Printf("%s failed to load blob content %s: %s", opts.RepoDir, key.ID, err)
+					continue
+				}
 				return err
 			}
-			if err := builder.Add(zoekt.Document{
+			doc := zoekt.Document{
 				SubRepositoryPath: key.SubRepoPath,
 				Name:              key.FullPath(),
 				Content:           contents,
 				Branches:          brs,
-			}); err != nil {
+			}
+			err = builder.Add(doc)
+			if err != nil {
+				if opts.ContinueOnError {
+					log.Printf("%s failed to add document %s : %s", opts.RepoDir, doc.Name, err)
+					continue
+				}
 				return err
 			}
+			if opts.Debug {
+				log.Printf("added document %s", doc.Name)
+			}
 		}
 	}
 	return builder.Finish()