Merge branch 'nd/sparse'

* nd/sparse: (25 commits)
  t7002: test for not using external grep on skip-worktree paths
  t7002: set test prerequisite "external-grep" if supported
  grep: do not do external grep on skip-worktree entries
  commit: correctly respect skip-worktree bit
  ie_match_stat(): do not ignore skip-worktree bit with CE_MATCH_IGNORE_VALID
  tests: rename duplicate t1009
  sparse checkout: inhibit empty worktree
  Add tests for sparse checkout
  read-tree: add --no-sparse-checkout to disable sparse checkout support
  unpack-trees(): ignore worktree check outside checkout area
  unpack_trees(): apply $GIT_DIR/info/sparse-checkout to the final index
  unpack-trees(): "enable" sparse checkout and load $GIT_DIR/info/sparse-checkout
  unpack-trees.c: generalize verify_* functions
  unpack-trees(): add CE_WT_REMOVE to remove on worktree alone
  Introduce "sparse checkout"
  dir.c: export excluded_1() and add_excludes_from_file_1()
  excluded_1(): support exclude files in index
  unpack-trees(): carry skip-worktree bit over in merged_entry()
  Read .gitignore from index if it is skip-worktree
  Avoid writing to buffer in add_excludes_from_file_1()
  ...

Conflicts:
	.gitignore
	Documentation/config.txt
	Documentation/git-update-index.txt
	Makefile
	entry.c
	t/t7002-grep.sh

diff --cc .gitignore
index 5d32289664bbff40f85a450cff7f2261384204b0,e3a864c0ad54754de437b9d5bbbdba89211fe339..0746a4d2090c5387d7dbe6e9e40b529064e18a8d
--- 1/.gitignore
--- 2/.gitignore
+++ b/.gitignore
@@@ -1,169 -1,165 +1,170 @@@
 -GIT-BUILD-OPTIONS
 -GIT-CFLAGS
 -GIT-GUI-VARS
 -GIT-VERSION-FILE
 -git
 -git-add
 -git-add--interactive
 -git-am
 -git-annotate
 -git-apply
 -git-archimport
 -git-archive
 -git-bisect
 -git-bisect--helper
 -git-blame
 -git-branch
 -git-bundle
 -git-cat-file
 -git-check-attr
 -git-check-ref-format
 -git-checkout
 -git-checkout-index
 -git-cherry
 -git-cherry-pick
 -git-clean
 -git-clone
 -git-commit
 -git-commit-tree
 -git-config
 -git-count-objects
 -git-cvsexportcommit
 -git-cvsimport
 -git-cvsserver
 -git-daemon
 -git-diff
 -git-diff-files
 -git-diff-index
 -git-diff-tree
 -git-difftool
 -git-difftool--helper
 -git-describe
 -git-fast-export
 -git-fast-import
 -git-fetch
 -git-fetch--tool
 -git-fetch-pack
 -git-filter-branch
 -git-fmt-merge-msg
 -git-for-each-ref
 -git-format-patch
 -git-fsck
 -git-fsck-objects
 -git-gc
 -git-get-tar-commit-id
 -git-grep
 -git-hash-object
 -git-help
 -git-http-fetch
 -git-http-push
 -git-imap-send
 -git-index-pack
 -git-init
 -git-init-db
 -git-instaweb
 -git-log
 -git-lost-found
 -git-ls-files
 -git-ls-remote
 -git-ls-tree
 -git-mailinfo
 -git-mailsplit
 -git-merge
 -git-merge-base
 -git-merge-index
 -git-merge-file
 -git-merge-tree
 -git-merge-octopus
 -git-merge-one-file
 -git-merge-ours
 -git-merge-recursive
 -git-merge-resolve
 -git-merge-subtree
 -git-mergetool
 -git-mergetool--lib
 -git-mktag
 -git-mktree
 -git-name-rev
 -git-mv
 -git-pack-redundant
 -git-pack-objects
 -git-pack-refs
 -git-parse-remote
 -git-patch-id
 -git-peek-remote
 -git-prune
 -git-prune-packed
 -git-pull
 -git-push
 -git-quiltimport
 -git-read-tree
 -git-rebase
 -git-rebase--interactive
 -git-receive-pack
 -git-reflog
 -git-relink
 -git-remote
 -git-repack
 -git-repo-config
 -git-request-pull
 -git-rerere
 -git-reset
 -git-rev-list
 -git-rev-parse
 -git-revert
 -git-rm
 -git-send-email
 -git-send-pack
 -git-sh-setup
 -git-shell
 -git-shortlog
 -git-show
 -git-show-branch
 -git-show-index
 -git-show-ref
 -git-stage
 -git-stash
 -git-status
 -git-stripspace
 -git-submodule
 -git-svn
 -git-symbolic-ref
 -git-tag
 -git-tar-tree
 -git-unpack-file
 -git-unpack-objects
 -git-update-index
 -git-update-ref
 -git-update-server-info
 -git-upload-archive
 -git-upload-pack
 -git-var
 -git-verify-pack
 -git-verify-tag
 -git-web--browse
 -git-whatchanged
 -git-write-tree
 -git-core-*/?*
 -gitk-wish
 -gitweb/gitweb.cgi
 -test-chmtime
 -test-ctype
 -test-date
 -test-delta
 -test-dump-cache-tree
 -test-genrandom
 -test-index-version
 -test-match-trees
 -test-parse-options
 -test-path-utils
 -test-sha1
 -test-sigchain
 -common-cmds.h
 +/GIT-BUILD-OPTIONS
 +/GIT-CFLAGS
 +/GIT-GUI-VARS
 +/GIT-VERSION-FILE
 +/bin-wrappers/
 +/git
 +/git-add
 +/git-add--interactive
 +/git-am
 +/git-annotate
 +/git-apply
 +/git-archimport
 +/git-archive
 +/git-bisect
 +/git-bisect--helper
 +/git-blame
 +/git-branch
 +/git-bundle
 +/git-cat-file
 +/git-check-attr
 +/git-check-ref-format
 +/git-checkout
 +/git-checkout-index
 +/git-cherry
 +/git-cherry-pick
 +/git-clean
 +/git-clone
 +/git-commit
 +/git-commit-tree
 +/git-config
 +/git-count-objects
 +/git-cvsexportcommit
 +/git-cvsimport
 +/git-cvsserver
 +/git-daemon
 +/git-diff
 +/git-diff-files
 +/git-diff-index
 +/git-diff-tree
 +/git-difftool
 +/git-difftool--helper
 +/git-describe
 +/git-fast-export
 +/git-fast-import
 +/git-fetch
 +/git-fetch--tool
 +/git-fetch-pack
 +/git-filter-branch
 +/git-fmt-merge-msg
 +/git-for-each-ref
 +/git-format-patch
 +/git-fsck
 +/git-fsck-objects
 +/git-gc
 +/git-get-tar-commit-id
 +/git-grep
 +/git-hash-object
 +/git-help
 +/git-http-backend
 +/git-http-fetch
 +/git-http-push
 +/git-imap-send
 +/git-index-pack
 +/git-init
 +/git-init-db
 +/git-instaweb
 +/git-log
 +/git-lost-found
 +/git-ls-files
 +/git-ls-remote
 +/git-ls-tree
 +/git-mailinfo
 +/git-mailsplit
 +/git-merge
 +/git-merge-base
 +/git-merge-index
 +/git-merge-file
 +/git-merge-tree
 +/git-merge-octopus
 +/git-merge-one-file
 +/git-merge-ours
 +/git-merge-recursive
 +/git-merge-resolve
 +/git-merge-subtree
 +/git-mergetool
 +/git-mergetool--lib
 +/git-mktag
 +/git-mktree
 +/git-name-rev
 +/git-mv
 +/git-notes
 +/git-pack-redundant
 +/git-pack-objects
 +/git-pack-refs
 +/git-parse-remote
 +/git-patch-id
 +/git-peek-remote
 +/git-prune
 +/git-prune-packed
 +/git-pull
 +/git-push
 +/git-quiltimport
 +/git-read-tree
 +/git-rebase
 +/git-rebase--interactive
 +/git-receive-pack
 +/git-reflog
 +/git-relink
 +/git-remote
 +/git-remote-curl
 +/git-repack
 +/git-replace
 +/git-repo-config
 +/git-request-pull
 +/git-rerere
 +/git-reset
 +/git-rev-list
 +/git-rev-parse
 +/git-revert
 +/git-rm
 +/git-send-email
 +/git-send-pack
 +/git-sh-setup
 +/git-shell
 +/git-shortlog
 +/git-show
 +/git-show-branch
 +/git-show-index
 +/git-show-ref
 +/git-stage
 +/git-stash
 +/git-status
 +/git-stripspace
 +/git-submodule
 +/git-svn
 +/git-symbolic-ref
 +/git-tag
 +/git-tar-tree
 +/git-unpack-file
 +/git-unpack-objects
 +/git-update-index
 +/git-update-ref
 +/git-update-server-info
 +/git-upload-archive
 +/git-upload-pack
 +/git-var
 +/git-verify-pack
 +/git-verify-tag
 +/git-web--browse
 +/git-whatchanged
 +/git-write-tree
 +/git-core-*/?*
 +/gitk-git/gitk-wish
 +/gitweb/gitweb.cgi
 +/test-chmtime
 +/test-ctype
 +/test-date
 +/test-delta
 +/test-dump-cache-tree
 +/test-genrandom
++/test-index-version
 +/test-match-trees
 +/test-parse-options
 +/test-path-utils
 +/test-sha1
 +/test-sigchain
 +/common-cmds.h
  *.tar.gz
  *.dsc
  *.deb

diff --cc Documentation/config.txt
index 9f40955f8e74a0263019eb228a548ffd2b1fd301,5825c914fbe9bbf3d8d9317b67648ca27703e60d..8acb613ec3a56dbe76459075a31fd4e66c218991
--- 1/Documentation/config.txt
--- 2/Documentation/config.txt
+++ b/Documentation/config.txt
@@@ -489,19 -439,10 +489,23 @@@ On some file system/operating system co
  Set this config setting to 'rename' there; However, This will remove the
  check that makes sure that existing object files will not get overwritten.
  
 +core.notesRef::
 +      When showing commit messages, also show notes which are stored in
 +      the given ref.  This ref is expected to contain files named
 +      after the full SHA-1 of the commit they annotate.
 ++
 +If such a file exists in the given ref, the referenced blob is read, and
 +appended to the commit message, separated by a "Notes:" line.  If the
 +given ref itself does not exist, it is not an error, but means that no
 +notes should be printed.
 ++
 +This setting defaults to "refs/notes/commits", and can be overridden by
 +the `GIT_NOTES_REF` environment variable.
 +
+ core.sparseCheckout::
+       Enable "sparse checkout" feature. See section "Sparse checkout" in
+       linkgit:git-read-tree[1] for more information.
+ 
  add.ignore-errors::
        Tells 'git-add' to continue adding files when some files cannot be
        added due to indexing errors. Equivalent to the '--ignore-errors'

diff --cc Documentation/git-ls-files.txt
Simple merge

diff --cc Documentation/git-read-tree.txt
Simple merge

diff --cc Documentation/git-update-index.txt
index 6052484ab9e28d5565067885b1c4ed5667e8e90f,a10f355b7cf6a11f936131a03fb9efab9ee5e2ea..8d88018eedd08511e2676013d88c8d44b10e97ac
--- 1/Documentation/git-update-index.txt
--- 2/Documentation/git-update-index.txt
+++ b/Documentation/git-update-index.txt
@@@ -99,10 -100,13 +100,17 @@@ in the index e.g. when merging in a com
  thus, in case the assumed-untracked file is changed upstream,
  you will need to handle the situation manually.
  
 +--really-refresh::
 +      Like '--refresh', but checks stat information unconditionally,
 +      without regard to the "assume unchanged" setting.
 +
+ --skip-worktree::
+ --no-skip-worktree::
+       When one of these flags is specified, the object name recorded
+       for the paths are not updated. Instead, these options
+       set and unset the "skip-worktree" bit for the paths. See
+       section "Skip-worktree bit" below for more information.
+ 
  -g::
  --again::
        Runs 'git-update-index' itself on the paths whose index

diff --cc Makefile
index be1838953451f78443ab1a97f569428b2001fcfb,3c5b890223f3795e6db120a725dc7f35d5df3d2c..412dda83ac6a9cf76682153b5d66c274e2a362fd
--- 1/Makefile
--- 2/Makefile
+++ b/Makefile
@@@ -1771,30 -1569,20 +1771,31 @@@ endi
  
  ### Testing rules
  
 -TEST_PROGRAMS += test-chmtime$X
 -TEST_PROGRAMS += test-ctype$X
 -TEST_PROGRAMS += test-date$X
 -TEST_PROGRAMS += test-delta$X
 -TEST_PROGRAMS += test-dump-cache-tree$X
 -TEST_PROGRAMS += test-genrandom$X
 -TEST_PROGRAMS += test-match-trees$X
 -TEST_PROGRAMS += test-parse-options$X
 -TEST_PROGRAMS += test-path-utils$X
 -TEST_PROGRAMS += test-sha1$X
 -TEST_PROGRAMS += test-sigchain$X
 -TEST_PROGRAMS += test-index-version$X
 -
 -all:: $(TEST_PROGRAMS)
 +TEST_PROGRAMS_NEED_X += test-chmtime
 +TEST_PROGRAMS_NEED_X += test-ctype
 +TEST_PROGRAMS_NEED_X += test-date
 +TEST_PROGRAMS_NEED_X += test-delta
 +TEST_PROGRAMS_NEED_X += test-dump-cache-tree
 +TEST_PROGRAMS_NEED_X += test-genrandom
 +TEST_PROGRAMS_NEED_X += test-match-trees
 +TEST_PROGRAMS_NEED_X += test-parse-options
 +TEST_PROGRAMS_NEED_X += test-path-utils
 +TEST_PROGRAMS_NEED_X += test-sha1
 +TEST_PROGRAMS_NEED_X += test-sigchain
++TEST_PROGRAMS_NEED_X += test-index-version
 +
 +TEST_PROGRAMS = $(patsubst %,%$X,$(TEST_PROGRAMS_NEED_X))
 +
 +test_bindir_programs := $(patsubst %,bin-wrappers/%,$(BINDIR_PROGRAMS_NEED_X) $(BINDIR_PROGRAMS_NO_X) $(TEST_PROGRAMS_NEED_X))
 +
 +all:: $(TEST_PROGRAMS) $(test_bindir_programs)
 +
 +bin-wrappers/%: wrap-for-bin.sh
 +      @mkdir -p bin-wrappers
 +      $(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \
 +           -e 's|@@BUILD_DIR@@|$(shell pwd)|' \
 +           -e 's|@@PROG@@|$(@F)|' < $< > $@ && \
 +      chmod +x $@
  
  # GNU make supports exporting all variables by "export" without parameters.
  # However, the environment gets quite big, and some programs have problems

diff --cc builtin-apply.c
Simple merge

diff --cc builtin-clean.c
index 28cdcd0274dd5891c7ad3dc3804cb0acead883f2,e424b77e6b7367a5704d586ab38d34908684de55..3a70fa81bdd8f0e087476560f7c2f668a3cda0ea
--- 1/builtin-clean.c
--- 2/builtin-clean.c
+++ b/builtin-clean.c
@@@ -70,11 -69,11 +70,14 @@@ int cmd_clean(int argc, const char **ar
                die("clean.requireForce%s set and -n or -f not given; "
                    "refusing to clean", config_set ? "" : " not");
  
 +      if (force > 1)
 +              rm_flags = 0;
 +
        dir.flags |= DIR_SHOW_OTHER_DIRECTORIES;
  
+       if (read_cache() < 0)
+               die("index file corrupt");
+ 
        if (!ignored)
                setup_standard_excludes(&dir);
  

diff --cc builtin-commit.c
Simple merge

diff --cc builtin-grep.c
Simple merge

diff --cc builtin-ls-files.c
Simple merge

diff --cc builtin-read-tree.c
Simple merge

diff --cc builtin-update-index.c
index a6b7f2d6361fc1e8ea9713082affc72362f08d76,97b9ea61f7605970875776bd5c84c00abbb0f2f6..a64a752990b85b69590cdbeed3901154642007b8
--- 1/builtin-update-index.c
--- 2/builtin-update-index.c
+++ b/builtin-update-index.c
@@@ -24,10 -24,10 +24,11 @@@ static int info_only
  static int force_remove;
  static int verbose;
  static int mark_valid_only;
- #define MARK_VALID 1
- #define UNMARK_VALID 2
+ static int mark_skip_worktree_only;
+ #define MARK_FLAG 1
+ #define UNMARK_FLAG 2
  
 +__attribute__((format (printf, 1, 2)))
  static void report(const char *fmt, ...)
  {
        va_list vp;

diff --cc cache.h
index 3f9ee861477883a5a9f27aea768a8c070559a891,9dcaf41a6c7aa4cd97f37eae33dd0aa7cc45753f..46606477b626c9bdb6a60f9c0cf826c8fc435e00
--- 1/cache.h
--- 2/cache.h
+++ b/cache.h
@@@ -526,9 -526,9 +533,10 @@@ extern size_t packed_git_window_size
  extern size_t packed_git_limit;
  extern size_t delta_base_cache_limit;
  extern int auto_crlf;
 +extern int read_replace_refs;
  extern int fsync_object_files;
  extern int core_preload_index;
+ extern int core_apply_sparse_checkout;
  
  enum safe_crlf {
        SAFE_CRLF_FALSE = 0,

diff --cc config.c
Simple merge

diff --cc diff-lib.c
Simple merge

diff --cc diff.c
Simple merge

diff --cc dir.c
Simple merge

diff --cc dir.h
Simple merge

diff --cc entry.c
index 06d24f14c6ba9401637aebfb11659ae747796c06,efee21fe199768ac1e69578e2c3f4d210f484aa4..9d5b232781c6760930c354c1792ad6b924f8a72d
--- 1/entry.c
--- 2/entry.c
+++ b/entry.c
@@@ -205,8 -201,8 +205,8 @@@ int checkout_entry(struct cache_entry *
        strcpy(path + len, ce->name);
        len += ce_namelen(ce);
  
 -      if (!check_path(path, len, &st)) {
 +      if (!check_path(path, len, &st, state->base_dir_len)) {
-               unsigned changed = ce_match_stat(ce, &st, CE_MATCH_IGNORE_VALID);
+               unsigned changed = ce_match_stat(ce, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
                if (!changed)
                        return 0;
                if (!state->force) {

diff --cc environment.c
index 5171d9f9a4a30054983638e9e19e61f422724373,020422c03430a7bab3c69c36710671dd93e0839b..739ec2704031f0dd9d4a380e07f001c09742a7be
--- 1/environment.c
--- 2/environment.c
+++ b/environment.c
@@@ -49,8 -47,8 +49,9 @@@ enum push_default_type push_default = P
  #define OBJECT_CREATION_MODE OBJECT_CREATION_USES_HARDLINKS
  #endif
  enum object_creation_mode object_creation_mode = OBJECT_CREATION_MODE;
 +char *notes_ref_name;
  int grafts_replace_parents = 1;
+ int core_apply_sparse_checkout;
  
  /* Parallel index stat data preload? */
  int core_preload_index = 0;

diff --cc read-cache.c
Simple merge

diff --cc t/t7002-grep.sh
index abd14bf819f5c60fc1b9dc758c04974bd24b10a4,99142fd6bda0e9c241bad1c18a18583b30fda700..76c5e091b7d45bde38f34dc9726c51034ee7f9eb
--- 1/t/t7002-grep.sh
--- 2/t/t7002-grep.sh
+++ b/t/t7002-grep.sh
@@@ -400,30 -340,16 +412,42 @@@ test_expect_success 'grep -p -B5' 
        test_cmp expected actual
  '
  
 +test_expect_success 'grep from a subdirectory to search wider area (1)' '
 +      mkdir -p s &&
 +      (
 +              cd s && git grep "x x x" ..
 +      )
 +'
 +
 +test_expect_success 'grep from a subdirectory to search wider area (2)' '
 +      mkdir -p s &&
 +      (
 +              cd s || exit 1
 +              ( git grep xxyyzz .. >out ; echo $? >status )
 +              ! test -s out &&
 +              test 1 = $(cat status)
 +      )
 +'
 +
 +cat >expected <<EOF
 +hello.c:int main(int argc, const char **argv)
 +EOF
 +
 +test_expect_success 'grep -Fi' '
 +      git grep -Fi "CHAR *" >actual &&
 +      test_cmp expected actual
 +'
 +
+ test_expect_success EXTGREP 'external grep is called' '
+       GIT_TRACE=2 git grep foo >/dev/null 2>actual &&
+       grep "trace: grep:.*foo" actual >/dev/null
+ '
+ 
+ test_expect_success EXTGREP 'no external grep when skip-worktree entries exist' '
+       git update-index --skip-worktree file &&
+       GIT_TRACE=2 git grep foo >/dev/null 2>actual &&
+       ! grep "trace: grep:" actual >/dev/null &&
+       git update-index --no-skip-worktree file
+ '
+ 
  test_done

diff --cc t/t7300-clean.sh
Simple merge

diff --cc unpack-trees.c
Simple merge