fetch/pull: Add the --recurse-submodules option

Until now you had to call "git submodule update" (without -N|--no-fetch
option) or something like "git submodule foreach git fetch" to fetch
new commits in populated submodules from their remote.

This could lead to "(commits not present)" messages in the output of
"git diff --submodule" (which is used by "git gui" and "gitk") after
fetching or pulling new commits in the superproject and is an obstacle for
implementing recursive checkout of submodules. Also "git submodule
update" cannot fetch changes when disconnected, so it was very easy to
forget to fetch the submodule changes before disconnecting only to
discover later that they are needed.

This patch adds the "--recurse-submodules" option to recursively fetch
each populated submodule from the url configured in the .git/config of the
submodule at the end of each "git fetch" or during "git pull" in the
superproject. The submodule paths are taken from the index.

The hidden option "--submodule-prefix" is added to "git fetch" to be able
to print out the full paths of nested submodules.

Signed-off-by: Jens Lehmann <Jens.Lehmann@web.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 9333c42c558464ce2430df39e9864f932e786b36..7f8d562efc2bde302ca23445a616d1a9b5802e32 100644 (file)
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -60,6 +60,16 @@ endif::git-pull[]
        flag lets all tags and their associated objects be
        downloaded.
 
+--recurse-submodules::
+       Use this option to fetch new commits of all populated submodules too.
+
+ifndef::git-pull[]
+--submodule-prefix=<path>::
+       Prepend <path> to paths printed in informative messages
+       such as "Fetching submodule foo".  This option is used
+       internally when recursing over submodules.
+endif::git-pull[]
+
 -u::
 --update-head-ok::
        By default 'git fetch' refuses to update the head which

diff --git a/builtin/fetch.c b/builtin/fetch.c
index ea14d5dc6b00b475b8817768b7d35d0bb54d5236..cbbde2d0ee15623299ded2c74d06a4be5bdcf548 100644 (file)
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -12,6 +12,7 @@
 #include "parse-options.h"
 #include "sigchain.h"
 #include "transport.h"
+#include "submodule.h"
 
 static const char * const builtin_fetch_usage[] = {
        "git fetch [<options>] [<repository> [<refspec>...]]",
@@ -28,12 +29,13 @@ enum {
 };
 
 static int all, append, dry_run, force, keep, multiple, prune, update_head_ok, verbosity;
-static int progress;
+static int progress, recurse_submodules;
 static int tags = TAGS_DEFAULT;
 static const char *depth;
 static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
 static struct transport *transport;
+static const char *submodule_prefix = "";
 
 static struct option builtin_fetch_options[] = {
        OPT__VERBOSITY(&verbosity),
@@ -53,6 +55,8 @@ static struct option builtin_fetch_options[] = {
                    "do not fetch all tags (--no-tags)", TAGS_UNSET),
        OPT_BOOLEAN('p', "prune", &prune,
                    "prune tracking branches no longer on remote"),
+       OPT_BOOLEAN(0, "recurse-submodules", &recurse_submodules,
+                   "control recursive fetching of submodules"),
        OPT_BOOLEAN(0, "dry-run", &dry_run,
                    "dry run"),
        OPT_BOOLEAN('k', "keep", &keep, "keep downloaded pack"),
@@ -61,6 +65,8 @@ static struct option builtin_fetch_options[] = {
        OPT_BOOLEAN(0, "progress", &progress, "force progress reporting"),
        OPT_STRING(0, "depth", &depth, "DEPTH",
                   "deepen history of shallow clone"),
+       { OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, "DIR",
+                  "prepend this to submodule path output", PARSE_OPT_HIDDEN },
        OPT_END()
 };
 
@@ -777,28 +783,36 @@ static int add_remote_or_group(const char *name, struct string_list *list)
        return 1;
 }
 
-static int fetch_multiple(struct string_list *list)
+static void add_options_to_argv(int *argc, const char **argv)
 {
-       int i, result = 0;
-       const char *argv[11] = { "fetch", "--append" };
-       int argc = 2;
-
        if (dry_run)
-               argv[argc++] = "--dry-run";
+               argv[(*argc)++] = "--dry-run";
        if (prune)
-               argv[argc++] = "--prune";
+               argv[(*argc)++] = "--prune";
        if (update_head_ok)
-               argv[argc++] = "--update-head-ok";
+               argv[(*argc)++] = "--update-head-ok";
        if (force)
-               argv[argc++] = "--force";
+               argv[(*argc)++] = "--force";
        if (keep)
-               argv[argc++] = "--keep";
+               argv[(*argc)++] = "--keep";
+       if (recurse_submodules)
+               argv[(*argc)++] = "--recurse-submodules";
        if (verbosity >= 2)
-               argv[argc++] = "-v";
+               argv[(*argc)++] = "-v";
        if (verbosity >= 1)
-               argv[argc++] = "-v";
+               argv[(*argc)++] = "-v";
        else if (verbosity < 0)
-               argv[argc++] = "-q";
+               argv[(*argc)++] = "-q";
+
+}
+
+static int fetch_multiple(struct string_list *list)
+{
+       int i, result = 0;
+       const char *argv[12] = { "fetch", "--append" };
+       int argc = 2;
+
+       add_options_to_argv(&argc, argv);
 
        if (!append && !dry_run) {
                int errcode = truncate_fetch_head();
@@ -919,6 +933,17 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
                }
        }
 
+       if (!result && recurse_submodules) {
+               const char *options[10];
+               int num_options = 0;
+               gitmodules_config();
+               git_config(submodule_config, NULL);
+               add_options_to_argv(&num_options, options);
+               result = fetch_populated_submodules(num_options, options,
+                                                   submodule_prefix,
+                                                   verbosity < 0);
+       }
+
        /* All names were strdup()ed or strndup()ed */
        list.strdup_strings = 1;
        string_list_clear(&list, 0);

diff --git a/git-pull.sh b/git-pull.sh
index 8eb74d45debe7ab2bd2cc9b5ed57b1ba49d94bdb..5804c627cae7d81c321821b123387ddd5a21d7a7 100755 (executable)
--- a/git-pull.sh
+++ b/git-pull.sh
@@ -38,7 +38,7 @@ test -z "$(git ls-files -u)" || die_conflict
 test -f "$GIT_DIR/MERGE_HEAD" && die_merge
 
 strategy_args= diffstat= no_commit= squash= no_ff= ff_only=
-log_arg= verbosity= progress=
+log_arg= verbosity= progress= recurse_submodules=
 merge_args=
 curr_branch=$(git symbolic-ref -q HEAD)
 curr_branch_short="${curr_branch#refs/heads/}"
@@ -105,6 +105,9 @@ do
        --no-r|--no-re|--no-reb|--no-reba|--no-rebas|--no-rebase)
                rebase=false
                ;;
+       --recurse-submodules)
+               recurse_submodules=--recurse-submodules
+               ;;
        --d|--dr|--dry|--dry-|--dry-r|--dry-ru|--dry-run)
                dry_run=--dry-run
                ;;
@@ -220,7 +223,7 @@ test true = "$rebase" && {
        done
 }
 orig_head=$(git rev-parse -q --verify HEAD)
-git fetch $verbosity $progress $dry_run --update-head-ok "$@" || exit 1
+git fetch $verbosity $progress $dry_run $recurse_submodules --update-head-ok "$@" || exit 1
 test -z "$dry_run" || exit 0
 
 curr_head=$(git rev-parse -q --verify HEAD)

diff --git a/submodule.c b/submodule.c
index 91a47587478ae0550be8f41c00cb1749c85834f4..4d9b774cd107b5e9225c28620d2a2dc9efb665cc 100644 (file)
--- a/submodule.c
+++ b/submodule.c
@@ -63,7 +63,7 @@ void set_diffopt_flags_from_submodule_config(struct diff_options *diffopt,
        }
 }
 
-static int submodule_config(const char *var, const char *value, void *cb)
+int submodule_config(const char *var, const char *value, void *cb)
 {
        if (!prefixcmp(var, "submodule."))
                return parse_submodule_config_option(var, value);
@@ -229,6 +229,70 @@ void show_submodule_summary(FILE *f, const char *path,
        strbuf_release(&sb);
 }
 
+int fetch_populated_submodules(int num_options, const char **options,
+                              const char *prefix, int quiet)
+{
+       int i, result = 0, argc = 0;
+       struct child_process cp;
+       const char **argv;
+       struct string_list_item *name_for_path;
+       const char *work_tree = get_git_work_tree();
+       if (!work_tree)
+               return 0;
+
+       if (!the_index.initialized)
+               if (read_cache() < 0)
+                       die("index file corrupt");
+
+       argv = xcalloc(num_options + 5, sizeof(const char *));
+       argv[argc++] = "fetch";
+       for (i = 0; i < num_options; i++)
+               argv[argc++] = options[i];
+       argv[argc++] = "--submodule-prefix";
+
+       memset(&cp, 0, sizeof(cp));
+       cp.argv = argv;
+       cp.env = local_repo_env;
+       cp.git_cmd = 1;
+       cp.no_stdin = 1;
+
+       for (i = 0; i < active_nr; i++) {
+               struct strbuf submodule_path = STRBUF_INIT;
+               struct strbuf submodule_git_dir = STRBUF_INIT;
+               struct strbuf submodule_prefix = STRBUF_INIT;
+               struct cache_entry *ce = active_cache[i];
+               const char *git_dir, *name;
+
+               if (!S_ISGITLINK(ce->ce_mode))
+                       continue;
+
+               name = ce->name;
+               name_for_path = unsorted_string_list_lookup(&config_name_for_path, ce->name);
+               if (name_for_path)
+                       name = name_for_path->util;
+
+               strbuf_addf(&submodule_path, "%s/%s", work_tree, ce->name);
+               strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
+               strbuf_addf(&submodule_prefix, "%s%s/", prefix, ce->name);
+               git_dir = read_gitfile_gently(submodule_git_dir.buf);
+               if (!git_dir)
+                       git_dir = submodule_git_dir.buf;
+               if (is_directory(git_dir)) {
+                       if (!quiet)
+                               printf("Fetching submodule %s%s\n", prefix, ce->name);
+                       cp.dir = submodule_path.buf;
+                       argv[argc] = submodule_prefix.buf;
+                       if (run_command(&cp))
+                               result = 1;
+               }
+               strbuf_release(&submodule_path);
+               strbuf_release(&submodule_git_dir);
+               strbuf_release(&submodule_prefix);
+       }
+       free(argv);
+       return result;
+}
+
 unsigned is_submodule_modified(const char *path, int ignore_untracked)
 {
        ssize_t len;

diff --git a/submodule.h b/submodule.h
index 386f410a66d9c431a61c0e26c773f1b3452f0d09..b39d7a12ff7841d9ae2412e04cc5e1d1209236f9 100644 (file)
--- a/submodule.h
+++ b/submodule.h
@@ -5,6 +5,7 @@ struct diff_options;
 
 void set_diffopt_flags_from_submodule_config(struct diff_options *diffopt,
                const char *path);
+int submodule_config(const char *var, const char *value, void *cb);
 void gitmodules_config();
 int parse_submodule_config_option(const char *var, const char *value);
 void handle_ignore_submodules_arg(struct diff_options *diffopt, const char *);
@@ -12,6 +13,8 @@ void show_submodule_summary(FILE *f, const char *path,
                unsigned char one[20], unsigned char two[20],
                unsigned dirty_submodule,
                const char *del, const char *add, const char *reset);
+int fetch_populated_submodules(int num_options, const char **options,
+                              const char *prefix, int quiet);
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int merge_submodule(unsigned char result[20], const char *path, const unsigned char base[20],
                    const unsigned char a[20], const unsigned char b[20]);

diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
new file mode 100755 (executable)
index 0000000..9c1d3a0
--- /dev/null
+++ b/t/t5526-fetch-submodules.sh
@@ -0,0 +1,109 @@
+#!/bin/sh
+# Copyright (c) 2010, Jens Lehmann
+
+test_description='Recursive "git fetch" for submodules'
+
+. ./test-lib.sh
+
+pwd=$(pwd)
+
+add_upstream_commit() {
+       (
+               cd submodule &&
+               head1=$(git rev-parse --short HEAD) &&
+               echo new >> subfile &&
+               test_tick &&
+               git add subfile &&
+               git commit -m new subfile &&
+               head2=$(git rev-parse --short HEAD) &&
+               echo "From $pwd/submodule" > ../expect.err &&
+               echo "   $head1..$head2  master     -> origin/master" >> ../expect.err
+       ) &&
+       (
+               cd deepsubmodule &&
+               head1=$(git rev-parse --short HEAD) &&
+               echo new >> deepsubfile &&
+               test_tick &&
+               git add deepsubfile &&
+               git commit -m new deepsubfile &&
+               head2=$(git rev-parse --short HEAD) &&
+               echo "From $pwd/deepsubmodule" >> ../expect.err &&
+               echo "   $head1..$head2  master     -> origin/master" >> ../expect.err
+       )
+}
+
+test_expect_success setup '
+       mkdir deepsubmodule &&
+       (
+               cd deepsubmodule &&
+               git init &&
+               echo deepsubcontent > deepsubfile &&
+               git add deepsubfile &&
+               git commit -m new deepsubfile
+       ) &&
+       mkdir submodule &&
+       (
+               cd submodule &&
+               git init &&
+               echo subcontent > subfile &&
+               git add subfile &&
+               git submodule add "$pwd/deepsubmodule" deepsubmodule &&
+               git commit -a -m new
+       ) &&
+       git submodule add "$pwd/submodule" submodule &&
+       git commit -am initial &&
+       git clone . downstream &&
+       (
+               cd downstream &&
+               git submodule update --init --recursive
+       ) &&
+       echo "Fetching submodule submodule" > expect.out &&
+       echo "Fetching submodule submodule/deepsubmodule" >> expect.out
+'
+
+test_expect_success "fetch --recurse-submodules recurses into submodules" '
+       add_upstream_commit &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules >../actual.out 2>../actual.err
+       ) &&
+       test_cmp expect.out actual.out &&
+       test_cmp expect.err actual.err
+'
+
+test_expect_success "fetch alone only fetches superproject" '
+       add_upstream_commit &&
+       (
+               cd downstream &&
+               git fetch >../actual.out 2>../actual.err
+       ) &&
+       ! test -s actual.out &&
+       ! test -s actual.err
+'
+
+test_expect_success "--quiet propagates to submodules" '
+       (
+               cd downstream &&
+               git fetch --recurse-submodules --quiet >../actual.out 2>../actual.err
+       ) &&
+       ! test -s actual.out &&
+       ! test -s actual.err
+'
+
+test_expect_success "--dry-run propagates to submodules" '
+       add_upstream_commit &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules --dry-run >../actual.out 2>../actual.err
+       ) &&
+       test_cmp expect.out actual.out &&
+       test_cmp expect.err actual.err &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules >../actual.out 2>../actual.err
+       ) &&
+       test_cmp expect.out actual.out &&
+       test_cmp expect.err actual.err
+'
+
+test_done