Create object subdirectories on demand

This makes it possible to have a "sparse" git object subdirectory
structure, something that has become much more attractive now that people
use pack-files all the time.

As a result of pack-files, a git object directory doesn't necessarily have
any individual objects lying around, and in that case it's just wasting
space to keep the empty first-level object directories around: on many
filesystems the 256 empty directories will be aboue 1MB of diskspace.

Even more importantly, after you re-pack a project that _used_ to be
unpacked, you could be left with huge directories that no longer contain
anything, but that waste space and take time to look through.

With this change, "git prune-packed" can just do an rmdir() on the
directories, and they'll get removed if empty, and re-created on demand.

This patch also tries to fix up "write_sha1_from_fd()" to use the new
common infrastructure for creating the object files, closing a hole where
we might otherwise leave half-written objects in the object database.

[jc: I unoptimized the part that really removes the fan-out directories
 to ease transition.  init-db still wastes 1MB of diskspace to hold 256
 empty fan-outs, and prune-packed rmdir()'s the grown but empty directories,
 but runs mkdir() immediately after that -- reducing the saving from 150KB
 to 146KB.  These parts will be re-introduced when everybody has the
 on-demand capability.]

Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/daemon.c b/daemon.c
index f285a8c98ca9fbfc421b735f76f9d9f2248bd749..11fa3ed11f1468d391ae24953ae2a378e75eb74a 100644 (file)
--- a/daemon.c
+++ b/daemon.c
@@ -142,7 +142,7 @@ static int upload(char *dir, int dirlen)
         * is ok with us doing this.
         */
        if ((!export_all_trees && access("git-daemon-export-ok", F_OK)) ||
-           access("objects/00", X_OK) ||
+           access("objects/", X_OK) ||
            access("HEAD", R_OK)) {
                logerror("Not a valid git-daemon-enabled repository: '%s'", dir);
                return -1;

diff --git a/fsck-objects.c b/fsck-objects.c
index 65cec7d12b6fecc95e842069d2afa1e87bceebc1..17d05363e08b48275cb38631e82f0f48321fa728 100644 (file)
--- a/fsck-objects.c
+++ b/fsck-objects.c
@@ -329,9 +329,8 @@ static int fsck_dir(int i, char *path)
        DIR *dir = opendir(path);
        struct dirent *de;
 
-       if (!dir) {
-               return error("missing sha1 directory '%s'", path);
-       }
+       if (!dir)
+               return 0;
 
        while ((de = readdir(dir)) != NULL) {
                char name[100];

diff --git a/git-rename.perl b/git-rename.perl
index a28c8c83bb98e81635ce5cb629921e13eca1e3c2..3b1127b1b20280cc4c6c657168dba963f31fa067 100755 (executable)
--- a/git-rename.perl
+++ b/git-rename.perl
@@ -15,7 +15,7 @@ sub usage($);
 my $GIT_DIR = $ENV{'GIT_DIR'} || ".git";
 
 unless ( -d $GIT_DIR && -d $GIT_DIR . "/objects" && 
-       -d $GIT_DIR . "/objects/00" && -d $GIT_DIR . "/refs") {
+       -d $GIT_DIR . "/objects/" && -d $GIT_DIR . "/refs") {
        usage("Git repository not found.");
 }
 

diff --git a/git-sh-setup.sh b/git-sh-setup.sh
index a0172686a90c58a454656f9e6d49d6c694aec2fc..dbb98842bf6327210c604ae65a128cdfc785b32c 100755 (executable)
--- a/git-sh-setup.sh
+++ b/git-sh-setup.sh
@@ -22,4 +22,4 @@ refs/*)       : ;;
 *)     false ;;
 esac &&
 [ -d "$GIT_DIR/refs" ] &&
-[ -d "$GIT_OBJECT_DIRECTORY/00" ]
+[ -d "$GIT_OBJECT_DIRECTORY/" ]

diff --git a/prune-packed.c b/prune-packed.c
index 5306e8e5ef411bfa7e6bcf4ba0639e640592818b..73f0f3a46277e9053d97590f8e2200477fe247e3 100644 (file)
--- a/prune-packed.c
+++ b/prune-packed.c
@@ -26,6 +26,9 @@ static void prune_dir(int i, DIR *dir, char *pathname, int len)
                else if (unlink(pathname) < 0)
                        error("unable to unlink %s", pathname);
        }
+       pathname[len] = 0;
+       if (rmdir(pathname))
+               mkdir(pathname, 0777);
 }
 
 static void prune_packed_objects(void)
@@ -46,7 +49,7 @@ static void prune_packed_objects(void)
                sprintf(pathname + len, "%02x/", i);
                d = opendir(pathname);
                if (!d)
-                       die("unable to open %s", pathname);
+                       continue;
                prune_dir(i, d, pathname, len + 3);
                closedir(d);
        }

diff --git a/sha1_file.c b/sha1_file.c
index 287f618827d9b2fd472c06add42ead65de291822..baaa4c00da50342a7db8064dadf7f963b0e2e679 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1248,6 +1248,73 @@ char *write_sha1_file_prepare(void *buf,
        return sha1_file_name(sha1);
 }
 
+/*
+ * Link the tempfile to the final place, possibly creating the
+ * last directory level as you do so.
+ *
+ * Returns the errno on failure, 0 on success.
+ */
+static int link_temp_to_file(const char *tmpfile, char *filename)
+{
+       int ret;
+
+       if (!link(tmpfile, filename))
+               return 0;
+
+       /*
+        * Try to mkdir the last path component if that failed
+        * with an ENOENT.
+        *
+        * Re-try the "link()" regardless of whether the mkdir
+        * succeeds, since a race might mean that somebody
+        * else succeeded.
+        */
+       ret = errno;
+       if (ret == ENOENT) {
+               char *dir = strrchr(filename, '/');
+               if (dir) {
+                       *dir = 0;
+                       mkdir(filename, 0777);
+                       *dir = '/';
+                       if (!link(tmpfile, filename))
+                               return 0;
+                       ret = errno;
+               }
+       }
+       return ret;
+}
+
+/*
+ * Move the just written object into its final resting place
+ */
+static int move_temp_to_file(const char *tmpfile, char *filename)
+{
+       int ret = link_temp_to_file(tmpfile, filename);
+       if (ret) {
+               /*
+                * Coda hack - coda doesn't like cross-directory links,
+                * so we fall back to a rename, which will mean that it
+                * won't be able to check collisions, but that's not a
+                * big deal.
+                *
+                * When this succeeds, we just return 0. We have nothing
+                * left to unlink.
+                */
+               if (ret == EXDEV && !rename(tmpfile, filename))
+                       return 0;
+       }
+       unlink(tmpfile);
+       if (ret) {
+               if (ret != EEXIST) {
+                       fprintf(stderr, "unable to write sha1 filename %s: %s", filename, strerror(ret));
+                       return -1;
+               }
+               /* FIXME!!! Collision check here ? */
+       }
+
+       return 0;
+}
+
 int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
 {
        int size;
@@ -1257,7 +1324,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
        char *filename;
        static char tmpfile[PATH_MAX];
        unsigned char hdr[50];
-       int fd, hdrlen, ret;
+       int fd, hdrlen;
 
        /* Normally if we have it in the pack then we do not bother writing
         * it out into .git/objects/??/?{38} file.
@@ -1320,32 +1387,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
        close(fd);
        free(compressed);
 
-       ret = link(tmpfile, filename);
-       if (ret < 0) {
-               ret = errno;
-
-               /*
-                * Coda hack - coda doesn't like cross-directory links,
-                * so we fall back to a rename, which will mean that it
-                * won't be able to check collisions, but that's not a
-                * big deal.
-                *
-                * When this succeeds, we just return 0. We have nothing
-                * left to unlink.
-                */
-               if (ret == EXDEV && !rename(tmpfile, filename))
-                       return 0;
-       }
-       unlink(tmpfile);
-       if (ret) {
-               if (ret != EEXIST) {
-                       fprintf(stderr, "unable to write sha1 filename %s: %s", filename, strerror(ret));
-                       return -1;
-               }
-               /* FIXME!!! Collision check here ? */
-       }
-
-       return 0;
+       return move_temp_to_file(tmpfile, filename);
 }
 
 int write_sha1_to_fd(int fd, const unsigned char *sha1)
@@ -1420,8 +1462,7 @@ int write_sha1_to_fd(int fd, const unsigned char *sha1)
 int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
                       size_t bufsize, size_t *bufposn)
 {
-       char *filename = sha1_file_name(sha1);
-
+       char tmpfile[PATH_MAX];
        int local;
        z_stream stream;
        unsigned char real_sha1[20];
@@ -1429,10 +1470,11 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
        int ret;
        SHA_CTX c;
 
-       local = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0666);
+       snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX", get_object_directory());
 
+       local = mkstemp(tmpfile);
        if (local < 0)
-               return error("Couldn't open %s\n", filename);
+               return error("Couldn't open %s for %s\n", tmpfile, sha1_to_hex(sha1));
 
        memset(&stream, 0, sizeof(stream));
 
@@ -1462,7 +1504,7 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
                size = read(fd, buffer + *bufposn, bufsize - *bufposn);
                if (size <= 0) {
                        close(local);
-                       unlink(filename);
+                       unlink(tmpfile);
                        if (!size)
                                return error("Connection closed?");
                        perror("Reading from connection");
@@ -1475,15 +1517,15 @@ int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
        close(local);
        SHA1_Final(real_sha1, &c);
        if (ret != Z_STREAM_END) {
-               unlink(filename);
+               unlink(tmpfile);
                return error("File %s corrupted", sha1_to_hex(sha1));
        }
        if (memcmp(sha1, real_sha1, 20)) {
-               unlink(filename);
+               unlink(tmpfile);
                return error("File %s has bad hash\n", sha1_to_hex(sha1));
        }
-       
-       return 0;
+
+       return move_temp_to_file(tmpfile, sha1_file_name(sha1));
 }
 
 int has_pack_index(const unsigned char *sha1)

diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh
index bd940bd09b3df6ff6cecae4fa8a9db739ef000e4..5c5f854858522f15f929d9a9320ee9c310d0b634 100755 (executable)
--- a/t/t0000-basic.sh
+++ b/t/t0000-basic.sh
@@ -28,7 +28,7 @@ test_expect_success \
     '.git/objects should be empty after git-init-db in an empty repo.' \
     'cmp -s /dev/null should-be-empty' 
 
-# also it should have 258 subdirectories; 256 fan-out, pack, and info.
+# also it should have 258 subdirectories; 256 fan-out anymore, pack, and info.
 # 259 is counting "objects" itself
 find .git/objects -type d -print >full-of-directories
 test_expect_success \