index-pack --verify: read anomalous offsets from v2 idx file

A pack v2 .idx file usually records offset using 64-bit representation
only when the offset does not fit within 31-bit, but you can handcraft
your .idx file to record smaller offset using 64-bit, storing all zero
in the upper 4-byte.  By inspecting the original idx file when running
index-pack --verify, encode such low offsets that do not need to be in
64-bit but are encoded using 64-bit just like the original idx file so
that we can still validate the pack/idx pair by comparing the idx file
recomputed with the original.

Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index 24a9a162202aec2c1481ab75b565cd4a73b60be6..513fbbc55f0ed438ab11e0bc23c2057690aefd36 100644 (file)
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -891,6 +891,51 @@ static int git_index_pack_config(const char *k, const char *v, void *cb)
        return git_default_config(k, v, cb);
 }
 
+static int cmp_uint32(const void *a_, const void *b_)
+{
+       uint32_t a = *((uint32_t *)a_);
+       uint32_t b = *((uint32_t *)b_);
+
+       return (a < b) ? -1 : (a != b);
+}
+
+static void read_v2_anomalous_offsets(struct packed_git *p,
+                                     struct pack_idx_option *opts)
+{
+       const uint32_t *idx1, *idx2;
+       uint32_t i;
+
+       /* The address of the 4-byte offset table */
+       idx1 = (((const uint32_t *)p->index_data)
+               + 2 /* 8-byte header */
+               + 256 /* fan out */
+               + 5 * p->num_objects /* 20-byte SHA-1 table */
+               + p->num_objects /* CRC32 table */
+               );
+
+       /* The address of the 8-byte offset table */
+       idx2 = idx1 + p->num_objects;
+
+       for (i = 0; i < p->num_objects; i++) {
+               uint32_t off = ntohl(idx1[i]);
+               if (!(off & 0x80000000))
+                       continue;
+               off = off & 0x7fffffff;
+               if (idx2[off * 2])
+                       continue;
+               /*
+                * The real offset is ntohl(idx2[off * 2]) in high 4
+                * octets, and ntohl(idx2[off * 2 + 1]) in low 4
+                * octets.  But idx2[off * 2] is Zero!!!
+                */
+               ALLOC_GROW(opts->anomaly, opts->anomaly_nr + 1, opts->anomaly_alloc);
+               opts->anomaly[opts->anomaly_nr++] = ntohl(idx2[off * 2 + 1]);
+       }
+
+       if (1 < opts->anomaly_nr)
+               qsort(opts->anomaly, opts->anomaly_nr, sizeof(uint32_t), cmp_uint32);
+}
+
 static void read_idx_option(struct pack_idx_option *opts, const char *pack_name)
 {
        struct packed_git *p = add_packed_git(pack_name, strlen(pack_name), 1);
@@ -903,6 +948,9 @@ static void read_idx_option(struct pack_idx_option *opts, const char *pack_name)
        /* Read the attributes from the existing idx file */
        opts->version = p->index_version;
 
+       if (opts->version == 2)
+               read_v2_anomalous_offsets(p, opts);
+
        /*
         * Get rid of the idx file as we do not need it anymore.
         * NEEDSWORK: extract this bit from free_pack_by_name() in

diff --git a/pack-write.c b/pack-write.c
index 92e7eefb405c1df9a91dbe003a0b2fe1c29cfc69..9cd3bfbb4b3859cbbdc1b9375ea95f511fffc94e 100644 (file)
--- a/pack-write.c
+++ b/pack-write.c
@@ -16,9 +16,25 @@ static int sha1_compare(const void *_a, const void *_b)
        return hashcmp(a->sha1, b->sha1);
 }
 
+static int cmp_uint32(const void *a_, const void *b_)
+{
+       uint32_t a = *((uint32_t *)a_);
+       uint32_t b = *((uint32_t *)b_);
+
+       return (a < b) ? -1 : (a != b);
+}
+
 static int need_large_offset(off_t offset, const struct pack_idx_option *opts)
 {
-       return (offset >> 31) || (opts->off32_limit < offset);
+       uint32_t ofsval;
+
+       if ((offset >> 31) || (opts->off32_limit < offset))
+               return 1;
+       if (!opts->anomaly_nr)
+               return 0;
+       ofsval = offset;
+       return !!bsearch(&ofsval, opts->anomaly, opts->anomaly_nr,
+                        sizeof(ofsval), cmp_uint32);
 }
 
 /*

diff --git a/pack.h b/pack.h
index dddafdd160a9ea18a627e24a5c29458e0805c132..722a54e00a2cb7d9514c12f799fb1ec15930cf5d 100644 (file)
--- a/pack.h
+++ b/pack.h
@@ -41,6 +41,14 @@ struct pack_idx_option {
 
        uint32_t version;
        uint32_t off32_limit;
+
+       /*
+        * List of offsets that would fit within off32_limit but
+        * need to be written out as 64-bit entity for byte-for-byte
+        * verification.
+        */
+       int anomaly_alloc, anomaly_nr;
+       uint32_t *anomaly;
 };
 
 extern void reset_pack_idx_option(struct pack_idx_option *);

diff --git a/t/t5302-pack-index.sh b/t/t5302-pack-index.sh
index 7c5fa03920e4ead778568af562fa5fe97d9550bb..76bcaca9889b701f1fe490421be977b5e5875f1d 100755 (executable)
--- a/t/t5302-pack-index.sh
+++ b/t/t5302-pack-index.sh
@@ -107,7 +107,7 @@ test_expect_success OFF64_T 'index-pack --verify on 64-bit offset v2 (cheat)' '
        git index-pack --verify --index-version=2,0x40000 "test-3-${pack3}.pack"
 '
 
-test_expect_failure OFF64_T 'index-pack --verify on 64-bit offset v2' '
+test_expect_success OFF64_T 'index-pack --verify on 64-bit offset v2' '
        git index-pack --verify "test-3-${pack3}.pack"
 '