seanhalle@25: /* seanhalle@25: -------------------------------------------------------------------- seanhalle@25: lookup8.c, by Bob Jenkins, January 4 1997, Public Domain. seanhalle@25: hash(), hash2(), hash3, and mix() are externally useful functions. seanhalle@25: Routines to test the hash are included if SELF_TEST is defined. seanhalle@25: You can use this free for any purpose. It has no warranty. seanhalle@25: seanhalle@25: 2009: This is obsolete. I recently timed lookup3.c as being faster seanhalle@25: at producing 64-bit results. seanhalle@25: -------------------------------------------------------------------- seanhalle@25: */ seanhalle@25: //#define SELF_TEST seanhalle@25: seanhalle@25: #include seanhalle@25: #include seanhalle@25: #include seanhalle@25: typedef unsigned long long ub8; /* unsigned 8-byte quantities */ seanhalle@25: typedef unsigned long int ub4; /* unsigned 4-byte quantities */ seanhalle@25: typedef unsigned char ub1; seanhalle@25: seanhalle@25: #define hashsize(n) ((ub8)1<<(n)) seanhalle@25: #define hashmask(n) (hashsize(n)-1) seanhalle@25: seanhalle@25: /* seanhalle@25: -------------------------------------------------------------------- seanhalle@25: mix -- mix 3 64-bit values reversibly. seanhalle@25: mix() takes 48 machine instructions, but only 24 cycles on a superscalar seanhalle@25: machine (like Intel's new MMX architecture). It requires 4 64-bit seanhalle@25: registers for 4::2 parallelism. seanhalle@25: All 1-bit deltas, all 2-bit deltas, all deltas composed of top bits of seanhalle@25: (a,b,c), and all deltas of bottom bits were tested. All deltas were seanhalle@25: tested both on random keys and on keys that were nearly all zero. seanhalle@25: These deltas all cause every bit of c to change between 1/3 and 2/3 seanhalle@25: of the time (well, only 113/400 to 287/400 of the time for some seanhalle@25: 2-bit delta). These deltas all cause at least 80 bits to change seanhalle@25: among (a,b,c) when the mix is run either forward or backward (yes it seanhalle@25: is reversible). seanhalle@25: This implies that a hash using mix64 has no funnels. There may be seanhalle@25: characteristics with 3-bit deltas or bigger, I didn't test for seanhalle@25: those. seanhalle@25: -------------------------------------------------------------------- seanhalle@25: */ seanhalle@25: #define mix64(a,b,c) \ seanhalle@25: { \ seanhalle@25: a -= b; a -= c; a ^= (c>>43); \ seanhalle@25: b -= c; b -= a; b ^= (a<<9); \ seanhalle@25: c -= a; c -= b; c ^= (b>>8); \ seanhalle@25: a -= b; a -= c; a ^= (c>>38); \ seanhalle@25: b -= c; b -= a; b ^= (a<<23); \ seanhalle@25: c -= a; c -= b; c ^= (b>>5); \ seanhalle@25: a -= b; a -= c; a ^= (c>>35); \ seanhalle@25: b -= c; b -= a; b ^= (a<<49); \ seanhalle@25: c -= a; c -= b; c ^= (b>>11); \ seanhalle@25: a -= b; a -= c; a ^= (c>>12); \ seanhalle@25: b -= c; b -= a; b ^= (a<<18); \ seanhalle@25: c -= a; c -= b; c ^= (b>>22); \ seanhalle@25: } seanhalle@25: seanhalle@25: /* seanhalle@25: -------------------------------------------------------------------- seanhalle@25: hash() -- hash a variable-length key into a 64-bit value seanhalle@25: k : the key (the unaligned variable-length array of bytes) seanhalle@25: len : the length of the key, counting by bytes seanhalle@25: level : can be any 8-byte value seanhalle@25: Returns a 64-bit value. Every bit of the key affects every bit of seanhalle@25: the return value. No funnels. Every 1-bit and 2-bit delta achieves seanhalle@25: avalanche. About 41+5len instructions. seanhalle@25: seanhalle@25: The best hash table sizes are powers of 2. There is no need to do seanhalle@25: mod a prime (mod is sooo slow!). If you need less than 64 bits, seanhalle@25: use a bitmask. For example, if you need only 10 bits, do seanhalle@25: h = (h & hashmask(10)); seanhalle@25: In which case, the hash table should have hashsize(10) elements. seanhalle@25: seanhalle@25: If you are hashing n strings (ub1 **)k, do it like this: seanhalle@25: for (i=0, h=0; i= 24) seanhalle@25: { seanhalle@25: a += (k[0] +((ub8)k[ 1]<< 8)+((ub8)k[ 2]<<16)+((ub8)k[ 3]<<24) seanhalle@25: +((ub8)k[4 ]<<32)+((ub8)k[ 5]<<40)+((ub8)k[ 6]<<48)+((ub8)k[ 7]<<56)); seanhalle@25: b += (k[8] +((ub8)k[ 9]<< 8)+((ub8)k[10]<<16)+((ub8)k[11]<<24) seanhalle@25: +((ub8)k[12]<<32)+((ub8)k[13]<<40)+((ub8)k[14]<<48)+((ub8)k[15]<<56)); seanhalle@25: c += (k[16] +((ub8)k[17]<< 8)+((ub8)k[18]<<16)+((ub8)k[19]<<24) seanhalle@25: +((ub8)k[20]<<32)+((ub8)k[21]<<40)+((ub8)k[22]<<48)+((ub8)k[23]<<56)); seanhalle@25: mix64(a,b,c); seanhalle@25: k += 24; len -= 24; seanhalle@25: } seanhalle@25: seanhalle@25: /*------------------------------------- handle the last 23 bytes */ seanhalle@25: c += length; seanhalle@25: switch(len) /* all the case statements fall through */ seanhalle@25: { seanhalle@25: case 23: c+=((ub8)k[22]<<56); seanhalle@25: case 22: c+=((ub8)k[21]<<48); seanhalle@25: case 21: c+=((ub8)k[20]<<40); seanhalle@25: case 20: c+=((ub8)k[19]<<32); seanhalle@25: case 19: c+=((ub8)k[18]<<24); seanhalle@25: case 18: c+=((ub8)k[17]<<16); seanhalle@25: case 17: c+=((ub8)k[16]<<8); seanhalle@25: /* the first byte of c is reserved for the length */ seanhalle@25: case 16: b+=((ub8)k[15]<<56); seanhalle@25: case 15: b+=((ub8)k[14]<<48); seanhalle@25: case 14: b+=((ub8)k[13]<<40); seanhalle@25: case 13: b+=((ub8)k[12]<<32); seanhalle@25: case 12: b+=((ub8)k[11]<<24); seanhalle@25: case 11: b+=((ub8)k[10]<<16); seanhalle@25: case 10: b+=((ub8)k[ 9]<<8); seanhalle@25: case 9: b+=((ub8)k[ 8]); seanhalle@25: case 8: a+=((ub8)k[ 7]<<56); seanhalle@25: case 7: a+=((ub8)k[ 6]<<48); seanhalle@25: case 6: a+=((ub8)k[ 5]<<40); seanhalle@25: case 5: a+=((ub8)k[ 4]<<32); seanhalle@25: case 4: a+=((ub8)k[ 3]<<24); seanhalle@25: case 3: a+=((ub8)k[ 2]<<16); seanhalle@25: case 2: a+=((ub8)k[ 1]<<8); seanhalle@25: case 1: a+=((ub8)k[ 0]); seanhalle@25: /* case 0: nothing left to add */ seanhalle@25: } seanhalle@25: mix64(a,b,c); seanhalle@25: /*-------------------------------------------- report the result */ seanhalle@25: return c; seanhalle@25: } seanhalle@25: seanhalle@25: /* seanhalle@25: -------------------------------------------------------------------- seanhalle@25: This works on all machines, is identical to hash() on little-endian seanhalle@25: machines, and it is much faster than hash(), but it requires seanhalle@25: -- that the key be an array of ub8's, and seanhalle@25: -- that all your machines have the same endianness, and seanhalle@25: -- that the length be the number of ub8's in the key seanhalle@25: -------------------------------------------------------------------- seanhalle@25: */ seanhalle@25: ub8 hash2( k, length, level) seanhalle@25: register ub8 *k; /* the key */ seanhalle@25: register ub8 length; /* the length of the key */ seanhalle@25: register ub8 level; /* the previous hash, or an arbitrary value */ seanhalle@25: { seanhalle@25: register ub8 a,b,c,len; seanhalle@25: seanhalle@25: /* Set up the internal state */ seanhalle@25: len = length; seanhalle@25: a = b = level; /* the previous hash value */ seanhalle@25: c = 0x9e3779b97f4a7c13LL; /* the golden ratio; an arbitrary value */ seanhalle@25: seanhalle@25: /*---------------------------------------- handle most of the key */ seanhalle@25: while (len >= 3) seanhalle@25: { seanhalle@25: a += k[0]; seanhalle@25: b += k[1]; seanhalle@25: c += k[2]; seanhalle@25: mix64(a,b,c); seanhalle@25: k += 3; len -= 3; seanhalle@25: } seanhalle@25: seanhalle@25: /*-------------------------------------- handle the last 2 ub8's */ seanhalle@25: c += (length<<3); seanhalle@25: switch(len) /* all the case statements fall through */ seanhalle@25: { seanhalle@25: /* c is reserved for the length */ seanhalle@25: case 2: b+=k[1]; seanhalle@25: case 1: a+=k[0]; seanhalle@25: /* case 0: nothing left to add */ seanhalle@25: } seanhalle@25: mix64(a,b,c); seanhalle@25: /*-------------------------------------------- report the result */ seanhalle@25: return c; seanhalle@25: } seanhalle@25: seanhalle@25: /* seanhalle@25: -------------------------------------------------------------------- seanhalle@25: This is identical to hash() on little-endian machines, and it is much seanhalle@25: faster than hash(), but a little slower than hash2(), and it requires seanhalle@25: -- that all your machines be little-endian, for example all Intel x86 seanhalle@25: chips or all VAXen. It gives wrong results on big-endian machines. seanhalle@25: -------------------------------------------------------------------- seanhalle@25: */ seanhalle@25: seanhalle@25: ub8 hash3( k, length, level) seanhalle@25: register ub1 *k; /* the key */ seanhalle@25: register ub8 length; /* the length of the key */ seanhalle@25: register ub8 level; /* the previous hash, or an arbitrary value */ seanhalle@25: { seanhalle@25: register ub8 a,b,c,len; seanhalle@25: seanhalle@25: /* Set up the internal state */ seanhalle@25: len = length; seanhalle@25: a = b = level; /* the previous hash value */ seanhalle@25: c = 0x9e3779b97f4a7c13LL; /* the golden ratio; an arbitrary value */ seanhalle@25: seanhalle@25: /*---------------------------------------- handle most of the key */ seanhalle@25: if (((size_t)k)&7) seanhalle@25: { seanhalle@25: while (len >= 24) seanhalle@25: { seanhalle@25: a += (k[0] +((ub8)k[ 1]<< 8)+((ub8)k[ 2]<<16)+((ub8)k[ 3]<<24) seanhalle@25: +((ub8)k[4 ]<<32)+((ub8)k[ 5]<<40)+((ub8)k[ 6]<<48)+((ub8)k[ 7]<<56)); seanhalle@25: b += (k[8] +((ub8)k[ 9]<< 8)+((ub8)k[10]<<16)+((ub8)k[11]<<24) seanhalle@25: +((ub8)k[12]<<32)+((ub8)k[13]<<40)+((ub8)k[14]<<48)+((ub8)k[15]<<56)); seanhalle@25: c += (k[16] +((ub8)k[17]<< 8)+((ub8)k[18]<<16)+((ub8)k[19]<<24) seanhalle@25: +((ub8)k[20]<<32)+((ub8)k[21]<<40)+((ub8)k[22]<<48)+((ub8)k[23]<<56)); seanhalle@25: mix64(a,b,c); seanhalle@25: k += 24; len -= 24; seanhalle@25: } seanhalle@25: } seanhalle@25: else seanhalle@25: { seanhalle@25: while (len >= 24) /* aligned */ seanhalle@25: { seanhalle@25: a += *(ub8 *)(k+0); seanhalle@25: b += *(ub8 *)(k+8); seanhalle@25: c += *(ub8 *)(k+16); seanhalle@25: mix64(a,b,c); seanhalle@25: k += 24; len -= 24; seanhalle@25: } seanhalle@25: } seanhalle@25: seanhalle@25: /*------------------------------------- handle the last 23 bytes */ seanhalle@25: c += length; seanhalle@25: switch(len) /* all the case statements fall through */ seanhalle@25: { seanhalle@25: case 23: c+=((ub8)k[22]<<56); seanhalle@25: case 22: c+=((ub8)k[21]<<48); seanhalle@25: case 21: c+=((ub8)k[20]<<40); seanhalle@25: case 20: c+=((ub8)k[19]<<32); seanhalle@25: case 19: c+=((ub8)k[18]<<24); seanhalle@25: case 18: c+=((ub8)k[17]<<16); seanhalle@25: case 17: c+=((ub8)k[16]<<8); seanhalle@25: /* the first byte of c is reserved for the length */ seanhalle@25: case 16: b+=((ub8)k[15]<<56); seanhalle@25: case 15: b+=((ub8)k[14]<<48); seanhalle@25: case 14: b+=((ub8)k[13]<<40); seanhalle@25: case 13: b+=((ub8)k[12]<<32); seanhalle@25: case 12: b+=((ub8)k[11]<<24); seanhalle@25: case 11: b+=((ub8)k[10]<<16); seanhalle@25: case 10: b+=((ub8)k[ 9]<<8); seanhalle@25: case 9: b+=((ub8)k[ 8]); seanhalle@25: case 8: a+=((ub8)k[ 7]<<56); seanhalle@25: case 7: a+=((ub8)k[ 6]<<48); seanhalle@25: case 6: a+=((ub8)k[ 5]<<40); seanhalle@25: case 5: a+=((ub8)k[ 4]<<32); seanhalle@25: case 4: a+=((ub8)k[ 3]<<24); seanhalle@25: case 3: a+=((ub8)k[ 2]<<16); seanhalle@25: case 2: a+=((ub8)k[ 1]<<8); seanhalle@25: case 1: a+=((ub8)k[ 0]); seanhalle@25: /* case 0: nothing left to add */ seanhalle@25: } seanhalle@25: mix64(a,b,c); seanhalle@25: /*-------------------------------------------- report the result */ seanhalle@25: return c; seanhalle@25: } seanhalle@25: seanhalle@25: #ifdef SELF_TEST seanhalle@25: seanhalle@25: /* used for timings */ seanhalle@25: void driver1() seanhalle@25: { seanhalle@25: ub8 buf[256]; seanhalle@25: ub8 i; seanhalle@25: ub8 h=0; seanhalle@25: seanhalle@25: for (i=0; i<256; ++i) seanhalle@25: { seanhalle@25: h = hash(buf,i,h); seanhalle@25: } seanhalle@25: } seanhalle@25: seanhalle@25: /* check that every input bit changes every output bit half the time */ seanhalle@25: #define HASHSTATE 1 seanhalle@25: #define HASHLEN 1 seanhalle@25: #define MAXPAIR 80 seanhalle@25: #define MAXLEN 5 seanhalle@25: void driver2() seanhalle@25: { seanhalle@25: ub1 qa[MAXLEN+1], qb[MAXLEN+2], *a = &qa[0], *b = &qb[1]; seanhalle@25: ub8 c[HASHSTATE], d[HASHSTATE], i, j=0, k, l, m, z; seanhalle@25: ub8 e[HASHSTATE],f[HASHSTATE],g[HASHSTATE],h[HASHSTATE]; seanhalle@25: ub8 x[HASHSTATE],y[HASHSTATE]; seanhalle@25: ub8 hlen; seanhalle@25: seanhalle@25: printf("No more than %d trials should ever be needed \n",MAXPAIR/2); seanhalle@25: for (hlen=0; hlen < MAXLEN; ++hlen) seanhalle@25: { seanhalle@25: z=0; seanhalle@25: for (i=0; i>(8-j)); seanhalle@25: c[0] = hash(a, hlen, m); seanhalle@25: b[i] ^= ((k+1)<>(8-j)); seanhalle@25: d[0] = hash(b, hlen, m); seanhalle@25: /* check every bit is 1, 0, set, and not set at least once */ seanhalle@25: for (l=0; lz) z=k; seanhalle@25: if (k==MAXPAIR) seanhalle@25: { seanhalle@25: seanhalle@25: printf("Some bit didn't change: "); seanhalle@25: printf("%.8lx %.8lx %.8lx %.8lx %.8lx %.8lx ", seanhalle@25: e[0],f[0],g[0],h[0],x[0],y[0]); seanhalle@25: printf("i %ld j %ld m %ld len %ld\n", seanhalle@25: (ub4)i,(ub4)j,(ub4)m,(ub4)hlen); seanhalle@25: seanhalle@25: } seanhalle@25: if (z==MAXPAIR) goto done; seanhalle@25: } seanhalle@25: } seanhalle@25: } seanhalle@25: done: seanhalle@25: if (z < MAXPAIR) seanhalle@25: { seanhalle@25: printf("Mix success %2ld bytes %2ld levels ",(ub4)i,(ub4)m); seanhalle@25: printf("required %ld trials\n",(ub4)(z/2)); seanhalle@25: } seanhalle@25: } seanhalle@25: printf("\n"); seanhalle@25: } seanhalle@25: seanhalle@25: /* Check for reading beyond the end of the buffer and alignment problems */ seanhalle@25: void driver3() seanhalle@25: { seanhalle@25: ub1 buf[MAXLEN+20], *b; seanhalle@25: ub8 len; seanhalle@25: ub1 q[] = "This is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 qq[] = "xThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 qqq[] = "xxThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 qqqq[] = "xxxThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 o[] = "xxxxThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 oo[] = "xxxxxThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 ooo[] = "xxxxxxThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub1 oooo[] = "xxxxxxxThis is the time for all good men to come to the aid of their country"; seanhalle@25: ub8 h,i,j,ref,x,y; seanhalle@25: seanhalle@25: printf("Endianness. These should all be the same:\n"); seanhalle@25: h = hash(q+0, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(qq+1, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(qqq+2, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(qqqq+3, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(o+4, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(oo+5, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(ooo+6, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: h = hash(oooo+7, (ub8)(sizeof(q)-1), (ub8)0); seanhalle@25: printf("%.8lx%.8lx\n", (ub4)h, (ub4)(h>>32)); seanhalle@25: printf("\n"); seanhalle@25: for (h=0, b=buf+1; h<8; ++h, ++b) seanhalle@25: { seanhalle@25: for (i=0; i>32)); seanhalle@25: } seanhalle@25: } seanhalle@25: seanhalle@25: seanhalle@25: int test_lookup8() seanhalle@25: { seanhalle@25: driver1(); /* test that the key is hashed: used for timings */ seanhalle@25: driver2(); /* test that whole key is hashed thoroughly */ seanhalle@25: driver3(); /* test that nothing but the key is hashed */ seanhalle@25: driver4(); /* test hashing multiple buffers (all buffers are null) */ seanhalle@25: return 1; seanhalle@25: } seanhalle@25: seanhalle@25: #endif /* SELF_TEST */