caleb-brown/redict
1
0
Fork 0
mirror of https://codeberg.org/redict/redict.git synced 2025-01-23 00:28:26 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'zipmap' of git://github.com/pietern/redis

Browse Source
This commit is contained in:
antirez 2010-04-01 18:48:14 +02:00
commit 14ae26d89d
2 changed files with 98 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
redis.c
View File

@ -6000,10 +6000,8 @@ static void hsetCommand(redisClient *c) {
decrRefCount(valobj); decrRefCount(valobj);
o->ptr = zm; o->ptr = zm;
/* And here there is the second check for hash conversion... /* And here there is the second check for hash conversion. */
* we want to do it only if the operation was not just an update as if (zipmapLen(zm) > server.hash_max_zipmap_entries)
* zipmapLen() is O(N). */
if (!update && zipmapLen(zm) > server.hash_max_zipmap_entries)
convertToRealHash(o); convertToRealHash(o);
} else { } else {
tryObjectEncoding(c->argv[2]); tryObjectEncoding(c->argv[2]);
@ -6021,7 +6019,6 @@ static void hsetCommand(redisClient *c) {
} }
static void hincrbyCommand(redisClient *c) { static void hincrbyCommand(redisClient *c) {
int update = 0;
long long value = 0, incr = 0; long long value = 0, incr = 0;
robj *o = lookupKeyWrite(c->db,c->argv[1]); robj *o = lookupKeyWrite(c->db,c->argv[1]);
@ -6058,13 +6055,12 @@ static void hincrbyCommand(redisClient *c) {
value += incr; value += incr;
sds svalue = sdscatprintf(sdsempty(),"%lld",value); sds svalue = sdscatprintf(sdsempty(),"%lld",value);
zm = zipmapSet(zm,c->argv[2]->ptr,sdslen(c->argv[2]->ptr), zm = zipmapSet(zm,c->argv[2]->ptr,sdslen(c->argv[2]->ptr),
(unsigned char*)svalue,sdslen(svalue),&update); (unsigned char*)svalue,sdslen(svalue),NULL);
sdsfree(svalue); sdsfree(svalue);
o->ptr = zm; o->ptr = zm;
/* Check if the zipmap needs to be converted /* Check if the zipmap needs to be converted. */
* if this was not an update. */ if (zipmapLen(zm) > server.hash_max_zipmap_entries)
if (!update && zipmapLen(zm) > server.hash_max_zipmap_entries)
convertToRealHash(o); convertToRealHash(o);
} else { } else {
robj *hval; robj *hval;

202
zipmap.c
View File

@ -42,10 +42,11 @@
/* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world": /* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
* *
* <status><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world" * <zmlen><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
* *
* <status> is 1 byte status. Currently only 1 bit is used: if the least * <zmlen> is 1 byte length that holds the current size of the zipmap.
* significant bit is set, it means the zipmap needs to be defragmented. * When the zipmap length is greater than or equal to 254, this value
* is not used and the zipmap needs to be traversed to find out the length.
* *
* <len> is the length of the following string (key or value). * <len> is the length of the following string (key or value).
* <len> lengths are encoded in a single value or in a 5 bytes value. * <len> lengths are encoded in a single value or in a 5 bytes value.
@ -62,22 +63,15 @@
* or even in order to add a key/value pair if it fits. * or even in order to add a key/value pair if it fits.
* *
* <free> is always an unsigned 8 bit number, because if after an * <free> is always an unsigned 8 bit number, because if after an
* update operation there are more than a few free bytes, they'll be converted * update operation there are more than a few free bytes, the zipmap will be
* into empty space prefixed by the special value 254. * reallocated to make sure it is as small as possible.
* *
* The most compact representation of the above two elements hash is actually: * The most compact representation of the above two elements hash is actually:
* *
* "\x00\x03foo\x03\x00bar\x05hello\x05\x00world\xff" * "\x02\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
* *
* Empty space is marked using a 254 bytes + a <len> (coded as already * Note that because keys and values are prefixed length "objects",
* specified). The length includes the 254 bytes in the count and the * the lookup will take O(N) where N is the number of elements
* space taken by the <len> field. So for instance removing the "foo" key
* from the zipmap above will lead to the following representation:
*
* "\x00\xfd\x10........\x05hello\x05\x00world\xff"
*
* Note that because empty space, keys, values, are all prefixed length
* "objects", the lookup will take O(N) where N is the numeber of elements
* in the zipmap and *not* the number of bytes needed to represent the zipmap. * in the zipmap and *not* the number of bytes needed to represent the zipmap.
* This lowers the constant times considerably. * This lowers the constant times considerably.
*/ */
@ -87,15 +81,12 @@
#include <assert.h> #include <assert.h>
#include "zmalloc.h" #include "zmalloc.h"
#define ZIPMAP_BIGLEN 253 #define ZIPMAP_BIGLEN 254
#define ZIPMAP_EMPTY 254
#define ZIPMAP_END 255 #define ZIPMAP_END 255
#define ZIPMAP_STATUS_FRAGMENTED 1
/* The following defines the max value for the <free> field described in the /* The following defines the max value for the <free> field described in the
* comments above, that is, the max number of trailing bytes in a value. */ * comments above, that is, the max number of trailing bytes in a value. */
#define ZIPMAP_VALUE_MAX_FREE 5 #define ZIPMAP_VALUE_MAX_FREE 4
/* The following macro returns the number of bytes needed to encode the length /* The following macro returns the number of bytes needed to encode the length
* for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and * for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
@ -106,7 +97,7 @@
unsigned char *zipmapNew(void) { unsigned char *zipmapNew(void) {
unsigned char *zm = zmalloc(2); unsigned char *zm = zmalloc(2);
zm[0] = 0; /* Status */ zm[0] = 0; /* Length */
zm[1] = ZIPMAP_END; zm[1] = ZIPMAP_END;
return zm; return zm;
} }
@ -142,50 +133,34 @@ static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
* *
* If NULL is returned, and totlen is not NULL, it is set to the entire * If NULL is returned, and totlen is not NULL, it is set to the entire
* size of the zimap, so that the calling function will be able to * size of the zimap, so that the calling function will be able to
* reallocate the original zipmap to make room for more entries. * reallocate the original zipmap to make room for more entries. */
* static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
* If NULL is returned, and freeoff and freelen are not NULL, they are set unsigned char *p = zm+1, *k = NULL;
* to the offset of the first empty space that can hold '*freelen' bytes
* (freelen is an integer pointer used both to signal the required length
* and to get the reply from the function). If there is not a suitable
* free space block to hold the requested bytes, *freelen is set to 0. */
static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen, unsigned int *freeoff, unsigned int *freelen) {
unsigned char *p = zm+1;
unsigned int l; unsigned int l;
unsigned int reqfreelen = 0; /* initialized just to prevent warning */
if (freelen) {
reqfreelen = *freelen;
*freelen = 0;
assert(reqfreelen != 0);
}
while(*p != ZIPMAP_END) { while(*p != ZIPMAP_END) {
if (*p == ZIPMAP_EMPTY) { unsigned char free;
l = zipmapDecodeLength(p+1);
/* if the user want a free space report, and this space is
* enough, and we did't already found a suitable space... */
if (freelen && l >= reqfreelen && *freelen == 0) {
*freelen = l;
*freeoff = p-zm;
}
p += l;
zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
} else {
unsigned char free;
/* Match or skip the key */ /* Match or skip the key */
l = zipmapDecodeLength(p); l = zipmapDecodeLength(p);
if (l == klen && !memcmp(p+1,key,l)) return p; if (k == NULL && l == klen && !memcmp(p+1,key,l)) {
p += zipmapEncodeLength(NULL,l) + l; /* Only return when the user doesn't care
/* Skip the value as well */ * for the total length of the zipmap. */
l = zipmapDecodeLength(p); if (totlen != NULL) {
p += zipmapEncodeLength(NULL,l); k = p;
free = p[0]; } else {
p += l+1+free; /* +1 to skip the free byte */ return p;
}
} }
p += zipmapEncodeLength(NULL,l) + l;
/* Skip the value as well */
l = zipmapDecodeLength(p);
p += zipmapEncodeLength(NULL,l);
free = p[0];
p += l+1+free; /* +1 to skip the free byte */
} }
if (totlen != NULL) *totlen = (unsigned int)(p-zm)+1; if (totlen != NULL) *totlen = (unsigned int)(p-zm)+1;
return NULL; return k;
} }
static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen) { static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen) {
@ -220,64 +195,70 @@ static unsigned int zipmapRawValueLength(unsigned char *p) {
* free space if any). */ * free space if any). */
static unsigned int zipmapRawEntryLength(unsigned char *p) { static unsigned int zipmapRawEntryLength(unsigned char *p) {
unsigned int l = zipmapRawKeyLength(p); unsigned int l = zipmapRawKeyLength(p);
return l + zipmapRawValueLength(p+l); return l + zipmapRawValueLength(p+l);
} }
static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
zm = zrealloc(zm, len);
zm[len-1] = ZIPMAP_END;
return zm;
}
/* Set key to value, creating the key if it does not already exist. /* Set key to value, creating the key if it does not already exist.
* If 'update' is not NULL, *update is set to 1 if the key was * If 'update' is not NULL, *update is set to 1 if the key was
* already preset, otherwise to 0. */ * already preset, otherwise to 0. */
unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) { unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) {
unsigned int oldlen = 0, freeoff = 0, freelen; unsigned int zmlen, offset;
unsigned int reqlen = zipmapRequiredLength(klen,vlen); unsigned int freelen, reqlen = zipmapRequiredLength(klen,vlen);
unsigned int empty, vempty; unsigned int empty, vempty;
unsigned char *p; unsigned char *p;
freelen = reqlen; freelen = reqlen;
if (update) *update = 0; if (update) *update = 0;
p = zipmapLookupRaw(zm,key,klen,&oldlen,&freeoff,&freelen); p = zipmapLookupRaw(zm,key,klen,&zmlen);
if (p == NULL && freelen == 0) { if (p == NULL) {
/* Key not found, and not space for the new key. Enlarge */ /* Key not found: enlarge */
zm = zrealloc(zm,oldlen+reqlen); zm = zipmapResize(zm, zmlen+reqlen);
p = zm+oldlen-1; p = zm+zmlen-1;
zm[oldlen+reqlen-1] = ZIPMAP_END; zmlen = zmlen+reqlen;
freelen = reqlen;
} else if (p == NULL) {
/* Key not found, but there is enough free space. */
p = zm+freeoff;
/* note: freelen is already set in this case */
} else {
unsigned char *b = p;
/* Increase zipmap length (this is an insert) */
if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
} else {
/* Key found. Is there enough space for the new value? */ /* Key found. Is there enough space for the new value? */
/* Compute the total length: */ /* Compute the total length: */
if (update) *update = 1; if (update) *update = 1;
freelen = zipmapRawKeyLength(b); freelen = zipmapRawEntryLength(p);
b += freelen;
freelen += zipmapRawValueLength(b);
if (freelen < reqlen) { if (freelen < reqlen) {
/* Mark this entry as free and recurse */ /* Store the offset of this key within the current zipmap, so
p[0] = ZIPMAP_EMPTY; * it can be resized. Then, move the tail backwards so this
zipmapEncodeLength(p+1,freelen); * pair fits at the current position. */
zm[0] |= ZIPMAP_STATUS_FRAGMENTED; offset = p-zm;
return zipmapSet(zm,key,klen,val,vlen,NULL); zm = zipmapResize(zm, zmlen-freelen+reqlen);
p = zm+offset;
/* The +1 in the number of bytes to be moved is caused by the
* end-of-zipmap byte. Note: the *original* zmlen is used. */
memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
zmlen = zmlen-freelen+reqlen;
freelen = reqlen;
} }
} }
/* Ok we have a suitable block where to write the new key/value /* We now have a suitable block where the key/value entry can
* entry. */ * be written. If there is too much free space, move the tail
* of the zipmap a few bytes to the front and shrink the zipmap,
* as we want zipmaps to be very space efficient. */
empty = freelen-reqlen; empty = freelen-reqlen;
/* If there is too much free space mark it as a free block instead if (empty >= ZIPMAP_VALUE_MAX_FREE) {
* of adding it as trailing empty space for the value, as we want /* First, move the tail <empty> bytes to the front, then resize
* zipmaps to be very space efficient. */ * the zipmap to be <empty> bytes smaller. */
if (empty > ZIPMAP_VALUE_MAX_FREE) { offset = p-zm;
unsigned char *e; memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
zmlen -= empty;
e = p+reqlen; zm = zipmapResize(zm, zmlen);
e[0] = ZIPMAP_EMPTY; p = zm+offset;
zipmapEncodeLength(e+1,empty);
vempty = 0; vempty = 0;
zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
} else { } else {
vempty = empty; vempty = empty;
} }
@ -297,13 +278,16 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
/* Remove the specified key. If 'deleted' is not NULL the pointed integer is /* Remove the specified key. If 'deleted' is not NULL the pointed integer is
* set to 0 if the key was not found, to 1 if it was found and deleted. */ * set to 0 if the key was not found, to 1 if it was found and deleted. */
unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int klen, int *deleted) { unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int klen, int *deleted) {
unsigned char *p = zipmapLookupRaw(zm,key,klen,NULL,NULL,NULL); unsigned int zmlen, freelen;
unsigned char *p = zipmapLookupRaw(zm,key,klen,&zmlen);
if (p) { if (p) {
unsigned int freelen = zipmapRawEntryLength(p); freelen = zipmapRawEntryLength(p);
memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
zm = zipmapResize(zm, zmlen-freelen);
/* Decrease zipmap length */
if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;
p[0] = ZIPMAP_EMPTY;
zipmapEncodeLength(p+1,freelen);
zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
if (deleted) *deleted = 1; if (deleted) *deleted = 1;
} else { } else {
if (deleted) *deleted = 0; if (deleted) *deleted = 0;
@ -328,8 +312,6 @@ unsigned char *zipmapRewind(unsigned char *zm) {
* } * }
*/ */
unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) { unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
while(zm[0] == ZIPMAP_EMPTY)
zm += zipmapDecodeLength(zm+1);
if (zm[0] == ZIPMAP_END) return NULL; if (zm[0] == ZIPMAP_END) return NULL;
if (key) { if (key) {
*key = zm; *key = zm;
@ -351,7 +333,7 @@ unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *
int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char **value, unsigned int *vlen) { int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char **value, unsigned int *vlen) {
unsigned char *p; unsigned char *p;
if ((p = zipmapLookupRaw(zm,key,klen,NULL,NULL,NULL)) == NULL) return 0; if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
p += zipmapRawKeyLength(p); p += zipmapRawKeyLength(p);
*vlen = zipmapDecodeLength(p); *vlen = zipmapDecodeLength(p);
*value = p + ZIPMAP_LEN_BYTES(*vlen) + 1; *value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
@ -360,15 +342,21 @@ int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned
/* Return 1 if the key exists, otherwise 0 is returned. */ /* Return 1 if the key exists, otherwise 0 is returned. */
int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) { int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
return zipmapLookupRaw(zm,key,klen,NULL,NULL,NULL) != NULL; return zipmapLookupRaw(zm,key,klen,NULL) != NULL;
} }
/* Return the number of entries inside a zipmap */ /* Return the number of entries inside a zipmap */
unsigned int zipmapLen(unsigned char *zm) { unsigned int zipmapLen(unsigned char *zm) {
unsigned char *p = zipmapRewind(zm);
unsigned int len = 0; unsigned int len = 0;
if (zm[0] < ZIPMAP_BIGLEN) {
len = zm[0];
} else {
unsigned char *p = zipmapRewind(zm);
while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++; /* Re-store length if small enough */
if (len < ZIPMAP_BIGLEN) zm[0] = len;
}
return len; return len;
} }
@ -380,10 +368,6 @@ void zipmapRepr(unsigned char *p) {
if (p[0] == ZIPMAP_END) { if (p[0] == ZIPMAP_END) {
printf("{end}"); printf("{end}");
break; break;
} else if (p[0] == ZIPMAP_EMPTY) {
l = zipmapDecodeLength(p+1);
printf("{%u empty block}", l);
p += l;
} else { } else {
unsigned char e; unsigned char e;