mirror of
https://codeberg.org/redict/redict.git
synced 2025-01-23 00:28:26 -05:00
c7dc17fc0f
This caused a crash when adding elements larger than 2GB to a set (same goes for hash keys). See #8455. Details: * The fix makes the dict hash functions receive a `size_t` instead of an `int`. In practice the dict hash functions call siphash which receives a `size_t` and the callers to the hash function pass a `size_t` to it so the fix is trivial. * The issue was recreated by attempting to add a >2gb value to a set. Appropriate tests were added where I create a set with large elements and check basic functionality on it (SADD, SCARD, SPOP, etc...). * When I added the tests I also refactored a bit all the tests code which is run under the `--large-memory` flag. This removed code duplication for the test framework's `write_big_bulk` and `write_big_bulk` code and also takes care of not allocating the test frameworks helper huge string used by these tests when not run under `--large-memory`. * I also added the _violoations.tcl_ unit tests to be part of the entire test suite and leaned up non relevant list related tests that were in there. This was done in this PR because most of the _violations_ tests are "large memory" tests.
213 lines
8.1 KiB
C
213 lines
8.1 KiB
C
/* Hash Tables Implementation.
|
|
*
|
|
* This file implements in-memory hash tables with insert/del/replace/find/
|
|
* get-random-element operations. Hash tables will auto-resize if needed
|
|
* tables of power of two in size are used, collisions are handled by
|
|
* chaining. See the source code for more information... :)
|
|
*
|
|
* Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright notice,
|
|
* this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* * Neither the name of Redis nor the names of its contributors may be used
|
|
* to endorse or promote products derived from this software without
|
|
* specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#ifndef __DICT_H
|
|
#define __DICT_H
|
|
|
|
#include "mt19937-64.h"
|
|
#include <limits.h>
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
|
|
#define DICT_OK 0
|
|
#define DICT_ERR 1
|
|
|
|
typedef struct dictEntry {
|
|
void *key;
|
|
union {
|
|
void *val;
|
|
uint64_t u64;
|
|
int64_t s64;
|
|
double d;
|
|
} v;
|
|
struct dictEntry *next; /* Next entry in the same hash bucket. */
|
|
void *metadata[]; /* An arbitrary number of bytes (starting at a
|
|
* pointer-aligned address) of size as returned
|
|
* by dictType's dictEntryMetadataBytes(). */
|
|
} dictEntry;
|
|
|
|
typedef struct dict dict;
|
|
|
|
typedef struct dictType {
|
|
uint64_t (*hashFunction)(const void *key);
|
|
void *(*keyDup)(dict *d, const void *key);
|
|
void *(*valDup)(dict *d, const void *obj);
|
|
int (*keyCompare)(dict *d, const void *key1, const void *key2);
|
|
void (*keyDestructor)(dict *d, void *key);
|
|
void (*valDestructor)(dict *d, void *obj);
|
|
int (*expandAllowed)(size_t moreMem, double usedRatio);
|
|
/* Allow a dictEntry to carry extra caller-defined metadata. The
|
|
* extra memory is initialized to 0 when a dictEntry is allocated. */
|
|
size_t (*dictEntryMetadataBytes)(dict *d);
|
|
} dictType;
|
|
|
|
#define DICTHT_SIZE(exp) ((exp) == -1 ? 0 : (unsigned long)1<<(exp))
|
|
#define DICTHT_SIZE_MASK(exp) ((exp) == -1 ? 0 : (DICTHT_SIZE(exp))-1)
|
|
|
|
struct dict {
|
|
dictType *type;
|
|
|
|
dictEntry **ht_table[2];
|
|
unsigned long ht_used[2];
|
|
|
|
long rehashidx; /* rehashing not in progress if rehashidx == -1 */
|
|
|
|
/* Keep small vars at end for optimal (minimal) struct padding */
|
|
int16_t pauserehash; /* If >0 rehashing is paused (<0 indicates coding error) */
|
|
signed char ht_size_exp[2]; /* exponent of size. (size = 1<<exp) */
|
|
};
|
|
|
|
/* If safe is set to 1 this is a safe iterator, that means, you can call
|
|
* dictAdd, dictFind, and other functions against the dictionary even while
|
|
* iterating. Otherwise it is a non safe iterator, and only dictNext()
|
|
* should be called while iterating. */
|
|
typedef struct dictIterator {
|
|
dict *d;
|
|
long index;
|
|
int table, safe;
|
|
dictEntry *entry, *nextEntry;
|
|
/* unsafe iterator fingerprint for misuse detection. */
|
|
unsigned long long fingerprint;
|
|
} dictIterator;
|
|
|
|
typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
|
|
typedef void (dictScanBucketFunction)(dict *d, dictEntry **bucketref);
|
|
|
|
/* This is the initial size of every hash table */
|
|
#define DICT_HT_INITIAL_EXP 2
|
|
#define DICT_HT_INITIAL_SIZE (1<<(DICT_HT_INITIAL_EXP))
|
|
|
|
/* ------------------------------- Macros ------------------------------------*/
|
|
#define dictFreeVal(d, entry) \
|
|
if ((d)->type->valDestructor) \
|
|
(d)->type->valDestructor((d), (entry)->v.val)
|
|
|
|
#define dictSetVal(d, entry, _val_) do { \
|
|
if ((d)->type->valDup) \
|
|
(entry)->v.val = (d)->type->valDup((d), _val_); \
|
|
else \
|
|
(entry)->v.val = (_val_); \
|
|
} while(0)
|
|
|
|
#define dictSetSignedIntegerVal(entry, _val_) \
|
|
do { (entry)->v.s64 = _val_; } while(0)
|
|
|
|
#define dictSetUnsignedIntegerVal(entry, _val_) \
|
|
do { (entry)->v.u64 = _val_; } while(0)
|
|
|
|
#define dictSetDoubleVal(entry, _val_) \
|
|
do { (entry)->v.d = _val_; } while(0)
|
|
|
|
#define dictFreeKey(d, entry) \
|
|
if ((d)->type->keyDestructor) \
|
|
(d)->type->keyDestructor((d), (entry)->key)
|
|
|
|
#define dictSetKey(d, entry, _key_) do { \
|
|
if ((d)->type->keyDup) \
|
|
(entry)->key = (d)->type->keyDup((d), _key_); \
|
|
else \
|
|
(entry)->key = (_key_); \
|
|
} while(0)
|
|
|
|
#define dictCompareKeys(d, key1, key2) \
|
|
(((d)->type->keyCompare) ? \
|
|
(d)->type->keyCompare((d), key1, key2) : \
|
|
(key1) == (key2))
|
|
|
|
#define dictMetadata(entry) (&(entry)->metadata)
|
|
#define dictMetadataSize(d) ((d)->type->dictEntryMetadataBytes \
|
|
? (d)->type->dictEntryMetadataBytes(d) : 0)
|
|
|
|
#define dictHashKey(d, key) (d)->type->hashFunction(key)
|
|
#define dictGetKey(he) ((he)->key)
|
|
#define dictGetVal(he) ((he)->v.val)
|
|
#define dictGetSignedIntegerVal(he) ((he)->v.s64)
|
|
#define dictGetUnsignedIntegerVal(he) ((he)->v.u64)
|
|
#define dictGetDoubleVal(he) ((he)->v.d)
|
|
#define dictSlots(d) (DICTHT_SIZE((d)->ht_size_exp[0])+DICTHT_SIZE((d)->ht_size_exp[1]))
|
|
#define dictSize(d) ((d)->ht_used[0]+(d)->ht_used[1])
|
|
#define dictIsRehashing(d) ((d)->rehashidx != -1)
|
|
#define dictPauseRehashing(d) (d)->pauserehash++
|
|
#define dictResumeRehashing(d) (d)->pauserehash--
|
|
|
|
/* If our unsigned long type can store a 64 bit number, use a 64 bit PRNG. */
|
|
#if ULONG_MAX >= 0xffffffffffffffff
|
|
#define randomULong() ((unsigned long) genrand64_int64())
|
|
#else
|
|
#define randomULong() random()
|
|
#endif
|
|
|
|
/* API */
|
|
dict *dictCreate(dictType *type);
|
|
int dictExpand(dict *d, unsigned long size);
|
|
int dictTryExpand(dict *d, unsigned long size);
|
|
int dictAdd(dict *d, void *key, void *val);
|
|
dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing);
|
|
dictEntry *dictAddOrFind(dict *d, void *key);
|
|
int dictReplace(dict *d, void *key, void *val);
|
|
int dictDelete(dict *d, const void *key);
|
|
dictEntry *dictUnlink(dict *d, const void *key);
|
|
void dictFreeUnlinkedEntry(dict *d, dictEntry *he);
|
|
void dictRelease(dict *d);
|
|
dictEntry * dictFind(dict *d, const void *key);
|
|
void *dictFetchValue(dict *d, const void *key);
|
|
int dictResize(dict *d);
|
|
dictIterator *dictGetIterator(dict *d);
|
|
dictIterator *dictGetSafeIterator(dict *d);
|
|
dictEntry *dictNext(dictIterator *iter);
|
|
void dictReleaseIterator(dictIterator *iter);
|
|
dictEntry *dictGetRandomKey(dict *d);
|
|
dictEntry *dictGetFairRandomKey(dict *d);
|
|
unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count);
|
|
void dictGetStats(char *buf, size_t bufsize, dict *d);
|
|
uint64_t dictGenHashFunction(const void *key, size_t len);
|
|
uint64_t dictGenCaseHashFunction(const unsigned char *buf, size_t len);
|
|
void dictEmpty(dict *d, void(callback)(dict*));
|
|
void dictEnableResize(void);
|
|
void dictDisableResize(void);
|
|
int dictRehash(dict *d, int n);
|
|
int dictRehashMilliseconds(dict *d, int ms);
|
|
void dictSetHashFunctionSeed(uint8_t *seed);
|
|
uint8_t *dictGetHashFunctionSeed(void);
|
|
unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, dictScanBucketFunction *bucketfn, void *privdata);
|
|
uint64_t dictGetHash(dict *d, const void *key);
|
|
dictEntry **dictFindEntryRefByPtrAndHash(dict *d, const void *oldptr, uint64_t hash);
|
|
|
|
#ifdef REDIS_TEST
|
|
int dictTest(int argc, char *argv[], int flags);
|
|
#endif
|
|
|
|
#endif /* __DICT_H */
|