相信不少朋友在遷移數據、擴容存儲時需要將memcached中的數據導一份出來,然后在新的memcached中寫入,操作步驟如下:
第一步:導出所有key
方法一:memdump工具
memdump --servers="xxx.xxx.xxx.xxx:port" > keys.txt
方法二:
echo "stats cachedump slabid 0" | nc xxx.xxx.xxx.xxx port > keys.txt
第二步:獲取所有key對應的value
第三步:將所有的key-value寫入新的memcached
問題出來了,當我們在執行第一步時,發現導出的key的數量並不和memcached的number數一樣多,如下圖:
看了一下memdump命令幫助,發現文檔里面描述了不能保證獲取到所有的keys,另外查看memdump工具的源碼,發現同樣是使用stats cachedump命令實現的
於是就看了一下memcached的源碼,發現有緩沖區大小限制,最大2M,汗
修改方案:
1、加大緩沖區,但該方法有局限性,當cache中的keys數太大時又面臨緩沖區不夠用的情況
2、修改stats cachedump命令,使其支持范圍導出, stats cachedump slabid start limit
下面修改的代碼,僅供參考:
Index: items.c =================================================================== --- items.c (revision 793) +++ items.c (working copy) @@ -276,18 +276,23 @@ } /*@null@*/ -char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) { +char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes) { unsigned int memlimit = 2 * 1024 * 1024; /* 2MB max response size */ char *buffer; unsigned int bufcurr; item *it; + int i; unsigned int len; unsigned int shown = 0; char temp[512]; if (slabs_clsid > LARGEST_ID) return NULL; it = heads[slabs_clsid]; - + i = 0; + while (it != NULL && i < start) { + it = it->next; + i++; + } buffer = malloc((size_t)memlimit); if (buffer == 0) return NULL; bufcurr = 0; Index: items.h =================================================================== --- items.h (revision 793) +++ items.h (working copy) @@ -12,7 +12,7 @@ int do_item_replace(item *it, item *new_it); /*@null@*/ -char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes); +char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes); char *do_item_stats(int *bytes); /*@null@*/ Index: memcached.c =================================================================== --- memcached.c (revision 793) +++ memcached.c (working copy) @@ -1115,7 +1115,7 @@ if (strcmp(subcommand, "cachedump") == 0) { char *buf; - unsigned int bytes, id, limit = 0; + unsigned int bytes, id, limit = 0, start = 0; if(ntokens < 5) { out_string(c, "CLIENT_ERROR bad command line"); @@ -1123,14 +1123,21 @@ } id = strtoul(tokens[2].value, NULL, 10); - limit = strtoul(tokens[3].value, NULL, 10); + if(ntokens == 5){ + start = 0; + limit = strtoul(tokens[3].value, NULL, 10); + } + else if(ntokens == 6){ + start = strtoul(tokens[3].value, NULL, 10); + limit = strtoul(tokens[4].value, NULL, 10); + } if(errno == ERANGE) { out_string(c, "CLIENT_ERROR bad command line format"); return; } - buf = item_cachedump(id, limit, &bytes); + buf = item_cachedump(id, start, limit, &bytes); write_and_free(c, buf, bytes); return; } Index: memcached.h =================================================================== --- memcached.h (revision 793) +++ memcached.h (working copy) @@ -280,7 +280,7 @@ char *mt_defer_delete(item *it, time_t exptime); int mt_is_listen_thread(void); item *mt_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes); -char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes); +char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes); void mt_item_flush_expired(void); item *mt_item_get_notedeleted(const char *key, const size_t nkey, bool *delete_locked); int mt_item_link(item *it); @@ -309,7 +309,7 @@ # define defer_delete(x,y) mt_defer_delete(x,y) # define is_listen_thread() mt_is_listen_thread() # define item_alloc(x,y,z,a,b) mt_item_alloc(x,y,z,a,b) -# define item_cachedump(x,y,z) mt_item_cachedump(x,y,z) +# define item_cachedump(x,y,z,a) mt_item_cachedump(x,y,z,a) # define item_flush_expired() mt_item_flush_expired() # define item_get_notedeleted(x,y,z) mt_item_get_notedeleted(x,y,z) # define item_link(x) mt_item_link(x) @@ -342,7 +342,7 @@ # define dispatch_event_add(t,c) event_add(&(c)->event, 0) # define is_listen_thread() 1 # define item_alloc(x,y,z,a,b) do_item_alloc(x,y,z,a,b) -# define item_cachedump(x,y,z) do_item_cachedump(x,y,z) +# define item_cachedump(x,y,z,a) do_item_cachedump(x,y,z,a) # define item_flush_expired() do_item_flush_expired() # define item_get_notedeleted(x,y,z) do_item_get_notedeleted(x,y,z) # define item_link(x) do_item_link(x) Index: thread.c =================================================================== --- thread.c (revision 793) +++ thread.c (working copy) @@ -528,11 +528,11 @@ /* * Dumps part of the cache */ -char *mt_item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int *bytes) { +char *mt_item_cachedump(unsigned int slabs_clsid, const unsigned int start, unsigned int limit, unsigned int *bytes) { char *ret; pthread_mutex_lock(&cache_lock); - ret = do_item_cachedump(slabs_clsid, limit, bytes); + ret = do_item_cachedump(slabs_clsid, start, limit, bytes); pthread_mutex_unlock(&cache_lock); return ret; }