memcached數據遷移問題及解決方案


相信不少朋友在遷移數據、擴容存儲時需要將memcached中的數據導一份出來,然后在新的memcached中寫入,操作步驟如下:

第一步:導出所有key

  方法一:memdump工具

  memdump --servers="xxx.xxx.xxx.xxx:port" > keys.txt

  方法二:

  echo "stats cachedump slabid 0" | nc xxx.xxx.xxx.xxx port > keys.txt 

第二步:獲取所有key對應的value

第三步:將所有的key-value寫入新的memcached

問題出來了,當我們在執行第一步時,發現導出的key的數量並不和memcached的number數一樣多,如下圖:

看了一下memdump命令幫助,發現文檔里面描述了不能保證獲取到所有的keys,另外查看memdump工具的源碼,發現同樣是使用stats cachedump命令實現的

於是就看了一下memcached的源碼,發現有緩沖區大小限制,最大2M,汗

 

修改方案:

  1、加大緩沖區,但該方法有局限性,當cache中的keys數太大時又面臨緩沖區不夠用的情況

  2、修改stats cachedump命令,使其支持范圍導出, stats cachedump slabid start limit

下面修改的代碼,僅供參考:

Index: items.c
===================================================================
--- items.c    (revision 793)
+++ items.c    (working copy)
@@ -276,18 +276,23 @@
 }
 
 /*@null@*/
-char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) {
+char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes) {
     unsigned int memlimit = 2 * 1024 * 1024;   /* 2MB max response size */
     char *buffer;
     unsigned int bufcurr;
     item *it;
+    int i;
     unsigned int len;
     unsigned int shown = 0;
     char temp[512];
 
     if (slabs_clsid > LARGEST_ID) return NULL;
     it = heads[slabs_clsid];
-
+    i = 0;    
+    while (it != NULL && i < start) {
+        it = it->next;
+        i++;
+        }
     buffer = malloc((size_t)memlimit);
     if (buffer == 0) return NULL;
     bufcurr = 0;
Index: items.h
===================================================================
--- items.h    (revision 793)
+++ items.h    (working copy)
@@ -12,7 +12,7 @@
 int  do_item_replace(item *it, item *new_it);
 
 /*@null@*/
-char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
+char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes);
 char *do_item_stats(int *bytes);
 
 /*@null@*/
Index: memcached.c
===================================================================
--- memcached.c    (revision 793)
+++ memcached.c    (working copy)
@@ -1115,7 +1115,7 @@
     if (strcmp(subcommand, "cachedump") == 0) {
 
         char *buf;
-        unsigned int bytes, id, limit = 0;
+        unsigned int bytes, id, limit = 0, start = 0;
 
         if(ntokens < 5) {
             out_string(c, "CLIENT_ERROR bad command line");
@@ -1123,14 +1123,21 @@
         }
 
         id = strtoul(tokens[2].value, NULL, 10);
-        limit = strtoul(tokens[3].value, NULL, 10);
+    if(ntokens == 5){
+        start = 0;
+        limit = strtoul(tokens[3].value, NULL, 10);
+        }
+        else if(ntokens == 6){
+            start = strtoul(tokens[3].value, NULL, 10);
+            limit = strtoul(tokens[4].value, NULL, 10);
+            }
 
         if(errno == ERANGE) {
             out_string(c, "CLIENT_ERROR bad command line format");
             return;
         }
 
-        buf = item_cachedump(id, limit, &bytes);
+        buf = item_cachedump(id, start, limit, &bytes);
         write_and_free(c, buf, bytes);
         return;
     }
Index: memcached.h
===================================================================
--- memcached.h    (revision 793)
+++ memcached.h    (working copy)
@@ -280,7 +280,7 @@
 char *mt_defer_delete(item *it, time_t exptime);
 int   mt_is_listen_thread(void);
 item *mt_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes);
-char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
+char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int start, const unsigned int limit, unsigned int *bytes);
 void  mt_item_flush_expired(void);
 item *mt_item_get_notedeleted(const char *key, const size_t nkey, bool *delete_locked);
 int   mt_item_link(item *it);
@@ -309,7 +309,7 @@
 # define defer_delete(x,y)           mt_defer_delete(x,y)
 # define is_listen_thread()          mt_is_listen_thread()
 # define item_alloc(x,y,z,a,b)       mt_item_alloc(x,y,z,a,b)
-# define item_cachedump(x,y,z)       mt_item_cachedump(x,y,z)
+# define item_cachedump(x,y,z,a)       mt_item_cachedump(x,y,z,a)
 # define item_flush_expired()        mt_item_flush_expired()
 # define item_get_notedeleted(x,y,z) mt_item_get_notedeleted(x,y,z)
 # define item_link(x)                mt_item_link(x)
@@ -342,7 +342,7 @@
 # define dispatch_event_add(t,c)     event_add(&(c)->event, 0)
 # define is_listen_thread()          1
 # define item_alloc(x,y,z,a,b)       do_item_alloc(x,y,z,a,b)
-# define item_cachedump(x,y,z)       do_item_cachedump(x,y,z)
+# define item_cachedump(x,y,z,a)       do_item_cachedump(x,y,z,a)
 # define item_flush_expired()        do_item_flush_expired()
 # define item_get_notedeleted(x,y,z) do_item_get_notedeleted(x,y,z)
 # define item_link(x)                do_item_link(x)
Index: thread.c
===================================================================
--- thread.c    (revision 793)
+++ thread.c    (working copy)
@@ -528,11 +528,11 @@
 /*
  * Dumps part of the cache
  */
-char *mt_item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int *bytes) {
+char *mt_item_cachedump(unsigned int slabs_clsid, const unsigned int start, unsigned int limit, unsigned int *bytes) {
     char *ret;
 
     pthread_mutex_lock(&cache_lock);
-    ret = do_item_cachedump(slabs_clsid, limit, bytes);
+    ret = do_item_cachedump(slabs_clsid, start, limit, bytes);
     pthread_mutex_unlock(&cache_lock);
     return ret;
 }

 


免責聲明!

本站轉載的文章為個人學習借鑒使用,本站對版權不負任何法律責任。如果侵犯了您的隱私權益,請聯系本站郵箱yoyou2525@163.com刪除。



 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM