使用Curl進行http請求及下載器設計

    Curl是一個跨平台的網絡協議庫。本文記錄一些常見的請求http用法。

一、基本介紹

    curl_easy_setopt選項

1. CURLOPT_URL 訪問的url

2. CURLOPT_WRITEFUNCTION 有數據來時，默認執行的function

    CURLOPT_WRITEDATA 可以設置一個文件流sink，有數據來時，默認寫到這個sink里邊；

3. CURLOPT_HEADERFUNCTION 與write同理，有header數據來時，默認執行的function

    CURLOPT_HEADERDATA header文件流sink

4. CURLOPT_TIMEOUT 傳輸timeout時間

    CURLOPT_CONNECTIONTIMEOUT 連接timeout時間

5. CURLOPT_FOLLOWLOCATION 允許重定向

    CURLOPT_FOLLOWLOCATION 允許重定向的次數

6. CURLOPT_RESUME_FROM 

    斷點續傳，傳遞一個long參數，指定開始傳遞的偏移量

二、基本的http GET/POST操作

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
 
#include <curl/curl.h>
//#include <curl/types.h>
#include <curl/easy.h>
 
FILE *fp;

size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
    printf("%s", ptr);
    int written = fwrite(ptr, size, nmemb, (FILE *)fp);
    return written;
}

size_t header_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
    printf("%s", ptr);
    int written = fwrite(ptr, size, nmemb, (FILE *)fp);
    return written;
}
 
int main(int argc, char *argv[])
{
    CURL *curl;
 
    curl_global_init(CURL_GLOBAL_ALL);
    curl=curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_URL,"http://www.baidu.com");
 
    if((fp=fopen("curltest.txt","w"))==NULL)
    {
        curl_easy_cleanup(curl);
        exit(1);
    }
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_data);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    exit(0);
}

三、進度條顯示下載文件的進度

CURLOPT_XFERINFOFUNCTION獲取文件傳輸進度，其中CURLOPT_NOPROGRESS必須設置為false；

#include <stdio.h>
#include <curl/curl.h>
 
#define TIME_IN_US 1 /* microseconds */
#define TIMETYPE curl_off_t
#define TIMEOPT CURLINFO_TOTAL_TIME_T
#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL     3000000
#define STOP_DOWNLOAD_AFTER_THIS_MANY_BYTES         6000
 
struct myprogress {
  TIMETYPE lastruntime; /* type depends on version, see above */
  CURL *curl;
};
 
/* this is how the CURLOPT_XFERINFOFUNCTION callback works */
static int xferinfo(void *p,
                    curl_off_t dltotal, curl_off_t dlnow,
                    curl_off_t ultotal, curl_off_t ulnow)
{
  struct myprogress *myp = (struct myprogress *)p;
  CURL *curl = myp->curl;
  TIMETYPE curtime = 0;
 
  curl_easy_getinfo(curl, TIMEOPT, &curtime);
 
  /* under certain circumstances it may be desirable for certain functionality
     to only run every N seconds, in order to do this the transaction time can
     be used */
  if((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) {
    myp->lastruntime = curtime;
#ifdef TIME_IN_US
    fprintf(stderr, "TOTAL TIME: %" CURL_FORMAT_CURL_OFF_T ".%06ld\r\n",
            (curtime / 1000000), (long)(curtime % 1000000));
#else
    fprintf(stderr, "TOTAL TIME: %f \r\n", curtime);
#endif
  }
 
  fprintf(stderr, "UP: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
          "  DOWN: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
          "\r\n",
          ulnow, ultotal, dlnow, dltotal);
 
  if(dlnow > STOP_DOWNLOAD_AFTER_THIS_MANY_BYTES)
    return 1;
  return 0;
}
 
int main(void)
{
  CURL *curl;
  CURLcode res = CURLE_OK;
  struct myprogress prog;
 
  curl = curl_easy_init();
  if(curl) {
    prog.lastruntime = 0;
    prog.curl = curl;
 
    curl_easy_setopt(curl, CURLOPT_URL, "https://www.baidu.com");
    curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xferinfo);
    /* pass the struct pointer into the xferinfo function, note that this is
       an alias to CURLOPT_PROGRESSDATA */
    curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &prog);

 
    curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
    res = curl_easy_perform(curl);
 
    if(res != CURLE_OK)
      fprintf(stderr, "%s\n", curl_easy_strerror(res));
 
    /* always cleanup */
    curl_easy_cleanup(curl);
  }
  return (int)res;
}

四、斷點續傳

CURLOPT_RESUME_FROM_LARGE 實現斷點續傳

//采用CURLOPT_RESUME_FROM_LARGE 實現文件斷點續傳功能
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
 
#include <curl/curl.h>
//這個函數為CURLOPT_HEADERFUNCTION參數構造
/* 從http頭部獲取文件size*/
size_t getcontentlengthfunc(void *ptr, size_t size, size_t nmemb, void *stream) {
       int r;
       long len = 0;
 
       /* _snscanf() is Win32 specific */
       // r = _snscanf(ptr, size * nmemb, "Content-Length: %ld\n", &len);
 r = sscanf((const char*)ptr, "Content-Length: %ld\n", &len);
       if (r) /* Microsoft: we don't read the specs */
              *((long *) stream) = len;
 
       return size * nmemb;
}
 
/* 保存下載文件 */
size_t wirtefunc(void *ptr, size_t size, size_t nmemb, void *stream)
{
        return fwrite(ptr, size, nmemb, (FILE*)stream);
}
 
/*讀取上傳文件 */
size_t readfunc(void *ptr, size_t size, size_t nmemb, void *stream)
{
       FILE *f = (FILE*)stream;
       size_t n;
 
       if (ferror(f))
              return CURL_READFUNC_ABORT;
 
       n = fread(ptr, size, nmemb, f) * size;
 
       return n;
}
 
// 下載 或者上傳文件函數
int download(CURL *curlhandle, const char * remotepath, const char * localpath,
           long timeout, long tries)
{
    FILE *f;
    curl_off_t local_file_len = -1 ;
    long filesize =0 ;

    CURLcode r = CURLE_GOT_NOTHING;
    int c;
    struct stat file_info;
    int use_resume = 0;
    if(stat(localpath, &file_info) == 0)
    {
        local_file_len =  file_info.st_size;
        use_resume  = 1;
    }
    //采用追加方式打開文件，便於實現文件斷點續傳工作
    f = fopen(localpath, "ab+");
    if (f == NULL) {
          perror(NULL);
          return 0;
    }

    curl_easy_setopt(curlhandle, CURLOPT_URL, remotepath);

    curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, timeout);  // 設置連接超時，單位秒
    //設置http 頭部處理函數
    curl_easy_setopt(curlhandle, CURLOPT_HEADERFUNCTION, getcontentlengthfunc);
    curl_easy_setopt(curlhandle, CURLOPT_HEADERDATA, &filesize);
    // 設置文件續傳的位置給libcurl
    curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM_LARGE, use_resume?local_file_len:0);

    curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, f);
    curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, wirtefunc);

    curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1L);
    curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, 1L);

    r = curl_easy_perform(curlhandle);
    fclose(f);
    if (r == CURLE_OK)
          return 1;
    else {
          fprintf(stderr, "%s\n", curl_easy_strerror(r));
          return 0;
    }
}
 
int main(int c, char **argv) {
       CURL *curlhandle = NULL;
 
       curl_global_init(CURL_GLOBAL_ALL);
       curlhandle = curl_easy_init();
 
       download(curlhandle , "http://www.baidu.com","curltest.txt",1,3);
       curl_easy_cleanup(curlhandle);
       curl_global_cleanup();
 
       return 0;
}

五、下載器設計

1. 封裝curl；

2. 設計Source類，內部包含線程，負責讀取數據到內部隊列；

3. Source內部線程每讀到128字節（實際可酌情增減）就組裝成一個Block，push到本地Block隊列中； 

4. Source包含Read接口，供外部讀取數據；