netty的http客戶端與apache的http客戶端比較

本文轉載自查看原文 2019-09-20 18:57 1237

　　之前學習了netty和http異步連接池，跟倉頡大神問的結果是netty的http客戶端性能比apache的好。

　　咱今兒就用三種http連接池進行測試。

　　首先是pom.xml：

 1 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 3   <modelVersion>4.0.0</modelVersion>
 4 
 5   <groupId>com.company</groupId>
 6   <artifactId>websocket_demo</artifactId>
 7   <version>0.0.1-SNAPSHOT</version>
 8   <packaging>jar</packaging>
 9 
10   <name>websocket_demo</name>
11   <url>http://maven.apache.org</url>
12 
13   <properties>
14     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
15     <junit.version>4.12</junit.version>
16     <log4j.version>1.2.17</log4j.version>
17     <netty.version>4.1.36.Final</netty.version>
18     <google.gson.version>2.3</google.gson.version>
19     <guava.version>19.0</guava.version>
20   </properties>
21 
22   <dependencies>
23      <!-- junit -->
24      <dependency>
25          <groupId>junit</groupId>
26          <artifactId>junit</artifactId>
27          <version>${junit.version}</version>
28          <scope>test</scope>
29      </dependency>
30 
31      <!-- log4j -->
32      <dependency>
33          <groupId>log4j</groupId>
34          <artifactId>log4j</artifactId>
35          <version>${log4j.version}</version>
36      </dependency>
37      
38      <!-- netty -->
39      <dependency>
40         <groupId>io.netty</groupId>
41         <artifactId>netty-all</artifactId>
42         <version>${netty.version}</version>
43      </dependency>
44      
45      <!-- gson -->
46      <dependency>
47          <groupId>com.google.code.gson</groupId>
48          <artifactId>gson</artifactId>
49          <version>${google.gson.version}</version>
50      </dependency>
51      
52      <!-- guava -->
53      <dependency>
54          <groupId>com.google.guava</groupId>
55          <artifactId>guava</artifactId>
56          <version>${guava.version}</version>
57      </dependency>
58      
59      
60 
61 <dependency>
62     <groupId>org.apache.httpcomponents</groupId>
63     <artifactId>httpclient</artifactId>
64     <version>4.5.8</version>
65 </dependency>
66 
67      <dependency>
68     <groupId>org.apache.commons</groupId>
69     <artifactId>commons-io</artifactId>
70     <version>1.3.2</version>
71 </dependency>
72      
73   </dependencies>
74   
75   <build>
76       <finalName>Netty_WebSocket</finalName>
77       <plugins>
78           <!--編譯版本-->
79           <plugin>
80               <artifactId>maven-compiler-plugin</artifactId>
81               <version>2.3.1</version>
82               <configuration>
83                   <source>1.8</source>
84                   <target>1.8</target>
85                   <encoding>UTF-8</encoding>
86                   <compilerArguments>
87                       <extdirs>src/main/webapp/WEB-INF/lib</extdirs>
88                   </compilerArguments>
89               </configuration>
90           </plugin>
91        </plugins>
92   </build>
93   
94 </project>

pom.xml

　　導入好包之后，創建一個用於測試http連接池連接時間的基類BaseHttpClientTest（在倉頡大神的基礎之上做的測試，測試時發現某些數據有問題，就再觀察下載的網頁代碼長度）：

 1 package com.company.client;
 2 
 3 import java.util.ArrayList;
 4 import java.util.List;
 5 import java.util.concurrent.atomic.AtomicInteger;
 6 
 7 /**
 8  * 連接池基類
 9  * 
10  * @author 五月的倉頡https://www.cnblogs.com/xrq730/p/10963689.html
11  */
12 public class BaseHttpClientTest {
13 
14     protected static final int REQUEST_COUNT = 15;
15 
16     protected static final String SEPERATOR = "   ";
17     
18     protected static final AtomicInteger NOW_COUNT = new AtomicInteger(0);
19     
20     protected static final StringBuilder EVERY_REQ_COST = new StringBuilder(200);
21     
22     protected static final String TEST_URL = "http://sohu.com";
23     
24     protected static long averageLength = 0;
25     
26     /**
27      * 獲取待運行的線程
28      */
29     protected List<Thread> getRunThreads(Runnable runnable) {
30         List<Thread> tList = new ArrayList<Thread>(REQUEST_COUNT);
31         
32         for (int i = 0; i < REQUEST_COUNT; i++) {
33             tList.add(new Thread(runnable));
34         }
35         
36         return tList;
37     }
38     
39     /**
40      * 啟動所有線程
41      */
42     protected void startUpAllThreads(List<Thread> tList) {
43         for (Thread t : tList) {
44             t.start();
45             // 這里需要加一點延遲，保證請求按順序發出去
46             try {
47                 Thread.sleep(300);
48             } catch (InterruptedException e) {
49                 e.printStackTrace();
50             }
51         }
52     }
53     
54     protected synchronized void addContentLength(long len) {
55         averageLength += len;
56     }
57     
58     protected synchronized void addCost(long cost) {
59         EVERY_REQ_COST.append(cost);
60         EVERY_REQ_COST.append("ms");
61         EVERY_REQ_COST.append(SEPERATOR);
62     }
63     
64 }

View Code

　　不用連接池的子類HttpClientWithoutPoolTest：

 1 package com.company.client;
 2 
 3 import java.io.InputStream;
 4 
 5 import org.apache.commons.io.IOUtils;
 6 import org.apache.http.client.methods.CloseableHttpResponse;
 7 import org.apache.http.client.methods.HttpGet;
 8 import org.apache.http.impl.client.CloseableHttpClient;
 9 import org.apache.http.impl.client.HttpClients;
10 import org.junit.Test;
11 
12 /**
13  * 不使用連接池測試
14  * 
15  * @author 五月的倉頡https://www.cnblogs.com/xrq730/p/10963689.html
16  */
17 public class HttpClientWithoutPoolTest extends BaseHttpClientTest {
18 
19     @Test
20     public void test() throws Exception {
21         startUpAllThreads(getRunThreads(new HttpThread()));
22         // 等待線程運行
23         for (;;);
24     }
25     
26     private class HttpThread implements Runnable {
27 
28         @Override
29         public void run() {
30             /**
31              * HttpClient是線程安全的，因此HttpClient正常使用應當做成全局變量，但是一旦全局共用一個，HttpClient內部構建的時候會new一個連接池
32              * 出來，這樣就體現不出使用連接池的效果，因此這里每次new一個HttpClient，保證每次都不通過連接池請求對端
33              */
34             CloseableHttpClient httpClient = HttpClients.custom().build();
35             HttpGet httpGet = new HttpGet(TEST_URL);
36             
37             long startTime = System.currentTimeMillis();
38             try {
39                 CloseableHttpResponse response = httpClient.execute(httpGet);
40                 if (response != null) {
41                     addContentLength(IOUtils.toString(response.getEntity().getContent()).length());
42                     response.close();
43                 }
44             } catch (Exception e) {
45                 e.printStackTrace();
46             } finally {
47                 addCost(System.currentTimeMillis() - startTime);
48                 
49                 if (NOW_COUNT.incrementAndGet() == REQUEST_COUNT) {
50                     System.out.println(EVERY_REQ_COST.toString());
51                     System.out.println(averageLength / REQUEST_COUNT);
52                 }
53             }
54         }
55         
56     }
57     
58 }

View Code

　　異步連接池的子類HttpclientWithPoolTest：

package com.company.client;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHost;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.junit.Before;
import org.junit.Test;

/**
 * 使用連接池測試
 * 
 * @author 五月的倉頡https://www.cnblogs.com/xrq730/p/10963689.html
 */
public class HttpclientWithPoolTest extends BaseHttpClientTest {

    private CloseableHttpClient httpClient = null;
    
    PoolingHttpClientConnectionManager connectionManager = null;
    
    @Before
    public void before() {
        initHttpClient();
    }
    
    @Test
    public void test() throws Exception {
        startUpAllThreads(getRunThreads(new HttpThread()));
        connectionManager.shutdown();
    }
    
    private class HttpThread implements Runnable {

        @Override
        public void run() {
            HttpGet httpGet = new HttpGet(TEST_URL);
            // 長連接標識，不加也沒事，HTTP1.1默認都是Connection: keep-alive的
            httpGet.addHeader("Connection", "keep-alive");
            
            long startTime = System.currentTimeMillis();
            try {
                CloseableHttpResponse response = httpClient.execute(httpGet);
                if (response != null) {
                	addContentLength(IOUtils.toString(response.getEntity().getContent()).length());
                    response.close();
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                addCost(System.currentTimeMillis() - startTime);
                
                if (NOW_COUNT.incrementAndGet() == REQUEST_COUNT) {
                    System.out.println(EVERY_REQ_COST.toString());
                    System.out.println(averageLength / REQUEST_COUNT);
                }
            }
        }
        
    }
    
    private void initHttpClient() {
        connectionManager = new PoolingHttpClientConnectionManager();
        // 總連接池數量
        connectionManager.setMaxTotal(1);
        // 可為每個域名設置單獨的連接池數量
        try {
			connectionManager.setMaxPerRoute(new HttpRoute(new HttpHost(new URL(TEST_URL).getHost())), 1);
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
        // setConnectTimeout表示設置建立連接的超時時間
        // setConnectionRequestTimeout表示從連接池中拿連接的等待超時時間
        // setSocketTimeout表示發出請求后等待對端應答的超時時間
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(10000).setConnectionRequestTimeout(20000)
                .setSocketTimeout(30000).build();
        // 重試處理器，StandardHttpRequestRetryHandler這個是官方提供的，看了下感覺比較挫，很多錯誤不能重試，可自己實現HttpRequestRetryHandler接口去做
        HttpRequestRetryHandler retryHandler = new StandardHttpRequestRetryHandler();
        
        httpClient = HttpClients.custom().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig)
                .setRetryHandler(retryHandler).build();
        
        // 服務端假設關閉了連接，對客戶端是不透明的，HttpClient為了緩解這一問題，在某個連接使用前會檢測這個連接是否過時，如果過時則連接失效，但是這種做法會為每個請求
        // 增加一定額外開銷，因此有一個定時任務專門回收長時間不活動而被判定為失效的連接，可以某種程度上解決這個問題
        Timer timer = new Timer();
        timer.schedule(new TimerTask() {
            @Override
            public void run() {
                try {
                    // 關閉失效連接並從連接池中移除
                    connectionManager.closeExpiredConnections();
                    // 關閉30秒鍾內不活動的連接並從連接池中移除，空閑時間從交還給連接管理器時開始
                    connectionManager.closeIdleConnections(20, TimeUnit.SECONDS);
                } catch (Throwable t) {
                    t.printStackTrace();
                }
            }
        }, 0 , 1000 * 5);
    }
    
}

View Code

　　使用netty管理http客戶端連接的HttpclientNettyTest：

  1 package com.company.client;
  2 
  3 import java.net.MalformedURLException;
  4 import java.net.URL;
  5 
  6 import org.apache.commons.io.IOUtils;
  7 import org.junit.Before;
  8 import org.junit.Test;
  9 
 10 import io.netty.bootstrap.Bootstrap;
 11 import io.netty.channel.Channel;
 12 import io.netty.channel.ChannelHandlerContext;
 13 import io.netty.channel.ChannelInitializer;
 14 import io.netty.channel.ChannelOption;
 15 import io.netty.channel.ChannelPipeline;
 16 import io.netty.channel.EventLoopGroup;
 17 import io.netty.channel.SimpleChannelInboundHandler;
 18 import io.netty.channel.nio.NioEventLoopGroup;
 19 import io.netty.channel.socket.SocketChannel;
 20 import io.netty.channel.socket.nio.NioSocketChannel;
 21 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 22 import io.netty.handler.codec.http.DefaultHttpContent;
 23 import io.netty.handler.codec.http.DefaultHttpResponse;
 24 import io.netty.handler.codec.http.FullHttpRequest;
 25 import io.netty.handler.codec.http.HttpClientCodec;
 26 import io.netty.handler.codec.http.HttpContentDecompressor;
 27 import io.netty.handler.codec.http.HttpHeaderNames;
 28 import io.netty.handler.codec.http.HttpHeaderValues;
 29 import io.netty.handler.codec.http.HttpMethod;
 30 import io.netty.handler.codec.http.HttpObjectAggregator;
 31 import io.netty.handler.codec.http.HttpVersion;
 32 import io.netty.handler.codec.http.LastHttpContent;
 33 
 34 /**
 35  * 使用連接池測試
 36  * 
 37  * @author 五月的倉頡https://www.cnblogs.com/xrq730/p/10963689.html
 38  */
 39 public class HttpclientNettyTest extends BaseHttpClientTest {
 40 
 41     //線程組
 42     private static EventLoopGroup bossGroup = null;
 43 
 44     //啟動類
 45     private static Bootstrap bootstrap = null;
 46     
 47     @Before
 48     public void before() {
 49         initHttpClient();
 50     }
 51     
 52     public static void closePool() {
 53         if(null != bossGroup) {
 54             //優雅退出，釋放線程池資源
 55             bossGroup.shutdownGracefully();
 56         }
 57         else {
 58             System.out.println("is null");
 59         }
 60     }
 61     
 62     public static Channel getChannel(String url) throws MalformedURLException, InterruptedException {
 63         URL urlObj  = new URL(url);
 64         String host = urlObj.getHost();
 65         int port = urlObj.getPort();
 66         if(port == -1 && urlObj.getProtocol().equals("http")) {
 67             port = 80;
 68         }
 69         if(port == -1 && urlObj.getProtocol().equals("https")) {
 70             port = 443;
 71         }
 72         return bootstrap.connect(host, port).sync().channel();
 73     }
 74     
 75     public static void get(Channel channel, String url) throws MalformedURLException, InterruptedException {
 76         URL urlObj  = new URL(url);
 77         String path = urlObj.getFile();
 78         if(path == "") {
 79             path = "/";
 80         }
 81         FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 82         request.headers().set(HttpHeaderNames.HOST, urlObj.getHost());
 83         request.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
 84         request.headers().set(HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderValues.GZIP + ", deflate");
 85         channel.writeAndFlush(request);
 86     }
 87     
 88     @Test
 89     public void test() throws Exception {
 90         startUpAllThreads(getRunThreads(new HttpThread()));
 91         closePool();
 92     }
 93     
 94     private class HttpThread implements Runnable {
 95 
 96         @Override
 97         public void run() {
 98             Channel channel;
 99             long startTime = System.currentTimeMillis();
100             try {
101                 channel = getChannel(TEST_URL);
102                 get(channel, TEST_URL);
103                 channel.closeFuture().sync();
104             } catch (Exception e) {
105                 e.printStackTrace();
106             } finally {
107                 addCost(System.currentTimeMillis() - startTime);
108                 if (NOW_COUNT.incrementAndGet() == REQUEST_COUNT) {
109                     System.out.println(EVERY_REQ_COST.toString());
110                     System.out.println(averageLength / REQUEST_COUNT);
111                 }
112             }
113         }
114         
115     }
116     
117     private void initHttpClient() {
118         try {
119             bossGroup = new NioEventLoopGroup();
120             bootstrap = new Bootstrap();
121             bootstrap.group(bossGroup)
122                 .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30000)
123                 .option(ChannelOption.SO_KEEPALIVE, true)
124                 .channel(NioSocketChannel.class)
125                 .handler(new ChannelInitializer<SocketChannel>() {
126                     @Override
127                     protected void initChannel(SocketChannel ch) throws Exception {
128                         ChannelPipeline p = ch.pipeline();
129                         p.addLast(new HttpClientCodec());
130                         p.addLast(new HttpContentDecompressor());//這里要添加解壓，不然打印時會亂碼
131                         p.addLast(new HttpObjectAggregator(1234330));//添加HttpObjectAggregator， HttpClientMsgHandler才會收到FullHttpResponse
132                         p.addLast("handler", new SimpleChannelInboundHandler<Object>() {
133                             @Override
134                             protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
135                                 LastHttpContent d = (LastHttpContent)msg;
136                                 addContentLength(d.content().toString(io.netty.util.CharsetUtil.UTF_8).length());
137                             }
138                         });
139                     }
140                 });
141         }catch(Exception e) {
142             e.printStackTrace();
143         }
144     }
145     
146 }

View Code

　　在測試不同網址時，觀察到一些想不到的情況。

442ms   443ms   449ms   460ms   178ms   153ms   131ms   146ms   161ms   145ms   135ms   128ms   142ms   145ms   140ms   
23149

無連接池的cnblogs連接

384ms   249ms   165ms   161ms   158ms   103ms   128ms   129ms   89ms   147ms   143ms   144ms   126ms   146ms   155ms   
23149

有連接池的cnblogs連接

372ms   73ms   67ms   61ms   69ms   64ms   107ms   61ms   68ms   61ms   137ms   68ms   72ms   59ms   57ms   
256

netty的cnblogs連接

　　可以看到，在連接cnblogs這種https站時，異步連接的優勢明顯，下載性能從優到劣是：netty>異步連接池>不用連接池。

　　但是如果是http站呢？

　　咱們把基類第22行的網頁url改成"http://mini.eastday.com"

101ms   100ms   102ms   27ms   22ms   23ms   21ms   20ms   21ms   21ms   20ms   21ms   29ms   20ms   22ms   
31126

無連接池的東方頭條連接

106ms   44ms   34ms   506ms   359ms   157ms   60ms   44ms   37ms   38ms   34ms   36ms   37ms   36ms   40ms   
31126

有連接池的東方頭條連接

539ms   239ms   43ms   317ms   82ms   46ms   38ms   100ms   42ms   35ms   41ms   40ms   43ms   35ms   48ms   
31126

netty的東方頭條連接

　　對於東方頭條門戶這種加載速度本身就快的網頁，使用異步機制對整體加載速度影響不大。

　　但是如果需要加載大一點的網頁呢？

　　咱們把基類第22行的網頁url改成"https://mini.eastday.com/a/190919091345041.html"

　　無連接池的東方頭條新聞網頁：

510ms   374ms   526ms   539ms   124ms   74ms   72ms   62ms   70ms   75ms   70ms   75ms   207ms   86ms   74ms   
65325

　　有連接池的東方頭條新聞網頁：

424ms   185ms   92ms   63ms   74ms   77ms   77ms   66ms   92ms   64ms   107ms   65ms   70ms   51ms   90ms   
65325

　　netty的東方頭條新聞網頁：

318ms   44ms   33ms   41ms   42ms   41ms   47ms   34ms   49ms   38ms   30ms   34ms   47ms   40ms   44ms   
264

　　如果下載比較大的網頁，異步機制確實能讓系統總體的加載速度加快。

免責聲明！

本站轉載的文章為個人學習借鑒使用，本站對版權不負任何法律責任。如果侵犯了您的隱私權益，請聯系本站郵箱yoyou2525@163.com刪除。

猜您在找 Netty實現Http客戶端 Netty 5 獲取客戶端IP（非HTTP） Netty實現Http客戶端【支持https請求】 apache httpclient cache 實現可緩存的http客戶端 python網絡編程 — HTTP客戶端 IDEA自帶的http客戶端（轉） HTTP/FTP客戶端開發庫 C++的 HTTP 客戶端庫客戶端用httpurlconnection來進行http連接的 java中的HTTP客戶端