せかいろぐ

ほぼ自分用備忘録

JavaのHttpClientをマルチスレッドで使う

JavaのHttpClientをマルチスレッドで使いたい。(HttpClient 4.2.5)


複数のスレッドから同じインスタンスを普通に使ってみる。(ダメな例)

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

public class HttpTest extends Thread {
	private DefaultHttpClient client;

	public HttpTest(DefaultHttpClient client){
		this.client = client;
	}

	public static void main(String[] args) throws Exception {
		DefaultHttpClient client = new DefaultHttpClient();

		HttpTest[] test = new HttpTest[100];

		for(int i = 0; i < test.length; i++){
			test[i] = new HttpTest(client);
			test[i].start();
		}
		for(int i = 0; i < test.length; i++){
			test[i].join();
		}

		client.getConnectionManager().shutdown();
	}

	public void run() {
		try {
			HttpEntity entity = client.execute(new HttpGet("http://localhost/sleep.php")).getEntity();
			System.out.println(getId() + " : " + EntityUtils.toString(entity));
			EntityUtils.consume(entity);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}


結果はコチラ

java.lang.IllegalStateException: Invalid use of BasicClientConnManager: connection still allocated.
java.lang.IllegalStateException: Invalid use of BasicClientConnManager: connection still allocated.
java.lang.IllegalStateException: Invalid use of BasicClientConnManager: connection still allocated.
(以下延々と)


デフォルトで使われるBasicClientConnectionManagerでマルチスレッドな通信は無理な様子。日本語の資料は古いモノしかなくて、 MultiThreadedHttpConnectionManagerとかThreadSafeClientConnManagerを使え!って書いてあるんだけど、MultiThreadedHttpConnectionManagerはなくなってるし、ThreadSafeClientConnManagerはDeprecatedになってる。で、4.2以降のHttpClientを複数のスレッドから使うには、PoolingClientConnectionManagerを使えばいいらしい。やってみる。(△な例)

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.util.EntityUtils;

public class HttpTest extends Thread {
	private DefaultHttpClient client;

	public HttpTest(DefaultHttpClient client){
		this.client = client;
	}

	public static void main(String[] args) throws Exception {
		PoolingClientConnectionManager mgr = new PoolingClientConnectionManager();
		DefaultHttpClient client = new DefaultHttpClient(mgr);

		HttpTest[] test = new HttpTest[100];

		for(int i = 0; i < test.length; i++){
			test[i] = new HttpTest(client);
			test[i].start();
		}
		for(int i = 0; i < test.length; i++){
			test[i].join();
		}

		mgr.shutdown();
	}

	public void run() {
		try {
			HttpEntity entity = client.execute(new HttpGet("http://localhost/sleep.php")).getEntity();
			System.out.println(getId() + " : " + EntityUtils.toString(entity));
			EntityUtils.consume(entity);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}


ここでGETしている http://localhost/sleep.php はこんなスクリプト。

<?php
	sleep(5);
	echo("OK");
?>


うまく並列にアクセス出来ていれば、5秒と少しくらいで終了するはずが……遅い。どうやら、デフォルトでは同時接続数が一つの経路につき2、全体で20までに制限されている様子。ということで、上限を適当に変えてもう一度試す。(OKな例)

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.util.EntityUtils;

public class HttpTest extends Thread {
	private DefaultHttpClient client;

	public HttpTest(DefaultHttpClient client){
		this.client = client;
	}

	public static void main(String[] args) throws Exception {
		PoolingClientConnectionManager mgr = new PoolingClientConnectionManager();
		DefaultHttpClient client = new DefaultHttpClient(mgr);

		mgr.setDefaultMaxPerRoute(100);
		mgr.setMaxTotal(100);

		HttpTest[] test = new HttpTest[100];

		for(int i = 0; i < test.length; i++){
			test[i] = new HttpTest(client);
			test[i].start();
		}
		for(int i = 0; i < test.length; i++){
			test[i].join();
		}

		mgr.shutdown();
	}

	public void run() {
		try {
			HttpEntity entity = client.execute(new HttpGet("http://localhost/sleep.php")).getEntity();
			System.out.println(getId() + " : " + EntityUtils.toString(entity));
			EntityUtils.consume(entity);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}


うまくいった!