ElasticSearchのJavaクライアントでhttpログをlogbackで出力する方法
ElasticSearchに直接HTTPでAPIを投入し動作確認をした後、Javaクライアント側で動かない場合に原因究明が難しいため、HTTPログを出力するやり方です。
elasticsearch-javaの8.2.0をmavenで取り込むと、httpclient-4.5.10.jarを使っているようなので、httpclient-4.5.10.jarでHTTPログを出せばいいことがわかりました。
pom
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>elasticsearch_sample3</groupId>
<artifactId>elasticsearch_sample3</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>11</java.version>
<maven.compiler.target>${java.version}</maven.compiler.target>
<maven.compiler.source>${java.version}</maven.compiler.source>
</properties>
<dependencies>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>8.2.0</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.12.3</version>
</dependency>
<!--
https://www.baeldung.com/apache-httpclient-enable-logging
httpclient-4.5.10.jarでログ出力を行う
-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.11</version>
</dependency>
</dependencies>
</project>https://www.baeldung.com/apache-httpclient-enable-logging
にHTTPログを出す方法が書いてあるのですが、おそらくjul-to-slf4jは間違いで、jcl-over-slf4jを使うべきだったようです。
これが分かるまでにかなり時間がかかりました。
jcl-over-slf4jでcommons-logging側の処理を行うので、commons-loggingはexclusionしています。
logback.xml
<configuration debug="false">
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%date [%level] %logger - %msg %n</pattern>
</encoder>
</appender>
<!-- 以下の設定でHTTPログを出力させる -->
<logger name="org.apache.http.wire" level="debug"/>
<root level="INFO">
<appender-ref ref="stdout"/>
</root>
</configuration>テストコード
package elasticsearch_sample3;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.Time;
import co.elastic.clients.elasticsearch._types.mapping.DateProperty;
import co.elastic.clients.elasticsearch._types.mapping.IndexOptions;
import co.elastic.clients.elasticsearch._types.mapping.KeywordProperty;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
import co.elastic.clients.elasticsearch.indices.IndexSettings;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
public class TestElasticsearchClient {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
public static void main(String args[]) {
new TestElasticsearchClient().exe();
}
public void exe() {
logger.info("接続開始");
// Create the low-level client
RestClient restClient = RestClient.builder(new HttpHost("100.64.1.25", 9200)).build();
// Create the transport with a Jackson mapper
ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
// And create the API client
ElasticsearchClient client = new ElasticsearchClient(transport);
logger.info("接続完了");
//**************************************
//**************************************
//**************************************
Map<String, Property> map = new HashMap<>();
//.index(true).store(true)
//のようにstore(true)とすると、
//GET /mytestindes/_search
//{
// "stored_fields": [ "xxx" ]
//}
//のような検索をするときにstore指定した部分を指定すると検索できる。
//store(false)のデフォルト設定だと、上記をやってもHITしない
//べつにこれはデフォルトのfalseのままでよく、どのような時につかうのか不明
// map.put("xxx", new Property(new KeywordProperty.Builder().index(false).store(false).norms(false).docValues(false).indexOptions(IndexOptions.Freqs).build()));
map.put("xxx", new Property(new KeywordProperty.Builder().index(false).store(false).norms(false).indexOptions(IndexOptions.Freqs).build()));
map.put("datetime", new Property(new DateProperty.Builder().index(false).build()));
TypeMapping typeMapping = new TypeMapping.Builder().properties(map).build();
IndexSettings indexSettings = new IndexSettings.Builder()
.numberOfShards(String.valueOf(1))
.numberOfReplicas(String.valueOf(0))
.refreshInterval( new Time.Builder().time("60s").build() )
.build();
co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder cirb = new co.elastic.clients.elasticsearch.indices.CreateIndexRequest.Builder();
co.elastic.clients.elasticsearch.indices.CreateIndexRequest cir =
cirb.index("mytestindes")
.mappings(typeMapping)
.settings(indexSettings)
.build();
try {
CreateIndexResponse response = client.indices().create(cir);
logger.info("response.acknowledged():" + response.acknowledged());
} catch (ElasticsearchException e) {
logger.error("Elasticsearchで問題発生1!", e);
} catch (IOException e) {
logger.error("Elasticsearchで問題発生2!", e);
}
try {
//transport側で読んでいるのはrestClient.close();なので
//どちらでもよいのかもしれない。
// restClient.close();
transport.close();
} catch (IOException e) {
logger.error("Elasticsearchへのコネクション切断処理でERROR発生!", e);
}
}
}
上記の処理の流れとしては、
1)ElasticsearchClientでElasticsearchとの接続を開始
2)ElasticsearchClientのindices().create()でインデックス作成
3)Elasticsearchとの接続を切断。
です。
ログ
23:40:19,096 [INFO] elasticsearch_sample3.TestElasticsearchClient - 接続開始
23:40:20,751 [INFO] elasticsearch_sample3.TestElasticsearchClient - 接続完了
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "PUT /mytestindes HTTP/1.1[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "User-Agent: elastic-java/8.2.0 (Java/11.0.2)[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "X-Elastic-Client-Meta: es=8.2.0,jv=11,hl=2,t=8.2.0,hc=4.1.4[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "Accept: application/vnd.elasticsearch+json; compatible-with=8[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "Content-Length: 247[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "Content-Type: application/vnd.elasticsearch+json; compatible-with=8[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "Host: 100.64.1.25:9200[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "Connection: Keep-Alive[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "[\r][\n]"
23:40:20,997 [DEBUG] org.apache.http.wire - http-outgoing-0 >> "{"mappings":{"properties":{"datetime":{"type":"date","index":false},"xxx":{"type":"keyword","store":false,"index":false,"index_options":"freqs","norms":false}}},"settings":{"number_of_shards":"1","number_of_replicas":"0","refresh_interval":"60s"}}"
23:40:21,216 [DEBUG] org.apache.http.wire - http-outgoing-0 << "HTTP/1.1 200 OK[\r][\n]"
23:40:21,216 [DEBUG] org.apache.http.wire - http-outgoing-0 << "X-elastic-product: Elasticsearch[\r][\n]"
23:40:21,216 [DEBUG] org.apache.http.wire - http-outgoing-0 << "content-type: application/vnd.elasticsearch+json;compatible-with=8[\r][\n]"
23:40:21,216 [DEBUG] org.apache.http.wire - http-outgoing-0 << "content-length: 70[\r][\n]"
23:40:21,216 [DEBUG] org.apache.http.wire - http-outgoing-0 << "[\r][\n]"
23:40:21,216 [DEBUG] org.apache.http.wire - http-outgoing-0 << "{"acknowledged":true,"shards_acknowledged":true,"index":"mytestindes"}"
23:40:21,309 [INFO] elasticsearch_sample3.TestElasticsearchClient - response.acknowledged():true
HTTPログが出力されており、以下のjsonをElasticsearchへ送信していることが確認できます。
{
"mappings": {
"properties": {
"datetime": {
"type": "date",
"index": false
},
"xxx": {
"type": "keyword",
"store": false,
"index": false,
"index_options": "freqs",
"norms": false
}
}
},
"settings": {
"number_of_shards": "1",
"number_of_replicas": "0",
"refresh_interval": "60s"
}
}logbackを使わない方法もある。
以下のパラメータをJAVAの引数に指定することでHTTPログを出力することが出来る。
-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.SimpleLog -Dorg.apache.commons.logging.simplelog.showdatetime=true -Dorg.apache.commons.logging.simplelog.log.org.apache.http=DEBUG