Comments (12)
官方示例代码(https://github.com/apache/incubator-doris/blob/master/samples/stream_load/java/DorisStreamLoad.java)
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
* This class is a java demo for doris stream load
*
* The pom.xml dependency:
*
* <dependency>
* <groupId>org.apache.httpcomponents</groupId>
* <artifactId>httpclient</artifactId>
* <version>4.5.3</version>
* </dependency>
*
* How to use:
*
* 1 create a table in doris with any mysql client
*
* CREATE TABLE `stream_test` (
* `id` bigint(20) COMMENT "",
* `id2` bigint(20) COMMENT "",
* `username` varchar(32) COMMENT ""
* ) ENGINE=OLAP
* DUPLICATE KEY(`id`)
* DISTRIBUTED BY HASH(`id`) BUCKETS 20;
*
*
* 2 change the Doris cluster, db, user config in this class
*
* 3 run this class, you should see the following output:
*
* {
* "TxnId": 27,
* "Label": "39c25a5c-7000-496e-a98e-348a264c81de",
* "Status": "Success",
* "Message": "OK",
* "NumberTotalRows": 10,
* "NumberLoadedRows": 10,
* "NumberFilteredRows": 0,
* "NumberUnselectedRows": 0,
* "LoadBytes": 50,
* "LoadTimeMs": 151
* }
*
* Attention:
*
* 1 wrong dependency version(such as 4.4) of httpclient may cause shaded.org.apache.http.ProtocolException
* Caused by: shaded.org.apache.http.ProtocolException: Content-Length header already present
* at shaded.org.apache.http.protocol.RequestContent.process(RequestContent.java:96)
* at shaded.org.apache.http.protocol.ImmutableHttpProcessor.process(ImmutableHttpProcessor.java:132)
* at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:182)
* at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
* at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
* at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
*
*2 run this class more than once, the status code for http response is still ok, and you will see
* the following output:
*
* {
* "TxnId": -1,
* "Label": "39c25a5c-7000-496e-a98e-348a264c81de",
* "Status": "Label Already Exists",
* "ExistingJobStatus": "FINISHED",
* "Message": "Label [39c25a5c-7000-496e-a98e-348a264c81de"] has already been used.",
* "NumberTotalRows": 0,
* "NumberLoadedRows": 0,
* "NumberFilteredRows": 0,
* "NumberUnselectedRows": 0,
* "LoadBytes": 0,
* "LoadTimeMs": 0
* }
* 3 when the response statusCode is 200, that doesn't mean your stream load is ok, there may be still
* some stream problem unless you see the output with 'ok' message
*/
public class DorisStreamLoad {
private final static String DORIS_HOST = "xxx.com";
private final static String DORIS_DB = "test";
private final static String DORIS_TABLE = "stream_test";
private final static String DORIS_USER = "root";
private final static String DORIS_PASSWORD = "xxx";
private final static int DORIS_HTTP_PORT = 8410;
private void sendData(String content) throws Exception {
final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load",
DORIS_HOST,
DORIS_HTTP_PORT,
DORIS_DB,
DORIS_TABLE);
final HttpClientBuilder httpClientBuilder = HttpClients
.custom()
.setRedirectStrategy(new DefaultRedirectStrategy() {
@Override
protected boolean isRedirectable(String method) {
return true;
}
});
try (CloseableHttpClient client = httpClientBuilder.build()) {
HttpPut put = new HttpPut(loadUrl);
StringEntity entity = new StringEntity(content, "UTF-8");
put.setHeader(HttpHeaders.EXPECT, "100-continue");
put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(DORIS_USER, DORIS_PASSWORD));
// the label header is optional, not necessary
// use label header can ensure at most once semantics
put.setHeader("label", "39c25a5c-7000-496e-a98e-348a264c81de");
put.setEntity(entity);
try (CloseableHttpResponse response = client.execute(put)) {
String loadResult = "";
if (response.getEntity() != null) {
loadResult = EntityUtils.toString(response.getEntity());
}
final int statusCode = response.getStatusLine().getStatusCode();
// statusCode 200 just indicates that doris be service is ok, not stream load
// you should see the output content to find whether stream load is success
if (statusCode != 200) {
throw new IOException(
String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult));
}
System.out.println(loadResult);
}
}
}
private String basicAuthHeader(String username, String password) {
final String tobeEncode = username + ":" + password;
byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8));
return "Basic " + new String(encoded);
}
public static void main(String[] args) throws Exception {
int id1 = 1;
int id2 = 10;
String id3 = "张三";
int rowNumber = 10;
String oneRow = id1 + "\t" + id2 + "\t" + id3 + "\n";
StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < rowNumber; i++) {
stringBuilder.append(oneRow);
}
//in doris 0.9 version, you need to comment this line
//refer to https://github.com/apache/incubator-doris/issues/783
stringBuilder.deleteCharAt(stringBuilder.length() - 1);
String loadData = stringBuilder.toString();
DorisStreamLoad dorisStreamLoad = new DorisStreamLoad();
dorisStreamLoad.sendData(loadData);
}
}
from addax.
如果放在插件里应该如何处理
from addax.
@zhbdesign 我是把示例代码贴在这里作为一个参考,插件今天开始动手写
from addax.
@zhbdesign 我是把示例代码贴在这里作为一个参考,插件今天开始动手写
老师辛苦!etl又变得更强了
from addax.
@zhbdesign 我是把示例代码贴在这里作为一个参考,插件今天开始动手写
老师辛苦!etl又变得更强了
光搭建环境就头大,官方没有提供二进制包,还得自己编译
from addax.
@zhbdesign 我是把示例代码贴在这里作为一个参考,插件今天开始动手写
老师辛苦!etl又变得更强了
光搭建环境就头大,官方没有提供二进制包,还得自己编译
你要不要使用dorisDB进行搭建,1.12.1 版本发布:
http://dorisdb-release.oss-cn-zhangjiakou.aliyuncs.com/DorisDB-SE-1.12.1.tar.gz?Expires=1614578905&OSSAccessKeyId=LTAI4GFYjbX9e7QmFnAAvkt8&Signature=zr95gaMaHG0Hure%2FMPkNjOlu1W0%3D
文档:http://doc.dorisdb.com/2140965
from addax.
@zhbdesign 我是把示例代码贴在这里作为一个参考,插件今天开始动手写
老师辛苦!etl又变得更强了
光搭建环境就头大,官方没有提供二进制包,还得自己编译
你要不要使用dorisDB进行搭建,1.12.1 版本发布:
http://dorisdb-release.oss-cn-zhangjiakou.aliyuncs.com/DorisDB-SE-1.12.1.tar.gz?Expires=1614578905&OSSAccessKeyId=LTAI4GFYjbX9e7QmFnAAvkt8&Signature=zr95gaMaHG0Hure%2FMPkNjOlu1W0%3D
文档:http://doc.dorisdb.com/2140965
这个是二进制包,启动单实例很简单方便
from addax.
@zhbdesign 我是把示例代码贴在这里作为一个参考,插件今天开始动手写
老师辛苦!etl又变得更强了
光搭建环境就头大,官方没有提供二进制包,还得自己编译
你要不要使用dorisDB进行搭建,1.12.1 版本发布:
http://dorisdb-release.oss-cn-zhangjiakou.aliyuncs.com/DorisDB-SE-1.12.1.tar.gz?Expires=1614578905&OSSAccessKeyId=LTAI4GFYjbX9e7QmFnAAvkt8&Signature=zr95gaMaHG0Hure%2FMPkNjOlu1W0%3D
文档:http://doc.dorisdb.com/2140965这个是二进制包,启动单实例很简单方便
from addax.
@zhbdesign 感谢,我试下
from addax.
@zhbdesign 已经初步实现并通过简单测试,如果你使用的是3.2.1版本,可以直接下载 doriswriter-3.2.2-SNAPSHOT.zip 并测试
from addax.
官方示例代码(https://github.com/apache/incubator-doris/blob/master/samples/stream_load/java/DorisStreamLoad.java)
// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. import org.apache.commons.codec.binary.Base64; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultRedirectStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.nio.charset.StandardCharsets; /** * This class is a java demo for doris stream load * * The pom.xml dependency: * * <dependency> * <groupId>org.apache.httpcomponents</groupId> * <artifactId>httpclient</artifactId> * <version>4.5.3</version> * </dependency> * * How to use: * * 1 create a table in doris with any mysql client * * CREATE TABLE `stream_test` ( * `id` bigint(20) COMMENT "", * `id2` bigint(20) COMMENT "", * `username` varchar(32) COMMENT "" * ) ENGINE=OLAP * DUPLICATE KEY(`id`) * DISTRIBUTED BY HASH(`id`) BUCKETS 20; * * * 2 change the Doris cluster, db, user config in this class * * 3 run this class, you should see the following output: * * { * "TxnId": 27, * "Label": "39c25a5c-7000-496e-a98e-348a264c81de", * "Status": "Success", * "Message": "OK", * "NumberTotalRows": 10, * "NumberLoadedRows": 10, * "NumberFilteredRows": 0, * "NumberUnselectedRows": 0, * "LoadBytes": 50, * "LoadTimeMs": 151 * } * * Attention: * * 1 wrong dependency version(such as 4.4) of httpclient may cause shaded.org.apache.http.ProtocolException * Caused by: shaded.org.apache.http.ProtocolException: Content-Length header already present * at shaded.org.apache.http.protocol.RequestContent.process(RequestContent.java:96) * at shaded.org.apache.http.protocol.ImmutableHttpProcessor.process(ImmutableHttpProcessor.java:132) * at shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:182) * at shaded.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) * at shaded.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) * at shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) * *2 run this class more than once, the status code for http response is still ok, and you will see * the following output: * * { * "TxnId": -1, * "Label": "39c25a5c-7000-496e-a98e-348a264c81de", * "Status": "Label Already Exists", * "ExistingJobStatus": "FINISHED", * "Message": "Label [39c25a5c-7000-496e-a98e-348a264c81de"] has already been used.", * "NumberTotalRows": 0, * "NumberLoadedRows": 0, * "NumberFilteredRows": 0, * "NumberUnselectedRows": 0, * "LoadBytes": 0, * "LoadTimeMs": 0 * } * 3 when the response statusCode is 200, that doesn't mean your stream load is ok, there may be still * some stream problem unless you see the output with 'ok' message */ public class DorisStreamLoad { private final static String DORIS_HOST = "xxx.com"; private final static String DORIS_DB = "test"; private final static String DORIS_TABLE = "stream_test"; private final static String DORIS_USER = "root"; private final static String DORIS_PASSWORD = "xxx"; private final static int DORIS_HTTP_PORT = 8410; private void sendData(String content) throws Exception { final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load", DORIS_HOST, DORIS_HTTP_PORT, DORIS_DB, DORIS_TABLE); final HttpClientBuilder httpClientBuilder = HttpClients .custom() .setRedirectStrategy(new DefaultRedirectStrategy() { @Override protected boolean isRedirectable(String method) { return true; } }); try (CloseableHttpClient client = httpClientBuilder.build()) { HttpPut put = new HttpPut(loadUrl); StringEntity entity = new StringEntity(content, "UTF-8"); put.setHeader(HttpHeaders.EXPECT, "100-continue"); put.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(DORIS_USER, DORIS_PASSWORD)); // the label header is optional, not necessary // use label header can ensure at most once semantics put.setHeader("label", "39c25a5c-7000-496e-a98e-348a264c81de"); put.setEntity(entity); try (CloseableHttpResponse response = client.execute(put)) { String loadResult = ""; if (response.getEntity() != null) { loadResult = EntityUtils.toString(response.getEntity()); } final int statusCode = response.getStatusLine().getStatusCode(); // statusCode 200 just indicates that doris be service is ok, not stream load // you should see the output content to find whether stream load is success if (statusCode != 200) { throw new IOException( String.format("Stream load failed, statusCode=%s load result=%s", statusCode, loadResult)); } System.out.println(loadResult); } } } private String basicAuthHeader(String username, String password) { final String tobeEncode = username + ":" + password; byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8)); return "Basic " + new String(encoded); } public static void main(String[] args) throws Exception { int id1 = 1; int id2 = 10; String id3 = "张三"; int rowNumber = 10; String oneRow = id1 + "\t" + id2 + "\t" + id3 + "\n"; StringBuilder stringBuilder = new StringBuilder(); for (int i = 0; i < rowNumber; i++) { stringBuilder.append(oneRow); } //in doris 0.9 version, you need to comment this line //refer to https://github.com/apache/incubator-doris/issues/783 stringBuilder.deleteCharAt(stringBuilder.length() - 1); String loadData = stringBuilder.toString(); DorisStreamLoad dorisStreamLoad = new DorisStreamLoad(); dorisStreamLoad.sendData(loadData); } }
你好,请问有群,或者其他联系方式吗,沟通交流的
from addax.
@zhbdesign 你可以在 https://github.com/wgzhao/DataX/discussions 这里发起讨论,我暂时不考虑使用QQ或者微信或者钉钉等任何IM群。
from addax.
Related Issues (20)
- [Bug]: windows调试源码包方法抛出'java.lang.NoClassDefFoundError' HOT 1
- 请问一下postgresqlreader支持querySql吗? HOT 1
- [Bug]: 各个组件中的libs目录下的jar包与根目录lib包下的jar包冲突
- [Bug]: ftp文件过多程序崩溃 HOT 3
- [Bug]: java.lang.NoClassDefFoundError: org/apache/logging/log4j/spi/LoggerAdapter HOT 5
- [Bug]: 4.1.4版本中是否把Record中的removeColumn方法移除
- [Bug]: 数据从rdbms 写入到Doris 报错 HOT 5
- [Bug]: log4j-core-2.17.1.jar HOT 3
- [Bug]: MongoDB写入starrocks报错 HOT 2
- [Bug]: java往数据库中直接存为clob字段时,oracle会报ORA-01704问题:字符串文字过长。 HOT 9
- [Bug]: mongoDB 写入 mongoDB 报错:Command failed with error 18 (AuthenticationFailed) HOT 2
- [help wanted]: 任务结果上报使用详细描述 HOT 1
- 关于influxdb2同步influxdb2问题
- 关于influxdb2同步到influxdb2问题 HOT 1
- [Bug]: MongoDBReader连接复制集类型mongo报错连接超时 HOT 2
- [question]: Support for deployment via k8s HOT 1
- [Bug]: KafkaReader无法读取 HOT 4
- [Bug]: 通过hdfswriter写入orc格式数据异常 HOT 10
- [Bug]: 写入hive数据为null HOT 2
- [Bug]: 使用hdfswriter时kerberos认证不成功 HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from addax.