-
Version: ES 5.02 with elasticsearch-jdbc-2.3.4.1, openjdk version "1.8.0_111"
-
Operating System: ubuntu16.04.1, Oracle 12c on linux
-
Config File (if you have sensitive info, please remove it):
input {
jdbc {
# Oracle jdbc connection string to Oracle database
jdbc_connection_string => "jdbc:oracle:thin:@//host:port/sid"
# The user we wish to execute our statement as
jdbc_user => "xxx"
jdbc_password=> "xxx"
# The path to our downloaded jdbc driver
jdbc_driver_library => "PATH/ojdbc7.jar"
# The name of the driver class for Oracle
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
# our query
statement => "select t1.id,t1.name,t1.description, t2.date_of_birth from t1 inner join t2 on t1.ID=t2.T1_ID"
}
}
filter {
aggregate {
task_id => "%{id}"
code => "
map['id'] ||= event.get('id')
map['name'] ||= event.get('name')
map['description'] ||= event.get('description')
map['date_of_birth'] ||= []
map['date_of_birth'] <<= event.get('date_of_birth')
"
push_previous_map_as_event => true
timeout => 5
}
}
output {
elasticsearch {
index => "test"
document_type => "aggregate"
document_id => "%{id}"
hosts => "host"
}
}
-
Sample Data:
DROP TABLE t1;
CREATE TABLE t1 (id NUMBER (10) NOT NULL, name VARCHAR2 (30 BYTE) NOT NULL, description VARCHAR2 (200 BYTE));
ALTER TABLE t1 ADD (
CONSTRAINT t1_pk
PRIMARY KEY
(id)
ENABLE VALIDATE);
COMMIT;
INSERT INTO t1
VALUES (1, 'name1', '1st name');
INSERT INTO t1
VALUES (2, 'name2', '2nd name');
INSERT INTO t1
VALUES (3, 'name3', '3rd name');
COMMIT;
drop table t2;
CREATE TABLE t2 (id NUMBER (10) NOT NULL, t1_id NUMBER (10) NOT NULL, date_of_birth DATE NOT NULL);
ALTER TABLE t2 ADD (
CONSTRAINT t2_pk
PRIMARY KEY
(id)
ENABLE VALIDATE);
COMMIT;
INSERT INTO t2
VALUES (1, 1, to_date('1/1/2000','mm/dd/yyyy'));
INSERT INTO t2
VALUES (2, 1, to_date('2/1/2000','mm/dd/yyyy'));
INSERT INTO t2
VALUES (3, 1, to_date('3/1/2000','mm/dd/yyyy'));
INSERT INTO t2
VALUES (4, 2, to_date('4/1/2000','mm/dd/yyyy'));
INSERT INTO t2
VALUES (5, 3, to_date('5/1/2000','mm/dd/yyyy'));
INSERT INTO t2
VALUES (6, 3, to_date('6/1/2000','mm/dd/yyyy'));
commit;
- Steps to Reproduce:
Create the table in Oracle 12C as script provide above
Create the config file as provided above as oracle-out_test.conf
run:
sudo ./bin/logstash -f oracle-out_test.conf --path.settings=/etc/logstash
Results are as follows:
curl -XGET localhost:9200/test/_search?pretty
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 3,
"max_score" : 1.0,
"hits" : [
{
"_index" : "test",
"_type" : "aggregate",
"_id" : "2",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2016-12-06T22:54:35.107Z",
"date_of_birth" : [
"2000-04-01T08:00:00.000Z"
],
"name" : "name2",
"@Version" : "1",
"description" : "2nd name",
"id" : 2,
"tags" : [ ]
}
},
{
"_index" : "test",
"_type" : "aggregate",
"_id" : "1",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2016-12-06T22:54:35.107Z",
"date_of_birth" : [
"2000-01-01T08:00:00.000Z",
"2000-02-01T08:00:00.000Z",
"2000-03-01T08:00:00.000Z"
],
"name" : "name1",
"@Version" : "1",
"description" : "1st name",
"id" : 1,
"tags" : [ ]
}
},
{
"_index" : "test",
"_type" : "aggregate",
"_id" : "3",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2016-12-06T22:54:34.993Z",
"date_of_birth" : "2000-06-01T07:00:00.000Z",
"name" : "name3",
"@Version" : "1",
"description" : "3rd name",
"id" : 3,
"tags" : [ ]
}
}
]
}
}
Did you see that the last doc date_of_birth is: "date_of_birth" : "2000-06-01T07:00:00.000Z",
while the first doc is: "date_of_birth" : [
"2000-04-01T08:00:00.000Z"
],
Last doc does not aggregate.
Also, why is the mapping like this:
curl -XGET localhost:9200/test/_mapping?pretty
{
"test" : {
"mappings" : {
"aggregate" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@Version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"date_of_birth" : {
"type" : "date"
},
"description" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"id" : {
"type" : "long"
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
I would expect to have "type":"nested" somewhere. Did I do anything wrong?
Thanks