[Bug] Unable to correctly recognize time partition fields about doris-spark-connector

[tuoluzhe8521]

i can correctly query data from doris like this:
select * from dwd_test where date_format(dt,'yyyyMMdd') ='20240102' limit 3;
But it may not recognize partitions and scan the entire table

[gnehil]

Please post the create table statement, I'll try to reproduce it.

[tuoluzhe8521]

Please post the create table statement, I'll try to reproduce it.
CREATE TABLE dwd_test (
dt date NULL COMMENT '处理日期',
id varchar(64) NULL COMMENT '充值订单ID',
pay_success_time datetime NULL COMMENT '订单支付成功时间',
user_id varchar(64) NULL COMMENT '用户ID',
amount int(11) NULL COMMENT '金额',
app_id varchar(255) NULL COMMENT 'appId',
body varchar(255) NULL COMMENT 'body',
channel varchar(64) NULL COMMENT '支付渠道',
currency varchar(32) NULL COMMENT '币种',
description varchar(255) NULL COMMENT '充值描述',
extra varchar(512) NULL,
metadata varchar(255) NULL,
subject varchar(255) NULL,
ip varchar(64) NULL COMMENT '用户IP',
order_no varchar(64) NULL COMMENT '订单ID',
pay_dts int(11) NULL COMMENT '充值订单创建时间',
pay_id varchar(128) NULL COMMENT '支付关联id',
is_deleted int(11) NULL COMMENT '是否删除',
is_test int(11) NULL COMMENT '是否测试'
) ENGINE=OLAP
UNIQUE KEY(dt, id)
COMMENT '交易域充值成功事实表'
PARTITION BY RANGE(dt)
(PARTITION p202312 VALUES [('0000-01-01'), ('2024-01-01')),
PARTITION p202401 VALUES [('2024-01-01'), ('2024-02-01')),
PARTITION p202402 VALUES [('2024-02-01'), ('2024-03-01')),
PARTITION p202403 VALUES [('2024-03-01'), ('2024-04-01')),
PARTITION p202404 VALUES [('2024-04-01'), ('2024-05-01')),
PARTITION p202405 VALUES [('2024-05-01'), ('2024-06-01')),
PARTITION p202406 VALUES [('2024-06-01'), ('2024-07-01')),
PARTITION p202407 VALUES [('2024-07-01'), ('2024-08-01')))
DISTRIBUTED BY HASH(id) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 3",
"is_being_synced" = "false",
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "MONTH",
"dynamic_partition.time_zone" = "Asia/Shanghai",
"dynamic_partition.start" = "-2147483648",
"dynamic_partition.end" = "1",
"dynamic_partition.prefix" = "p",
"dynamic_partition.replication_allocation" = "tag.location.default: 3",
"dynamic_partition.buckets" = "3",
"dynamic_partition.create_history_partition" = "false",
"dynamic_partition.history_partition_num" = "-1",
"dynamic_partition.hot_partition_num" = "0",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.storage_policy" = "",
"dynamic_partition.storage_medium" = "HDD",
"dynamic_partition.start_day_of_month" = "1",
"storage_format" = "V2",
"compression" = "ZSTD",
"enable_unique_key_merge_on_write" = "true",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false"
);

[tuoluzhe8521]

Please post the create table statement, I'll try to reproduce it.

can you help me solve this problem? thank you

[gnehil]

Please post the create table statement, I'll try to reproduce it.

can you help me solve this problem? thank you

I haven't reproduced it, but you can search the keyword "receive SQL statement" in fe.log to see what the specific query obtained by FE is.

[tuoluzhe8521]

this can solve it : https://github.com/apache/doris-spark-connector/pull/209/files