YQL-17364: [tpcds-49] Неоптимальный план подзапроса (не фильтруется таблица перед джойном),about ydb-platform/ydb

Comments (6)

qrort commented on May 30, 2024

Problem type: yt
Username: @psushin
Url: https://yql-test.yandex-team.ru/Operations/6572edd3074f35528d6c9847
Timestamp: Fri Dec 08 2023 13:21:38 GMT+0300 (Moscow Standard Time)
User description: <[Хотим избавиться от full-scan большой таблицы]>

<{Query text
%%(yql)

-- START PARAMETERS
use ada;
pragma TablePathPrefix = "home/tpcds/3Tb";
pragma CostBasedOptimizer="Native";

pragma yt.OperationSpec='{time_limit=3600000}';
pragma yt.MaxReplicationFactorToFuseOperations="30";
pragma CompactGroupBy;
-- END PARAMETERS

-- START COMMON HEADER
pragma TablePathPrefix = "home/tpcds/3Tb";
pragma yt.TemporaryPrimaryMedium = "ssd_blobs_tpcds_bench";
pragma yt.PublishedPrimaryMedium = "ssd_blobs_tpcds_bench";
pragma yt.IntermediateDataMedium = "ssd_blobs_tpcds_bench";
pragma yt.Pool = "tpcds";
pragma yt.SchedulingTagFilter = "%true";
pragma yt.QueryCacheMode = "disable";
--pragma dq.WorkerFilter='ClusterName: "ada"';
--pragma DqEngine="auto";
pragma yt.HybridDqExecution = "true";
pragma yt.AutoMerge = "disabled";
pragma yt.DataSizePerJob = "16M";
pragma yt.DataSizePerMapJob = "24M";
--pragma yt.DataSizePerSortJob = "1G";
pragma yt.DataSizePerPartition = "64M";
pragma AnsiOptionalAs;
pragma AnsiInForEmptyOrNullableItemsCollections;
pragma yt.MapJoinLimit = "33554432";
-- END COMMON HEADER

-- NB: Subquerys
-- start query 1 in stream 0 using template query49.tpl and seed 1819994127

select *
	from
	 web_sales ws left join web_returns wr on (ws.ws_order_number = wr.wr_order_number 
     and	ws.ws_item_sk = wr.wr_item_sk)
             cross join date_dim
	where
		wr.wr_return_amt > 10000
		and ws.ws_net_profit > 1
                     and ws.ws_net_paid > 0
                     and ws.ws_quantity > 0
                     and ws_sold_date_sk = d_date_sk
                     and d_year = 2000
                     and d_moy = 12

%%}>
<{Debug info
##>> 2023-12-08 13:21:03.406 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=1000000&columns_preset=false 200 47803##
##<< 2023-12-08 13:21:03.567##
//Response data//
%%(json) {
"ast": "(\n(let $1 (DataSource '"yt" '"$all"))\n(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))\n(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))\n(let $4 '"ssd_blobs_tpcds_bench")\n(let $5 (YtConfigure! $3 $1 '"Attr" '"temporaryprimarymedium" $4))\n(let $6 (YtConfigure! $5 $1 '"Attr" '"publishedprimarymedium" $4))\n(let $7 (YtConfigure! $6 $1 '"Attr" '"intermediatedatamedium" $4))\n(let $8 (YtConfigure! $7 $1 '"Attr" '"pool" '"tpcds"))\n(let $9 (YtConfigure! $8 $1 '"Attr" '"schedulingtagfilter" '"%true"))\n(let $10 (YtConfigure! $9 $1 '"Attr" '"querycachemode" '"disable"))\n(let $11 (YtConfigure! $10 $1 '"Attr" '"hybriddqexecution" 'true))\n(let $12 (YtConfigure! $11 $1 '"Attr" '"automerge" '"disabled"))\n(let $13 (YtConfigure! $12 $1 '"Attr" '"datasizeperjob" '"16M"))\n(let $14 (YtConfigure! $13 $1 '"Attr" '"datasizepermapjob" '"24M"))\n(let $15 (YtConfigure! $14 $1 '"Attr" '"datasizeperpartition" '"64M"))\n(let $16 (YtConfigure! $15 $1 '"Attr" '"mapjoinlimit" '"33554432"))\n(let $17 (DataSource '"config"))\n(let $18 (Configure! world $17 '"DqEngine" '"auto"))\n(let $19 (Configure! $18 $17 '"CostBasedOptimizer" '"native"))\n(let $20 (DataSink 'result))\n(let $21 (DataSink '"yt" '"ada"))\n(let $22 '('NativeYtTypeFlags '0))\n(let $23 '('StrictSchema '1))\n(let $24 '('UniqueKeys '0))\n(let $25 '@@["StructType";[["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";
...%%

##>> 2023-12-08 13:21:03.396 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=999999&columns_preset=false 200 47803##
##<< 2023-12-08 13:21:03.548##
//Response data//
%%(json) {
"ast": "(\n(let $1 (DataSource '"yt" '"$all"))\n(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))\n(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))\n(let $4 '"ssd_blobs_tpcds_bench")\n(let $5 (YtConfigure! $3 $1 '"Attr" '"temporaryprimarymedium" $4))\n(let $6 (YtConfigure! $5 $1 '"Attr" '"publishedprimarymedium" $4))\n(let $7 (YtConfigure! $6 $1 '"Attr" '"intermediatedatamedium" $4))\n(let $8 (YtConfigure! $7 $1 '"Attr" '"pool" '"tpcds"))\n(let $9 (YtConfigure! $8 $1 '"Attr" '"schedulingtagfilter" '"%true"))\n(let $10 (YtConfigure! $9 $1 '"Attr" '"querycachemode" '"disable"))\n(let $11 (YtConfigure! $10 $1 '"Attr" '"hybriddqexecution" 'true))\n(let $12 (YtConfigure! $11 $1 '"Attr" '"automerge" '"disabled"))\n(let $13 (YtConfigure! $12 $1 '"Attr" '"datasizeperjob" '"16M"))\n(let $14 (YtConfigure! $13 $1 '"Attr" '"datasizepermapjob" '"24M"))\n(let $15 (YtConfigure! $14 $1 '"Attr" '"datasizeperpartition" '"64M"))\n(let $16 (YtConfigure! $15 $1 '"Attr" '"mapjoinlimit" '"33554432"))\n(let $17 (DataSource '"config"))\n(let $18 (Configure! world $17 '"DqEngine" '"auto"))\n(let $19 (Configure! $18 $17 '"CostBasedOptimizer" '"native"))\n(let $20 (DataSink 'result))\n(let $21 (DataSink '"yt" '"ada"))\n(let $22 '('NativeYtTypeFlags '0))\n(let $23 '('StrictSchema '1))\n(let $24 '('UniqueKeys '0))\n(let $25 '@@["StructType";[["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";
...%%

##>> 2023-12-08 13:21:03.395 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847?version=1000000 200 2429##
##<< 2023-12-08 13:21:03.551##
//Response data//
%%(json) {
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryData": {
"attributes": {
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.1064 YaBrowser/23.9.1.1064 (beta) Yowser/2.5 Safari/537.36"
},
"clusterType": "UNKNOWN",
"content": "\n\n\n-- START PARAMETERS\nuse ada;\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma CostBasedOptimizer="Native";\n\npragma yt.OperationSpec='{time_limit=3600000}';\npragma yt.MaxReplicationFactorToFuseOperations="30";\npragma CompactGroupBy;\n-- END PARAMETERS\n\n\n\n\n-- START COMMON HEADER\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma yt.TemporaryPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.PublishedPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.IntermediateDataMedium = "ssd_blobs_tpcds_bench";\npragma yt.Pool = "tpcds";\npragma yt.SchedulingTagFilter = "%true";\npragma yt.QueryCacheMode = "disable";\n--pragma dq.WorkerFilter='ClusterName: "ada"';\n--pragma DqEngine="auto";\npragma yt.HybridDqExecution = "true";\npragma yt.AutoMerge = "disabled";\npragma yt.DataSizePerJob = "16M";\npragma yt.DataSizePerMapJob = "24M";\n--pragma yt.DataSizePerSortJob = "1G";\npragma yt.DataSizePerPartition = "64M";\npragma AnsiOptionalAs;\npragma AnsiInForEmptyOrNullableItemsCollections;\npragma yt.MapJoinLimit = "33554432";\n-- END COMMON HEADER\n\n-- NB: Subquerys\n-- start query 1 in stream 0 using template query49.tpl and seed 1819994127\n\n\tselect *\n \t\tfrom\n \t\t web_sales ws left join web_returns wr on (ws.ws_order_number = wr.wr_order_number \n and\tws.ws_item_sk = wr.wr_item_sk)\n cross join date_dim\n \t\twhere\n \t\t\twr.wr_return_amt > 10000\n \t\t\tand ws.ws_net_profit > 1\n and ws.ws_net_paid > 0\n and ws.ws_quantity > 0\n and ws_sold_date_sk = d_date_sk\n and d_year = 2000\n and d_moy = 12\n\n \t",
"files": [],
"parameters": {},
"type": "SQLv1"
},
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:21:00.696Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
}%%

##>> 2023-12-08 13:21:03.216 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=&columns_preset=false 200 400##
##<< 2023-12-08 13:21:03.395##
//Response data//
%%(json) {
"errors": [],
"id": "6572edd3074f35528d6c9847",
"issues": [
{
"code": 1070,
"column": 0,
"file": "

",
"issues": [
{
"code": 0,
"column": 0,
"file": "",
"issues": [],
"message": "DQ cannot execute the query. Cause: unsupported yt pragma: operationspec",
"row": 0,
"severity": "S_INFO"
}
],
"message": "Optimization",
"row": 0,
"severity": "S_INFO"
}
],
"status": "ABORTED",
"updatedAt": "2023-12-08T10:21:00.664Z",
"version": 1000000
}%%

##>> 2023-12-08 13:21:01.288 GET https://yql-api-test.yandex-team.ru/api/v2/operations?project_service_slug=psushin&page_size=50&sort=-createdAt&page=0&filters=(username%3Dpsushin) 200 23934##
##<< 2023-12-08 13:21:01.466##
//Response data//
%%(json) {
"page": {
"count": 2,
"current": 0,
"next": 1
},
"result": [
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:21:00.696Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:20.290Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572eda87543d45a39ed9366",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:19:53.003Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "241554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:17.165Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572eda57543d45a39ed9355",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:19:18.185Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "591881",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:46.294Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed86074f35528d6c9778",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:18:47.391Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "590834",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:38.958Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed7e7543d45a39ed92e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:39.692Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test01.search.yandex.net",
"workerId": "83d38b4b-b006ffed-ecbf3042-da205ff3",
"workerPid": "689554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:29.235Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed757543d45a39ed92c7",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:30.013Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test04.search.yandex.net",
"workerId": "dbfd9c86-b594da57-6dede183-3d364e46",
"workerPid": "947935",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T09:51:12.251Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572e710074f35528d6c8682",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-49",
"updatedAt": "2023-12-08T10:02:27.527Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "170780",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:19:48.836Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637e34ced8c52a70f77897",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-7",
"updatedAt": "2023-11-26T17:28:42.704Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "cd7e9331-587d0fee-fa09ea24-c9766bf9",
"workerPid": "743831",
"workerVersion": "12966812"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:17:58.794Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637dc6ced8c52a70f777e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-11",
"updatedAt": "2023-11-26T17:41:17.751Z",
"username
...%%

##>> 2023-12-08 13:20:59.257 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=999999&columns_preset=false 200 44373##
##<< 2023-12-08 13:20:59.419##
//Response data//
%%(json) {
"ast": "(\n(let $1 (DataSource '"yt" '"$all"))\n(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))\n(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))\n(let $4 '"ssd_blobs_tpcds_bench")\n(let $5 (YtConfigure! $3 $1 '"Attr" '"temporaryprimarymedium" $4))\n(let $6 (YtConfigure! $5 $1 '"Attr" '"publishedprimarymedium" $4))\n(let $7 (YtConfigure! $6 $1 '"Attr" '"intermediatedatamedium" $4))\n(let $8 (YtConfigure! $7 $1 '"Attr" '"pool" '"tpcds"))\n(let $9 (YtConfigure! $8 $1 '"Attr" '"schedulingtagfilter" '"%true"))\n(let $10 (YtConfigure! $9 $1 '"Attr" '"querycachemode" '"disable"))\n(let $11 (YtConfigure! $10 $1 '"Attr" '"hybriddqexecution" 'true))\n(let $12 (YtConfigure! $11 $1 '"Attr" '"automerge" '"disabled"))\n(let $13 (YtConfigure! $12 $1 '"Attr" '"datasizeperjob" '"16M"))\n(let $14 (YtConfigure! $13 $1 '"Attr" '"datasizepermapjob" '"24M"))\n(let $15 (YtConfigure! $14 $1 '"Attr" '"datasizeperpartition" '"64M"))\n(let $16 (YtConfigure! $15 $1 '"Attr" '"mapjoinlimit" '"33554432"))\n(let $17 (DataSource '"config"))\n(let $18 (Configure! world $17 '"DqEngine" '"auto"))\n(let $19 (Configure! $18 $17 '"CostBasedOptimizer" '"native"))\n(let $20 (DataSink 'result))\n(let $21 (DataSink '"yt" '"ada"))\n(let $22 '('NativeYtTypeFlags '0))\n(let $23 '('StrictSchema '1))\n(let $24 '('UniqueKeys '0))\n(let $25 '@@["StructType";[["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";
...%%

##>> 2023-12-08 13:20:59.249 GET https://yql-api-test.yandex-team.ru/api/v2/operations?project_service_slug=psushin&page_size=50&sort=-createdAt&page=0&filters=(username%3Dpsushin) 200 23933##
##<< 2023-12-08 13:20:59.430##
//Response data//
%%(json) {
"page": {
"count": 2,
"current": 0,
"next": 1
},
"result": [
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:20:59.194Z",
"username": "psushin",
"version": 999999,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:20.290Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572eda87543d45a39ed9366",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:19:53.003Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "241554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:17.165Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572eda57543d45a39ed9355",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:19:18.185Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "591881",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:46.294Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed86074f35528d6c9778",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:18:47.391Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "590834",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:38.958Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed7e7543d45a39ed92e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:39.692Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test01.search.yandex.net",
"workerId": "83d38b4b-b006ffed-ecbf3042-da205ff3",
"workerPid": "689554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:29.235Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed757543d45a39ed92c7",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:30.013Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test04.search.yandex.net",
"workerId": "dbfd9c86-b594da57-6dede183-3d364e46",
"workerPid": "947935",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T09:51:12.251Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572e710074f35528d6c8682",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-49",
"updatedAt": "2023-12-08T10:02:27.527Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "170780",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:19:48.836Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637e34ced8c52a70f77897",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-7",
"updatedAt": "2023-11-26T17:28:42.704Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "cd7e9331-587d0fee-fa09ea24-c9766bf9",
"workerPid": "743831",
"workerVersion": "12966812"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:17:58.794Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637dc6ced8c52a70f777e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-11",
"updatedAt": "2023-11-26T17:41:17.751Z",
"username"
...%%

##>> 2023-12-08 13:20:59.249 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847?version=3 200 2428##
##<< 2023-12-08 13:20:59.435##
//Response data//
%%(json) {
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryData": {
"attributes": {
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.1064 YaBrowser/23.9.1.1064 (beta) Yowser/2.5 Safari/537.36"
},
"clusterType": "UNKNOWN",
"content": "\n\n\n-- START PARAMETERS\nuse ada;\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma CostBasedOptimizer="Native";\n\npragma yt.OperationSpec='{time_limit=3600000}';\npragma yt.MaxReplicationFactorToFuseOperations="30";\npragma CompactGroupBy;\n-- END PARAMETERS\n\n\n\n\n-- START COMMON HEADER\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma yt.TemporaryPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.PublishedPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.IntermediateDataMedium = "ssd_blobs_tpcds_bench";\npragma yt.Pool = "tpcds";\npragma yt.SchedulingTagFilter = "%true";\npragma yt.QueryCacheMode = "disable";\n--pragma dq.WorkerFilter='ClusterName: "ada"';\n--pragma DqEngine="auto";\npragma yt.HybridDqExecution = "true";\npragma yt.AutoMerge = "disabled";\npragma yt.DataSizePerJob = "16M";\npragma yt.DataSizePerMapJob = "24M";\n--pragma yt.DataSizePerSortJob = "1G";\npragma yt.DataSizePerPartition = "64M";\npragma AnsiOptionalAs;\npragma AnsiInForEmptyOrNullableItemsCollections;\npragma yt.MapJoinLimit = "33554432";\n-- END COMMON HEADER\n\n-- NB: Subquerys\n-- start query 1 in stream 0 using template query49.tpl and seed 1819994127\n\n\tselect *\n \t\tfrom\n \t\t web_sales ws left join web_returns wr on (ws.ws_order_number = wr.wr_order_number \n and\tws.ws_item_sk = wr.wr_item_sk)\n cross join date_dim\n \t\twhere\n \t\t\twr.wr_return_amt > 10000\n \t\t\tand ws.ws_net_profit > 1\n and ws.ws_net_paid > 0\n and ws.ws_quantity > 0\n and ws_sold_date_sk = d_date_sk\n and d_year = 2000\n and d_moy = 12\n\n \t",
"files": [],
"parameters": {},
"type": "SQLv1"
},
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:20:59.194Z",
"username": "psushin",
"version": 999999,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
}%%

##>> 2023-12-08 13:20:58.983 POST https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847 200 432##
##<< 2023-12-08 13:20:59.248##
//Request data//
%%(json) {
"action": "ABORT"
}%%
//Response data//
%%(json) {
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:20:59.194621Z",
"username": "psushin",
"version": 999999,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
}%%

##>> 2023-12-08 13:20:33.233 GET https://yql-api-test.yandex-team.ru/api/v2/operations?project_service_slug=psushin&page_size=50&sort=-createdAt&page=0&filters=(status%3DPENDING%7Cstatus%3DRUNNING%7Cstatus%3DABORTING),(username%3Dpsushin) 200 468##
##<< 2023-12-08 13:20:33.478##
//Response data//
%%(json) {
"page": {
"count": 1,
"current": 0
},
"result": [
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:04.881Z",
"username": "psushin",
"version": 3,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
}
]
}%%

##>> 2023-12-08 13:20:33.215 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=&columns_preset=false 200 394##
##<< 2023-12-08 13:20:33.366##
//Response data//
%%(json) {
"errors": [],
"id": "6572edd3074f35528d6c9847",
"issues": [
{
"code": 1070,
"column": 0,
"file": "

",
"issues": [
{
"code": 0,
"column": 0,
"file": "",
"issues": [],
"message": "DQ cannot execute the query. Cause: unsupported yt pragma: operationspec",
"row": 0,
"severity": "S_INFO"
}
],
"message": "Optimization",
"row": 0,
"severity": "S_INFO"
}
],
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:04.845Z",
"version": 3
}%%

##>> 2023-12-08 13:20:04.887 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=3&columns_preset=false 200 44368##
##<< 2023-12-08 13:20:05.047##
//Response data//
%%(json) {
"ast": "(\n(let $1 (DataSource '"yt" '"$all"))\n(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))\n(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))\n(let $4 '"ssd_blobs_tpcds_bench")\n(let $5 (YtConfigure! $3 $1 '"Attr" '"temporaryprimarymedium" $4))\n(let $6 (YtConfigure! $5 $1 '"Attr" '"publishedprimarymedium" $4))\n(let $7 (YtConfigure! $6 $1 '"Attr" '"intermediatedatamedium" $4))\n(let $8 (YtConfigure! $7 $1 '"Attr" '"pool" '"tpcds"))\n(let $9 (YtConfigure! $8 $1 '"Attr" '"schedulingtagfilter" '"%true"))\n(let $10 (YtConfigure! $9 $1 '"Attr" '"querycachemode" '"disable"))\n(let $11 (YtConfigure! $10 $1 '"Attr" '"hybriddqexecution" 'true))\n(let $12 (YtConfigure! $11 $1 '"Attr" '"automerge" '"disabled"))\n(let $13 (YtConfigure! $12 $1 '"Attr" '"datasizeperjob" '"16M"))\n(let $14 (YtConfigure! $13 $1 '"Attr" '"datasizepermapjob" '"24M"))\n(let $15 (YtConfigure! $14 $1 '"Attr" '"datasizeperpartition" '"64M"))\n(let $16 (YtConfigure! $15 $1 '"Attr" '"mapjoinlimit" '"33554432"))\n(let $17 (DataSource '"config"))\n(let $18 (Configure! world $17 '"DqEngine" '"auto"))\n(let $19 (Configure! $18 $17 '"CostBasedOptimizer" '"native"))\n(let $20 (DataSink 'result))\n(let $21 (DataSink '"yt" '"ada"))\n(let $22 '('NativeYtTypeFlags '0))\n(let $23 '('StrictSchema '1))\n(let $24 '('UniqueKeys '0))\n(let $25 '@@["StructType";[["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";
...%%

##>> 2023-12-08 13:20:04.880 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=2&columns_preset=false 200 44368##
##<< 2023-12-08 13:20:05.045##
//Response data//
%%(json) {
"ast": "(\n(let $1 (DataSource '"yt" '"$all"))\n(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))\n(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))\n(let $4 '"ssd_blobs_tpcds_bench")\n(let $5 (YtConfigure! $3 $1 '"Attr" '"temporaryprimarymedium" $4))\n(let $6 (YtConfigure! $5 $1 '"Attr" '"publishedprimarymedium" $4))\n(let $7 (YtConfigure! $6 $1 '"Attr" '"intermediatedatamedium" $4))\n(let $8 (YtConfigure! $7 $1 '"Attr" '"pool" '"tpcds"))\n(let $9 (YtConfigure! $8 $1 '"Attr" '"schedulingtagfilter" '"%true"))\n(let $10 (YtConfigure! $9 $1 '"Attr" '"querycachemode" '"disable"))\n(let $11 (YtConfigure! $10 $1 '"Attr" '"hybriddqexecution" 'true))\n(let $12 (YtConfigure! $11 $1 '"Attr" '"automerge" '"disabled"))\n(let $13 (YtConfigure! $12 $1 '"Attr" '"datasizeperjob" '"16M"))\n(let $14 (YtConfigure! $13 $1 '"Attr" '"datasizepermapjob" '"24M"))\n(let $15 (YtConfigure! $14 $1 '"Attr" '"datasizeperpartition" '"64M"))\n(let $16 (YtConfigure! $15 $1 '"Attr" '"mapjoinlimit" '"33554432"))\n(let $17 (DataSource '"config"))\n(let $18 (Configure! world $17 '"DqEngine" '"auto"))\n(let $19 (Configure! $18 $17 '"CostBasedOptimizer" '"native"))\n(let $20 (DataSink 'result))\n(let $21 (DataSink '"yt" '"ada"))\n(let $22 '('NativeYtTypeFlags '0))\n(let $23 '('StrictSchema '1))\n(let $24 '('UniqueKeys '0))\n(let $25 '@@["StructType";[["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";
...%%

##>> 2023-12-08 13:20:04.879 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847?version=3 200 2423##
##<< 2023-12-08 13:20:05.054##
//Response data//
%%(json) {
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryData": {
"attributes": {
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.1064 YaBrowser/23.9.1.1064 (beta) Yowser/2.5 Safari/537.36"
},
"clusterType": "UNKNOWN",
"content": "\n\n\n-- START PARAMETERS\nuse ada;\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma CostBasedOptimizer="Native";\n\npragma yt.OperationSpec='{time_limit=3600000}';\npragma yt.MaxReplicationFactorToFuseOperations="30";\npragma CompactGroupBy;\n-- END PARAMETERS\n\n\n\n\n-- START COMMON HEADER\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma yt.TemporaryPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.PublishedPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.IntermediateDataMedium = "ssd_blobs_tpcds_bench";\npragma yt.Pool = "tpcds";\npragma yt.SchedulingTagFilter = "%true";\npragma yt.QueryCacheMode = "disable";\n--pragma dq.WorkerFilter='ClusterName: "ada"';\n--pragma DqEngine="auto";\npragma yt.HybridDqExecution = "true";\npragma yt.AutoMerge = "disabled";\npragma yt.DataSizePerJob = "16M";\npragma yt.DataSizePerMapJob = "24M";\n--pragma yt.DataSizePerSortJob = "1G";\npragma yt.DataSizePerPartition = "64M";\npragma AnsiOptionalAs;\npragma AnsiInForEmptyOrNullableItemsCollections;\npragma yt.MapJoinLimit = "33554432";\n-- END COMMON HEADER\n\n-- NB: Subquerys\n-- start query 1 in stream 0 using template query49.tpl and seed 1819994127\n\n\tselect *\n \t\tfrom\n \t\t web_sales ws left join web_returns wr on (ws.ws_order_number = wr.wr_order_number \n and\tws.ws_item_sk = wr.wr_item_sk)\n cross join date_dim\n \t\twhere\n \t\t\twr.wr_return_amt > 10000\n \t\t\tand ws.ws_net_profit > 1\n and ws.ws_net_paid > 0\n and ws.ws_quantity > 0\n and ws_sold_date_sk = d_date_sk\n and d_year = 2000\n and d_moy = 12\n\n \t",
"files": [],
"parameters": {},
"type": "SQLv1"
},
"queryType": "SQLv1",
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:04.881Z",
"username": "psushin",
"version": 3,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
}%%

##>> 2023-12-08 13:20:04.224 GET https://yql-api-test.yandex-team.ru/api/v2/operations?project_service_slug=psushin&page_size=50&sort=-createdAt&page=0&filters=(username%3Dpsushin) 200 23928##
##<< 2023-12-08 13:20:04.513##
//Response data//
%%(json) {
"page": {
"count": 2,
"current": 0,
"next": 1
},
"result": [
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:03.654Z",
"username": "psushin",
"version": 2,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:20.290Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572eda87543d45a39ed9366",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:19:53.003Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "241554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:17.165Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572eda57543d45a39ed9355",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:19:18.185Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "591881",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:46.294Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed86074f35528d6c9778",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:18:47.391Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "590834",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:38.958Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed7e7543d45a39ed92e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:39.692Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test01.search.yandex.net",
"workerId": "83d38b4b-b006ffed-ecbf3042-da205ff3",
"workerPid": "689554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:29.235Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed757543d45a39ed92c7",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:30.013Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test04.search.yandex.net",
"workerId": "dbfd9c86-b594da57-6dede183-3d364e46",
"workerPid": "947935",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T09:51:12.251Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572e710074f35528d6c8682",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-49",
"updatedAt": "2023-12-08T10:02:27.527Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "170780",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:19:48.836Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637e34ced8c52a70f77897",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-7",
"updatedAt": "2023-11-26T17:28:42.704Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "cd7e9331-587d0fee-fa09ea24-c9766bf9",
"workerPid": "743831",
"workerVersion": "12966812"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:17:58.794Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637dc6ced8c52a70f777e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-11",
"updatedAt": "2023-11-26T17:41:17.751Z",
"username": "ps
...%%

##>> 2023-12-08 13:20:04.208 GET https://yql-api-test.yandex-team.ru/api/v2/operations?project_service_slug=psushin&page_size=50&sort=-createdAt&page=0&filters=(username%3Dpsushin) 200 23928##
##<< 2023-12-08 13:20:04.361##
//Response data//
%%(json) {
"page": {
"count": 2,
"current": 0,
"next": 1
},
"result": [
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:03.654Z",
"username": "psushin",
"version": 2,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:20.290Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572eda87543d45a39ed9366",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:19:53.003Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "241554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:17.165Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572eda57543d45a39ed9355",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:19:18.185Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "591881",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:46.294Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed86074f35528d6c9778",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:18:47.391Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "590834",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:38.958Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed7e7543d45a39ed92e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:39.692Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test01.search.yandex.net",
"workerId": "83d38b4b-b006ffed-ecbf3042-da205ff3",
"workerPid": "689554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:29.235Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed757543d45a39ed92c7",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:30.013Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test04.search.yandex.net",
"workerId": "dbfd9c86-b594da57-6dede183-3d364e46",
"workerPid": "947935",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T09:51:12.251Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572e710074f35528d6c8682",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-49",
"updatedAt": "2023-12-08T10:02:27.527Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "170780",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:19:48.836Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637e34ced8c52a70f77897",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-7",
"updatedAt": "2023-11-26T17:28:42.704Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "cd7e9331-587d0fee-fa09ea24-c9766bf9",
"workerPid": "743831",
"workerVersion": "12966812"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:17:58.794Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637dc6ced8c52a70f777e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-11",
"updatedAt": "2023-11-26T17:41:17.751Z",
"username": "ps
...%%

##>> 2023-12-08 13:20:04.062 GET https://yql-api-test.yandex-team.ru/api/v2/operations?project_service_slug=psushin&page_size=50&sort=-createdAt&page=0&filters=(username%3Dpsushin) 200 23928##
##<< 2023-12-08 13:20:04.217##
//Response data//
%%(json) {
"page": {
"count": 2,
"current": 0,
"next": 1
},
"result": [
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:03.654Z",
"username": "psushin",
"version": 2,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:20.290Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572eda87543d45a39ed9366",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ABORTED",
"updatedAt": "2023-12-08T10:19:53.003Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "241554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:19:17.165Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572eda57543d45a39ed9355",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:19:18.185Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "591881",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:46.294Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed86074f35528d6c9778",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"updatedAt": "2023-12-08T10:18:47.391Z",
"username": "psushin",
"version": 1000000,
"workerHost": "kikimr0002.search.yandex.net",
"workerId": "c94d28c1-624f4680-6fa1c5f1-c763d0f0",
"workerPid": "590834",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:38.958Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed7e7543d45a39ed92e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:39.692Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test01.search.yandex.net",
"workerId": "83d38b4b-b006ffed-ecbf3042-da205ff3",
"workerPid": "689554",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:18:29.235Z",
"execMode": "VALIDATE",
"externalQueryIds": [],
"id": "6572ed757543d45a39ed92c7",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "ERROR",
"updatedAt": "2023-12-08T10:18:30.013Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test04.search.yandex.net",
"workerId": "dbfd9c86-b594da57-6dede183-3d364e46",
"workerPid": "947935",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T09:51:12.251Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572e710074f35528d6c8682",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-49",
"updatedAt": "2023-12-08T10:02:27.527Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "5c552805-b714de48-ceeed83a-4c5fd849",
"workerPid": "170780",
"workerVersion": "13048753"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:19:48.836Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637e34ced8c52a70f77897",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-7",
"updatedAt": "2023-11-26T17:28:42.704Z",
"username": "psushin",
"version": 1000000,
"workerHost": "yql-front-test03.search.yandex.net",
"workerId": "cd7e9331-587d0fee-fa09ea24-c9766bf9",
"workerPid": "743831",
"workerVersion": "12966812"
},
{
"clusterType": "UNKNOWN",
"createdAt": "2023-11-26T17:17:58.794Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "65637dc6ced8c52a70f777e4",
"projectId": "6001d201dd0e49e90533b83a",
"queryType": "SQLv1",
"status": "COMPLETED",
"title": "[YQL] YT benchmark tpcds-11",
"updatedAt": "2023-11-26T17:41:17.751Z",
"username": "ps
...%%

##>> 2023-12-08 13:20:03.725 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=2&columns_preset=false 200 127##
##<< 2023-12-08 13:20:03.881##
//Response data//
%%(json) {
"errors": [],
"id": "6572edd3074f35528d6c9847",
"issues": [],
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:03.625Z",
"version": 2
}%%

##>> 2023-12-08 13:20:03.713 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847/results?filters=DATA,EXEC_STATISTICS,PLAN,PROGRESS,AST&version=1&columns_preset=false 200 127##
##<< 2023-12-08 13:20:03.873##
//Response data//
%%(json) {
"errors": [],
"id": "6572edd3074f35528d6c9847",
"issues": [],
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:03.625Z",
"version": 2
}%%

##>> 2023-12-08 13:20:03.706 GET https://yql-api-test.yandex-team.ru/api/v2/operations/6572edd3074f35528d6c9847?version=1 200 2423##
##<< 2023-12-08 13:20:03.872##
//Response data//
%%(json) {
"clusterType": "UNKNOWN",
"createdAt": "2023-12-08T10:20:03.002Z",
"execMode": "RUN",
"externalQueryIds": [],
"id": "6572edd3074f35528d6c9847",
"projectId": "6001d201dd0e49e90533b83a",
"queryData": {
"attributes": {
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.1064 YaBrowser/23.9.1.1064 (beta) Yowser/2.5 Safari/537.36"
},
"clusterType": "UNKNOWN",
"content": "\n\n\n-- START PARAMETERS\nuse ada;\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma CostBasedOptimizer="Native";\n\npragma yt.OperationSpec='{time_limit=3600000}';\npragma yt.MaxReplicationFactorToFuseOperations="30";\npragma CompactGroupBy;\n-- END PARAMETERS\n\n\n\n\n-- START COMMON HEADER\npragma TablePathPrefix = "home/tpcds/3Tb";\npragma yt.TemporaryPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.PublishedPrimaryMedium = "ssd_blobs_tpcds_bench";\npragma yt.IntermediateDataMedium = "ssd_blobs_tpcds_bench";\npragma yt.Pool = "tpcds";\npragma yt.SchedulingTagFilter = "%true";\npragma yt.QueryCacheMode = "disable";\n--pragma dq.WorkerFilter='ClusterName: "ada"';\n--pragma DqEngine="auto";\npragma yt.HybridDqExecution = "true";\npragma yt.AutoMerge = "disabled";\npragma yt.DataSizePerJob = "16M";\npragma yt.DataSizePerMapJob = "24M";\n--pragma yt.DataSizePerSortJob = "1G";\npragma yt.DataSizePerPartition = "64M";\npragma AnsiOptionalAs;\npragma AnsiInForEmptyOrNullableItemsCollections;\npragma yt.MapJoinLimit = "33554432";\n-- END COMMON HEADER\n\n-- NB: Subquerys\n-- start query 1 in stream 0 using template query49.tpl and seed 1819994127\n\n\tselect *\n \t\tfrom\n \t\t web_sales ws left join web_returns wr on (ws.ws_order_number = wr.wr_order_number \n and\tws.ws_item_sk = wr.wr_item_sk)\n cross join date_dim\n \t\twhere\n \t\t\twr.wr_return_amt > 10000\n \t\t\tand ws.ws_net_profit > 1\n and ws.ws_net_paid > 0\n and ws.ws_quantity > 0\n and ws_sold_date_sk = d_date_sk\n and d_year = 2000\n and d_moy = 12\n\n \t",
"files": [],
"parameters": {},
"type": "SQLv1"
},
"queryType": "SQLv1",
"status": "RUNNING",
"updatedAt": "2023-12-08T10:20:03.654Z",
"username": "psushin",
"version": 2,
"workerHost": "kikimr0005.search.yandex.net",
"workerId": "bc166f56-18848ff-c4638de8-8574f33",
"workerPid": "861219",
"workerVersion": "13048753"
}%%
}>
<{Additional info
%%(json){
"version": "0cf2ff096c80a6192b9050d5697b04f09ae3ec3b",
"requestId": "1702029720312231-16706068908272401021",
"url": "https://yql-test.yandex-team.ru/Operations/6572edd3074f35528d6c9847",
"language": "en",
"widowSize": {
"width": 1850,
"height": 1053
},
"screenSize": {
"width": 1920,
"height": 1080
},
"userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.1064 YaBrowser/23.9.1.1064 (beta) Yowser/2.5 Safari/537.36",
"userAgentData": {
"architecture": "x86",
"bitness": "64",
"brands": [
{
"brand": "Chromium",
"version": "116"
},
{
"brand": "Not)A;Brand",
"version": "24"
},
{
"brand": "YaBrowser",
"version": "23"
}
],
"mobile": false,
"model": "",
"platform": "Linux",
"platformVersion": "6.2.0",
"uaFullVersion": "23.9.1.1064"
}
}%%}>

from ydb.

qrort commented on May 30, 2024

(
(let $1 (DataSource '"yt" '"$all"))
(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))
(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))
(let $4 (YtConfigure! $3 $1 '"Attr" '"pool" '"tpcds"))
(let $5 (YtConfigure! $4 $1 '"Attr" '"schedulingtagfilter" '"%true"))
(let $6 (YtConfigure! $5 $1 '"Attr" '"querycachemode" '"disable"))
(let $7 (YtConfigure! $6 $1 '"Attr" '"hybriddqexecution" '"true"))
(let $8 (YtConfigure! $7 $1 '"Attr" '"automerge" '"disabled"))
(let $9 (YtConfigure! $8 $1 '"Attr" '"datasizeperjob" '"16M"))
(let $10 (YtConfigure! $9 $1 '"Attr" '"datasizepermapjob" '"24M"))
(let $11 (YtConfigure! $10 $1 '"Attr" '"datasizeperpartition" '"64M"))
(let $12 (YtConfigure! $11 $1 '"Attr" '"mapjoinlimit" '"33554432"))
(let $13 (Configure! world (DataSource '"config") '"CostBasedOptimizer" '"native"))
(let $14 (DataSink 'result))
(let $15 (DataSource '"yt" '"dirac"))
(let $16 '('NativeYtTypeFlags '16))
(let $17 '('StrictSchema '1))
(let $18 '('UniqueKeys '0))
(let $19 '@@["StructType";[["ws_sold_date_sk";["OptionalType";["DataType";"Int64"]]];["ws_sold_time_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_date_sk";["OptionalType";["DataType";"Int64"]]];["ws_item_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_web_site_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_mode_sk";["OptionalType";["DataType";"Int64"]]];["ws_warehouse_sk";["OptionalType";["DataType";"Int64"]]];["ws_promo_sk";["OptionalType";["DataType";"Int64"]]];["ws_order_number";["OptionalType";["DataType";"Int64"]]];["ws_quantity";["OptionalType";["DataType";"Int64"]]];["ws_wholesale_cost";["OptionalType";["DataType";"Float"]]];["ws_list_price";["OptionalType";["DataType";"Float"]]];["ws_sales_price";["OptionalType";["DataType";"Float"]]];["ws_ext_discount_amt";["OptionalType";["DataType";"Float"]]];["ws_ext_sales_price";["OptionalType";["DataType";"Float"]]];["ws_ext_wholesale_cost";["OptionalType";["DataType";"Float"]]];["ws_ext_list_price";["OptionalType";["DataType";"Float"]]];["ws_ext_tax";["OptionalType";["DataType";"Float"]]];["ws_coupon_amt";["OptionalType";["DataType";"Float"]]];["ws_ext_ship_cost";["OptionalType";["DataType";"Float"]]];["ws_net_paid";["OptionalType";["DataType";"Float"]]];["ws_net_paid_inc_tax";["OptionalType";["DataType";"Float"]]];["ws_net_paid_inc_ship";["OptionalType";["DataType";"Float"]]];["ws_net_paid_inc_ship_tax";["OptionalType";["DataType";"Float"]]];["ws_net_profit";["OptionalType";["DataType";"Float"]]]]]@@)
(let $20 (YqlRowSpec $16 $17 $18 '('Type $19)))
(let $21 (YtMeta '('CanWrite '1) '('DoesExist '1) '('YqlCompatibleScheme '1) '('InferredScheme '0) '('IsDynamic '0)))
(let $22 '('ChunkCount '"1000"))
(let $23 '('ModifyTime '"1707907334"))
(let $24 '('Revision '"7481833077937"))
(let $25 (YtStat '('Id '"6cd-bde8-13440191-b4afb85b") '('RecordsCount '"2211809044") '('DataSize '"603695506444") $22 $23 $24))
(let $26 (YtTable '"home/tpcds/3Tb/web_sales" $20 $21 $25 '() (Void) (Void) '"dirac"))
(let $27 '('('"syscolumns" '('"path" '"record"))))
(let $28 '"ws_bill_customer_sk")
(let $29 '"ws_ext_discount_amt")
(let $30 '"ws_ext_list_price")
(let $31 '"ws_ext_sales_price")
(let $32 '"ws_ext_wholesale_cost")
(let $33 '"ws_net_paid_inc_ship")
(let $34 '"ws_net_paid_inc_ship_tax")
(let $35 '"ws_net_paid_inc_tax")
(let $36 '"ws_ship_customer_sk")
(let $37 '"ws_wholesale_cost")
(let $38 '('"ws_bill_addr_sk" '"ws_bill_cdemo_sk" $28 '"ws_bill_hdemo_sk" '"ws_coupon_amt" $29 $30 $31 '"ws_ext_ship_cost" '"ws_ext_tax" $32 '"ws_item_sk" '"ws_list_price" '"ws_net_paid" $33 $34 $35 '"ws_net_profit" '"ws_order_number" '"ws_promo_sk" '"ws_quantity" '"ws_sales_price" '"ws_ship_addr_sk" '"ws_ship_cdemo_sk" $36 '"ws_ship_date_sk" '"ws_ship_hdemo_sk" '"ws_ship_mode_sk" '"ws_sold_date_sk" '"ws_sold_time_sk" '"ws_warehouse_sk" '"ws_web_page_sk" '"ws_web_site_sk" $37))
(let $39 (Bool 'false))
(let $40 '@@["StructType";[["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]]]]@@)
(let $41 (YqlRowSpec $16 $17 $18 '('Type $40)))
(let $42 (YtStat '('Id '"6cd-bdec-13440191-2de00700") '('RecordsCount '"221188884") '('DataSize '"40937892044") $22 $23 $24))
(let $43 (YtTable '"home/tpcds/3Tb/web_returns" $41 $21 $42 '() (Void) (Void) '"dirac"))
(let $44 '"wr_account_credit")
(let $45 '"wr_refunded_addr_sk")
(let $46 '"wr_refunded_cdemo_sk")
(let $47 '"wr_refunded_customer_sk")
(let $48 '"wr_refunded_hdemo_sk")
(let $49 '"wr_return_amt_inc_tax")
(let $50 '"wr_return_quantity")
(let $51 '"wr_return_ship_cost")
(let $52 '"wr_returned_date_sk")
(let $53 '"wr_returned_time_sk")
(let $54 '"wr_returning_addr_sk")
(let $55 '"wr_returning_cdemo_sk")
(let $56 '"wr_returning_customer_sk")
(let $57 '"wr_returning_hdemo_sk")
(let $58 '"wr_reversed_charge")
(let $59 '($44 '"wr_fee" '"wr_item_sk" '"wr_net_loss" '"wr_order_number" '"wr_reason_sk" $45 '"wr_refunded_cash" $46 $47 $48 '"wr_return_amt" $49 $50 $51 '"wr_return_tax" $52 $53 $54 $55 $56 $57 $58 '"wr_web_page_sk"))
(let $60 '@@["StructType";[["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]]]]@@)
(let $61 (YqlRowSpec '('NativeYtTypeFlags '2) $17 $18 '('Type $60)))
(let $62 (YtStat '('Id '"6cd-be05-13440191-b0161bf0") '('RecordsCount '"73049") '('DataSize '"12720962") '('ChunkCount '1) $23 $24))
(let $63 (YtTable '"home/tpcds/3Tb/date_dim" $61 $21 $62 '() (Void) (Void) '"dirac"))
(let $64 '"d_current_quarter")
(let $65 '"d_following_holiday")
(let $66 '('"d_current_day" '"d_current_month" $64 '"d_current_week" '"d_current_year" '"d_date" '"d_date_id" '"d_date_sk" '"d_day_name" '"d_dom" '"d_dow" '"d_first_dom" $65 '"d_fy_quarter_seq" '"d_fy_week_seq" '"d_fy_year" '"d_holiday" '"d_last_dom" '"d_month_seq" '"d_moy" '"d_qoy" '"d_quarter_name" '"d_quarter_seq" '"d_same_day_lq" '"d_same_day_ly" '"d_week_seq" '"d_weekend" '"d_year"))
(let $67 '('"ws" '"ws_order_number" '"ws" '"ws_item_sk"))
(let $68 '('"wr" '"wr_order_number" '"wr" '"wr_item_sk"))
(let $69 '('Left '"ws" '"wr" $67 $68 '()))
(let $70 '('Cross $69 '"date_dim" '() '() '()))
(let $71 (EquiJoin '((OrderedFlatMap (ExtractMembers (Right! (YtReadTable! $12 $15 '((YtSection '((YtPath $26 (Void) (Void) (Void))) $27)))) $38) (lambda '($73) (OptionalIf (Coalesce (> (Member $73 '"ws_net_profit") (Int32 '1)) $39) $73))) '"ws") '((ExtractMembers (Right! (YtReadTable! $12 $15 '((YtSection '((YtPath $43 (Void) (Void) (Void))) $27)))) $59) '"wr") '((ExtractMembers (Right! (YtReadTable! $12 $15 '((YtSection '((YtPath $63 (Void) (Void) (Void))) $27)))) $66) '"date_dim") $70 '()))
(let $72 (ResWrite! (Sync! $12 $13) $14 (Key) (Unordered (OrderedFlatMap $71 (lambda '($74) (block '(
  (let $75 (Member $74 '"wr.wr_return_amt"))
  (let $76 (Member $74 '"ws.ws_net_paid"))
  (let $77 (Int32 '0))
  (let $78 (Member $74 '"ws.ws_quantity"))
  (let $79 (Member $74 '"ws.ws_sold_date_sk"))
  (let $80 (Member $74 '"date_dim.d_date_sk"))
  (let $81 (Member $74 '"date_dim.d_year"))
  (let $82 (Member $74 '"date_dim.d_moy"))
  (let $83 (And (Coalesce (> $75 (Int32 '"10000")) $39) (Coalesce (> $76 $77) $39) (Coalesce (> $78 $77) $39) (Coalesce (== $79 $80) $39) (Coalesce (== $81 (Int32 '"2000")) $39) (Coalesce (== $82 (Int32 '"12")) $39)))
  (let $84 (AsStruct '('"d_current_day" (Member $74 '"date_dim.d_current_day")) '('"d_current_month" (Member $74 '"date_dim.d_current_month")) '($64 (Member $74 '"date_dim.d_current_quarter")) '('"d_current_week" (Member $74 '"date_dim.d_current_week")) '('"d_current_year" (Member $74 '"date_dim.d_current_year")) '('"d_date" (Member $74 '"date_dim.d_date")) '('"d_date_id" (Member $74 '"date_dim.d_date_id")) '('"d_date_sk" $80) '('"d_day_name" (Member $74 '"date_dim.d_day_name")) '('"d_dom" (Member $74 '"date_dim.d_dom")) '('"d_dow" (Member $74 '"date_dim.d_dow")) '('"d_first_dom" (Member $74 '"date_dim.d_first_dom")) '($65 (Member $74 '"date_dim.d_following_holiday")) '('"d_fy_quarter_seq" (Member $74 '"date_dim.d_fy_quarter_seq")) '('"d_fy_week_seq" (Member $74 '"date_dim.d_fy_week_seq")) '('"d_fy_year" (Member $74 '"date_dim.d_fy_year")) '('"d_holiday" (Member $74 '"date_dim.d_holiday")) '('"d_last_dom" (Member $74 '"date_dim.d_last_dom")) '('"d_month_seq" (Member $74 '"date_dim.d_month_seq")) '('"d_moy" $82) '('"d_qoy" (Member $74 '"date_dim.d_qoy")) '('"d_quarter_name" (Member $74 '"date_dim.d_quarter_name")) '('"d_quarter_seq" (Member $74 '"date_dim.d_quarter_seq")) '('"d_same_day_lq" (Member $74 '"date_dim.d_same_day_lq")) '('"d_same_day_ly" (Member $74 '"date_dim.d_same_day_ly")) '('"d_week_seq" (Member $74 '"date_dim.d_week_seq")) '('"d_weekend" (Member $74 '"date_dim.d_weekend")) '('"d_year" $81) '($44 (Member $74 '"wr.wr_account_credit")) '('"wr_fee" (Member $74 '"wr.wr_fee")) '('"wr_item_sk" (Member $74 '"wr.wr_item_sk")) '('"wr_net_loss" (Member $74 '"wr.wr_net_loss")) '('"wr_order_number" (Member $74 '"wr.wr_order_number")) '('"wr_reason_sk" (Member $74 '"wr.wr_reason_sk")) '($45 (Member $74 '"wr.wr_refunded_addr_sk")) '('"wr_refunded_cash" (Member $74 '"wr.wr_refunded_cash")) '($46 (Member $74 '"wr.wr_refunded_cdemo_sk")) '($47 (Member $74 '"wr.wr_refunded_customer_sk")) '($48 (Member $74 '"wr.wr_refunded_hdemo_sk")) '('"wr_return_amt" $75) '($49 (Member $74 '"wr.wr_return_amt_inc_tax")) '($50 (Member $74 '"wr.wr_return_quantity")) '($51 (Member $74 '"wr.wr_return_ship_cost")) '('"wr_return_tax" (Member $74 '"wr.wr_return_tax")) '($52 (Member $74 '"wr.wr_returned_date_sk")) '($53 (Member $74 '"wr.wr_returned_time_sk")) '($54 (Member $74 '"wr.wr_returning_addr_sk")) '($55 (Member $74 '"wr.wr_returning_cdemo_sk")) '($56 (Member $74 '"wr.wr_returning_customer_sk")) '($57 (Member $74 '"wr.wr_returning_hdemo_sk")) '($58 (Member $74 '"wr.wr_reversed_charge")) '('"wr_web_page_sk" (Member $74 '"wr.wr_web_page_sk")) '('"ws_bill_addr_sk" (Member $74 '"ws.ws_bill_addr_sk")) '('"ws_bill_cdemo_sk" (Member $74 '"ws.ws_bill_cdemo_sk")) '($28 (Member $74 '"ws.ws_bill_customer_sk")) '('"ws_bill_hdemo_sk" (Member $74 '"ws.ws_bill_hdemo_sk")) '('"ws_coupon_amt" (Member $74 '"ws.ws_coupon_amt")) '($29 (Member $74 '"ws.ws_ext_discount_amt")) '($30 (Member $74 '"ws.ws_ext_list_price")) '($31 (Member $74 '"ws.ws_ext_sales_price")) '('"ws_ext_ship_cost" (Member $74 '"ws.ws_ext_ship_cost")) '('"ws_ext_tax" (Member $74 '"ws.ws_ext_tax")) '($32 (Member $74 '"ws.ws_ext_wholesale_cost")) '('"ws_item_sk" (Member $74 '"ws.ws_item_sk")) '('"ws_list_price" (Member $74 '"ws.ws_list_price")) '('"ws_net_paid" $76) '($33 (Member $74 '"ws.ws_net_paid_inc_ship")) '($34 (Member $74 '"ws.ws_net_paid_inc_ship_tax")) '($35 (Member $74 '"ws.ws_net_paid_inc_tax")) '('"ws_net_profit" (Member $74 '"ws.ws_net_profit")) '('"ws_order_number" (Member $74 '"ws.ws_order_number")) '('"ws_promo_sk" (Member $74 '"ws.ws_promo_sk")) '('"ws_quantity" $78) '('"ws_sales_price" (Member $74 '"ws.ws_sales_price")) '('"ws_ship_addr_sk" (Member $74 '"ws.ws_ship_addr_sk")) '('"ws_ship_cdemo_sk" (Member $74 '"ws.ws_ship_cdemo_sk")) '($36 (Member $74 '"ws.ws_ship_customer_sk")) '('"ws_ship_date_sk" (Member $74 '"ws.ws_ship_date_sk")) '('"ws_ship_hdemo_sk" (Member $74 '"ws.ws_ship_hdemo_sk")) '('"ws_ship_mode_sk" (Member $74 '"ws.ws_ship_mode_sk")) '('"ws_sold_date_sk" $79) '('"ws_sold_time_sk" (Member $74 '"ws.ws_sold_time_sk")) '('"ws_warehouse_sk" (Member $74 '"ws.ws_warehouse_sk")) '('"ws_web_page_sk" (Member $74 '"ws.ws_web_page_sk")) '('"ws_web_site_sk" (Member $74 '"ws.ws_web_site_sk")) '($37 (Member $74 '"ws.ws_wholesale_cost"))))
  (return (OptionalIf $83 $84))
))))) '('('type) '('autoref))))
(return (Commit! (Commit! $72 $14) (DataSink '"yt" '"dirac") '('('"epoch" '1))))
)

2024-03-06 15:03:41.816 DEBUG mrrun(pid=2454930, tid=0x00007F0865BF7D80) [core] yql_flatmap_over_join.cpp:736: {8705dd46-8c5b9f07-1c12db31-3ff5fc3} SingleInputPredicatePushdownOverEquiJoin
2024-03-06 15:03:41.817 DEBUG mrrun(pid=2454930, tid=0x00007F0865BF7D80) [core] yql_co_flow2.cpp:1120: {8705dd46-8c5b9f07-1c12db31-3ff5fc3} OrderedFlatMapOverEquiJoin
2024-03-06 15:03:41.838 TRACE mrrun(pid=2454930, tid=0x00007F0865BF7D80) [core] yql_out_transformers.cpp:59: {8705dd46-8c5b9f07-1c12db31-3ff5fc3} OptimizedExpr:
(
(let $1 (DataSource '"yt" '"$all"))
(let $2 (YtConfigure! world $1 '"Attr" '"operationspec" '"{time_limit=3600000}"))
(let $3 (YtConfigure! $2 $1 '"Attr" '"maxreplicationfactortofuseoperations" '"30"))
(let $4 (YtConfigure! $3 $1 '"Attr" '"pool" '"tpcds"))
(let $5 (YtConfigure! $4 $1 '"Attr" '"schedulingtagfilter" '"%true"))
(let $6 (YtConfigure! $5 $1 '"Attr" '"querycachemode" '"disable"))
(let $7 (YtConfigure! $6 $1 '"Attr" '"hybriddqexecution" '"true"))
(let $8 (YtConfigure! $7 $1 '"Attr" '"automerge" '"disabled"))
(let $9 (YtConfigure! $8 $1 '"Attr" '"datasizeperjob" '"16M"))
(let $10 (YtConfigure! $9 $1 '"Attr" '"datasizepermapjob" '"24M"))
(let $11 (YtConfigure! $10 $1 '"Attr" '"datasizeperpartition" '"64M"))
(let $12 (YtConfigure! $11 $1 '"Attr" '"mapjoinlimit" '"33554432"))
(let $13 (Configure! world (DataSource '"config") '"CostBasedOptimizer" '"native"))
(let $14 (DataSink 'result))
(let $15 (DataSource '"yt" '"dirac"))
(let $16 '('NativeYtTypeFlags '16))
(let $17 '('StrictSchema '1))
(let $18 '('UniqueKeys '0))
(let $19 '@@["StructType";[["ws_sold_date_sk";["OptionalType";["DataType";"Int64"]]];["ws_sold_time_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_date_sk";["OptionalType";["DataType";"Int64"]]];["ws_item_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_bill_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_customer_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_addr_sk";["OptionalType";["DataType";"Int64"]]];["ws_web_page_sk";["OptionalType";["DataType";"Int64"]]];["ws_web_site_sk";["OptionalType";["DataType";"Int64"]]];["ws_ship_mode_sk";["OptionalType";["DataType";"Int64"]]];["ws_warehouse_sk";["OptionalType";["DataType";"Int64"]]];["ws_promo_sk";["OptionalType";["DataType";"Int64"]]];["ws_order_number";["OptionalType";["DataType";"Int64"]]];["ws_quantity";["OptionalType";["DataType";"Int64"]]];["ws_wholesale_cost";["OptionalType";["DataType";"Float"]]];["ws_list_price";["OptionalType";["DataType";"Float"]]];["ws_sales_price";["OptionalType";["DataType";"Float"]]];["ws_ext_discount_amt";["OptionalType";["DataType";"Float"]]];["ws_ext_sales_price";["OptionalType";["DataType";"Float"]]];["ws_ext_wholesale_cost";["OptionalType";["DataType";"Float"]]];["ws_ext_list_price";["OptionalType";["DataType";"Float"]]];["ws_ext_tax";["OptionalType";["DataType";"Float"]]];["ws_coupon_amt";["OptionalType";["DataType";"Float"]]];["ws_ext_ship_cost";["OptionalType";["DataType";"Float"]]];["ws_net_paid";["OptionalType";["DataType";"Float"]]];["ws_net_paid_inc_tax";["OptionalType";["DataType";"Float"]]];["ws_net_paid_inc_ship";["OptionalType";["DataType";"Float"]]];["ws_net_paid_inc_ship_tax";["OptionalType";["DataType";"Float"]]];["ws_net_profit";["OptionalType";["DataType";"Float"]]]]]@@)
(let $20 (YqlRowSpec $16 $17 $18 '('Type $19)))
(let $21 (YtMeta '('CanWrite '1) '('DoesExist '1) '('YqlCompatibleScheme '1) '('InferredScheme '0) '('IsDynamic '0)))
(let $22 '('ChunkCount '"1000"))
(let $23 '('ModifyTime '"1707907334"))
(let $24 '('Revision '"7481833077937"))
(let $25 (YtStat '('Id '"6cd-bde8-13440191-b4afb85b") '('RecordsCount '"2211809044") '('DataSize '"603695506444") $22 $23 $24))
(let $26 (YtTable '"home/tpcds/3Tb/web_sales" $20 $21 $25 '() (Void) (Void) '"dirac"))
(let $27 '('('"syscolumns" '('"path" '"record"))))
(let $28 '"ws_bill_customer_sk")
(let $29 '"ws_ext_discount_amt")
(let $30 '"ws_ext_list_price")
(let $31 '"ws_ext_sales_price")
(let $32 '"ws_ext_wholesale_cost")
(let $33 '"ws_net_paid_inc_ship")
(let $34 '"ws_net_paid_inc_ship_tax")
(let $35 '"ws_net_paid_inc_tax")
(let $36 '"ws_ship_customer_sk")
(let $37 '"ws_wholesale_cost")
(let $38 '('"ws_bill_addr_sk" '"ws_bill_cdemo_sk" $28 '"ws_bill_hdemo_sk" '"ws_coupon_amt" $29 $30 $31 '"ws_ext_ship_cost" '"ws_ext_tax" $32 '"ws_item_sk" '"ws_list_price" '"ws_net_paid" $33 $34 $35 '"ws_net_profit" '"ws_order_number" '"ws_promo_sk" '"ws_quantity" '"ws_sales_price" '"ws_ship_addr_sk" '"ws_ship_cdemo_sk" $36 '"ws_ship_date_sk" '"ws_ship_hdemo_sk" '"ws_ship_mode_sk" '"ws_sold_date_sk" '"ws_sold_time_sk" '"ws_warehouse_sk" '"ws_web_page_sk" '"ws_web_site_sk" $37))
(let $39 (Bool 'false))
(let $40 '"ws.ws_bill_addr_sk")
(let $41 '"ws.ws_bill_cdemo_sk")
(let $42 '"ws.ws_bill_customer_sk")
(let $43 '"ws.ws_bill_hdemo_sk")
(let $44 '"ws.ws_ext_discount_amt")
(let $45 '"ws.ws_ext_list_price")
(let $46 '"ws.ws_ext_sales_price")
(let $47 '"ws.ws_ext_ship_cost")
(let $48 '"ws.ws_ext_wholesale_cost")
(let $49 '"ws.ws_net_paid_inc_ship")
(let $50 '"ws.ws_net_paid_inc_ship_tax")
(let $51 '"ws.ws_net_paid_inc_tax")
(let $52 '"ws.ws_order_number")
(let $53 '"ws.ws_sales_price")
(let $54 '"ws.ws_ship_addr_sk")
(let $55 '"ws.ws_ship_cdemo_sk")
(let $56 '"ws.ws_ship_customer_sk")
(let $57 '"ws.ws_ship_date_sk")
(let $58 '"ws.ws_ship_hdemo_sk")
(let $59 '"ws.ws_ship_mode_sk")
(let $60 '"ws.ws_sold_date_sk")
(let $61 '"ws.ws_sold_time_sk")
(let $62 '"ws.ws_warehouse_sk")
(let $63 '"ws.ws_web_page_sk")
(let $64 '"ws.ws_web_site_sk")
(let $65 '"ws.ws_wholesale_cost")
(let $66 (Int32 '0))
(let $67 '@@["StructType";[["wr_returned_date_sk";["OptionalType";["DataType";"Int64"]]];["wr_returned_time_sk";["OptionalType";["DataType";"Int64"]]];["wr_item_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_refunded_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_customer_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_cdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_hdemo_sk";["OptionalType";["DataType";"Int64"]]];["wr_returning_addr_sk";["OptionalType";["DataType";"Int64"]]];["wr_web_page_sk";["OptionalType";["DataType";"Int64"]]];["wr_reason_sk";["OptionalType";["DataType";"Int64"]]];["wr_order_number";["OptionalType";["DataType";"Int64"]]];["wr_return_quantity";["OptionalType";["DataType";"Int64"]]];["wr_return_amt";["OptionalType";["DataType";"Float"]]];["wr_return_tax";["OptionalType";["DataType";"Float"]]];["wr_return_amt_inc_tax";["OptionalType";["DataType";"Float"]]];["wr_fee";["OptionalType";["DataType";"Float"]]];["wr_return_ship_cost";["OptionalType";["DataType";"Float"]]];["wr_refunded_cash";["OptionalType";["DataType";"Float"]]];["wr_reversed_charge";["OptionalType";["DataType";"Float"]]];["wr_account_credit";["OptionalType";["DataType";"Float"]]];["wr_net_loss";["OptionalType";["DataType";"Float"]]]]]@@)
(let $68 (YqlRowSpec $16 $17 $18 '('Type $67)))
(let $69 (YtStat '('Id '"6cd-bdec-13440191-2de00700") '('RecordsCount '"221188884") '('DataSize '"40937892044") $22 $23 $24))
(let $70 (YtTable '"home/tpcds/3Tb/web_returns" $68 $21 $69 '() (Void) (Void) '"dirac"))
(let $71 '"wr_account_credit")
(let $72 '"wr_refunded_addr_sk")
(let $73 '"wr_refunded_cdemo_sk")
(let $74 '"wr_refunded_customer_sk")
(let $75 '"wr_refunded_hdemo_sk")
(let $76 '"wr_return_amt_inc_tax")
(let $77 '"wr_return_quantity")
(let $78 '"wr_return_ship_cost")
(let $79 '"wr_returned_date_sk")
(let $80 '"wr_returned_time_sk")
(let $81 '"wr_returning_addr_sk")
(let $82 '"wr_returning_cdemo_sk")
(let $83 '"wr_returning_customer_sk")
(let $84 '"wr_returning_hdemo_sk")
(let $85 '"wr_reversed_charge")
(let $86 '($71 '"wr_fee" '"wr_item_sk" '"wr_net_loss" '"wr_order_number" '"wr_reason_sk" $72 '"wr_refunded_cash" $73 $74 $75 '"wr_return_amt" $76 $77 $78 '"wr_return_tax" $79 $80 $81 $82 $83 $84 $85 '"wr_web_page_sk"))
(let $87 '@@["StructType";[["d_date_sk";["OptionalType";["DataType";"Int64"]]];["d_date_id";["OptionalType";["DataType";"String"]]];["d_date";["OptionalType";["DataType";"Date"]]];["d_month_seq";["OptionalType";["DataType";"Int64"]]];["d_week_seq";["OptionalType";["DataType";"Int64"]]];["d_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_year";["OptionalType";["DataType";"Int64"]]];["d_dow";["OptionalType";["DataType";"Int64"]]];["d_moy";["OptionalType";["DataType";"Int64"]]];["d_dom";["OptionalType";["DataType";"Int64"]]];["d_qoy";["OptionalType";["DataType";"Int64"]]];["d_fy_year";["OptionalType";["DataType";"Int64"]]];["d_fy_quarter_seq";["OptionalType";["DataType";"Int64"]]];["d_fy_week_seq";["OptionalType";["DataType";"Int64"]]];["d_day_name";["OptionalType";["DataType";"String"]]];["d_quarter_name";["OptionalType";["DataType";"String"]]];["d_holiday";["OptionalType";["DataType";"String"]]];["d_weekend";["OptionalType";["DataType";"String"]]];["d_following_holiday";["OptionalType";["DataType";"String"]]];["d_first_dom";["OptionalType";["DataType";"Int64"]]];["d_last_dom";["OptionalType";["DataType";"Int64"]]];["d_same_day_ly";["OptionalType";["DataType";"Int64"]]];["d_same_day_lq";["OptionalType";["DataType";"Int64"]]];["d_current_day";["OptionalType";["DataType";"String"]]];["d_current_week";["OptionalType";["DataType";"String"]]];["d_current_month";["OptionalType";["DataType";"String"]]];["d_current_quarter";["OptionalType";["DataType";"String"]]];["d_current_year";["OptionalType";["DataType";"String"]]]]]@@)
(let $88 (YqlRowSpec '('NativeYtTypeFlags '2) $17 $18 '('Type $87)))
(let $89 (YtStat '('Id '"6cd-be05-13440191-b0161bf0") '('RecordsCount '"73049") '('DataSize '"12720962") '('ChunkCount '1) $23 $24))
(let $90 (YtTable '"home/tpcds/3Tb/date_dim" $88 $21 $89 '() (Void) (Void) '"dirac"))
(let $91 '"d_current_quarter")
(let $92 '"d_following_holiday")
(let $93 '('"d_current_day" '"d_current_month" $91 '"d_current_week" '"d_current_year" '"d_date" '"d_date_id" '"d_date_sk" '"d_day_name" '"d_dom" '"d_dow" '"d_first_dom" $92 '"d_fy_quarter_seq" '"d_fy_week_seq" '"d_fy_year" '"d_holiday" '"d_last_dom" '"d_month_seq" '"d_moy" '"d_qoy" '"d_quarter_name" '"d_quarter_seq" '"d_same_day_lq" '"d_same_day_ly" '"d_week_seq" '"d_weekend" '"d_year"))
(let $94 '('"ws" '"ws_order_number" '"ws" '"ws_item_sk"))
(let $95 '('"wr" '"wr_order_number" '"wr" '"wr_item_sk"))
(let $96 '('Left '"ws" '"wr" $94 $95 '()))
(let $97 '('Cross $96 '"date_dim" '() '() '()))
(let $98 (EquiJoin '((OrderedFilter (OrderedFlatMap (ExtractMembers (Right! (YtReadTable! $12 $15 '((YtSection '((YtPath $26 (Void) (Void) (Void))) $27)))) $38) (lambda '($101) (OptionalIf (Coalesce (> (Member $101 '"ws_net_profit") (Int32 '1)) $39) $101))) (lambda '($102) (block '(
  (let $103 (AsStruct '($40 (Member $102 '"ws_bill_addr_sk")) '($41 (Member $102 '"ws_bill_cdemo_sk")) '($42 (Member $102 $28)) '($43 (Member $102 '"ws_bill_hdemo_sk")) '('"ws.ws_coupon_amt" (Member $102 '"ws_coupon_amt")) '($44 (Member $102 $29)) '($45 (Member $102 $30)) '($46 (Member $102 $31)) '($47 (Member $102 '"ws_ext_ship_cost")) '('"ws.ws_ext_tax" (Member $102 '"ws_ext_tax")) '($48 (Member $102 $32)) '('"ws.ws_item_sk" (Member $102 '"ws_item_sk")) '('"ws.ws_list_price" (Member $102 '"ws_list_price")) '('"ws.ws_net_paid" (Member $102 '"ws_net_paid")) '($49 (Member $102 $33)) '($50 (Member $102 $34)) '($51 (Member $102 $35)) '('"ws.ws_net_profit" (Member $102 '"ws_net_profit")) '($52 (Member $102 '"ws_order_number")) '('"ws.ws_promo_sk" (Member $102 '"ws_promo_sk")) '('"ws.ws_quantity" (Member $102 '"ws_quantity")) '($53 (Member $102 '"ws_sales_price")) '($54 (Member $102 '"ws_ship_addr_sk")) '($55 (Member $102 '"ws_ship_cdemo_sk")) '($56 (Member $102 $36)) '($57 (Member $102 '"ws_ship_date_sk")) '($58 (Member $102 '"ws_ship_hdemo_sk")) '($59 (Member $102 '"ws_ship_mode_sk")) '($60 (Member $102 '"ws_sold_date_sk")) '($61 (Member $102 '"ws_sold_time_sk")) '($62 (Member $102 '"ws_warehouse_sk")) '($63 (Member $102 '"ws_web_page_sk")) '($64 (Member $102 '"ws_web_site_sk")) '($65 (Member $102 $37))))
  (return (Coalesce (> (Member $103 '"ws.ws_net_paid") $66) $39))
)))) '"ws") '((ExtractMembers (Right! (YtReadTable! $12 $15 '((YtSection '((YtPath $70 (Void) (Void) (Void))) $27)))) $86) '"wr") '((ExtractMembers (Right! (YtReadTable! $12 $15 '((YtSection '((YtPath $90 (Void) (Void) (Void))) $27)))) $93) '"date_dim") $97 '()))
(let $99 '"date_dim.d_date_sk")
(let $100 (ResWrite! (Sync! $12 $13) $14 (Key) (Unordered (OrderedMap (OrderedFilter $98 (lambda '($104) (And (And (And (And (Coalesce (> (Member $104 '"wr.wr_return_amt") (Int32 '"10000")) $39) (Coalesce (> (Member $104 '"ws.ws_quantity") $66) $39)) (Coalesce (== (Member $104 $60) (Member $104 $99)) $39)) (Coalesce (== (Member $104 '"date_dim.d_year") (Int32 '"2000")) $39)) (Coalesce (== (Member $104 '"date_dim.d_moy") (Int32 '"12")) $39)))) (lambda '($105) (AsStruct '('"d_current_day" (Member $105 '"date_dim.d_current_day")) '('"d_current_month" (Member $105 '"date_dim.d_current_month")) '($91 (Member $105 '"date_dim.d_current_quarter")) '('"d_current_week" (Member $105 '"date_dim.d_current_week")) '('"d_current_year" (Member $105 '"date_dim.d_current_year")) '('"d_date" (Member $105 '"date_dim.d_date")) '('"d_date_id" (Member $105 '"date_dim.d_date_id")) '('"d_date_sk" (Member $105 $99)) '('"d_day_name" (Member $105 '"date_dim.d_day_name")) '('"d_dom" (Member $105 '"date_dim.d_dom")) '('"d_dow" (Member $105 '"date_dim.d_dow")) '('"d_first_dom" (Member $105 '"date_dim.d_first_dom")) '($92 (Member $105 '"date_dim.d_following_holiday")) '('"d_fy_quarter_seq" (Member $105 '"date_dim.d_fy_quarter_seq")) '('"d_fy_week_seq" (Member $105 '"date_dim.d_fy_week_seq")) '('"d_fy_year" (Member $105 '"date_dim.d_fy_year")) '('"d_holiday" (Member $105 '"date_dim.d_holiday")) '('"d_last_dom" (Member $105 '"date_dim.d_last_dom")) '('"d_month_seq" (Member $105 '"date_dim.d_month_seq")) '('"d_moy" (Member $105 '"date_dim.d_moy")) '('"d_qoy" (Member $105 '"date_dim.d_qoy")) '('"d_quarter_name" (Member $105 '"date_dim.d_quarter_name")) '('"d_quarter_seq" (Member $105 '"date_dim.d_quarter_seq")) '('"d_same_day_lq" (Member $105 '"date_dim.d_same_day_lq")) '('"d_same_day_ly" (Member $105 '"date_dim.d_same_day_ly")) '('"d_week_seq" (Member $105 '"date_dim.d_week_seq")) '('"d_weekend" (Member $105 '"date_dim.d_weekend")) '('"d_year" (Member $105 '"date_dim.d_year")) '($71 (Member $105 '"wr.wr_account_credit")) '('"wr_fee" (Member $105 '"wr.wr_fee")) '('"wr_item_sk" (Member $105 '"wr.wr_item_sk")) '('"wr_net_loss" (Member $105 '"wr.wr_net_loss")) '('"wr_order_number" (Member $105 '"wr.wr_order_number")) '('"wr_reason_sk" (Member $105 '"wr.wr_reason_sk")) '($72 (Member $105 '"wr.wr_refunded_addr_sk")) '('"wr_refunded_cash" (Member $105 '"wr.wr_refunded_cash")) '($73 (Member $105 '"wr.wr_refunded_cdemo_sk")) '($74 (Member $105 '"wr.wr_refunded_customer_sk")) '($75 (Member $105 '"wr.wr_refunded_hdemo_sk")) '('"wr_return_amt" (Member $105 '"wr.wr_return_amt")) '($76 (Member $105 '"wr.wr_return_amt_inc_tax")) '($77 (Member $105 '"wr.wr_return_quantity")) '($78 (Member $105 '"wr.wr_return_ship_cost")) '('"wr_return_tax" (Member $105 '"wr.wr_return_tax")) '($79 (Member $105 '"wr.wr_returned_date_sk")) '($80 (Member $105 '"wr.wr_returned_time_sk")) '($81 (Member $105 '"wr.wr_returning_addr_sk")) '($82 (Member $105 '"wr.wr_returning_cdemo_sk")) '($83 (Member $105 '"wr.wr_returning_customer_sk")) '($84 (Member $105 '"wr.wr_returning_hdemo_sk")) '($85 (Member $105 '"wr.wr_reversed_charge")) '('"wr_web_page_sk" (Member $105 '"wr.wr_web_page_sk")) '('"ws_bill_addr_sk" (Member $105 $40)) '('"ws_bill_cdemo_sk" (Member $105 $41)) '($28 (Member $105 $42)) '('"ws_bill_hdemo_sk" (Member $105 $43)) '('"ws_coupon_amt" (Member $105 '"ws.ws_coupon_amt")) '($29 (Member $105 $44)) '($30 (Member $105 $45)) '($31 (Member $105 $46)) '('"ws_ext_ship_cost" (Member $105 $47)) '('"ws_ext_tax" (Member $105 '"ws.ws_ext_tax")) '($32 (Member $105 $48)) '('"ws_item_sk" (Member $105 '"ws.ws_item_sk")) '('"ws_list_price" (Member $105 '"ws.ws_list_price")) '('"ws_net_paid" (Member $105 '"ws.ws_net_paid")) '($33 (Member $105 $49)) '($34 (Member $105 $50)) '($35 (Member $105 $51)) '('"ws_net_profit" (Member $105 '"ws.ws_net_profit")) '('"ws_order_number" (Member $105 $52)) '('"ws_promo_sk" (Member $105 '"ws.ws_promo_sk")) '('"ws_quantity" (Member $105 '"ws.ws_quantity")) '('"ws_sales_price" (Member $105 $53)) '('"ws_ship_addr_sk" (Member $105 $54)) '('"ws_ship_cdemo_sk" (Member $105 $55)) '($36 (Member $105 $56)) '('"ws_ship_date_sk" (Member $105 $57)) '('"ws_ship_hdemo_sk" (Member $105 $58)) '('"ws_ship_mode_sk" (Member $105 $59)) '('"ws_sold_date_sk" (Member $105 $60)) '('"ws_sold_time_sk" (Member $105 $61)) '('"ws_warehouse_sk" (Member $105 $62)) '('"ws_web_page_sk" (Member $105 $63)) '('"ws_web_site_sk" (Member $105 $64)) '($37 (Member $105 $65)))))) '('('type) '('autoref))))
(return (Commit! (Commit! $100 $14) (DataSink '"yt" '"dirac") '('('"epoch" '1))))
)

from ydb.

qrort commented on May 30, 2024

mrrun log: https://gist.github.com/qrort/15a681a6564a9e25f32bfe608c8636ed'
rowcount: 3601932372 (x1.5 от всего запроса в hive)

Вижу только, что wr.wr_return_amt > 10000 применяется после джойна, а не до него. Других проблем не видно.
И, видимо, это важная штука, так как статистика операции такая:

Name    Rows    Compressed    Uncompressed    Data    Weight    Slices
Estimated input	29 088 987	2.68 GiB	4.09 GiB	12.58 GiB	245	
input -> map	29 088 987	18.60 GiB	27.32 GiB	13.23 GiB	1 715	
map -> output	57 101	10.27 MiB	10.57 MiB	34.53 MiB	575

from ydb.

nepal commented on May 30, 2024

Да, тут проблема в том, что не пушдаунится фильтр на правую сторону left join. У нас такого оптимизатора в данный момент нет - мы умеем только пушдаунить фильм сквозь inner join или на левую сторону left join.

Напишу здесь алгоритм проброса фильтра на правую часть left join.

Рассмотрим запрос

select * from a left join b on (a.key1 = b.key1 ...) where p(b)
Здесь p(b) - произвольный предикат, зависящий только от правой стороны.

Данный запрос можно переписать в виде

select * from (
   <обрабатываем строчки левой стороны для которых есть ключи справа>
   union all
   <обрабатываем строчки левой стороны для которых нет ключей справа>
) where p(b)

Этот запрос в свою очередь эквивалентен

select * from (
    select * from a inner join b on (a.key1 = b.key1 ...) 
    union all
    select * from a left only join b on (a.key1 = b.key1 ...)
) where p(b)

Что в свою очередь эквивалентно

select * from (
    select * from a inner join (select <добавляем опциональность ко всем колонкам> from b where p(b)) as b on (a.key1 = b.key1 ...)  
    union all
    select * from a left only join b on (a.key1 = b.key1 ...) where p2
)

Здесь p2 - предикат полученный заменой исходных колонок в p(b) на null (т.е. это константа)

В простых случаях получается что вторая часть union all пропадает. Например, если предикат имеет вид (x > 0 ?? false), то имеем

select * from (
    select * from a inner join (select ... Just(x) as x from b  where (b.x > 0 ?? false)) as b on (a.key1 = b.key1 ...)
    union all
    select * from a left only join b on (a.key1 = b.key1 ...) where (null > x ?? false)
)

или

select * from (
    select * from a inner join (select ... Just(x) as x from b   where (b.x > 0 ?? false)) as b on (a.key1 = b.key1 ...)
    union all
    select * from a left only join b on (a.key1 = b.key1 ...) where false
)

или

select * from a inner join (select ... Just(x) as x from b where (b.x > 0 ?? false)) as b on (a.key1 = b.key1 ...)

Но, например, в случае предиката not exist(b.x) получается

select * from (
    select * from a inner join (select ... Just(x) as x from b where (not exists(b.x))) as b on (a.key1 = b.key1 ...)
    union all
    select * from a left only join b on (a.key1 = b.key1 ...) where (not exists (null))
)

или

select * from (
    select * from a inner join (select ... Just(x) as x from b) as b on (a.key1 = b.key1 ...)  where (not true)
    union all
    select * from a left only join b on (a.key1 = b.key1 ...) where (not exists (null))
)

или

select * from a left only join b on (a.key1 = b.key1 ...)

from ydb.

nepal commented on May 30, 2024

Summary:

Запрос
select * from a left join b on (a.key1 = b.key1 ...) where p(b)
Здесь p(b) - произвольный предикат, зависящий только от правой стороны.

Такой запрос можно переписать в виде

select * from (
    select * from a inner join (
        select * from (
            select <добавляем опциональность ко всем колонкам> from b
        ) where p(b)
    ) as b on (a.key1 = b.key1 ...)  
    union all
    select * from a left only join b on (a.key1 = b.key1 ...) where p2
)

Здесь p2 - предикат полученный заменой исходных колонок в p(b) на null (т.е. это константа)

from ydb.

qrort commented on May 30, 2024

unmodified subquery:

select *
	from
	 web_sales ws cross join date_dim left join web_returns wr on (ws.ws_order_number = wr.wr_order_number 
     and	ws.ws_item_sk = wr.wr_item_sk)
	where
		wr.wr_return_amt > 10000
		and ws.ws_net_profit > 1
                     and ws.ws_net_paid > 0
                     and ws.ws_quantity > 0
                     and ws_sold_date_sk = d_date_sk
                     and d_year = 2000
                     and d_moy = 12

rowcount: 3601932372, time: 00:18:14

wr filter:

$filtered = select * from web_returns where wr_return_amt > 10000;

select *
	from
	 web_sales ws cross join date_dim left join $filtered wr on (ws.ws_order_number = wr.wr_order_number 
     and	ws.ws_item_sk = wr.wr_item_sk)
	where
	    ws.ws_net_profit > 1
                     and ws.ws_net_paid > 0
                     and ws.ws_quantity > 0
                     and ws_sold_date_sk = d_date_sk
                     and d_year = 2000
                     and d_moy = 12;

row_count: 3295331031, time: 00:13:05

from ydb.

YQL-17364: [tpcds-49] Неоптимальный план подзапроса (не фильтруется таблица перед джойном) about ydb HOT 6 OPEN

Comments (6)

Related Issues (20)

Recommend Projects

React

Vue.js

Typescript

TensorFlow

Django

Laravel

D3

Recommend Topics

javascript

web

server

Machine learning

Visualization

Game

Recommend Org

Facebook

Microsoft

Google

Alibaba

D3

Tencent

Jobs