Skip to content

feat: pxf fdw support parallel scan #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MisterRaindrop wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: pxf fdw support parallel scan #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query01.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query01 for PXF parallel scan correctness - count with parallel

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT count(*) FROM pxf_parallel_enabled;
count
-------
10000
(1 row)
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query02.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query02 for PXF parallel scan correctness - sum aggregation

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT sum(id) FROM pxf_parallel_enabled;
sum
----------
50005000
(1 row)
21 changes: 21 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query03.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
-- @description query03 for PXF parallel scan correctness - cross-check parallel vs non-parallel count

SET optimizer = off;
SET
SET enable_parallel = false;
SET
SELECT count(*) AS non_parallel_count FROM pxf_parallel_disabled;
non_parallel_count
--------------------
10000
(1 row)

SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT count(*) AS parallel_count FROM pxf_parallel_enabled;
parallel_count
----------------
10000
(1 row)
22 changes: 22 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query04.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
-- @description query04 for PXF parallel scan correctness - ORDER BY with LIMIT

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT id, val FROM pxf_parallel_enabled ORDER BY id LIMIT 10;
id | val
----+-------
1 | row_1
2 | row_2
3 | row_3
4 | row_4
5 | row_5
6 | row_6
7 | row_7
8 | row_8
9 | row_9
10 | row_10
(10 rows)
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query05.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query05 for PXF parallel scan correctness - MIN/MAX/AVG aggregates

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT min(id), max(id), avg(id) FROM pxf_parallel_enabled;
min | max | avg
-----+-------+-----------------------
1 | 10000 | 5000.5000000000000000
(1 row)
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query06.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query06 for PXF parallel scan correctness - WHERE pushdown with parallel

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT count(*) FROM pxf_parallel_enabled WHERE id > 5000;
count
-------
5000
(1 row)
17 changes: 17 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query07.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
-- @description query07 for PXF parallel scan correctness - column projection with WHERE

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT val FROM pxf_parallel_enabled WHERE id <= 5 ORDER BY val;
val
-------
row_1
row_2
row_3
row_4
row_5
(5 rows)
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query08.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query08 for PXF parallel scan correctness - empty result edge case

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT count(*) FROM pxf_parallel_enabled WHERE id < 0;
count
-------
0
(1 row)
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query09.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query09 for PXF parallel scan correctness - COUNT DISTINCT no duplicates

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
SELECT count(DISTINCT id) FROM pxf_parallel_enabled;
count
-------
10000
(1 row)
13 changes: 13 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/expected/query10.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- @description query10 for PXF parallel scan correctness - workers=0 fallback on parallel table

SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 0;
SET
SELECT count(*) FROM pxf_parallel_enabled;
count
-------
10000
(1 row)
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query01.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query01 for PXF parallel scan correctness - count with parallel

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT count(*) FROM pxf_parallel_enabled;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query02.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query02 for PXF parallel scan correctness - sum aggregation

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT sum(id) FROM pxf_parallel_enabled;
9 changes: 9 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query03.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
-- @description query03 for PXF parallel scan correctness - cross-check parallel vs non-parallel count

SET optimizer = off;
SET enable_parallel = false;
SELECT count(*) AS non_parallel_count FROM pxf_parallel_disabled;

SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT count(*) AS parallel_count FROM pxf_parallel_enabled;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query04.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query04 for PXF parallel scan correctness - ORDER BY with LIMIT

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT id, val FROM pxf_parallel_enabled ORDER BY id LIMIT 10;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query05.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query05 for PXF parallel scan correctness - MIN/MAX/AVG aggregates

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT min(id), max(id), avg(id) FROM pxf_parallel_enabled;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query06.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query06 for PXF parallel scan correctness - WHERE pushdown with parallel

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT count(*) FROM pxf_parallel_enabled WHERE id > 5000;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query07.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query07 for PXF parallel scan correctness - column projection with WHERE

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT val FROM pxf_parallel_enabled WHERE id <= 5 ORDER BY val;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query08.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query08 for PXF parallel scan correctness - empty result edge case

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT count(*) FROM pxf_parallel_enabled WHERE id < 0;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query09.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query09 for PXF parallel scan correctness - COUNT DISTINCT no duplicates

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 4;
SELECT count(DISTINCT id) FROM pxf_parallel_enabled;
6 changes: 6 additions & 0 deletions automation/sqlrepo/features/parallel/correctness/sql/query10.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- @description query10 for PXF parallel scan correctness - workers=0 fallback on parallel table

SET optimizer = off;
SET enable_parallel = true;
SET max_parallel_workers_per_gather = 0;
SELECT count(*) FROM pxf_parallel_enabled;
32 changes: 32 additions & 0 deletions automation/sqlrepo/features/parallel/explain/expected/query01.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
-- @description query01 for PXF parallel scan EXPLAIN - Gather node present when parallel enabled
-- start_matchsubs
--
-- m/Workers Planned: \d+/
-- s/Workers Planned: \d+/Workers Planned: N/
--
-- m/cost=\d+\.\d+\.\.\d+\.\d+/
-- s/cost=\d+\.\d+\.\.\d+\.\d+/cost=XXX/
--
-- m/rows=\d+/
-- s/rows=\d+/rows=NNN/
--
-- m/width=\d+/
-- s/width=\d+/width=NN/
--
-- end_matchsubs
SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
EXPLAIN SELECT count(*) FROM pxf_parallel_enabled;
QUERY PLAN
----------------------------------------------------------------------------------------------
Gather Motion 1:1 (slice1; segments: 1) (cost=XXX rows=NNN width=NN)
Copy link
Contributor

@my-ship-it my-ship-it Mar 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The plan seems not correct?

-> Finalize Aggregate (cost=XXX rows=NNN width=NN)
-> Gather (cost=XXX rows=NNN width=NN)
Workers Planned: N
-> Partial Aggregate (cost=XXX rows=NNN width=NN)
-> Parallel Foreign Scan on pxf_parallel_enabled (cost=XXX rows=NNN width=NN)
(6 rows)
24 changes: 24 additions & 0 deletions automation/sqlrepo/features/parallel/explain/expected/query02.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
-- @description query02 for PXF parallel scan EXPLAIN - no Gather node when parallel disabled
-- start_matchsubs
--
-- m/cost=\d+\.\d+\.\.\d+\.\d+/
-- s/cost=\d+\.\d+\.\.\d+\.\d+/cost=XXX/
--
-- m/rows=\d+/
-- s/rows=\d+/rows=NNN/
--
-- m/width=\d+/
-- s/width=\d+/width=NN/
--
-- end_matchsubs
SET optimizer = off;
SET
SET enable_parallel = false;
SET
EXPLAIN SELECT count(*) FROM pxf_parallel_disabled;
QUERY PLAN
----------------------------------------------------------------------------------------
Gather Motion 1:1 (slice1; segments: 1) (cost=XXX rows=NNN width=NN)
-> Aggregate (cost=XXX rows=NNN width=NN)
-> Foreign Scan on pxf_parallel_disabled (cost=XXX rows=NNN width=NN)
(3 rows)
84 changes: 84 additions & 0 deletions automation/sqlrepo/features/parallel/explain/expected/query03.ans
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
-- @description query03 for PXF parallel scan EXPLAIN ANALYZE - Workers Launched present
-- start_matchsubs
--
-- m/Workers Planned: \d+/
-- s/Workers Planned: \d+/Workers Planned: N/
--
-- m/Workers Launched: \d+/
-- s/Workers Launched: \d+/Workers Launched: N/
--
-- m/cost=\d+\.\d+\.\.\d+\.\d+/
-- s/cost=\d+\.\d+\.\.\d+\.\d+/cost=XXX/
--
-- m/rows=\d+/
-- s/rows=\d+/rows=NNN/
--
-- m/width=\d+/
-- s/width=\d+/width=NN/
--
-- m/actual time=\d+\.\d+\.\.\d+\.\d+/
-- s/actual time=\d+\.\d+\.\.\d+\.\d+/actual time=XXX/
--
-- m/loops=\d+/
-- s/loops=\d+/loops=N/
--
-- m/Execution Time: \d+\.\d+ ms/
-- s/Execution Time: \d+\.\d+ ms/Execution Time: XXX ms/
--
-- m/Planning Time: \d+\.\d+ ms/
-- s/Planning Time: \d+\.\d+ ms/Planning Time: XXX ms/
--
-- m/Memory Usage: \d+kB/
-- s/Memory Usage: \d+kB/Memory Usage: NNkB/
--
-- m/Memory: \d+kB/
-- s/Memory: \d+kB/Memory: NNkB/
--
-- m/Buckets: \d+/
-- s/Buckets: \d+/Buckets: NNN/
--
-- m/Batches: \d+/
-- s/Batches: \d+/Batches: NNN/
--
-- m/Peak Memory Usage: \d+/
-- s/Peak Memory Usage: \d+/Peak Memory Usage: NNN/
--
-- m/Avg Peak Memory \(per process\): \d+/
-- s/Avg Peak Memory \(per process\): \d+/Avg Peak Memory (per process): NNN/
--
-- m/slice \d+; segments: \d+/
-- s/slice \d+; segments: \d+/slice N; segments: N/
--
-- m/Optimizer: .*/
-- s/Optimizer: .*/Optimizer: OPT/
--
-- end_matchsubs
-- start_matchignore
--
-- m/^\s+Slice statistics:/
-- m/^\s+\(slice\d+\)/
-- m/^\s+Statement statistics:/
-- m/^\s+Settings:/
-- m/^\s+Query Identifier:/
--
-- end_matchignore
SET optimizer = off;
SET
SET enable_parallel = true;
SET
SET max_parallel_workers_per_gather = 4;
SET
EXPLAIN ANALYZE SELECT count(*) FROM pxf_parallel_enabled;
QUERY PLAN
----------------------------------------------------------------------------------------------
Gather Motion 1:1 (slice N; segments: N) (cost=XXX rows=NNN width=NN) (actual time=XXX rows=NNN loops=N)
-> Finalize Aggregate (cost=XXX rows=NNN width=NN) (actual time=XXX rows=NNN loops=N)
-> Gather (cost=XXX rows=NNN width=NN) (actual time=XXX rows=NNN loops=N)
Workers Planned: N
Workers Launched: N
-> Partial Aggregate (cost=XXX rows=NNN width=NN) (actual time=XXX rows=NNN loops=N)
-> Parallel Foreign Scan on pxf_parallel_enabled (cost=XXX rows=NNN width=NN) (actual time=XXX rows=NNN loops=N)
Optimizer: OPT
Planning Time: XXX ms
Execution Time: XXX ms
(10 rows)
Loading
Loading