diff --git a/contrib/pax_storage/expected/cluster.out b/contrib/pax_storage/expected/cluster.out index c541db45bc2..82891bb9085 100644 --- a/contrib/pax_storage/expected/cluster.out +++ b/contrib/pax_storage/expected/cluster.out @@ -1,4 +1,5 @@ set pax.max_tuples_per_file to 131072; +set pax.enable_sync_collect_stats = on; -- cluster table using index -- start_ignore drop table if EXISTS t_index_cluster; @@ -295,3 +296,4 @@ select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_lexical_ (6 rows) drop table t_lexical_cluster; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/disable_autovacuum.out b/contrib/pax_storage/expected/disable_autovacuum.out new file mode 100644 index 00000000000..9e57e5edf42 --- /dev/null +++ b/contrib/pax_storage/expected/disable_autovacuum.out @@ -0,0 +1,10 @@ +alter system set autovacuum = off; +select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id'); + gp_segment_id | pg_reload_conf +---------------+---------------- + 2 | t + 1 | t + 0 | t + -1 | t +(4 rows) + diff --git a/contrib/pax_storage/expected/enable_autovacuum.out b/contrib/pax_storage/expected/enable_autovacuum.out new file mode 100644 index 00000000000..3d3d33f34f9 --- /dev/null +++ b/contrib/pax_storage/expected/enable_autovacuum.out @@ -0,0 +1,10 @@ +alter system set autovacuum = on; +select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id'); + gp_segment_id | pg_reload_conf +---------------+---------------- + 2 | t + 1 | t + 0 | t + -1 | t +(4 rows) + diff --git a/contrib/pax_storage/expected/filter.out b/contrib/pax_storage/expected/filter.out index 44f38a36038..989d59364fc 100644 --- a/contrib/pax_storage/expected/filter.out +++ b/contrib/pax_storage/expected/filter.out @@ -1,5 +1,6 @@ set pax.enable_debug to on; set pax.enable_sparse_filter = on; +set pax.enable_sync_collect_stats = on; create table pax_test.null_test_t(a int, b int, c text) using pax; insert into pax_test.null_test_t(a) select null from generate_series(1,2)i; insert into pax_test.null_test_t select 1, i, 'cc_' || i from generate_series(1,2)i; @@ -223,3 +224,4 @@ kind group, filter rate: 0 / 1 reset client_min_messages; drop table pax_test.in_test_t; reset pax.enable_sparse_filter; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/filter_1.out b/contrib/pax_storage/expected/filter_1.out index d2124c5491c..e54b6bfe2ce 100644 --- a/contrib/pax_storage/expected/filter_1.out +++ b/contrib/pax_storage/expected/filter_1.out @@ -1,5 +1,6 @@ set pax.enable_debug to on; set pax.enable_sparse_filter = on; +set pax.enable_sync_collect_stats = on; create table pax_test.null_test_t(a int, b int, c text) using pax; insert into pax_test.null_test_t(a) select null from generate_series(1,2)i; insert into pax_test.null_test_t select 1, i, 'cc_' || i from generate_series(1,2)i; @@ -223,3 +224,4 @@ kind group, filter rate: 0 / 1 reset client_min_messages; drop table pax_test.in_test_t; reset pax.enable_sparse_filter; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/filter_tree.out b/contrib/pax_storage/expected/filter_tree.out index fabf6729d71..9484f1816f3 100644 --- a/contrib/pax_storage/expected/filter_tree.out +++ b/contrib/pax_storage/expected/filter_tree.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create or replace function intrc(iint int) returns int as $$ begin return iint; end; @@ -735,6 +736,7 @@ LOG: statement: select count(*) from t1 where coalesce(v1, 2) != 1; reset client_min_messages; LOG: statement: reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; drop table t_allnull; diff --git a/contrib/pax_storage/expected/filter_tree_1.out b/contrib/pax_storage/expected/filter_tree_1.out index 3e158df074b..1d09c503736 100644 --- a/contrib/pax_storage/expected/filter_tree_1.out +++ b/contrib/pax_storage/expected/filter_tree_1.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create or replace function intrc(iint int) returns int as $$ begin return iint; end; @@ -742,6 +743,7 @@ LOG: statement: select count(*) from t1 where coalesce(v1, 2) != 1; reset client_min_messages; LOG: statement: reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; drop table t_allnull; diff --git a/contrib/pax_storage/expected/filter_tree_arithmetic.out b/contrib/pax_storage/expected/filter_tree_arithmetic.out index d1c49823f9b..5c94fe8625e 100644 --- a/contrib/pax_storage/expected/filter_tree_arithmetic.out +++ b/contrib/pax_storage/expected/filter_tree_arithmetic.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create table t_arithmetic(same int, v1 int, v2 int, v3 int) using pax with (minmax_columns='v1,v2,v3'); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'same' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. @@ -1063,5 +1064,6 @@ LOG: kind file, filter rate: 0 / 1 reset client_min_messages; LOG: statement: reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t_arithmetic; drop table ta_mul; diff --git a/contrib/pax_storage/expected/filter_tree_arithmetic_1.out b/contrib/pax_storage/expected/filter_tree_arithmetic_1.out index b3fb2cb8e92..cd942493e23 100644 --- a/contrib/pax_storage/expected/filter_tree_arithmetic_1.out +++ b/contrib/pax_storage/expected/filter_tree_arithmetic_1.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create table t_arithmetic(same int, v1 int, v2 int, v3 int) using pax with (minmax_columns='v1,v2,v3'); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'same' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. @@ -938,5 +939,6 @@ LOG: statement: select count(*) from ta_mul where v2 * v2 >= 10000; reset client_min_messages; LOG: statement: reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t_arithmetic; drop table ta_mul; diff --git a/contrib/pax_storage/expected/filter_tree_optimizer.out b/contrib/pax_storage/expected/filter_tree_optimizer.out index 437408e52b6..0b879e89338 100644 --- a/contrib/pax_storage/expected/filter_tree_optimizer.out +++ b/contrib/pax_storage/expected/filter_tree_optimizer.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create or replace function intrc(iint int) returns int as $$ begin return iint; end; @@ -738,6 +739,7 @@ LOG: statement: select count(*) from t1 where coalesce(v1, 2) != 1; reset client_min_messages; LOG: statement: reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; drop table t_allnull; diff --git a/contrib/pax_storage/expected/filter_tree_optimizer_1.out b/contrib/pax_storage/expected/filter_tree_optimizer_1.out index 19049ca4cd8..c1e994bf7fd 100644 --- a/contrib/pax_storage/expected/filter_tree_optimizer_1.out +++ b/contrib/pax_storage/expected/filter_tree_optimizer_1.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create or replace function intrc(iint int) returns int as $$ begin return iint; end; @@ -746,6 +747,7 @@ LOG: statement: select count(*) from t1 where coalesce(v1, 2) != 1; reset client_min_messages; LOG: statement: reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; drop table t_allnull; diff --git a/contrib/pax_storage/expected/filter_tree_root_quals.out b/contrib/pax_storage/expected/filter_tree_root_quals.out index 6ca467e7142..698a94d3522 100644 --- a/contrib/pax_storage/expected/filter_tree_root_quals.out +++ b/contrib/pax_storage/expected/filter_tree_root_quals.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create table t1(same int, v1 int, v2 int, v3 int, v4 int) using pax with (minmax_columns='v1,v2,v3,v4'); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'same' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. @@ -55,5 +56,6 @@ LOG: kind file, filter rate: 1 / 2 reset client_min_messages; LOG: statement: reset client_min_messages; reset optimizer; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; diff --git a/contrib/pax_storage/expected/filter_tree_root_quals_1.out b/contrib/pax_storage/expected/filter_tree_root_quals_1.out index d473544e73b..d7617839894 100644 --- a/contrib/pax_storage/expected/filter_tree_root_quals_1.out +++ b/contrib/pax_storage/expected/filter_tree_root_quals_1.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create table t1(same int, v1 int, v2 int, v3 int, v4 int) using pax with (minmax_columns='v1,v2,v3,v4'); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'same' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. @@ -55,5 +56,6 @@ LOG: kind file, filter rate: 2 / 2 reset client_min_messages; LOG: statement: reset client_min_messages; reset optimizer; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; diff --git a/contrib/pax_storage/expected/pax_vacuum.out b/contrib/pax_storage/expected/pax_vacuum.out new file mode 100644 index 00000000000..315f6828fa2 --- /dev/null +++ b/contrib/pax_storage/expected/pax_vacuum.out @@ -0,0 +1,154 @@ +show autovacuum; + autovacuum +------------ + off +(1 row) + +-- async collect stats in vacuum +set pax.enable_sync_collect_stats to off; +create table pax_vacuum (a int, b int, c int) using pax with(minmax_columns='a,b') distributed by (a); +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+---------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),None,None],[(false,false),(322),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 340 | [(false,false),(340),None,None],[(false,false),(340),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),None,None],[(false,false),(338),None,None],[(true,true),(0),None,None] | f | f | f +(3 rows) + +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-----------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),None,None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),None,None] | f | f | f +(3 rows) + +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------+----------------+-----------------+--------------- +(0 rows) + +-- sync collect stats in vacuum +set pax.enable_sync_collect_stats to on; +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(true,true),(0),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(true,true),(0),None,None] | f | f | f +(3 rows) + +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(true,true),(0),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(true,true),(0),None,None] | f | f | f +(3 rows) + +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------+----------------+-----------------+--------------- +(0 rows) + +-- update and vacuum +set pax.enable_sync_collect_stats to off; +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+---------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 340 | [(false,false),(340),None,None],[(false,false),(340),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),None,None],[(false,false),(338),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 322 | [(false,false),(322),None,None],[(false,false),(322),None,None],[(true,true),(0),None,None] | f | f | f +(3 rows) + +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-----------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),None,None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),None,None] | f | f | f +(3 rows) + +update pax_vacuum set b = b + 1; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-----------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),None,None] | t | f | f + 1 | 340 | [(false,false),(340),None,None],[(false,false),(340),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),None,None] | t | f | f + 1 | 322 | [(false,false),(322),None,None],[(false,false),(322),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),None,None] | t | f | f + 1 | 338 | [(false,false),(338),None,None],[(false,false),(338),None,None],[(true,true),(0),None,None] | f | f | f +(6 rows) + +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-----------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(3,1001),(167000)],[(false,false),(338),None,None] | f | f | f + 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(2,999),(161602)],[(false,false),(322),None,None] | f | f | f + 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(6,998),(172898)],[(false,false),(340),None,None] | f | f | f +(3 rows) + +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------+----------------+-----------------+--------------- +(0 rows) + +-- sync collect stats with vacuum +set pax.enable_sync_collect_stats to on; +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(true,true),(0),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(true,true),(0),None,None] | f | f | f +(3 rows) + +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(true,true),(0),None,None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(true,true),(0),None,None] | f | f | f +(3 rows) + +update pax_vacuum set b = b + 1; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(true,true),(0),None,None] | t | f | f + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(3,1001),(167000)],[(true,true),(0),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(true,true),(0),None,None] | t | f | f + 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(6,998),(172898)],[(true,true),(0),None,None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(true,true),(0),None,None] | t | f | f + 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(2,999),(161602)],[(true,true),(0),None,None] | f | f | f +(6 rows) + +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+-------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(2,999),(161602)],[(true,true),(0),None,None] | f | f | f + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(3,1001),(167000)],[(true,true),(0),None,None] | f | f | f + 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(6,998),(172898)],[(true,true),(0),None,None] | f | f | f +(3 rows) + +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------+----------------+-----------------+--------------- +(0 rows) + +drop table pax_vacuum; diff --git a/contrib/pax_storage/expected/statistics/min_max_bit_byte_types.out b/contrib/pax_storage/expected/statistics/min_max_bit_byte_types.out index aca12f8eb6a..1857f94c2c5 100644 --- a/contrib/pax_storage/expected/statistics/min_max_bit_byte_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_bit_byte_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the bool min/max types support -- @@ -612,3 +613,4 @@ drop table t_varbit; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_float_types.out b/contrib/pax_storage/expected/statistics/min_max_float_types.out index 620f8c1b9d4..ddc5bb23f96 100644 --- a/contrib/pax_storage/expected/statistics/min_max_float_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_float_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the float4 min/max types support -- @@ -812,3 +813,4 @@ drop table t_float8; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_geo_types.out b/contrib/pax_storage/expected/statistics/min_max_geo_types.out index 2a1bf609fde..602460d223a 100644 --- a/contrib/pax_storage/expected/statistics/min_max_geo_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_geo_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the box min/max types support -- @@ -840,3 +841,4 @@ drop table t_path; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_int_types.out b/contrib/pax_storage/expected/statistics/min_max_int_types.out index c60fa8f247d..8a500c692b2 100644 --- a/contrib/pax_storage/expected/statistics/min_max_int_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_int_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the int2 min/max types support -- @@ -2018,3 +2019,4 @@ drop table t_numeric; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_net_types.out b/contrib/pax_storage/expected/statistics/min_max_net_types.out index 801ab2ca69e..add6c4d99a9 100644 --- a/contrib/pax_storage/expected/statistics/min_max_net_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_net_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the inet min/max types support -- @@ -647,3 +648,4 @@ drop table t_mac8; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_other_types.out b/contrib/pax_storage/expected/statistics/min_max_other_types.out index bb79c1e3ee1..eca6370ec2c 100644 --- a/contrib/pax_storage/expected/statistics/min_max_other_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_other_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the oid min/max types support -- @@ -640,3 +641,4 @@ drop table t_uuid; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_text_types.out b/contrib/pax_storage/expected/statistics/min_max_text_types.out index 119396957c5..f1fdbe18915 100644 --- a/contrib/pax_storage/expected/statistics/min_max_text_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_text_types.out @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the text min/max types support -- @@ -3213,3 +3214,4 @@ drop table t_name; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/min_max_time_types.out b/contrib/pax_storage/expected/statistics/min_max_time_types.out index c279e8207cb..53a1f35e93d 100644 --- a/contrib/pax_storage/expected/statistics/min_max_time_types.out +++ b/contrib/pax_storage/expected/statistics/min_max_time_types.out @@ -7,6 +7,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the date min/max types support -- @@ -2488,3 +2489,4 @@ drop table t_interval; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics/statistics.out b/contrib/pax_storage/expected/statistics/statistics.out index 90a06b8cfd2..d2bb8f4e4c1 100644 --- a/contrib/pax_storage/expected/statistics/statistics.out +++ b/contrib/pax_storage/expected/statistics/statistics.out @@ -1,4 +1,5 @@ set default_table_access_method = pax; +set pax.enable_sync_collect_stats = on; -- -- Test with small group -- @@ -11,9 +12,9 @@ insert into t1 select i,i::text,i::float8, i % 2 > 0 from generate_series(1, 100 select * from get_pax_aux_table('t1'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(10,997),None],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(f,t),None] | f | f | f - 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1000,999),None],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(f,t),None] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),None],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(f,t),None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1000,999),None],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(f,t),None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(10,997),None],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(f,t),None] | f | f | f (3 rows) drop table t1; @@ -24,8 +25,8 @@ insert into t2 select i::bpchar,i::bpchar,i::varchar, i::varchar from generate_s select * from get_pax_aux_table('t2'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 322 | [(false,false),(322),(1,998),None],[(false,false),(322),(1 ,998 ),None],[(false,false),(322),(1,998),None],[(false,false),(322),(1,998),None] | f | f | f 0 | 340 | [(false,false),(340),(1000,999),None],[(false,false),(340),(1000 ,999 ),None],[(false,false),(340),(1000,999),None],[(false,false),(340),(1000,999),None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),None],[(false,false),(322),(1 ,998 ),None],[(false,false),(322),(1,998),None],[(false,false),(322),(1,998),None] | f | f | f 0 | 338 | [(false,false),(338),(101,997),None],[(false,false),(338),(101 ,997 ),None],[(false,false),(338),(101,997),None],[(false,false),(338),(101,997),None] | f | f | f (3 rows) @@ -70,9 +71,9 @@ insert into t1 SELECT (CASE WHEN i % 2 > 0 THEN NULL ELSE i END), (CASE WHEN i % select * from get_pax_aux_table('t1'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 160 | [(false,false),(160),None,None],[(true,true),(0),None,None] | f | f | f 0 | 670 | [(false,true),(170),None,None],[(false,true),(500),None,None] | f | f | f 0 | 170 | [(false,false),(170),None,None],[(true,true),(0),None,None] | f | f | f - 0 | 160 | [(false,false),(160),None,None],[(true,true),(0),None,None] | f | f | f (3 rows) truncate t1; @@ -86,9 +87,9 @@ insert into t1 values(generate_series(1, 1000), generate_series(1, 1000), genera select * from get_pax_aux_table('t1'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),($1.00,$998.00),($161,280.00)],[(false,false),(322),(@ 3 secs,@ 3 secs),(@ 16 mins 6 secs)] | f | f | f 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),($5.00,$997.00),($172,558.00)],[(false,false),(340),(@ 3 secs,@ 3 secs),(@ 17 mins)] | f | f | f 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),($2.00,$1,000.00),($166,662.00)],[(false,false),(338),(@ 3 secs,@ 3 secs),(@ 16 mins 54 secs)] | f | f | f - 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),($1.00,$998.00),($161,280.00)],[(false,false),(322),(@ 3 secs,@ 3 secs),(@ 16 mins 6 secs)] | f | f | f (3 rows) drop table t1; @@ -112,9 +113,9 @@ insert into t_int4 values(generate_series(1001, 2000)); select * from get_pax_aux_table('t_int4'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------+----------------+-----------------+--------------- + 0 | 331 | [(false,false),(331),(1001,1997),(499268)] | f | f | f 0 | 335 | [(false,false),(335),(1013,2000),(507879)] | f | f | f 0 | 334 | [(false,false),(334),(1002,1995),(493353)] | f | f | f - 0 | 331 | [(false,false),(331),(1001,1997),(499268)] | f | f | f (3 rows) drop table t_int4; @@ -125,9 +126,9 @@ insert into t_int8 values(generate_series(2001, 3000)); select * from get_pax_aux_table('t_int8'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------+----------------+-----------------+--------------- + 0 | 332 | [(false,false),(332),(2004,3000),(826908)] | f | f | f 0 | 340 | [(false,false),(340),(2003,2998),(857576)] | f | f | f 0 | 328 | [(false,false),(328),(2001,2997),(816016)] | f | f | f - 0 | 332 | [(false,false),(332),(2004,3000),(826908)] | f | f | f (3 rows) drop table t_int8; @@ -139,8 +140,8 @@ select * from get_pax_aux_table('t_float4'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------+----------------+-----------------+--------------- 0 | 341 | [(false,false),(341),(3001,3998),(1.194769e+06)] | f | f | f - 0 | 335 | [(false,false),(335),(3002,4000),(1.169653e+06)] | f | f | f 0 | 324 | [(false,false),(324),(3006,3996),(1.136078e+06)] | f | f | f + 0 | 335 | [(false,false),(335),(3002,4000),(1.169653e+06)] | f | f | f (3 rows) drop table t_float4; @@ -151,9 +152,9 @@ insert into t_float8 values(generate_series(4001, 5000)); select * from get_pax_aux_table('t_float8'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------+----------------+-----------------+--------------- - 0 | 349 | [(false,false),(349),(4003,4998),(1571016)] | f | f | f - 0 | 324 | [(false,false),(324),(4001,4999),(1450686)] | f | f | f 0 | 327 | [(false,false),(327),(4004,5000),(1478798)] | f | f | f + 0 | 324 | [(false,false),(324),(4001,4999),(1450686)] | f | f | f + 0 | 349 | [(false,false),(349),(4003,4998),(1571016)] | f | f | f (3 rows) drop table t_float8; @@ -164,9 +165,9 @@ insert into t_money values(generate_series(5001, 6000), generate_series(5001, 60 select * from get_pax_aux_table('t_money'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 341 | [(false,false),(341),(5001,5997),(1874732)],[(false,false),(341),($5,001.00,$5,997.00),($1,874,732.00)] | f | f | f 0 | 348 | [(false,false),(348),(5002,6000),(1913581)],[(false,false),(348),($5,002.00,$6,000.00),($1,913,581.00)] | f | f | f 0 | 311 | [(false,false),(311),(5007,5995),(1712187)],[(false,false),(311),($5,007.00,$5,995.00),($1,712,187.00)] | f | f | f - 0 | 341 | [(false,false),(341),(5001,5997),(1874732)],[(false,false),(341),($5,001.00,$5,997.00),($1,874,732.00)] | f | f | f (3 rows) drop table t_money; @@ -210,8 +211,8 @@ insert into t1 values(generate_series(1, 1000), NULL); select * from get_pax_aux_table('t1'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+-------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(338),None,None],[(true,true),(0),None,None] | f | f | f 0 | 340 | [(false,false),(340),None,None],[(true,true),(0),None,None] | f | f | f + 0 | 338 | [(false,false),(338),None,None],[(true,true),(0),None,None] | f | f | f 0 | 322 | [(false,false),(322),None,None],[(true,true),(0),None,None] | f | f | f (3 rows) @@ -244,8 +245,8 @@ select * from get_pax_aux_table('t1'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),(2,1000),(166662)],[(false,false),(338),($2.00,$1,000.00),($166,662.00)],[(false,false),(338),(@ 3 secs,@ 3 secs),(@ 16 mins 54 secs)] | f | f | f - 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),($1.00,$998.00),($161,280.00)],[(false,false),(322),(@ 3 secs,@ 3 secs),(@ 16 mins 6 secs)] | f | f | f 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),(5,997),(172558)],[(false,false),(340),($5.00,$997.00),($172,558.00)],[(false,false),(340),(@ 3 secs,@ 3 secs),(@ 17 mins)] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),(1,998),(161280)],[(false,false),(322),($1.00,$998.00),($161,280.00)],[(false,false),(322),(@ 3 secs,@ 3 secs),(@ 16 mins 6 secs)] | f | f | f (3 rows) drop table t1; @@ -256,9 +257,9 @@ insert into t_int2 values(generate_series(1, 1000)); select * from get_pax_aux_table('t_int2'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+-----------------------------------------+----------------+-----------------+--------------- + 0 | 340 | [(false,false),(340),(5,997),(172558)] | f | f | f 0 | 338 | [(false,false),(338),(2,1000),(166662)] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)] | f | f | f - 0 | 340 | [(false,false),(340),(5,997),(172558)] | f | f | f (3 rows) drop table t_int2; @@ -270,8 +271,8 @@ select * from get_pax_aux_table('t_int4'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------+----------------+-----------------+--------------- 0 | 335 | [(false,false),(335),(1013,2000),(507879)] | f | f | f - 0 | 331 | [(false,false),(331),(1001,1997),(499268)] | f | f | f 0 | 334 | [(false,false),(334),(1002,1995),(493353)] | f | f | f + 0 | 331 | [(false,false),(331),(1001,1997),(499268)] | f | f | f (3 rows) drop table t_int4; @@ -282,9 +283,9 @@ insert into t_int8 values(generate_series(2001, 3000)); select * from get_pax_aux_table('t_int8'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------+----------------+-----------------+--------------- - 0 | 332 | [(false,false),(332),(2004,3000),(826908)] | f | f | f 0 | 328 | [(false,false),(328),(2001,2997),(816016)] | f | f | f 0 | 340 | [(false,false),(340),(2003,2998),(857576)] | f | f | f + 0 | 332 | [(false,false),(332),(2004,3000),(826908)] | f | f | f (3 rows) drop table t_int8; @@ -296,8 +297,8 @@ select * from get_pax_aux_table('t_float4'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------+----------------+-----------------+--------------- 0 | 335 | [(false,false),(335),(3002,4000),(1.169653e+06)] | f | f | f - 0 | 341 | [(false,false),(341),(3001,3998),(1.194769e+06)] | f | f | f 0 | 324 | [(false,false),(324),(3006,3996),(1.136078e+06)] | f | f | f + 0 | 341 | [(false,false),(341),(3001,3998),(1.194769e+06)] | f | f | f (3 rows) drop table t_float4; @@ -308,9 +309,9 @@ insert into t_float8 values(generate_series(4001, 5000)); select * from get_pax_aux_table('t_float8'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------+----------------+-----------------+--------------- - 0 | 327 | [(false,false),(327),(4004,5000),(1478798)] | f | f | f 0 | 349 | [(false,false),(349),(4003,4998),(1571016)] | f | f | f 0 | 324 | [(false,false),(324),(4001,4999),(1450686)] | f | f | f + 0 | 327 | [(false,false),(327),(4004,5000),(1478798)] | f | f | f (3 rows) drop table t_float8; @@ -322,8 +323,8 @@ select * from get_pax_aux_table('t_money'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- 0 | 348 | [(false,false),(348),(5002,6000),(1913581)],[(false,false),(348),($5,002.00,$6,000.00),($1,913,581.00)] | f | f | f - 0 | 311 | [(false,false),(311),(5007,5995),(1712187)],[(false,false),(311),($5,007.00,$5,995.00),($1,712,187.00)] | f | f | f 0 | 341 | [(false,false),(341),(5001,5997),(1874732)],[(false,false),(341),($5,001.00,$5,997.00),($1,874,732.00)] | f | f | f + 0 | 311 | [(false,false),(311),(5007,5995),(1712187)],[(false,false),(311),($5,007.00,$5,995.00),($1,712,187.00)] | f | f | f (3 rows) drop table t_money; @@ -339,8 +340,8 @@ select * from get_pax_aux_table('t_interval'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+-------------------------------------------------------------------------+----------------+-----------------+--------------- 0 | 7 | [(false,false),(7),(@ 5 secs,@ 400 hours),(@ 400 hours 40 mins 5 secs)] | f | f | f - 0 | 4 | [(false,false),(4),(@ 3 secs,@ 11 hours),(@ 12 hours 7 secs)] | f | f | f 0 | 7 | [(false,false),(7),(@ 1 sec,@ 10 hours),(@ 34 hours 53 secs)] | f | f | f + 0 | 4 | [(false,false),(4),(@ 3 secs,@ 11 hours),(@ 12 hours 7 secs)] | f | f | f (3 rows) drop table t_interval; @@ -356,20 +357,29 @@ insert into t1_update_stats values(generate_series(1, 1000), generate_series(100 select * from get_pax_aux_table('t1_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f (3 rows) -- v1: ((1 + 1000) * 1000 / 2) - ((1 + 20) * 20 / 2) = 500290 -- v2: ((1001 + 2000) * 1000 / 2) - ((1001 + 1020) * 20 / 2) = 1480290 delete from t1_update_stats where v1 <= 20; +select * from get_pax_aux_table('t1_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f +(3 rows) + +vacuum t1_update_stats; select * from get_pax_aux_table('t1_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | t | f | f - 0 | 322 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | t | f | f - 0 | 340 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | t | f | f + 1 | 330 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | f | f | f + 1 | 332 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | f | f | f + 1 | 318 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | f | f | f (3 rows) drop table t1_update_stats; @@ -392,9 +402,18 @@ delete from t2_update_stats where v1 <= 400 and v1 > 300; select * from get_pax_aux_table('t2_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(303),(2,1000),(154512)],[(false,false),(303),(1002,2000),(457512)],[(false,false),(303),None,None] | t | f | f - 0 | 340 | [(false,false),(305),(5,997),(160201)],[(false,false),(305),(1005,1997),(465201)],[(false,false),(305),None,None] | t | f | f - 0 | 322 | [(false,false),(292),(1,998),(150737)],[(false,false),(292),(1001,1998),(442737)],[(false,false),(292),None,None] | t | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f +(3 rows) + +vacuum t2_update_stats; +select * from get_pax_aux_table('t2_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 303 | [(false,false),(303),(2,1000),(154512)],[(false,false),(303),(1002,2000),(457512)],[(false,false),(303),None,None] | f | f | f + 1 | 292 | [(false,false),(292),(1,998),(150737)],[(false,false),(292),(1001,1998),(442737)],[(false,false),(292),None,None] | f | f | f + 1 | 305 | [(false,false),(305),(5,997),(160201)],[(false,false),(305),(1005,1997),(465201)],[(false,false),(305),None,None] | f | f | f (3 rows) drop table t2_update_stats; @@ -407,19 +426,28 @@ select * from get_pax_aux_table('t3_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f - 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f (3 rows) -- v1: ((1 + 1000) * 1000 / 2) - ((1 + 900) * 900 / 2) = 95050 -- v2: ((1001 + 2000) * 1000 / 2) - ((1001 + 1900) * 900 / 2) = 195050 delete from t3_update_stats where v2 <= 1900; +select * from get_pax_aux_table('t3_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f +(3 rows) + +vacuum t3_update_stats; select * from get_pax_aux_table('t3_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+-----------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(30),(902,1000),(28591)],[(false,false),(30),(1902,2000),(58591)],[(false,false),(30),None,None] | t | f | f - 0 | 322 | [(false,false),(36),(904,998),(34328)],[(false,false),(36),(1904,1998),(70328)],[(false,false),(36),None,None] | t | f | f - 0 | 340 | [(false,false),(34),(901,997),(32131)],[(false,false),(34),(1901,1997),(66131)],[(false,false),(34),None,None] | t | f | f + 1 | 36 | [(false,false),(36),(904,998),(34328)],[(false,false),(36),(1904,1998),(70328)],[(false,false),(36),None,None] | f | f | f + 1 | 34 | [(false,false),(34),(901,997),(32131)],[(false,false),(34),(1901,1997),(66131)],[(false,false),(34),None,None] | f | f | f + 1 | 30 | [(false,false),(30),(902,1000),(28591)],[(false,false),(30),(1902,2000),(58591)],[(false,false),(30),None,None] | f | f | f (3 rows) drop table t3_update_stats; @@ -440,13 +468,19 @@ select * from get_pax_aux_table('t4_update_stats'); -- v2: 0 delete from t4_update_stats; select * from get_pax_aux_table('t4_update_stats'); - ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered --------------+------------+----------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(true,false),(0),None,None],[(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f - 0 | 322 | [(true,false),(0),None,None],[(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f - 0 | 340 | [(true,false),(0),None,None],[(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f (3 rows) +vacuum t4_update_stats; +select * from get_pax_aux_table('t4_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------+----------------+-----------------+--------------- +(0 rows) + drop table t4_update_stats; -- update part of data in the first group create table t5_update_stats(v1 int, v2 int, v3 int) with (minmax_columns='v1,v2'); @@ -456,8 +490,8 @@ insert into t5_update_stats values(generate_series(1, 1000), generate_series(100 select * from get_pax_aux_table('t5_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f (3 rows) @@ -466,15 +500,27 @@ select * from get_pax_aux_table('t5_update_stats'); -- v1(block 1): (1 + 20) * 20 / 2 + 20 = 230 -- v2(block 1): (1001 + 1020) * 20 / 2 + 20 = 20230 update t5_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v1 <= 20; +select * from get_pax_aux_table('t5_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 1 | 3 | [(false,false),(3),(12,20),(47)],[(false,false),(3),(1012,1020),(3047)],[(false,false),(3),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f + 1 | 9 | [(false,false),(9),(5,21),(106)],[(false,false),(9),(1005,1021),(9106)],[(false,false),(9),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 1 | 8 | [(false,false),(8),(2,19),(77)],[(false,false),(8),(1002,1019),(8077)],[(false,false),(8),None,None] | f | f | f +(6 rows) + +vacuum t5_update_stats; select * from get_pax_aux_table('t5_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | t | f | f 1 | 8 | [(false,false),(8),(2,19),(77)],[(false,false),(8),(1002,1019),(8077)],[(false,false),(8),None,None] | f | f | f - 0 | 322 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | t | f | f + 2 | 330 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | f | f | f 1 | 3 | [(false,false),(3),(12,20),(47)],[(false,false),(3),(1012,1020),(3047)],[(false,false),(3),None,None] | f | f | f - 0 | 340 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | t | f | f + 2 | 318 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | f | f | f 1 | 9 | [(false,false),(9),(5,21),(106)],[(false,false),(9),(1005,1021),(9106)],[(false,false),(9),None,None] | f | f | f + 2 | 332 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | f | f | f (6 rows) drop table t5_update_stats; @@ -499,14 +545,26 @@ update t6_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v1 <= 400 and v1 > 300 select * from get_pax_aux_table('t6_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(303),(2,1000),(154512)],[(false,false),(303),(1002,2000),(457512)],[(false,false),(303),None,None] | t | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f 1 | 34 | [(false,false),(34),(304,394),(11849)],[(false,false),(34),(1304,1394),(45849)],[(false,false),(34),None,None] | f | f | f - 0 | 340 | [(false,false),(305),(5,997),(160201)],[(false,false),(305),(1005,1997),(465201)],[(false,false),(305),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f 1 | 35 | [(false,false),(35),(306,399),(12357)],[(false,false),(35),(1306,1399),(47357)],[(false,false),(35),None,None] | f | f | f - 0 | 322 | [(false,false),(292),(1,998),(150737)],[(false,false),(292),(1001,1998),(442737)],[(false,false),(292),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f 1 | 31 | [(false,false),(31),(302,401),(10944)],[(false,false),(31),(1302,1401),(41944)],[(false,false),(31),None,None] | f | f | f (6 rows) +vacuum t6_update_stats; +select * from get_pax_aux_table('t6_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 34 | [(false,false),(34),(304,394),(11849)],[(false,false),(34),(1304,1394),(45849)],[(false,false),(34),None,None] | f | f | f + 2 | 303 | [(false,false),(303),(2,1000),(154512)],[(false,false),(303),(1002,2000),(457512)],[(false,false),(303),None,None] | f | f | f + 1 | 31 | [(false,false),(31),(302,401),(10944)],[(false,false),(31),(1302,1401),(41944)],[(false,false),(31),None,None] | f | f | f + 2 | 292 | [(false,false),(292),(1,998),(150737)],[(false,false),(292),(1001,1998),(442737)],[(false,false),(292),None,None] | f | f | f + 1 | 35 | [(false,false),(35),(306,399),(12357)],[(false,false),(35),(1306,1399),(47357)],[(false,false),(35),None,None] | f | f | f + 2 | 305 | [(false,false),(305),(5,997),(160201)],[(false,false),(305),(1005,1997),(465201)],[(false,false),(305),None,None] | f | f | f +(6 rows) + drop table t6_update_stats; -- update part of data in all group create table t7_update_stats(v1 int, v2 int, v3 int) with (minmax_columns='v1,v2'); @@ -529,12 +587,24 @@ update t7_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v2 <= 1900; select * from get_pax_aux_table('t7_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(30),(902,1000),(28591)],[(false,false),(30),(1902,2000),(58591)],[(false,false),(30),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 1 | 285 | [(false,false),(285),(12,900),(126951)],[(false,false),(285),(1012,1900),(411951)],[(false,false),(285),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f 1 | 308 | [(false,false),(308),(2,896),(138071)],[(false,false),(308),(1002,1896),(446071)],[(false,false),(308),None,None] | f | f | f - 0 | 340 | [(false,false),(34),(901,997),(32131)],[(false,false),(34),(1901,1997),(66131)],[(false,false),(34),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f 1 | 307 | [(false,false),(307),(5,901),(141328)],[(false,false),(307),(1005,1901),(448328)],[(false,false),(307),None,None] | f | f | f - 0 | 322 | [(false,false),(36),(904,998),(34328)],[(false,false),(36),(1904,1998),(70328)],[(false,false),(36),None,None] | t | f | f +(6 rows) + +vacuum t7_update_stats; +select * from get_pax_aux_table('t7_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 308 | [(false,false),(308),(2,896),(138071)],[(false,false),(308),(1002,1896),(446071)],[(false,false),(308),None,None] | f | f | f + 2 | 30 | [(false,false),(30),(902,1000),(28591)],[(false,false),(30),(1902,2000),(58591)],[(false,false),(30),None,None] | f | f | f 1 | 285 | [(false,false),(285),(12,900),(126951)],[(false,false),(285),(1012,1900),(411951)],[(false,false),(285),None,None] | f | f | f + 2 | 36 | [(false,false),(36),(904,998),(34328)],[(false,false),(36),(1904,1998),(70328)],[(false,false),(36),None,None] | f | f | f + 1 | 307 | [(false,false),(307),(5,901),(141328)],[(false,false),(307),(1005,1901),(448328)],[(false,false),(307),None,None] | f | f | f + 2 | 34 | [(false,false),(34),(901,997),(32131)],[(false,false),(34),(1901,1997),(66131)],[(false,false),(34),None,None] | f | f | f (6 rows) drop table t7_update_stats; @@ -546,9 +616,9 @@ insert into t8_update_stats values(generate_series(1, 1000), generate_series(100 select * from get_pax_aux_table('t8_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f (3 rows) -- v1(block 0): 0 @@ -559,14 +629,23 @@ update t8_update_stats set v1 = v1 + 1, v2 = v2 + 1; select * from get_pax_aux_table('t8_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(true,false),(0),None,None],[(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 1 | 321 | [(false,false),(321),(12,998),(161279)],[(false,false),(321),(1012,1998),(482279)],[(false,false),(321),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f - 0 | 340 | [(true,false),(0),None,None],[(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f 1 | 341 | [(false,false),(341),(5,1001),(173559)],[(false,false),(341),(1005,2001),(514559)],[(false,false),(341),None,None] | f | f | f - 0 | 322 | [(true,false),(0),None,None],[(true,false),(0),None,None],[(true,false),(0),None,None] | t | f | f - 1 | 321 | [(false,false),(321),(12,998),(161279)],[(false,false),(321),(1012,1998),(482279)],[(false,false),(321),None,None] | f | f | f (6 rows) +vacuum t8_update_stats; +select * from get_pax_aux_table('t8_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f + 1 | 321 | [(false,false),(321),(12,998),(161279)],[(false,false),(321),(1012,1998),(482279)],[(false,false),(321),None,None] | f | f | f + 1 | 341 | [(false,false),(341),(5,1001),(173559)],[(false,false),(341),(1005,2001),(514559)],[(false,false),(341),None,None] | f | f | f +(3 rows) + drop table t8_update_stats; -- delete cross multi files create table t9_update_stats(v1 int, v2 int, v3 int) with (minmax_columns='v1,v2'); @@ -579,18 +658,18 @@ insert into t9_update_stats values(generate_series(100, 1000), generate_series(1 select * from get_pax_aux_table('t9_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f - 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f - 2 | 38 | [(false,false),(38),(2,99),(1912)],[(false,false),(38),(102,199),(5712)],[(false,false),(38),None,None] | f | f | f - 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f - 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f - 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f - 2 | 25 | [(false,false),(25),(5,100),(1061)],[(false,false),(25),(105,200),(3561)],[(false,false),(25),None,None] | f | f | f - 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f 2 | 37 | [(false,false),(37),(1,98),(2077)],[(false,false),(37),(101,198),(5777)],[(false,false),(37),None,None] | f | f | f 3 | 285 | [(false,false),(285),(114,998),(159203)],[(false,false),(285),(1115,1999),(444488)],[(false,false),(285),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f + 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f + 2 | 25 | [(false,false),(25),(5,100),(1061)],[(false,false),(25),(105,200),(3561)],[(false,false),(25),None,None] | f | f | f + 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f + 2 | 38 | [(false,false),(38),(2,99),(1912)],[(false,false),(38),(102,199),(5712)],[(false,false),(38),None,None] | f | f | f + 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f (12 rows) delete from t9_update_stats where v1 <= 20; @@ -598,19 +677,31 @@ select * from get_pax_aux_table('t9_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f - 0 | 338 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | t | f | f - 1 | 338 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | t | f | f - 2 | 38 | [(false,false),(30),(22,99),(1835)],[(false,false),(30),(122,199),(4835)],[(false,false),(30),None,None] | t | f | f - 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f - 0 | 340 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | t | f | f - 1 | 340 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | t | f | f - 2 | 25 | [(false,false),(17),(21,100),(976)],[(false,false),(17),(121,200),(2676)],[(false,false),(17),None,None] | t | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 2 | 38 | [(false,false),(38),(2,99),(1912)],[(false,false),(38),(102,199),(5712)],[(false,false),(38),None,None] | t | f | f 3 | 285 | [(false,false),(285),(114,998),(159203)],[(false,false),(285),(1115,1999),(444488)],[(false,false),(285),None,None] | f | f | f - 0 | 322 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | t | f | f - 1 | 322 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | t | f | f - 2 | 37 | [(false,false),(33),(23,98),(2029)],[(false,false),(33),(123,198),(5329)],[(false,false),(33),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 2 | 37 | [(false,false),(37),(1,98),(2077)],[(false,false),(37),(101,198),(5777)],[(false,false),(37),None,None] | t | f | f + 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f + 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f + 2 | 25 | [(false,false),(25),(5,100),(1061)],[(false,false),(25),(105,200),(3561)],[(false,false),(25),None,None] | t | f | f (12 rows) +vacuum t9_update_stats; +select * from get_pax_aux_table('t9_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f + 4 | 690 | [(false,false),(690),(22,1000),(335005)],[(false,false),(690),(122,2000),(998005)],[(false,false),(690),None,None] | f | f | f + 3 | 285 | [(false,false),(285),(114,998),(159203)],[(false,false),(285),(1115,1999),(444488)],[(false,false),(285),None,None] | f | f | f + 4 | 669 | [(false,false),(669),(23,998),(324493)],[(false,false),(669),(123,1998),(963793)],[(false,false),(669),None,None] | f | f | f + 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f + 4 | 681 | [(false,false),(681),(21,997),(345922)],[(false,false),(681),(121,1997),(1011622)],[(false,false),(681),None,None] | f | f | f +(6 rows) + drop table t9_update_stats; -- update cross multi files create table t10_update_stats(v1 int, v2 int, v3 int) with (minmax_columns='v1,v2'); @@ -623,10 +714,6 @@ insert into t10_update_stats values(generate_series(100, 1000), generate_series( select * from get_pax_aux_table('t10_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f - 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f - 2 | 38 | [(false,false),(38),(2,99),(1912)],[(false,false),(38),(102,199),(5712)],[(false,false),(38),None,None] | f | f | f - 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | f | f | f 2 | 37 | [(false,false),(37),(1,98),(2077)],[(false,false),(37),(101,198),(5777)],[(false,false),(37),None,None] | f | f | f @@ -635,28 +722,47 @@ select * from get_pax_aux_table('t10_update_stats'); 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | f | f | f 2 | 25 | [(false,false),(25),(5,100),(1061)],[(false,false),(25),(105,200),(3561)],[(false,false),(25),None,None] | f | f | f 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | f | f | f + 2 | 38 | [(false,false),(38),(2,99),(1912)],[(false,false),(38),(102,199),(5712)],[(false,false),(38),None,None] | f | f | f + 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f (12 rows) update t10_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v1 <= 20; select * from get_pax_aux_table('t10_update_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f - 0 | 338 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | t | f | f - 1 | 338 | [(false,false),(330),(22,1000),(166585)],[(false,false),(330),(1022,2000),(496585)],[(false,false),(330),None,None] | t | f | f - 2 | 38 | [(false,false),(30),(22,99),(1835)],[(false,false),(30),(122,199),(4835)],[(false,false),(30),None,None] | t | f | f - 4 | 24 | [(false,false),(24),(2,19),(231)],[(false,false),(24),(102,1019),(17031)],[(false,false),(24),None,None] | f | f | f + 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f + 0 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f + 1 | 340 | [(false,false),(340),(5,997),(172558)],[(false,false),(340),(1005,1997),(512558)],[(false,false),(340),None,None] | t | f | f + 2 | 25 | [(false,false),(25),(5,100),(1061)],[(false,false),(25),(105,200),(3561)],[(false,false),(25),None,None] | t | f | f + 4 | 27 | [(false,false),(27),(5,21),(318)],[(false,false),(27),(105,1021),(19218)],[(false,false),(27),None,None] | f | f | f 3 | 285 | [(false,false),(285),(114,998),(159203)],[(false,false),(285),(1115,1999),(444488)],[(false,false),(285),None,None] | f | f | f - 0 | 322 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | t | f | f - 1 | 322 | [(false,false),(318),(23,998),(161232)],[(false,false),(318),(1023,1998),(479232)],[(false,false),(318),None,None] | t | f | f - 2 | 37 | [(false,false),(33),(23,98),(2029)],[(false,false),(33),(123,198),(5329)],[(false,false),(33),None,None] | t | f | f + 0 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 1 | 322 | [(false,false),(322),(1,998),(161280)],[(false,false),(322),(1001,1998),(483280)],[(false,false),(322),None,None] | t | f | f + 2 | 37 | [(false,false),(37),(1,98),(2077)],[(false,false),(37),(101,198),(5777)],[(false,false),(37),None,None] | t | f | f 4 | 9 | [(false,false),(9),(12,20),(141)],[(false,false),(9),(112,1020),(6441)],[(false,false),(9),None,None] | f | f | f + 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f + 0 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 1 | 338 | [(false,false),(338),(2,1000),(166662)],[(false,false),(338),(1002,2000),(504662)],[(false,false),(338),None,None] | t | f | f + 2 | 38 | [(false,false),(38),(2,99),(1912)],[(false,false),(38),(102,199),(5712)],[(false,false),(38),None,None] | t | f | f + 4 | 24 | [(false,false),(24),(2,19),(231)],[(false,false),(24),(102,1019),(17031)],[(false,false),(24),None,None] | f | f | f +(15 rows) + +vacuum t10_update_stats; +select * from get_pax_aux_table('t10_update_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+---------------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- 3 | 316 | [(false,false),(316),(100,997),(171597)],[(false,false),(316),(1101,1998),(487913)],[(false,false),(316),None,None] | f | f | f - 0 | 340 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | t | f | f - 1 | 340 | [(false,false),(332),(21,997),(172473)],[(false,false),(332),(1021,1997),(504473)],[(false,false),(332),None,None] | t | f | f - 2 | 25 | [(false,false),(17),(21,100),(976)],[(false,false),(17),(121,200),(2676)],[(false,false),(17),None,None] | t | f | f 4 | 27 | [(false,false),(27),(5,21),(318)],[(false,false),(27),(105,1021),(19218)],[(false,false),(27),None,None] | f | f | f -(15 rows) + 5 | 681 | [(false,false),(681),(21,997),(345922)],[(false,false),(681),(121,1997),(1011622)],[(false,false),(681),None,None] | f | f | f + 3 | 285 | [(false,false),(285),(114,998),(159203)],[(false,false),(285),(1115,1999),(444488)],[(false,false),(285),None,None] | f | f | f + 4 | 9 | [(false,false),(9),(12,20),(141)],[(false,false),(9),(112,1020),(6441)],[(false,false),(9),None,None] | f | f | f + 5 | 669 | [(false,false),(669),(23,998),(324493)],[(false,false),(669),(123,1998),(963793)],[(false,false),(669),None,None] | f | f | f + 3 | 300 | [(false,false),(300),(101,1000),(164750)],[(false,true),(299),(1102,2000),(463049)],[(false,false),(300),None,None] | f | f | f + 4 | 24 | [(false,false),(24),(2,19),(231)],[(false,false),(24),(102,1019),(17031)],[(false,false),(24),None,None] | f | f | f + 5 | 690 | [(false,false),(690),(22,1000),(335005)],[(false,false),(690),(122,2000),(998005)],[(false,false),(690),None,None] | f | f | f +(9 rows) drop table t10_update_stats; -- delete twice @@ -680,9 +786,9 @@ select sum(v2), sum(v3) from t_delete_twice_stats; (1 row) select * from get_pax_aux_table('t_delete_twice_stats'); - ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered --------------+------------+------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 100 | [(false,false),(90),None,None],[(false,false),(90),(11,100),(4995)],[(false,false),(90),(111,200),(13995)] | t | f | f + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 100 | [(false,false),(100),None,None],[(false,false),(100),(1,100),(5050)],[(false,false),(100),(101,200),(15050)] | t | f | f (1 row) -- delete from group 2 @@ -693,10 +799,17 @@ select sum(v2), sum(v3) from t_delete_twice_stats; 4640 | 12640 (1 row) +select * from get_pax_aux_table('t_delete_twice_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 100 | [(false,false),(100),None,None],[(false,false),(100),(1,100),(5050)],[(false,false),(100),(101,200),(15050)] | t | f | f +(1 row) + +vacuum t_delete_twice_stats; select * from get_pax_aux_table('t_delete_twice_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 100 | [(false,false),(80),None,None],[(false,false),(80),(11,100),(4640)],[(false,false),(80),(111,200),(12640)] | t | f | f + 1 | 80 | [(false,false),(80),None,None],[(false,false),(80),(11,100),(4640)],[(false,false),(80),(111,200),(12640)] | f | f | f (1 row) drop table t_delete_twice_stats; @@ -720,10 +833,10 @@ select sum(v2), sum(v3) from t_update_twice_stats; (1 row) select * from get_pax_aux_table('t_update_twice_stats'); - ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered --------------+------------+------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- - 0 | 100 | [(false,false),(90),None,None],[(false,false),(90),(11,100),(4995)],[(false,false),(90),(111,200),(13995)] | t | f | f - 1 | 10 | [(false,false),(10),None,None],[(false,false),(10),(2,11),(65)],[(false,false),(10),(102,111),(1065)] | f | f | f + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 0 | 100 | [(false,false),(100),None,None],[(false,false),(100),(1,100),(5050)],[(false,false),(100),(101,200),(15050)] | t | f | f + 1 | 10 | [(false,false),(10),None,None],[(false,false),(10),(2,11),(65)],[(false,false),(10),(102,111),(1065)] | f | f | f (2 rows) -- delete from group 2 @@ -734,13 +847,23 @@ select sum(v2), sum(v3) from t_update_twice_stats; 5070 | 15070 (1 row) +select * from get_pax_aux_table('t_update_twice_stats'); + ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +-------------+------------+--------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- + 1 | 10 | [(false,false),(10),None,None],[(false,false),(10),(2,11),(65)],[(false,false),(10),(102,111),(1065)] | f | f | f + 0 | 100 | [(false,false),(100),None,None],[(false,false),(100),(1,100),(5050)],[(false,false),(100),(101,200),(15050)] | t | f | f + 2 | 10 | [(false,false),(10),None,None],[(false,false),(10),(32,41),(365)],[(false,false),(10),(132,141),(1365)] | f | f | f +(3 rows) + +vacuum t_update_twice_stats; select * from get_pax_aux_table('t_update_twice_stats'); ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+------------+------------------------------------------------------------------------------------------------------------+----------------+-----------------+--------------- 1 | 10 | [(false,false),(10),None,None],[(false,false),(10),(2,11),(65)],[(false,false),(10),(102,111),(1065)] | f | f | f - 0 | 100 | [(false,false),(80),None,None],[(false,false),(80),(11,100),(4640)],[(false,false),(80),(111,200),(12640)] | t | f | f 2 | 10 | [(false,false),(10),None,None],[(false,false),(10),(32,41),(365)],[(false,false),(10),(132,141),(1365)] | f | f | f + 3 | 80 | [(false,false),(80),None,None],[(false,false),(80),(11,100),(4640)],[(false,false),(80),(111,200),(12640)] | f | f | f (3 rows) drop table t_update_twice_stats; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/expected/statistics_bloom_filter.out b/contrib/pax_storage/expected/statistics_bloom_filter.out index ec96bcfbc97..2e52609484d 100644 --- a/contrib/pax_storage/expected/statistics_bloom_filter.out +++ b/contrib/pax_storage/expected/statistics_bloom_filter.out @@ -7,6 +7,7 @@ -- end_matchignore set default_table_access_method = pax; set pax.enable_debug to on; +set pax.enable_sync_collect_stats = on; -- -- Test with small group -- @@ -462,3 +463,4 @@ reset pax.bloom_filter_work_memory_bytes; reset pax.max_tuples_per_group; reset pax.max_tuples_per_file; reset pax.enable_sparse_filter; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/pax_schedule b/contrib/pax_storage/pax_schedule index f9818f8e0b7..72de4f94fcd 100644 --- a/contrib/pax_storage/pax_schedule +++ b/contrib/pax_storage/pax_schedule @@ -21,4 +21,8 @@ test: dictionary_encoding test: cluster test: db_size_functions +test: disable_autovacuum +test: pax_vacuum +test: enable_autovacuum + test: teardown diff --git a/contrib/pax_storage/sql/cluster.sql b/contrib/pax_storage/sql/cluster.sql index 9b44fbdc6be..153e6d615db 100644 --- a/contrib/pax_storage/sql/cluster.sql +++ b/contrib/pax_storage/sql/cluster.sql @@ -1,4 +1,5 @@ set pax.max_tuples_per_file to 131072; +set pax.enable_sync_collect_stats = on; -- cluster table using index -- start_ignore @@ -146,4 +147,6 @@ alter table t_lexical_cluster set(cluster_columns='c1,c2', cluster_type='lexical cluster t_lexical_cluster; select ptblockname,ptstatistics,ptisclustered from get_pax_aux_table('t_lexical_cluster'); -drop table t_lexical_cluster; \ No newline at end of file +drop table t_lexical_cluster; + +reset pax.enable_sync_collect_stats; \ No newline at end of file diff --git a/contrib/pax_storage/sql/disable_autovacuum.sql b/contrib/pax_storage/sql/disable_autovacuum.sql new file mode 100644 index 00000000000..ccf46b94d0e --- /dev/null +++ b/contrib/pax_storage/sql/disable_autovacuum.sql @@ -0,0 +1,2 @@ +alter system set autovacuum = off; +select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id'); \ No newline at end of file diff --git a/contrib/pax_storage/sql/enable_autovacuum.sql b/contrib/pax_storage/sql/enable_autovacuum.sql new file mode 100644 index 00000000000..a128f5f570c --- /dev/null +++ b/contrib/pax_storage/sql/enable_autovacuum.sql @@ -0,0 +1,2 @@ +alter system set autovacuum = on; +select gp_segment_id, pg_reload_conf() from gp_id union select gp_segment_id, pg_reload_conf() from gp_dist_random('gp_id'); diff --git a/contrib/pax_storage/sql/filter.sql b/contrib/pax_storage/sql/filter.sql index baf4d619c6a..db79049135c 100644 --- a/contrib/pax_storage/sql/filter.sql +++ b/contrib/pax_storage/sql/filter.sql @@ -1,5 +1,6 @@ set pax.enable_debug to on; set pax.enable_sparse_filter = on; +set pax.enable_sync_collect_stats = on; create table pax_test.null_test_t(a int, b int, c text) using pax; insert into pax_test.null_test_t(a) select null from generate_series(1,2)i; @@ -74,3 +75,4 @@ reset client_min_messages; drop table pax_test.in_test_t; reset pax.enable_sparse_filter; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/sql/filter_tree.sql b/contrib/pax_storage/sql/filter_tree.sql index 81b350daa1e..9edc298c2c4 100644 --- a/contrib/pax_storage/sql/filter_tree.sql +++ b/contrib/pax_storage/sql/filter_tree.sql @@ -9,6 +9,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create or replace function intrc(iint int) returns int as $$ @@ -162,6 +163,7 @@ select count(*) from t1 left join t2 on t1.v1 = t2.v1 where t1.v1 > 1 and t2.v1 select count(*) from t1 where coalesce(v1, 2) != 1; reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; diff --git a/contrib/pax_storage/sql/filter_tree_arithmetic.sql b/contrib/pax_storage/sql/filter_tree_arithmetic.sql index deb136c9e29..75a7d9ec954 100644 --- a/contrib/pax_storage/sql/filter_tree_arithmetic.sql +++ b/contrib/pax_storage/sql/filter_tree_arithmetic.sql @@ -9,7 +9,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; - +set pax.enable_sync_collect_stats = on; create table t_arithmetic(same int, v1 int, v2 int, v3 int) using pax with (minmax_columns='v1,v2,v3'); create table ta_mul(same int, v1 int, v2 int, v3 int) using pax with (minmax_columns='v1,v2,v3'); @@ -221,6 +221,7 @@ select count(*) from ta_mul where v2 * v2 > 10000; select count(*) from ta_mul where v2 * v2 >= 10000; reset client_min_messages; +reset pax.enable_sync_collect_stats; drop table t_arithmetic; drop table ta_mul; diff --git a/contrib/pax_storage/sql/filter_tree_root_quals.sql b/contrib/pax_storage/sql/filter_tree_root_quals.sql index f4d4880f3bf..d5f88b7885c 100644 --- a/contrib/pax_storage/sql/filter_tree_root_quals.sql +++ b/contrib/pax_storage/sql/filter_tree_root_quals.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; +set pax.enable_sync_collect_stats = on; create table t1(same int, v1 int, v2 int, v3 int, v4 int) using pax with (minmax_columns='v1,v2,v3,v4'); create table t2(same int, v1 int, v2 int, v3 int, v4 int) using pax with (minmax_columns='v1,v2,v3,v4'); @@ -41,6 +42,7 @@ WHERE reset client_min_messages; reset optimizer; +reset pax.enable_sync_collect_stats; drop table t1; drop table t2; diff --git a/contrib/pax_storage/sql/pax_vacuum.sql b/contrib/pax_storage/sql/pax_vacuum.sql new file mode 100644 index 00000000000..7404f7670a0 --- /dev/null +++ b/contrib/pax_storage/sql/pax_vacuum.sql @@ -0,0 +1,48 @@ +show autovacuum; +-- async collect stats in vacuum +set pax.enable_sync_collect_stats to off; +create table pax_vacuum (a int, b int, c int) using pax with(minmax_columns='a,b') distributed by (a); +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + +-- sync collect stats in vacuum +set pax.enable_sync_collect_stats to on; +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + +-- update and vacuum +set pax.enable_sync_collect_stats to off; +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); +update pax_vacuum set b = b + 1; +select * from get_pax_aux_table('pax_vacuum'); +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + +-- sync collect stats with vacuum +set pax.enable_sync_collect_stats to on; +insert into pax_vacuum select i, i from generate_series(1, 1000) i; +select * from get_pax_aux_table('pax_vacuum'); +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); +update pax_vacuum set b = b + 1; +select * from get_pax_aux_table('pax_vacuum'); +vacuum pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); +truncate pax_vacuum; +select * from get_pax_aux_table('pax_vacuum'); + + +drop table pax_vacuum; \ No newline at end of file diff --git a/contrib/pax_storage/sql/statistics/min_max_bit_byte_types.sql b/contrib/pax_storage/sql/statistics/min_max_bit_byte_types.sql index a9d5822c6e6..b5cabc2682a 100644 --- a/contrib/pax_storage/sql/statistics/min_max_bit_byte_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_bit_byte_types.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the bool min/max types support @@ -154,5 +155,5 @@ drop table t_varbit; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; - +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/sql/statistics/min_max_float_types.sql b/contrib/pax_storage/sql/statistics/min_max_float_types.sql index 1ece90d9e4e..a4ae2733523 100644 --- a/contrib/pax_storage/sql/statistics/min_max_float_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_float_types.sql @@ -8,7 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; - +set pax.enable_sync_collect_stats = on; -- -- Test the float4 min/max types support -- @@ -159,3 +159,4 @@ drop table t_float8; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; \ No newline at end of file diff --git a/contrib/pax_storage/sql/statistics/min_max_geo_types.sql b/contrib/pax_storage/sql/statistics/min_max_geo_types.sql index 575f0fe6721..a6c4a0c7016 100644 --- a/contrib/pax_storage/sql/statistics/min_max_geo_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_geo_types.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the box min/max types support @@ -197,3 +198,4 @@ drop table t_path; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; \ No newline at end of file diff --git a/contrib/pax_storage/sql/statistics/min_max_int_types.sql b/contrib/pax_storage/sql/statistics/min_max_int_types.sql index abcd1eb6374..95f061adc17 100644 --- a/contrib/pax_storage/sql/statistics/min_max_int_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_int_types.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the int2 min/max types support @@ -369,3 +370,4 @@ drop table t_numeric; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; \ No newline at end of file diff --git a/contrib/pax_storage/sql/statistics/min_max_net_types.sql b/contrib/pax_storage/sql/statistics/min_max_net_types.sql index ceb19783ca6..3f2928c5393 100644 --- a/contrib/pax_storage/sql/statistics/min_max_net_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_net_types.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the inet min/max types support @@ -157,3 +158,4 @@ drop table t_mac8; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; \ No newline at end of file diff --git a/contrib/pax_storage/sql/statistics/min_max_other_types.sql b/contrib/pax_storage/sql/statistics/min_max_other_types.sql index 79e93ccf83f..c2d93583971 100644 --- a/contrib/pax_storage/sql/statistics/min_max_other_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_other_types.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the oid min/max types support @@ -148,3 +149,4 @@ drop table t_uuid; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/sql/statistics/min_max_text_types.sql b/contrib/pax_storage/sql/statistics/min_max_text_types.sql index 0dfd4c9f11b..3614d505d58 100644 --- a/contrib/pax_storage/sql/statistics/min_max_text_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_text_types.sql @@ -9,6 +9,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the text min/max types support @@ -593,3 +594,4 @@ drop table t_name; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/sql/statistics/min_max_time_types.sql b/contrib/pax_storage/sql/statistics/min_max_time_types.sql index 710b5141926..95e3effb18e 100644 --- a/contrib/pax_storage/sql/statistics/min_max_time_types.sql +++ b/contrib/pax_storage/sql/statistics/min_max_time_types.sql @@ -8,6 +8,7 @@ set default_table_access_method to pax; set pax.enable_debug to on; set pax.enable_sparse_filter to on; set pax.max_tuples_per_group to 5; +set pax.enable_sync_collect_stats = on; -- -- Test the date min/max types support @@ -504,3 +505,4 @@ drop table t_interval; reset pax.enable_debug; reset pax.enable_sparse_filter; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/sql/statistics/statistics.sql b/contrib/pax_storage/sql/statistics/statistics.sql index 00a9337e91d..eac16dc247f 100644 --- a/contrib/pax_storage/sql/statistics/statistics.sql +++ b/contrib/pax_storage/sql/statistics/statistics.sql @@ -1,5 +1,5 @@ set default_table_access_method = pax; - +set pax.enable_sync_collect_stats = on; -- -- Test with small group @@ -172,6 +172,8 @@ select * from get_pax_aux_table('t1_update_stats'); -- v2: ((1001 + 2000) * 1000 / 2) - ((1001 + 1020) * 20 / 2) = 1480290 delete from t1_update_stats where v1 <= 20; select * from get_pax_aux_table('t1_update_stats'); +vacuum t1_update_stats; +select * from get_pax_aux_table('t1_update_stats'); drop table t1_update_stats; -- delete part of data in the second group @@ -182,6 +184,8 @@ select * from get_pax_aux_table('t2_update_stats'); -- v2: ((1001 + 2000) * 1000 / 2) - ((1301 + 1400) * 100 / 2) = 1365450 delete from t2_update_stats where v1 <= 400 and v1 > 300; select * from get_pax_aux_table('t2_update_stats'); +vacuum t2_update_stats; +select * from get_pax_aux_table('t2_update_stats'); drop table t2_update_stats; -- delete part of data in all group @@ -192,6 +196,8 @@ select * from get_pax_aux_table('t3_update_stats'); -- v2: ((1001 + 2000) * 1000 / 2) - ((1001 + 1900) * 900 / 2) = 195050 delete from t3_update_stats where v2 <= 1900; select * from get_pax_aux_table('t3_update_stats'); +vacuum t3_update_stats; +select * from get_pax_aux_table('t3_update_stats'); drop table t3_update_stats; -- delete all @@ -202,6 +208,8 @@ select * from get_pax_aux_table('t4_update_stats'); -- v2: 0 delete from t4_update_stats; select * from get_pax_aux_table('t4_update_stats'); +vacuum t4_update_stats; +select * from get_pax_aux_table('t4_update_stats'); drop table t4_update_stats; -- update part of data in the first group @@ -214,6 +222,8 @@ select * from get_pax_aux_table('t5_update_stats'); -- v2(block 1): (1001 + 1020) * 20 / 2 + 20 = 20230 update t5_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v1 <= 20; select * from get_pax_aux_table('t5_update_stats'); +vacuum t5_update_stats; +select * from get_pax_aux_table('t5_update_stats'); drop table t5_update_stats; -- update part of data in the second group @@ -226,6 +236,8 @@ select * from get_pax_aux_table('t6_update_stats'); -- v2(block 1): (1301 + 1400) * 100 / 2 + 100 = 135150 update t6_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v1 <= 400 and v1 > 300; select * from get_pax_aux_table('t6_update_stats'); +vacuum t6_update_stats; +select * from get_pax_aux_table('t6_update_stats'); drop table t6_update_stats; -- update part of data in all group @@ -238,6 +250,8 @@ select * from get_pax_aux_table('t7_update_stats'); -- v2(block 1): (1001 + 1900) * 900 / 2 + 900 = 1306350 update t7_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v2 <= 1900; select * from get_pax_aux_table('t7_update_stats'); +vacuum t7_update_stats; +select * from get_pax_aux_table('t7_update_stats'); drop table t7_update_stats; -- update all @@ -250,6 +264,8 @@ select * from get_pax_aux_table('t8_update_stats'); -- v2(block 1): (1002 + 2001) * 1000 / 2 = 1501500 update t8_update_stats set v1 = v1 + 1, v2 = v2 + 1; select * from get_pax_aux_table('t8_update_stats'); +vacuum t8_update_stats; +select * from get_pax_aux_table('t8_update_stats'); drop table t8_update_stats; -- delete cross multi files @@ -261,6 +277,8 @@ insert into t9_update_stats values(generate_series(100, 1000), generate_series(1 select * from get_pax_aux_table('t9_update_stats'); delete from t9_update_stats where v1 <= 20; select * from get_pax_aux_table('t9_update_stats'); +vacuum t9_update_stats; +select * from get_pax_aux_table('t9_update_stats'); drop table t9_update_stats; -- update cross multi files @@ -272,6 +290,8 @@ insert into t10_update_stats values(generate_series(100, 1000), generate_series( select * from get_pax_aux_table('t10_update_stats'); update t10_update_stats set v1 = v1 + 1, v2 = v2 + 1 where v1 <= 20; select * from get_pax_aux_table('t10_update_stats'); +vacuum t10_update_stats; +select * from get_pax_aux_table('t10_update_stats'); drop table t10_update_stats; @@ -288,6 +308,8 @@ select * from get_pax_aux_table('t_delete_twice_stats'); delete from t_delete_twice_stats where v2 > 30 and v2 <= 40; select sum(v2), sum(v3) from t_delete_twice_stats; select * from get_pax_aux_table('t_delete_twice_stats'); +vacuum t_delete_twice_stats; +select * from get_pax_aux_table('t_delete_twice_stats'); drop table t_delete_twice_stats; -- update twice @@ -302,6 +324,9 @@ select * from get_pax_aux_table('t_update_twice_stats'); update t_update_twice_stats set v2 = v2 + 1, v3 = v3 + 1 where v2 > 30 and v2 <= 40; select sum(v2), sum(v3) from t_update_twice_stats; select * from get_pax_aux_table('t_update_twice_stats'); +vacuum t_update_twice_stats; +select * from get_pax_aux_table('t_update_twice_stats'); drop table t_update_twice_stats; reset pax.max_tuples_per_group; +reset pax.enable_sync_collect_stats; diff --git a/contrib/pax_storage/sql/statistics_bloom_filter.sql b/contrib/pax_storage/sql/statistics_bloom_filter.sql index a19f8b1bd83..721fc6e8171 100644 --- a/contrib/pax_storage/sql/statistics_bloom_filter.sql +++ b/contrib/pax_storage/sql/statistics_bloom_filter.sql @@ -7,6 +7,7 @@ -- end_matchignore set default_table_access_method = pax; set pax.enable_debug to on; +set pax.enable_sync_collect_stats = on; -- -- Test with small group -- @@ -144,3 +145,4 @@ reset pax.bloom_filter_work_memory_bytes; reset pax.max_tuples_per_group; reset pax.max_tuples_per_file; reset pax.enable_sparse_filter; +reset pax.enable_sync_collect_stats; \ No newline at end of file diff --git a/contrib/pax_storage/src/cpp/access/pax_access_handle.cc b/contrib/pax_storage/src/cpp/access/pax_access_handle.cc index 6dcaecd3205..14ba2037c16 100644 --- a/contrib/pax_storage/src/cpp/access/pax_access_handle.cc +++ b/contrib/pax_storage/src/cpp/access/pax_access_handle.cc @@ -43,9 +43,15 @@ #include "comm/vec_numeric.h" #include "exceptions/CException.h" #include "storage/local_file_system.h" +#include "storage/pax_vacuum_compactor.h" #ifdef VEC_BUILD #include "storage/vec_parallel_pax.h" #endif +#include "access/pax_visimap.h" +#include "comm/bitmap.h" +#include "comm/iterator.h" +#include "storage/micro_partition_file_factory.h" +#include "storage/micro_partition_stats_updater.h" #include "storage/paxc_smgr.h" #include "storage/wal/pax_wal.h" #include "storage/wal/paxc_wal.h" @@ -410,7 +416,253 @@ void CCPaxAccessMethod::FinishBulkInsert(Relation relation, int options) { }); CBDB_END_TRY(); } +#ifdef USE_MANIFEST_API +void CCPaxAccessMethod::RelationVacuum(Relation rel, VacuumParams *params, + BufferAccessStrategy /*bstrategy*/) { + CBDB_TRY(); + { + std::vector minmax_cols = cbdb::GetMinMaxColumnIndexes(rel); + std::vector bf_cols = cbdb::GetBloomFilterColumnIndexes(rel); + + ManifestRelation mrel; + ManifestScan mscan; + ManifestTuple mtuple; + pax::PaxCatalogUpdater catalog = pax::PaxCatalogUpdater::Begin(rel); + std::vector batch_vacumm_list; + + mrel = manifest_open(rel); + mscan = manifest_beginscan(mrel, nullptr); + while ((mtuple = manifest_getnext(mscan, nullptr))) { + bool isnull; + int block_id = DatumGetInt32( + get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTBLOCKNAME, &isnull)); + Assert(!isnull); + + bool is_stats_valid = DatumGetBool(get_manifesttuple_value( + mtuple, mrel, PAX_AUX_PTISSTATSVALID, &isnull)); + Assert(!isnull); + + get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTVISIMAPNAME, &isnull); + + bool has_visimap = !isnull; + + bool need_scan = has_visimap || !is_stats_valid || + ((params->options & VACOPT_DISABLE_PAGE_SKIPPING) != 0); + if (!need_scan) { + cbdb::VacuumDelayPoint(); + continue; + } + + pax::MicroPartitionMetadata meta = + cbdb::GetMicroPartitionMetadata(rel, GetActiveSnapshot(), block_id); + + // if has visimap, collect the block metadata and batch vacuum later + if (has_visimap) { + batch_vacumm_list.push_back(meta); + continue; + } + + // update stats + { + // if no stats columns, skip + if (minmax_cols.empty() && bf_cols.empty()) { + cbdb::VacuumDelayPoint(); + continue; + } + + auto projection = std::make_shared(); + std::vector all_cols = minmax_cols; + // should merge minmax_cols and bf_cols and remove the duplicate + if (!bf_cols.empty()) { + all_cols.insert(all_cols.end(), bf_cols.begin(), bf_cols.end()); + std::sort(all_cols.begin(), all_cols.end()); + all_cols.erase(std::unique(all_cols.begin(), all_cols.end()), + all_cols.end()); + } + projection->SetColumnProjection(all_cols, rel->rd_att->natts); + + pax::MicroPartitionReader::ReaderOptions options; + std::unique_ptr toast_file; + int32 reader_flags = pax::FLAGS_EMPTY; + + options.filter = projection; + options.reused_buffer = nullptr; + options.visibility_bitmap = nullptr; + + if (meta.GetExistToast()) + toast_file = + pax::Singleton::GetInstance()->Open( + meta.GetFileName() + TOAST_FILE_SUFFIX, pax::fs::kReadMode); + + auto mp_reader = + pax::MicroPartitionFileFactory::CreateMicroPartitionReader( + std::move(options), reader_flags, + pax::Singleton::GetInstance()->Open( + meta.GetFileName(), pax::fs::kReadMode), + std::move(toast_file)); + + TupleTableSlot *slot = + cbdb::MakeSingleTupleTableSlot(rel->rd_att, &TTSOpsVirtual); + auto updated_stats = + pax::MicroPartitionStatsUpdater(mp_reader.get(), true, nullptr) + .Update(slot, minmax_cols, bf_cols); + catalog.UpdateStatistics(meta.GetMicroPartitionId(), + updated_stats->Serialize(), true); + + mp_reader->Close(); + cbdb::ExecDropSingleTupleTableSlot(slot); + + cbdb::VacuumDelayPoint(); + } + } + + if (!batch_vacumm_list.empty()) { + auto it = + std::make_unique >( + std::move(batch_vacumm_list)); + pax::PaxVacuumBatcher batcher(rel); + batcher.Process(std::move(it), minmax_cols, bf_cols); + } + + catalog.End(); + + manifest_endscan(mscan); + manifest_close(mrel); + } + CBDB_CATCH_DEFAULT(); + CBDB_FINALLY({}); + CBDB_END_TRY(); +} +#else +/** + * The vacuum process handles two types of MicroPartition files: + * 1. Files whose min/max and other statistics are outdated; for these, vacuum + * reads all tuples, asynchronously updates statistics. + * 2. Files that are marked for deletion (i.e., have visimap); for these, vacuum + * reads valid tuples, writes them to new files, and updates statistics. + */ +void CCPaxAccessMethod::RelationVacuum(Relation rel, VacuumParams *params, + BufferAccessStrategy /*bstrategy*/) { + CBDB_TRY(); + { + std::vector minmax_cols = cbdb::GetMinMaxColumnIndexes(rel); + std::vector bf_cols = cbdb::GetBloomFilterColumnIndexes(rel); + + Oid aux_oid = cbdb::GetPaxAuxRelid(RelationGetRelid(rel)); + Relation aux_rel = cbdb::TableOpen(aux_oid, RowExclusiveLock); + + SysScanDesc scan = + cbdb::SystableBeginScan(aux_rel, InvalidOid, false, NULL, 0, NULL); + HeapTuple tup; + + pax::PaxCatalogUpdater catalog = pax::PaxCatalogUpdater::Begin(rel); + std::vector batch_vacumm_list; + while (HeapTupleIsValid(tup = cbdb::SystableGetNext(scan))) { + bool isnull = false; + TupleDesc desc = RelationGetDescr(aux_rel); + + int block_id = DatumGetInt32(cbdb::HeapGetAttr( + tup, ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME, desc, &isnull)); + Assert(!isnull); + + bool is_stats_valid = DatumGetBool(cbdb::HeapGetAttr( + tup, ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID, desc, &isnull)); + Assert(!isnull); + + (void)cbdb::HeapGetAttr(tup, ANUM_PG_PAX_BLOCK_TABLES_PTVISIMAPNAME, desc, + &isnull); + bool has_visimap = !isnull; + + bool need_scan = has_visimap || !is_stats_valid || + ((params->options & VACOPT_DISABLE_PAGE_SKIPPING) != 0); + if (!need_scan) { + cbdb::VacuumDelayPoint(); + continue; + } + + pax::MicroPartitionMetadata meta = + cbdb::GetMicroPartitionMetadata(rel, GetActiveSnapshot(), block_id); + + // if has visimap, collect the block metadata and batch vacuum later + if (has_visimap) { + batch_vacumm_list.push_back(meta); + continue; + } + + // update stats + { + // if no stats columns, skip + if (minmax_cols.empty() && bf_cols.empty()) { + cbdb::VacuumDelayPoint(); + continue; + } + + auto projection = std::make_shared(); + std::vector all_cols = minmax_cols; + // should merge minmax_cols and bf_cols and remove the duplicate + if (!bf_cols.empty()) { + all_cols.insert(all_cols.end(), bf_cols.begin(), bf_cols.end()); + std::sort(all_cols.begin(), all_cols.end()); + all_cols.erase(std::unique(all_cols.begin(), all_cols.end()), + all_cols.end()); + } + projection->SetColumnProjection(all_cols, rel->rd_att->natts); + + pax::MicroPartitionReader::ReaderOptions options; + std::unique_ptr toast_file; + int32 reader_flags = pax::FLAGS_EMPTY; + + options.filter = projection; + options.reused_buffer = nullptr; + options.visibility_bitmap = nullptr; + + if (meta.GetExistToast()) + toast_file = + pax::Singleton::GetInstance()->Open( + meta.GetFileName() + TOAST_FILE_SUFFIX, pax::fs::kReadMode); + + auto mp_reader = + pax::MicroPartitionFileFactory::CreateMicroPartitionReader( + std::move(options), reader_flags, + pax::Singleton::GetInstance()->Open( + meta.GetFileName(), pax::fs::kReadMode), + std::move(toast_file)); + + TupleTableSlot *slot = + cbdb::MakeSingleTupleTableSlot(rel->rd_att, &TTSOpsVirtual); + auto updated_stats = + pax::MicroPartitionStatsUpdater(mp_reader.get(), true, nullptr) + .Update(slot, minmax_cols, bf_cols); + catalog.UpdateStatistics(meta.GetMicroPartitionId(), + updated_stats->Serialize(), true); + + mp_reader->Close(); + cbdb::ExecDropSingleTupleTableSlot(slot); + + cbdb::VacuumDelayPoint(); + } + } + + if (!batch_vacumm_list.empty()) { + auto it = + std::make_unique >( + std::move(batch_vacumm_list)); + pax::PaxVacuumBatcher batcher(rel); + batcher.Process(std::move(it), minmax_cols, bf_cols); + } + + catalog.End(); + + cbdb::SystableEndScan(scan); + cbdb::TableClose(aux_rel, RowExclusiveLock); + } + CBDB_CATCH_DEFAULT(); + CBDB_FINALLY({}); + CBDB_END_TRY(); +} + +#endif void CCPaxAccessMethod::ExtDmlInit(Relation rel, CmdType operation) { if (!RELATION_IS_PAX(rel)) return; @@ -538,12 +790,6 @@ TransactionId PaxAccessMethod::IndexDeleteTuples( return 0; } -void PaxAccessMethod::RelationVacuum(Relation /*onerel*/, - VacuumParams * /*params*/, - BufferAccessStrategy /*bstrategy*/) { - /* PAX: micro-partitions have no dead tuples, so vacuum is empty */ -} - BlockSequence *PaxAccessMethod::RelationGetBlockSequences(Relation rel, int *numSequences) { // PAX not support brin index yet @@ -771,7 +1017,7 @@ static const TableAmRoutine kPaxColumnMethods = { pax::CCPaxAccessMethod::RelationNontransactionalTruncate, .relation_copy_data = pax::CCPaxAccessMethod::RelationCopyData, .relation_copy_for_cluster = pax::CCPaxAccessMethod::RelationCopyForCluster, - .relation_vacuum = paxc::PaxAccessMethod::RelationVacuum, + .relation_vacuum = pax::CCPaxAccessMethod::RelationVacuum, .scan_analyze_next_block = pax::CCPaxAccessMethod::ScanAnalyzeNextBlock, .scan_analyze_next_tuple = pax::CCPaxAccessMethod::ScanAnalyzeNextTuple, .index_build_range_scan = paxc::PaxAccessMethod::IndexBuildRangeScan, diff --git a/contrib/pax_storage/src/cpp/access/pax_access_handle.h b/contrib/pax_storage/src/cpp/access/pax_access_handle.h index d541a400d2b..9c9f4fdde32 100644 --- a/contrib/pax_storage/src/cpp/access/pax_access_handle.h +++ b/contrib/pax_storage/src/cpp/access/pax_access_handle.h @@ -85,8 +85,6 @@ class PaxAccessMethod final { LockWaitPolicy wait_policy, uint8 flags, TM_FailureData *tmfd); - static void RelationVacuum(Relation onerel, VacuumParams *params, - BufferAccessStrategy bstrategy); static double IndexBuildRangeScan( Relation heap_relation, Relation index_relation, IndexInfo *index_info, bool allow_sync, bool anyvisible, bool progress, @@ -191,6 +189,9 @@ class CCPaxAccessMethod final { static void FinishBulkInsert(Relation relation, int options); + static void RelationVacuum(Relation onerel, VacuumParams *params, + BufferAccessStrategy bstrategy); + // DML init/fini hooks static void ExtDmlInit(Relation rel, CmdType operation); static void ExtDmlFini(Relation rel, CmdType operation); diff --git a/contrib/pax_storage/src/cpp/access/pax_inserter.cc b/contrib/pax_storage/src/cpp/access/pax_inserter.cc index 4f6cf7daa1e..01b056ca36f 100644 --- a/contrib/pax_storage/src/cpp/access/pax_inserter.cc +++ b/contrib/pax_storage/src/cpp/access/pax_inserter.cc @@ -45,6 +45,7 @@ CPaxInserter::CPaxInserter(Relation rel) writer_->SetWriteSummaryCallback(&cbdb::InsertOrUpdateMicroPartitionEntry) ->SetFileSplitStrategy(std::make_unique()) + ->SetEnableStats(pax::pax_enable_sync_collect_stats) ->Open(); } diff --git a/contrib/pax_storage/src/cpp/catalog/pax_aux_table.cc b/contrib/pax_storage/src/cpp/catalog/pax_aux_table.cc index baca9efe47d..e776c4990e6 100644 --- a/contrib/pax_storage/src/cpp/catalog/pax_aux_table.cc +++ b/contrib/pax_storage/src/cpp/catalog/pax_aux_table.cc @@ -114,6 +114,9 @@ void CPaxCreateMicroPartitionTable(Relation rel) { TupleDescInitEntry(tupdesc, (AttrNumber)ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED, PAX_AUX_PTISCLUSTERED, BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, + (AttrNumber)ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID, + PAX_AUX_PTISSTATSVALID, BOOLOID, -1, 0); { // Add constraints for the aux table auto attr = @@ -249,6 +252,10 @@ void InsertMicroPartitionPlaceHolder(Oid aux_relid, int block_id) { values[ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME - 1] = Int32GetDatum(block_id); nulls[ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME - 1] = false; + // initial placeholder, mark stats as not fresh + values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = BoolGetDatum(false); + nulls[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; + InsertTuple(aux_relid, values, nulls); CommandCounterIncrement(); } @@ -278,6 +285,11 @@ void UpdateVisimap(Oid aux_relid, int block_id, const char *visimap_filename) { NameGetDatum(&pt_visimap_name); } + // visimap only update in delete operation, and stats is always invalid + repls[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = true; + nulls[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; + values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; + newtuple = heap_modify_tuple(oldtuple, RelationGetDescr(aux_rel), values, nulls, repls); @@ -289,7 +301,8 @@ void UpdateVisimap(Oid aux_relid, int block_id, const char *visimap_filename) { } void UpdateStatistics(Oid aux_relid, int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats) { + pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid) { // NameData pt_visimap_name; Datum values[NATTS_PG_PAX_BLOCK_TABLES]; bool nulls[NATTS_PG_PAX_BLOCK_TABLES]; @@ -318,6 +331,11 @@ void UpdateStatistics(Oid aux_relid, int block_id, nulls[ANUM_PG_PAX_BLOCK_TABLES_PTSTATISITICS - 1] = false; values[ANUM_PG_PAX_BLOCK_TABLES_PTSTATISITICS - 1] = PointerGetDatum(stats_out); + repls[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = true; + nulls[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; + values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = + BoolGetDatum(is_stats_valid); + newtuple = heap_modify_tuple(oldtuple, RelationGetDescr(aux_rel), values, nulls, repls); @@ -336,6 +354,7 @@ void UpdateStatistics(Oid aux_relid, int block_id, void InsertOrUpdateMicroPartitionPlaceHolder( Oid aux_relid, int block_id, int num_tuples, int file_size, const ::pax::stats::MicroPartitionStatisticsInfo &mp_stats, + bool is_stats_valid, /* const char *visimap_filename, */ bool exist_ext_toast, bool is_clustered) { int stats_length = mp_stats.ByteSizeLong(); @@ -373,6 +392,9 @@ void InsertOrUpdateMicroPartitionPlaceHolder( values[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1] = BoolGetDatum(is_clustered); + values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = BoolGetDatum(is_stats_valid); + nulls[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; + ScanAuxContext context; context.BeginSearchMicroPartition(aux_relid, InvalidOid, NULL, RowExclusiveLock, block_id); @@ -629,8 +651,7 @@ static void FetchMicroPartitionAuxRowCallback(Datum *values, bool *isnull, void *arg) { auto ctx = reinterpret_cast(arg); auto rel = ctx->rel; - auto rel_path = cbdb::BuildPaxDirectoryPath( - rel->rd_node, rel->rd_backend); + auto rel_path = cbdb::BuildPaxDirectoryPath(rel->rd_node, rel->rd_backend); Assert(!isnull[ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME]); { @@ -679,6 +700,10 @@ static void FetchMicroPartitionAuxRowCallback(Datum *values, bool *isnull, Assert(!isnull[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1]); ctx->info.SetClustered( cbdb::DatumToBool(values[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1])); + + Assert(!isnull[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1]); + ctx->info.SetStatsValid( + cbdb::DatumToBool(values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1])); } static void FetchMicroPartitionAuxRowCallbackWrapper(Datum *values, @@ -712,9 +737,10 @@ void UpdateVisimap(Oid aux_relid, int block_id, const char *visimap_filename) { } void UpdateStatistics(Oid aux_relid, int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats) { + pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid) { CBDB_WRAP_START; - { paxc::UpdateStatistics(aux_relid, block_id, mp_stats); } + { paxc::UpdateStatistics(aux_relid, block_id, mp_stats, is_stats_valid); } CBDB_WRAP_END; } diff --git a/contrib/pax_storage/src/cpp/catalog/pax_aux_table.h b/contrib/pax_storage/src/cpp/catalog/pax_aux_table.h index 40095649057..03f1a3a16d9 100644 --- a/contrib/pax_storage/src/cpp/catalog/pax_aux_table.h +++ b/contrib/pax_storage/src/cpp/catalog/pax_aux_table.h @@ -41,7 +41,8 @@ #define ANUM_PG_PAX_BLOCK_TABLES_PTVISIMAPNAME 5 #define ANUM_PG_PAX_BLOCK_TABLES_PTEXISTEXTTOAST 6 #define ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED 7 -#define NATTS_PG_PAX_BLOCK_TABLES 7 +#define ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID 8 +#define NATTS_PG_PAX_BLOCK_TABLES 8 namespace paxc { void CPaxCreateMicroPartitionTable(Relation rel); @@ -52,10 +53,11 @@ void InsertMicroPartitionPlaceHolder(Oid aux_relid, int block_id); void InsertOrUpdateMicroPartitionPlaceHolder( Oid aux_relid, int block_id, int num_tuples, int file_size, const ::pax::stats::MicroPartitionStatisticsInfo &mp_stats, - bool exist_ext_toast, bool is_clustered); + bool is_stats_valid, bool exist_ext_toast, bool is_clustered); void UpdateVisimap(Oid aux_relid, int block_id, const char *visimap_filename); void UpdateStatistics(Oid aux_relid, int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats); + pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid); void DeleteMicroPartitionEntry(Oid pax_relid, Snapshot snapshot, int block_id); // Scan aux table // seqscan: MicroPartitionInfoIterator @@ -122,7 +124,8 @@ Oid FindAuxIndexOid(Oid aux_relid, Snapshot snapshot); void UpdateVisimap(Oid aux_relid, int block_id, const char *visimap_filename); void UpdateStatistics(Oid aux_relid, int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats); + pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid); bool IsMicroPartitionVisible(Relation pax_rel, BlockNumber block, Snapshot snapshot); diff --git a/contrib/pax_storage/src/cpp/catalog/pax_catalog.h b/contrib/pax_storage/src/cpp/catalog/pax_catalog.h index 9b1da99b2b1..3b77657a0ec 100644 --- a/contrib/pax_storage/src/cpp/catalog/pax_catalog.h +++ b/contrib/pax_storage/src/cpp/catalog/pax_catalog.h @@ -38,7 +38,7 @@ extern "C" { #define USE_OWN_MANIFEST_TUPLE typedef struct HeapTupleData *ManifestTuple; -#endif // end USE_PAX_CATALOG +#endif // end USE_PAX_CATALOG #include "catalog/manifest_api.h" } @@ -59,19 +59,19 @@ class PaxCatalogUpdater { void UpdateVisimap(int block_id, const char *visimap_filename); // update stats void UpdateStatistics(int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats); + pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid); private: PaxCatalogUpdater(Relation pax_rel) : pax_rel_(pax_rel) {} // disallow to allocate from heap - void* operator new(size_t) = delete; - void operator delete(void*) = delete; + void *operator new(size_t) = delete; + void operator delete(void *) = delete; PaxCatalogUpdater(const PaxCatalogUpdater &) = delete; PaxCatalogUpdater(PaxCatalogUpdater &&); PaxCatalogUpdater &operator=(const PaxCatalogUpdater &) = delete; PaxCatalogUpdater &operator=(PaxCatalogUpdater &&other); - private: Relation pax_rel_; #ifdef USE_MANIFEST_API @@ -83,7 +83,7 @@ class PaxCatalogUpdater { void PaxCopyAllDataFiles(Relation rel, const RelFileNode *newrnode, bool createnewpath); -} // namespace pax +} // namespace pax namespace cbdb { void InsertMicroPartitionPlaceHolder(Oid pax_relid, int block_id); @@ -94,15 +94,14 @@ bool IsMicroPartitionVisible(Relation pax_rel, BlockNumber block, pax::MicroPartitionMetadata GetMicroPartitionMetadata(Relation rel, Snapshot snapshot, int block_id); -} +} // namespace cbdb namespace paxc { #if !defined(USE_MANIFEST_API) || defined(USE_PAX_CATALOG) -void CPaxAuxSwapRelationFiles(Oid relid1, Oid relid2, - TransactionId frozen_xid, +void CPaxAuxSwapRelationFiles(Oid relid1, Oid relid2, TransactionId frozen_xid, MultiXactId cutoff_multi); #endif void CPaxCopyAllTuples(Relation old_rel, Relation new_rel, Snapshot snapshot); -} // namespace paxc \ No newline at end of file +} // namespace paxc \ No newline at end of file diff --git a/contrib/pax_storage/src/cpp/catalog/pax_catalog_columns.h b/contrib/pax_storage/src/cpp/catalog/pax_catalog_columns.h index dd72f06a8e9..065bf10fb70 100644 --- a/contrib/pax_storage/src/cpp/catalog/pax_catalog_columns.h +++ b/contrib/pax_storage/src/cpp/catalog/pax_catalog_columns.h @@ -34,7 +34,7 @@ #define PAX_AUX_PTVISIMAPNAME "ptvisimapname" #define PAX_AUX_PTEXISTEXTTOAST "ptexistexttoast" #define PAX_AUX_PTISCLUSTERED "ptisclustered" - +#define PAX_AUX_PTISSTATSVALID "ptisstatsvalid" #define ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME 1 #define ANUM_PG_PAX_BLOCK_TABLES_PTTUPCOUNT 2 @@ -43,4 +43,5 @@ #define ANUM_PG_PAX_BLOCK_TABLES_PTVISIMAPNAME 5 #define ANUM_PG_PAX_BLOCK_TABLES_PTEXISTEXTTOAST 6 #define ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED 7 -#define NATTS_PG_PAX_BLOCK_TABLES 7 \ No newline at end of file +#define ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID 8 +#define NATTS_PG_PAX_BLOCK_TABLES 8 \ No newline at end of file diff --git a/contrib/pax_storage/src/cpp/catalog/pax_manifest.cc b/contrib/pax_storage/src/cpp/catalog/pax_manifest.cc index eaf42c4d2cc..902b1026a11 100644 --- a/contrib/pax_storage/src/cpp/catalog/pax_manifest.cc +++ b/contrib/pax_storage/src/cpp/catalog/pax_manifest.cc @@ -25,9 +25,8 @@ *------------------------------------------------------------------------- */ -#include "catalog/pax_catalog.h" - #include "access/pax_visimap.h" +#include "catalog/pax_catalog.h" #include "comm/cbdb_wrappers.h" #include "exceptions/CException.h" #include "storage/file_system.h" @@ -35,8 +34,8 @@ #include "storage/micro_partition_metadata.h" #include "storage/micro_partition_stats.h" #include "storage/pax_itemptr.h" -#include "storage/wal/paxc_wal.h" #include "storage/wal/pax_wal.h" +#include "storage/wal/paxc_wal.h" namespace paxc { static inline bool TestVisimap(Relation rel, const char *visimap_name, @@ -73,8 +72,8 @@ void CPaxCopyAllTuples(Relation old_rel, Relation new_rel, Snapshot snapshot) { } #undef CMP_ATTR #endif - TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(old_rel), - table_slot_callbacks(old_rel)); + TupleTableSlot *slot = MakeSingleTupleTableSlot( + RelationGetDescr(old_rel), table_slot_callbacks(old_rel)); auto scan = table_beginscan(old_rel, snapshot, 0, nullptr); CommandId mycid = GetCurrentCommandId(true); @@ -85,7 +84,7 @@ void CPaxCopyAllTuples(Relation old_rel, Relation new_rel, Snapshot snapshot) { table_endscan(scan); ExecDropSingleTupleTableSlot(slot); } -} // namespace paxc +} // namespace paxc namespace pax { void PaxCopyAllDataFiles(Relation rel, const RelFileNode *newrnode, @@ -100,8 +99,7 @@ void PaxCopyAllDataFiles(Relation rel, const RelFileNode *newrnode, src_path = cbdb::BuildPaxDirectoryPath(rel->rd_node, rel->rd_backend); Assert(!src_path.empty()); - dst_path = - cbdb::BuildPaxDirectoryPath(*newrnode, rel->rd_backend); + dst_path = cbdb::BuildPaxDirectoryPath(*newrnode, rel->rd_backend); Assert(!dst_path.empty()); CBDB_CHECK(!src_path.empty() && !dst_path.empty(), @@ -168,14 +166,13 @@ void PaxCopyAllDataFiles(Relation rel, const RelFileNode *newrnode, cbdb::Pfree(buffer); } -} +} // namespace pax extern "C" { extern Datum MicroPartitionStatsCombineResult(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(MicroPartitionStatsCombineResult); } - #ifdef USE_MANIFEST_API extern "C" { PG_FUNCTION_INFO_V1(pax_get_catalog_rows); @@ -202,22 +199,21 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { oldctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); tupdesc = CreateTemplateTupleDesc(8); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segment_id", - INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)2, - PAX_AUX_PTBLOCKNAME, INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)3, - PAX_AUX_PTTUPCOUNT, INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)4, - PAX_AUX_PTBLOCKSIZE, INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)5, - PAX_AUX_PTSTATISITICS, PAX_AUX_STATS_TYPE_OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)6, - PAX_AUX_PTVISIMAPNAME, NAMEOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)7, - PAX_AUX_PTEXISTEXTTOAST, BOOLOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)8, - PAX_AUX_PTISCLUSTERED, BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "segment_id", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)2, PAX_AUX_PTBLOCKNAME, INT4OID, -1, + 0); + TupleDescInitEntry(tupdesc, (AttrNumber)3, PAX_AUX_PTTUPCOUNT, INT4OID, -1, + 0); + TupleDescInitEntry(tupdesc, (AttrNumber)4, PAX_AUX_PTBLOCKSIZE, INT4OID, -1, + 0); + TupleDescInitEntry(tupdesc, (AttrNumber)5, PAX_AUX_PTSTATISITICS, + PAX_AUX_STATS_TYPE_OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)6, PAX_AUX_PTVISIMAPNAME, NAMEOID, + -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)7, PAX_AUX_PTEXISTEXTTOAST, BOOLOID, + -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)8, PAX_AUX_PTISCLUSTERED, BOOLOID, + -1, 0); ctx = (struct fetch_catalog_rows_context *)palloc(sizeof(*ctx)); ctx->relation = table_open(relid, AccessShareLock); @@ -246,17 +242,24 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { values[0] = Int32GetDatum(GpIdentity.segindex); isnull[0] = false; - values[1] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTBLOCKNAME, &isnull[1]); - values[2] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTTUPCOUNT, &isnull[2]); - values[3] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTBLOCKSIZE, &isnull[3]); - values[4] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTSTATISITICS, &isnull[4]); - values[5] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTVISIMAPNAME, &isnull[5]); + values[1] = + get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTBLOCKNAME, &isnull[1]); + values[2] = + get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTTUPCOUNT, &isnull[2]); + values[3] = + get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTBLOCKSIZE, &isnull[3]); + values[4] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTSTATISITICS, + &isnull[4]); + values[5] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTVISIMAPNAME, + &isnull[5]); if (!isnull[5]) { namestrcpy(&visimap, DatumGetCString(values[5])); values[5] = NameGetDatum(&visimap); } - values[6] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTEXISTEXTTOAST, &isnull[6]); - values[7] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTISCLUSTERED, &isnull[7]); + values[6] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTEXISTEXTTOAST, + &isnull[6]); + values[7] = get_manifesttuple_value(mtuple, mrel, PAX_AUX_PTISCLUSTERED, + &isnull[7]); tuple = heap_form_tuple(fctx->tuple_desc, values, isnull); SRF_RETURN_NEXT(fctx, HeapTupleGetDatum(tuple)); @@ -274,8 +277,9 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { } namespace pax { -MicroPartitionMetadata ManifestTupleToValue( - const std::string &rel_path, ManifestRelation mrel, ManifestTuple tuple); +MicroPartitionMetadata ManifestTupleToValue(const std::string &rel_path, + ManifestRelation mrel, + ManifestTuple tuple); } namespace paxc { bool IndexUniqueCheck(Relation rel, ItemPointer tid, Snapshot snapshot, @@ -292,7 +296,8 @@ bool IndexUniqueCheck(Relation rel, ItemPointer tid, Snapshot snapshot, if (exists) { Datum datum; bool isnull; - datum = get_manifesttuple_value(tuple, mrel, PAX_AUX_PTVISIMAPNAME, &isnull); + datum = + get_manifesttuple_value(tuple, mrel, PAX_AUX_PTVISIMAPNAME, &isnull); if (!isnull) { exists = TestVisimap(rel, NameStr(*DatumGetName(datum)), pax::GetTupleOffset(*tid)); @@ -305,65 +310,71 @@ bool IndexUniqueCheck(Relation rel, ItemPointer tid, Snapshot snapshot, } namespace internal { -void InsertOrUpdateMicroPartitionEntry(const pax::WriteSummary &summary, - ::pax::stats::MicroPartitionStatisticsInfo *dummy_stats, - int block_id) { - void *stats_output; - - { - auto stats = summary.mp_stats ? summary.mp_stats : dummy_stats; - int stats_length = stats->ByteSizeLong(); - uint32 len = VARHDRSZ + stats_length; - - stats_output = palloc(len); - SET_VARSIZE(stats_output, len); - auto ok = stats->SerializeToArray(VARDATA(stats_output), stats_length); - if (!ok) - elog(ERROR, "corrupted pb stats, serialize failed"); - } +void InsertOrUpdateMicroPartitionEntry( + const pax::WriteSummary &summary, + ::pax::stats::MicroPartitionStatisticsInfo *dummy_stats, int block_id) { + void *stats_output = nullptr; + auto rel = table_open(summary.rel_oid, AccessShareLock); + auto mrel = manifest_open(rel); - MetaValue values[] = { - { - PAX_AUX_PTBLOCKNAME, - Int32GetDatum(block_id), - }, - { - PAX_AUX_PTTUPCOUNT, - Int32GetDatum(summary.num_tuples), - }, - { - PAX_AUX_PTBLOCKSIZE, - Int64GetDatum(summary.file_size), - }, + if (summary.num_tuples > 0) { { - PAX_AUX_PTSTATISITICS, - PointerGetDatum(stats_output), - }, - { - PAX_AUX_PTEXISTEXTTOAST, - BoolGetDatum(summary.exist_ext_toast), - }, - { - PAX_AUX_PTISCLUSTERED, - BoolGetDatum(summary.is_clustered), - }, - }; + auto stats = summary.mp_stats ? summary.mp_stats : dummy_stats; + int stats_length = stats->ByteSizeLong(); + uint32 len = VARHDRSZ + stats_length; + + stats_output = palloc(len); + SET_VARSIZE(stats_output, len); + auto ok = stats->SerializeToArray(VARDATA(stats_output), stats_length); + if (!ok) elog(ERROR, "corrupted pb stats, serialize failed"); + } - auto rel = table_open(summary.rel_oid, AccessShareLock); - auto mrel = manifest_open(rel); - manifest_update(mrel, block_id, values, lengthof(values)); + MetaValue values[] = { + { + PAX_AUX_PTBLOCKNAME, + Int32GetDatum(block_id), + }, + { + PAX_AUX_PTTUPCOUNT, + Int32GetDatum(summary.num_tuples), + }, + { + PAX_AUX_PTBLOCKSIZE, + Int64GetDatum(summary.file_size), + }, + { + PAX_AUX_PTSTATISITICS, + PointerGetDatum(stats_output), + }, + { + PAX_AUX_PTEXISTEXTTOAST, + BoolGetDatum(summary.exist_ext_toast), + }, + { + PAX_AUX_PTISCLUSTERED, + BoolGetDatum(summary.is_clustered), + }, + { + PAX_AUX_PTISSTATSVALID, + BoolGetDatum(summary.is_stats_valid), + }, + }; + manifest_update(mrel, block_id, values, lengthof(values)); + } else { + manifest_delete(mrel, block_id); + } manifest_close(mrel); table_close(rel, AccessShareLock); - pfree(stats_output); + if (stats_output) pfree(stats_output); } void InsertMicroPartitionPlaceHolder(Oid pax_relid, int block_id) { MetaValue values[] = { - { - PAX_AUX_PTBLOCKNAME, - Int32GetDatum(block_id), - }, + { + PAX_AUX_PTBLOCKNAME, + Int32GetDatum(block_id), + }, }; Relation pax_rel = table_open(pax_relid, AccessShareLock); auto rel = manifest_open(pax_rel); @@ -398,8 +409,7 @@ void GetMicroPartitionMetadata(Relation rel, Snapshot snapshot, int block_id, CBDB_TRY(); { - auto rel_path = cbdb::BuildPaxDirectoryPath(rel->rd_node, - rel->rd_backend); + auto rel_path = cbdb::BuildPaxDirectoryPath(rel->rd_node, rel->rd_backend); info = pax::ManifestTupleToValue(rel_path, mrel, tuple); } CBDB_CATCH_DEFAULT(); @@ -409,8 +419,8 @@ void GetMicroPartitionMetadata(Relation rel, Snapshot snapshot, int block_id, manifest_close(mrel); } -} // namespace internal -} // namespace paxc +} // namespace internal +} // namespace paxc namespace cbdb { void InsertOrUpdateMicroPartitionEntry(const pax::WriteSummary &summary) { @@ -420,32 +430,27 @@ void InsertOrUpdateMicroPartitionEntry(const pax::WriteSummary &summary) { block_id = std::stol(summary.block_id); CBDB_WRAP_START; { - paxc::internal::InsertOrUpdateMicroPartitionEntry(summary, &dummy_stats, block_id); + paxc::internal::InsertOrUpdateMicroPartitionEntry(summary, &dummy_stats, + block_id); } CBDB_WRAP_END; } void InsertMicroPartitionPlaceHolder(Oid pax_relid, int block_id) { CBDB_WRAP_START; - { - paxc::internal::InsertMicroPartitionPlaceHolder(pax_relid, block_id); - } + { paxc::internal::InsertMicroPartitionPlaceHolder(pax_relid, block_id); } CBDB_WRAP_END; } void DeleteMicroPartitionEntry(Oid pax_relid, Snapshot snapshot, int block_id) { CBDB_WRAP_START; - { - paxc::internal::DeleteMicroPartitionEntry(pax_relid, snapshot, block_id); - } + { paxc::internal::DeleteMicroPartitionEntry(pax_relid, snapshot, block_id); } CBDB_WRAP_END; } bool IsMicroPartitionVisible(Relation pax_rel, BlockNumber block, Snapshot snapshot) { CBDB_WRAP_START; - { - return paxc::internal::IsMicroPartitionVisible(pax_rel, block, snapshot); - } + { return paxc::internal::IsMicroPartitionVisible(pax_rel, block, snapshot); } CBDB_WRAP_END; } @@ -455,20 +460,16 @@ pax::MicroPartitionMetadata GetMicroPartitionMetadata(Relation rel, pax::MicroPartitionMetadata info; CBDB_WRAP_START; - { - paxc::internal::GetMicroPartitionMetadata(rel, snapshot, block_id, info); - } + { paxc::internal::GetMicroPartitionMetadata(rel, snapshot, block_id, info); } CBDB_WRAP_END; return info; } -} // namespace cbdb +} // namespace cbdb namespace pax { PaxCatalogUpdater::PaxCatalogUpdater(PaxCatalogUpdater &&other) - : pax_rel_(other.pax_rel_) - , mrel_(other.mrel_) - {} + : pax_rel_(other.pax_rel_), mrel_(other.mrel_) {} PaxCatalogUpdater &PaxCatalogUpdater::operator=(PaxCatalogUpdater &&other) { if (this != &other) { @@ -497,26 +498,26 @@ void PaxCatalogUpdater::End() { mrel_ = nullptr; } -void PaxCatalogUpdater::UpdateVisimap(int block_id, const char *visimap_filename) { +void PaxCatalogUpdater::UpdateVisimap(int block_id, + const char *visimap_filename) { Assert(block_id >= 0 && "block is is negative"); Assert(visimap_filename && "visimap file name is null"); MetaValue values[] = { - { - PAX_AUX_PTVISIMAPNAME, - CStringGetDatum(visimap_filename), - }, + { + PAX_AUX_PTVISIMAPNAME, + CStringGetDatum(visimap_filename), + }, }; CBDB_WRAP_START; - { - manifest_update(mrel_, block_id, values, lengthof(values)); - } + { manifest_update(mrel_, block_id, values, lengthof(values)); } CBDB_WRAP_END; } -void PaxCatalogUpdater::UpdateStatistics(int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats) { +void PaxCatalogUpdater::UpdateStatistics( + int block_id, pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid) { Assert(block_id >= 0 && "block is is negative"); Assert(mp_stats); @@ -528,16 +529,15 @@ void PaxCatalogUpdater::UpdateStatistics(int block_id, stats_len = VARHDRSZ + mp_stats->ByteSizeLong(); stats_out = palloc(stats_len); SET_VARSIZE(stats_out, stats_len); - auto ok = mp_stats->SerializeToArray(VARDATA(stats_out), - stats_len - VARHDRSZ); + auto ok = + mp_stats->SerializeToArray(VARDATA(stats_out), stats_len - VARHDRSZ); if (!ok) elog(ERROR, "failed to serialize stats"); - MetaValue values[] = { - { - PAX_AUX_PTSTATISITICS, - PointerGetDatum(stats_out), - }, + { + PAX_AUX_PTSTATISITICS, + PointerGetDatum(stats_out), + }, }; manifest_update(mrel_, block_id, values, lengthof(values)); pfree(stats_out); @@ -545,7 +545,7 @@ void PaxCatalogUpdater::UpdateStatistics(int block_id, CBDB_WRAP_END; } -} // namespace pax +} // namespace pax // CREATE OR REPLACE FUNCTION MicroPartitionStatsCombineResult(relid Oid) // RETURNS text @@ -570,8 +570,7 @@ Datum MicroPartitionStatsCombineResult(PG_FUNCTION_ARGS) { // get the tuple desc rel = table_open(relid, AccessShareLock); - if (!RelationIsPAX(rel)) - elog(ERROR, "non-pax table"); + if (!RelationIsPAX(rel)) elog(ERROR, "non-pax table"); rel_desc = CreateTupleDescCopy(RelationGetDescr(rel)); @@ -579,10 +578,12 @@ Datum MicroPartitionStatsCombineResult(PG_FUNCTION_ARGS) { mscan = manifest_beginscan(mrel, nullptr); while ((tuple = manifest_getnext(mscan, nullptr))) { Datum datum; - datum = get_manifesttuple_value(tuple, mrel, PAX_AUX_PTSTATISITICS, &isnull); + datum = + get_manifesttuple_value(tuple, mrel, PAX_AUX_PTSTATISITICS, &isnull); Assert(!isnull); - auto flat_stats = reinterpret_cast(DatumGetPointer(datum)); + auto flat_stats = + reinterpret_cast(DatumGetPointer(datum)); if (got_first) { ok = result.ParseFromArray(VARDATA_ANY(flat_stats), VARSIZE_ANY_EXHDR(flat_stats)); @@ -621,7 +622,7 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { FuncCallContext *fctx; paxc::ScanAuxContext *sctx; HeapTuple tuple; - + if (SRF_IS_FIRSTCALL()) { MemoryContext oldctx; TupleDesc tupdesc; @@ -637,22 +638,21 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { scan_context.BeginSearchMicroPartition(aux_relid, nullptr, AccessShareLock); tupdesc = CreateTemplateTupleDesc(8); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segment_id", - INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)2, - PAX_AUX_PTBLOCKNAME, INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)3, - PAX_AUX_PTTUPCOUNT, INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)4, - PAX_AUX_PTBLOCKSIZE, INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)5, - PAX_AUX_PTSTATISITICS, PAX_AUX_STATS_TYPE_OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)6, - PAX_AUX_PTVISIMAPNAME, NAMEOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)7, - PAX_AUX_PTEXISTEXTTOAST, BOOLOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber)8, - PAX_AUX_PTISCLUSTERED, BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "segment_id", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)2, PAX_AUX_PTBLOCKNAME, INT4OID, -1, + 0); + TupleDescInitEntry(tupdesc, (AttrNumber)3, PAX_AUX_PTTUPCOUNT, INT4OID, -1, + 0); + TupleDescInitEntry(tupdesc, (AttrNumber)4, PAX_AUX_PTBLOCKSIZE, INT4OID, -1, + 0); + TupleDescInitEntry(tupdesc, (AttrNumber)5, PAX_AUX_PTSTATISITICS, + PAX_AUX_STATS_TYPE_OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)6, PAX_AUX_PTVISIMAPNAME, NAMEOID, + -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)7, PAX_AUX_PTEXISTEXTTOAST, BOOLOID, + -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)8, PAX_AUX_PTISCLUSTERED, BOOLOID, + -1, 0); sctx = (paxc::ScanAuxContext *)palloc(sizeof(*sctx)); *sctx = scan_context; @@ -667,8 +667,8 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { tuple = sctx->SearchMicroPartitionEntry(); if (HeapTupleIsValid(tuple)) { - Datum values[8]; - bool isnull[8]; + Datum values[NATTS_PG_PAX_BLOCK_TABLES + 1]; + bool isnull[NATTS_PG_PAX_BLOCK_TABLES + 1]; Relation rel = sctx->GetRelation(); TupleDesc desc = RelationGetDescr(rel); @@ -676,13 +676,22 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { values[0] = Int32GetDatum(GpIdentity.segindex); isnull[0] = false; - values[1] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME, desc, &isnull[1]); - values[2] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTTUPCOUNT, desc, &isnull[2]); - values[3] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKSIZE, desc, &isnull[3]); - values[4] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTSTATISITICS, desc, &isnull[4]); - values[5] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTVISIMAPNAME, desc, &isnull[5]); - values[6] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTEXISTEXTTOAST, desc, &isnull[6]); - values[7] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED, desc, &isnull[7]); + values[1] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKNAME, desc, + &isnull[1]); + values[2] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTTUPCOUNT, desc, + &isnull[2]); + values[3] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTBLOCKSIZE, desc, + &isnull[3]); + values[4] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTSTATISITICS, + desc, &isnull[4]); + values[5] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTVISIMAPNAME, + desc, &isnull[5]); + values[6] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTEXISTEXTTOAST, + desc, &isnull[6]); + values[7] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED, + desc, &isnull[7]); + values[8] = heap_getattr(tuple, ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID, + desc, &isnull[8]); tuple = heap_form_tuple(fctx->tuple_desc, values, isnull); SRF_RETURN_NEXT(fctx, HeapTupleGetDatum(tuple)); } @@ -690,7 +699,7 @@ Datum pax_get_catalog_rows(PG_FUNCTION_ARGS) { sctx->EndSearchMicroPartition(AccessShareLock); SRF_RETURN_DONE(fctx); } -} // extern "C" +} // extern "C" namespace paxc { @@ -722,7 +731,7 @@ bool IndexUniqueCheck(Relation rel, ItemPointer tid, Snapshot snapshot, context.EndSearchMicroPartition(AccessShareLock); return exists; } -} // namespace paxc +} // namespace paxc namespace cbdb { void InsertMicroPartitionPlaceHolder(Oid pax_relid, int block_id) { @@ -748,7 +757,7 @@ void InsertOrUpdateMicroPartitionEntry(const pax::WriteSummary &summary) { summary.file_size, summary.mp_stats ? *summary.mp_stats : ::pax::stats::MicroPartitionStatisticsInfo(), - summary.exist_ext_toast, summary.is_clustered); + summary.is_stats_valid, summary.exist_ext_toast, summary.is_clustered); } CBDB_WRAP_END; } @@ -772,13 +781,11 @@ pax::MicroPartitionMetadata GetMicroPartitionMetadata(Relation rel, return cbdb::PaxGetMicroPartitionMetadata(rel, snapshot, block_id); } -} // namespace cbdb +} // namespace cbdb namespace pax { PaxCatalogUpdater::PaxCatalogUpdater(PaxCatalogUpdater &&other) - : pax_rel_(other.pax_rel_) - , aux_relid_(other.aux_relid_) - {} + : pax_rel_(other.pax_rel_), aux_relid_(other.aux_relid_) {} PaxCatalogUpdater &PaxCatalogUpdater::operator=(PaxCatalogUpdater &&other) { if (this != &other) { @@ -795,18 +802,20 @@ PaxCatalogUpdater PaxCatalogUpdater::Begin(Relation pax_rel) { return up; } -void PaxCatalogUpdater::End() { } +void PaxCatalogUpdater::End() {} -void PaxCatalogUpdater::UpdateVisimap(int block_id, const char *visimap_filename) { +void PaxCatalogUpdater::UpdateVisimap(int block_id, + const char *visimap_filename) { cbdb::UpdateVisimap(this->aux_relid_, block_id, visimap_filename); } -void PaxCatalogUpdater::UpdateStatistics(int block_id, - pax::stats::MicroPartitionStatisticsInfo *mp_stats) { - cbdb::UpdateStatistics(this->aux_relid_, block_id, mp_stats); +void PaxCatalogUpdater::UpdateStatistics( + int block_id, pax::stats::MicroPartitionStatisticsInfo *mp_stats, + bool is_stats_valid) { + cbdb::UpdateStatistics(this->aux_relid_, block_id, mp_stats, is_stats_valid); } -} // namespace pax +} // namespace pax // CREATE OR REPLACE FUNCTION MicroPartitionStatsCombineResult(relid Oid) // RETURNS text @@ -878,8 +887,7 @@ Datum MicroPartitionStatsCombineResult(PG_FUNCTION_ARGS) { namespace paxc { #if !defined(USE_MANIFEST_API) || defined(USE_PAX_CATALOG) -void CPaxAuxSwapRelationFiles(Oid relid1, Oid relid2, - TransactionId frozen_xid, +void CPaxAuxSwapRelationFiles(Oid relid1, Oid relid2, TransactionId frozen_xid, MultiXactId cutoff_multi) { HeapTuple old_tuple1; HeapTuple old_tuple2; @@ -1018,4 +1026,4 @@ void CPaxAuxSwapRelationFiles(Oid relid1, Oid relid2, #endif -} // namespace paxc +} // namespace paxc diff --git a/contrib/pax_storage/src/cpp/catalog/pax_manifest_impl.cc b/contrib/pax_storage/src/cpp/catalog/pax_manifest_impl.cc index f1571a96bf1..db5975fc800 100644 --- a/contrib/pax_storage/src/cpp/catalog/pax_manifest_impl.cc +++ b/contrib/pax_storage/src/cpp/catalog/pax_manifest_impl.cc @@ -77,6 +77,8 @@ static inline AttrNumber get_aux_name_attrno(const char *colname) { return ANUM_PG_PAX_BLOCK_TABLES_PTEXISTEXTTOAST; if (strcmp(colname, PAX_AUX_PTISCLUSTERED) == 0) return ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED; + if (strcmp(colname, PAX_AUX_PTISSTATSVALID) == 0) + return ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID; elog(ERROR, "unknown column name '%s'", colname); return 0; @@ -131,7 +133,12 @@ ManifestDesc manifest_init() { ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1, PAX_AUX_PTISCLUSTERED, Meta_Field_Type_Bool, - 0); + 0); + manifest_init_attribute(desc, + ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1, + PAX_AUX_PTISSTATSVALID, + Meta_Field_Type_Bool, + 0); return desc; } @@ -203,7 +210,7 @@ static void manifest_create_data_dir(Relation rel, const RelFileNode &newrnode) CBDB_END_TRY(); if (rc != 0) - elog(ERROR, "create data dir failed for %u/%u/%lu", + elog(ERROR, "create data dir failed for %u/%u/%u", newrnode.dbNode, newrnode.spcNode, newrnode.relNode); } @@ -310,6 +317,9 @@ void manifest_insert(ManifestRelation mrel, const MetaValue data[], int count) { } else if (AUX_CMP_NAME(col, PAX_AUX_PTISCLUSTERED)) { values[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1] = col.value; isnull[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1] = false; + } else if (AUX_CMP_NAME(col, PAX_AUX_PTISSTATSVALID)) { + values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = col.value; + isnull[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; } else { elog(ERROR, "unknown column name '%s'", col.field_name); } @@ -380,6 +390,10 @@ static void manifest_update_internal(ManifestRelation mrel, values[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1] = col.value; isnull[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1] = false; repl[ANUM_PG_PAX_BLOCK_TABLES_PTISCLUSTERED - 1] = true; + } else if (AUX_CMP_NAME(col, PAX_AUX_PTISSTATSVALID)) { + values[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = col.value; + isnull[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = false; + repl[ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID - 1] = true; } else { elog(ERROR, "unknown column name '%s'", col.field_name); } diff --git a/contrib/pax_storage/src/cpp/clustering/pax_clustering_writer.cc b/contrib/pax_storage/src/cpp/clustering/pax_clustering_writer.cc index 31469e58b87..b4c4af2e810 100644 --- a/contrib/pax_storage/src/cpp/clustering/pax_clustering_writer.cc +++ b/contrib/pax_storage/src/cpp/clustering/pax_clustering_writer.cc @@ -49,6 +49,7 @@ void PaxClusteringWriter::WriteTuple(TupleTableSlot *tuple) { writer_ = std::make_unique(rel_); writer_->SetWriteSummaryCallback(InsertOrUpdateClusteredMicroPartitionEntry) ->SetFileSplitStrategy(std::make_unique()) + ->SetEnableStats(true) ->Open(); } writer_->WriteTuple(tuple); diff --git a/contrib/pax_storage/src/cpp/cmake/pax.cmake b/contrib/pax_storage/src/cpp/cmake/pax.cmake index 099a66f30d8..b2674a203a3 100644 --- a/contrib/pax_storage/src/cpp/cmake/pax.cmake +++ b/contrib/pax_storage/src/cpp/cmake/pax.cmake @@ -85,8 +85,10 @@ set(pax_storage_src storage/orc/orc_writer.cc storage/pax_buffer.cc storage/pax_itemptr.cc + storage/pax_index_update.cc storage/proto/protobuf_stream.cc storage/pax.cc + storage/pax_vacuum_compactor.cc storage/paxc_smgr.cc storage/toast/pax_toast.cc storage/strategy.cc diff --git a/contrib/pax_storage/src/cpp/comm/cbdb_api.h b/contrib/pax_storage/src/cpp/comm/cbdb_api.h index 30db87143a1..a51f023c944 100644 --- a/contrib/pax_storage/src/cpp/comm/cbdb_api.h +++ b/contrib/pax_storage/src/cpp/comm/cbdb_api.h @@ -102,8 +102,10 @@ extern "C" { #ifndef BUILD_PAX_FORMAT #include "access/reloptions.h" #endif +#include "access/multixact.h" #include "catalog/storage.h" #include "cdb/cdbvars.h" +#include "commands/vacuum.h" #include "commands/cluster.h" #include "common/file_utils.h" #include "common/int128.h" diff --git a/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.cc b/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.cc index 3e54e965698..0be1f7f0669 100644 --- a/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.cc +++ b/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.cc @@ -300,8 +300,7 @@ std::string cbdb::BuildPaxDirectoryPath(RelFileNode rd_node, BackendId rd_backend) { CBDB_WRAP_START; { - char *tmp_str = - paxc::BuildPaxDirectoryPath(rd_node, rd_backend); + char *tmp_str = paxc::BuildPaxDirectoryPath(rd_node, rd_backend); std::string ret_str(tmp_str); pfree(tmp_str); return ret_str; @@ -628,3 +627,9 @@ void cbdb::ExecStoreVirtualTuple(TupleTableSlot *slot) { { ::ExecStoreVirtualTuple(slot); } CBDB_WRAP_END; } + +void cbdb::VacuumDelayPoint() { + CBDB_WRAP_START; + { ::vacuum_delay_point(); } + CBDB_WRAP_END; +} diff --git a/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.h b/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.h index 2031662357d..4a6dba15b7d 100644 --- a/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.h +++ b/contrib/pax_storage/src/cpp/comm/cbdb_wrappers.h @@ -292,6 +292,8 @@ void ExecClearTuple(TupleTableSlot *slot); void ExecStoreVirtualTuple(TupleTableSlot *slot); +void VacuumDelayPoint(); + } // namespace cbdb // clang-format off diff --git a/contrib/pax_storage/src/cpp/comm/guc.cc b/contrib/pax_storage/src/cpp/comm/guc.cc index 8a6497d1db3..811aac88391 100644 --- a/contrib/pax_storage/src/cpp/comm/guc.cc +++ b/contrib/pax_storage/src/cpp/comm/guc.cc @@ -72,6 +72,7 @@ int pax_min_size_of_external_toast = PAX_MIN_SIZE_MAKE_EXTERNAL_TOAST; char *pax_default_storage_format = nullptr; int pax_bloom_filter_work_memory_bytes = PAX_BLOOM_FILTER_WORK_MEMORY_BYTES; bool pax_log_filter_tree = false; +bool pax_enable_sync_collect_stats = false; } // namespace pax @@ -201,6 +202,10 @@ void DefineGUCs() { DefineCustomBoolVariable("pax.log_filter_tree", "Log the filter tree", NULL, &pax::pax_log_filter_tree, false, PGC_USERSET, 0, NULL, NULL, NULL); + + DefineCustomBoolVariable("pax.enable_sync_collect_stats", "collect column statistics in sync mode", NULL, + &pax::pax_enable_sync_collect_stats, false, PGC_USERSET, 0, + NULL, NULL, NULL); } } // namespace paxc diff --git a/contrib/pax_storage/src/cpp/comm/guc.h b/contrib/pax_storage/src/cpp/comm/guc.h index ce7ad7e30a5..ffdaf2d681e 100644 --- a/contrib/pax_storage/src/cpp/comm/guc.h +++ b/contrib/pax_storage/src/cpp/comm/guc.h @@ -31,6 +31,7 @@ namespace pax { extern bool pax_enable_debug; extern bool pax_enable_sparse_filter; extern bool pax_enable_row_filter; +extern bool pax_enable_sync_collect_stats; extern int pax_scan_reuse_buffer_size; extern int pax_max_tuples_per_group; diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition.h b/contrib/pax_storage/src/cpp/storage/micro_partition.h index 56d85b46a74..80549d011ff 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition.h +++ b/contrib/pax_storage/src/cpp/storage/micro_partition.h @@ -57,6 +57,7 @@ class MicroPartitionWriter { Oid rel_oid = InvalidOid; RelFileNode node; bool need_wal = false; + bool enable_stats = false; std::vector> encoding_opts; std::vector enable_min_max_col_idxs; std::vector enable_bf_col_idxs; diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition_iterator.cc b/contrib/pax_storage/src/cpp/storage/micro_partition_iterator.cc index 9c9a1f8c8cd..292e689783e 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition_iterator.cc +++ b/contrib/pax_storage/src/cpp/storage/micro_partition_iterator.cc @@ -249,6 +249,13 @@ MicroPartitionMetadata MicroPartitionInfoIterator::ToValue(HeapTuple tuple) { v.SetClustered(is_cluster); } + { + auto is_stats_valid = cbdb::HeapGetAttr( + tuple, ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID, tup_desc, &is_null); + CBDB_CHECK(!is_null, cbdb::CException::kExTypeLogicError); + v.SetStatsValid(cbdb::DatumToBool(is_stats_valid)); + } + // deserialize protobuf message return v; } @@ -446,6 +453,13 @@ MicroPartitionMetadata MicroPartitionInfoParallelIterator::ToValue( v.SetClustered(is_cluster); } + { + auto is_stats_valid = cbdb::HeapGetAttr( + tuple, ANUM_PG_PAX_BLOCK_TABLES_PTISSTATSVALID, tup_desc, &is_null); + CBDB_CHECK(!is_null, cbdb::CException::kExTypeLogicError); + v.SetStatsValid(cbdb::DatumToBool(is_stats_valid)); + } + // deserialize protobuf message return v; } diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition_metadata.h b/contrib/pax_storage/src/cpp/storage/micro_partition_metadata.h index 5e3357b8609..56a4d2389f5 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition_metadata.h +++ b/contrib/pax_storage/src/cpp/storage/micro_partition_metadata.h @@ -45,6 +45,7 @@ struct WriteSummary { bool is_clustered = false; pax::stats::MicroPartitionStatisticsInfo *mp_stats = nullptr; bool exist_ext_toast; + bool is_stats_valid = false; WriteSummary(); WriteSummary(const WriteSummary &summary) = default; }; @@ -96,6 +97,9 @@ struct MicroPartitionMetadata { inline void SetExistToast(bool exist) { exist_ext_toast_ = exist; } inline void SetClustered(bool clustered) { is_clustered_ = clustered; } + inline bool GetStatsValid() const { return is_stats_valid_; } + inline void SetStatsValid(bool valid) { is_stats_valid_ = valid; } + inline bool IsClustered() const { return is_clustered_; } private: @@ -111,6 +115,8 @@ struct MicroPartitionMetadata { bool exist_ext_toast_ = false; bool is_clustered_ = false; + bool is_stats_valid_ = false; + ::pax::stats::MicroPartitionStatisticsInfo stats_; }; // class MicroPartitionMetadata } // namespace pax diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition_stats.cc b/contrib/pax_storage/src/cpp/storage/micro_partition_stats.cc index b18ab952527..866fa37928b 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition_stats.cc +++ b/contrib/pax_storage/src/cpp/storage/micro_partition_stats.cc @@ -409,26 +409,6 @@ bool MicroPartitionStats::MicroPartitionStatisticsInfoCombine( return true; } -// We place the information of each column in a nearby layout, so that for -// each column status check, there is only one memory access, and other -// accesses can hit the cache. - -struct NullCountStats { - // whether the column has been set the has_null to true, if done, we should - // not set it again - bool has_null = false; - // whether the column has been set the all_null to false, if done, we should - // not set it again - bool all_null = true; - // the count of not null rows - int32 not_null_count = 0; - void Reset() { - has_null = false; - all_null = true; - not_null_count = 0; - } -}; - struct MinMaxStats { // status to indicate whether the oids are initialized // or the min-max values are initialized @@ -647,7 +627,19 @@ MicroPartitionStats *MicroPartitionStats::Reset() { return this; } -void MicroPartitionStats::AddRow(TupleTableSlot *slot) { +void MicroPartitionStats::UpdateNullStats(int column_index, + NullCountStats null_count_stats) { + Assert(column_index >= 0); + Assert(column_index < static_cast(column_mem_stats_.size())); + column_mem_stats_[column_index].null_count_stats_.all_null = + null_count_stats.all_null; + column_mem_stats_[column_index].null_count_stats_.has_null = + null_count_stats.has_null; + column_mem_stats_[column_index].null_count_stats_.not_null_count = + null_count_stats.not_null_count; +} + +void MicroPartitionStats::AddRow(TupleTableSlot *slot, bool update_null_stats) { auto n = tuple_desc_->natts; Assert(initialized_); @@ -656,10 +648,17 @@ void MicroPartitionStats::AddRow(TupleTableSlot *slot) { fmt("Current stats initialized [N=%lu], in tuple desc [natts=%d] ", column_mem_stats_.size(), n)); for (auto i = 0; i < n; i++) { - if (slot->tts_isnull[i]) - AddNullColumn(i); - else + if (slot->tts_isnull[i]) { + if (update_null_stats) { + AddNullColumn(i); + } + } else { + if (update_null_stats) { + column_mem_stats_[i].null_count_stats_.all_null = false; + ++column_mem_stats_[i].null_count_stats_.not_null_count; + } AddNonNullColumn(i, slot->tts_values[i]); + } } } @@ -750,8 +749,9 @@ void MicroPartitionStats::MergeRawInfo( // begin update sum auto sum_stat = &column_mem_stats_[column_index].sum_stats_; Assert(sum_stat->status != STATUS_UNINITIALIZED); - Assert(sum_stat->rettype == - stats_info->columnstats(column_index).info().prorettype()); + AssertImply(stats_info->columnstats(column_index).info().has_prorettype(), + sum_stat->rettype == + stats_info->columnstats(column_index).info().prorettype()); sum_stat->final_func_called = true; // no need call final function if (sum_stat->status == STATUS_NOT_SUPPORT) { @@ -1012,9 +1012,6 @@ inline void MicroPartitionStats::AddNonNullColumn(int column_index, int16 &typlen = att->attlen; bool &typbyval = att->attbyval; - column_mem_stats_[column_index].null_count_stats_.all_null = false; - ++column_mem_stats_[column_index].null_count_stats_.not_null_count; - // update min/max switch (column_mem_stats_[column_index].min_max_stats_.status) { case STATUS_NOT_SUPPORT: @@ -1314,7 +1311,6 @@ void MicroPartitionStats::Initialize(const std::vector &minmax_columns, info->set_typid(att->atttypid); info->set_collation(att->attcollation); - column_mem_stats_[i].min_max_stats_.status = STATUS_MISSING_INIT_VAL; init_sum_status: if (cbdb::SumAGGGetProcinfo( diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition_stats.h b/contrib/pax_storage/src/cpp/storage/micro_partition_stats.h index 6dc864b19e1..fae6e53be10 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition_stats.h +++ b/contrib/pax_storage/src/cpp/storage/micro_partition_stats.h @@ -47,6 +47,24 @@ class MicroPartitionStatsData; class BloomFilter; struct ColumnMemStats; +// We place the information of each column in a nearby layout, so that for +// each column status check, there is only one memory access, and other +// accesses can hit the cache. +struct NullCountStats { + // whether the column has been set the has_null to true, if done, we should + // not set it again + bool has_null = false; + // whether the column has been set the all_null to false, if done, we should + // not set it again + bool all_null = true; + // the count of not null rows + int32 not_null_count = 0; + void Reset() { + has_null = false; + all_null = true; + not_null_count = 0; + } +}; class MicroPartitionStats final { public: MicroPartitionStats(TupleDesc desc, bool allow_fallback_to_pg = false); @@ -54,7 +72,15 @@ class MicroPartitionStats final { void Initialize(const std::vector &minmax_columns, const std::vector &bf_columns); - void AddRow(TupleTableSlot *slot); + + // update_null_stats: whether to update the null stats for the column, in + // orc_writer, we don't need to update the null stats,because pax_column + // already updated the null stats + void AddRow(TupleTableSlot *slot, bool update_null_stats); + + // update the null stats for the column + void UpdateNullStats(int column_index, NullCountStats null_count_stats); + MicroPartitionStats *Reset(); ::pax::stats::MicroPartitionStatisticsInfo *Serialize(); diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.cc b/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.cc index 7237f3023f3..d095bb6cb7f 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.cc +++ b/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.cc @@ -36,10 +36,18 @@ namespace pax { MicroPartitionStatsUpdater::MicroPartitionStatsUpdater( - MicroPartitionReader *reader, std::shared_ptr visibility_bitmap) + MicroPartitionReader *reader, bool update_all_group, + std::shared_ptr visibility_bitmap) : reader_(reader) { Assert(reader); - Assert(visibility_bitmap); + AssertImply(!update_all_group, visibility_bitmap); + + if (update_all_group) { + exist_invisible_tuples_.resize(reader_->GetGroupNums()); + std::fill(exist_invisible_tuples_.begin(), exist_invisible_tuples_.end(), + true); + return; + } // O(n) here size_t group_tup_offset = 0; @@ -81,21 +89,10 @@ std::shared_ptr MicroPartitionStatsUpdater::Update( // already setup the visible map auto group = reader_->ReadGroup(group_index); -#ifdef ENABLE_DEBUG - size_t read_count = 0; -#endif while (group->ReadTuple(slot).first) { - group_stats->AddRow(slot); -#ifdef ENABLE_DEBUG - ++read_count; -#endif + group_stats->AddRow(slot, true); } -#ifdef ENABLE_DEBUG - // the read counts must less than the tuple counts in group - Assert(read_count < group->GetRows()); -#endif - } else { ::pax::stats::MicroPartitionStatisticsInfo stat_info; auto column_group_stats = reader_->GetGroupStatsInfo(group_index); diff --git a/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.h b/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.h index acfcfc0d6d2..09d711cf427 100644 --- a/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.h +++ b/contrib/pax_storage/src/cpp/storage/micro_partition_stats_updater.h @@ -36,6 +36,7 @@ class MicroPartitionStatsUpdater { // v1: 111000 // v2: 000100(updater) -> 111100(reader) MicroPartitionStatsUpdater(MicroPartitionReader *reader, + bool update_all_group, std::shared_ptr visibility_bitmap); std::shared_ptr Update( TupleTableSlot *slot, const std::vector &minmax_columns, diff --git a/contrib/pax_storage/src/cpp/storage/orc/orc_writer.cc b/contrib/pax_storage/src/cpp/storage/orc/orc_writer.cc index a33d2de0fc1..87b84e16566 100644 --- a/contrib/pax_storage/src/cpp/storage/orc/orc_writer.cc +++ b/contrib/pax_storage/src/cpp/storage/orc/orc_writer.cc @@ -248,9 +248,10 @@ OrcWriter::OrcWriter( post_script_.set_magic(PORC_MAGIC_ID); group_stats_.Initialize(writer_options.enable_min_max_col_idxs, - writer_options.enable_bf_col_idxs); + writer_options.enable_bf_col_idxs); - // Precompute slowpath indices for varlena columns (non-byval and typlen == -1) + // Precompute slowpath indices for varlena columns (non-byval and typlen == + // -1) varlena_slowpath_indices_.clear(); varlena_slowpath_indices_.reserve(writer_options.rel_tuple_desc->natts); for (int i = 0; i < writer_options.rel_tuple_desc->natts; ++i) { @@ -259,6 +260,8 @@ OrcWriter::OrcWriter( varlena_slowpath_indices_.push_back(i); } } + + summary_.is_stats_valid = writer_options.enable_stats; } OrcWriter::~OrcWriter() {} @@ -531,7 +534,9 @@ void OrcWriter::WriteTuple(TupleTableSlot *table_slot) { for (const auto &pair : detoast_map) table_slot->tts_values[pair.first] = pair.second; - group_stats_.AddRow(table_slot); + if (writer_options_.enable_stats) { + group_stats_.AddRow(table_slot, false); + } EndWriteTuple(table_slot); } @@ -801,6 +806,16 @@ bool OrcWriter::WriteStripe(BufferedOutputStream *buffer_mem_stream, data_len += stream.length(); } + // update the null stats for the column + for (size_t i = 0; i < pax_columns->GetColumns(); i++) { + auto pax_column = (*pax_columns)[i].get(); + NullCountStats null_count_stats; + null_count_stats.has_null = pax_column->HasNull(); + null_count_stats.all_null = pax_column->AllNull(); + null_count_stats.not_null_count = pax_column->GetNonNullRows(); + stripe_stats->UpdateNullStats(i, null_count_stats); + } + if (file_stats) { file_stats->MergeTo(stripe_stats); } @@ -821,21 +836,21 @@ bool OrcWriter::WriteStripe(BufferedOutputStream *buffer_mem_stream, *stripe_footer.add_pax_col_encodings() = encoding_kinds[i]; pb_stats->set_hastoast(pax_column->ToastCounts() > 0); - pb_stats->set_hasnull(col_stats.hasnull()); - pb_stats->set_allnull(col_stats.allnull()); - pb_stats->set_nonnullrows(col_stats.nonnullrows()); + pb_stats->set_hasnull(pax_column->HasNull()); + pb_stats->set_allnull(pax_column->AllNull()); + pb_stats->set_nonnullrows(pax_column->GetNonNullRows()); if (col_stats.has_bloomfilterinfo()) *pb_stats->mutable_bloomfilterinfo() = col_stats.bloomfilterinfo(); if (col_stats.has_columnbfstats()) *pb_stats->mutable_columnbfstats() = col_stats.columnbfstats(); *pb_stats->mutable_coldatastats() = col_stats.datastats(); - PAX_LOG_IF(pax_enable_debug, - "write group[%lu](allnull=%s, hasnull=%s, nonnullrows=%lu, " - "hastoast=%s, nrows=%lu)", - i, BOOL_TOSTRING(col_stats.allnull()), - BOOL_TOSTRING(col_stats.hasnull()), col_stats.nonnullrows(), - BOOL_TOSTRING(pax_column->ToastCounts() > 0), - pax_column->GetRows()); + PAX_LOG_IF( + pax_enable_debug, + "write group[%lu](allnull=%s, hasnull=%s, nonnullrows=%lu, " + "hastoast=%s, nrows=%lu)", + i, BOOL_TOSTRING(pax_column->AllNull()), + BOOL_TOSTRING(pax_column->HasNull()), pax_column->GetNonNullRows(), + BOOL_TOSTRING(pax_column->ToastCounts() > 0), pax_column->GetRows()); } stripe_stats->Reset(); diff --git a/contrib/pax_storage/src/cpp/storage/pax.cc b/contrib/pax_storage/src/cpp/storage/pax.cc index 69282738f4c..01536fc0f86 100644 --- a/contrib/pax_storage/src/cpp/storage/pax.cc +++ b/contrib/pax_storage/src/cpp/storage/pax.cc @@ -43,6 +43,7 @@ #include "storage/micro_partition_stats.h" #include "storage/micro_partition_stats_updater.h" #include "storage/orc/orc_dump_reader.h" +#include "storage/pax_index_update.h" #include "storage/wal/pax_wal.h" #include "storage/wal/paxc_wal.h" #ifdef VEC_BUILD @@ -51,87 +52,7 @@ #define PAX_SPLIT_STRATEGY_CHECK_INTERVAL (16) -namespace paxc { -class IndexUpdaterInternal { - public: - void Begin(Relation rel) { - Assert(rel); - rel_ = rel; - slot_ = MakeTupleTableSlot(rel->rd_att, &TTSOpsVirtual); - - if (HasIndex()) { - estate_ = CreateExecutorState(); - - relinfo_ = makeNode(ResultRelInfo); - relinfo_->ri_RelationDesc = rel; - ExecOpenIndices(relinfo_, false); - } - } - - void UpdateIndex(TupleTableSlot *slot) { - Assert(slot == slot_); - Assert(HasIndex()); - auto recheck_index = - ExecInsertIndexTuples(relinfo_, slot_, estate_, true, false, NULL, NIL); - list_free(recheck_index); - } - - void End() { - if (HasIndex()) { - Assert(relinfo_ && estate_); - - ExecCloseIndices(relinfo_); - pfree(relinfo_); - relinfo_ = nullptr; - - FreeExecutorState(estate_); - estate_ = nullptr; - } - Assert(relinfo_ == nullptr && estate_ == nullptr); - - ExecDropSingleTupleTableSlot(slot_); - slot_ = nullptr; - - rel_ = nullptr; - } - - inline TupleTableSlot *GetSlot() { return slot_; } - inline bool HasIndex() const { return rel_->rd_rel->relhasindex; } - - private: - Relation rel_ = nullptr; - TupleTableSlot *slot_ = nullptr; - EState *estate_ = nullptr; - ResultRelInfo *relinfo_ = nullptr; -}; -} // namespace paxc - -namespace pax { -class IndexUpdater final { - public: - void Begin(Relation rel) { - CBDB_WRAP_START; - { stub_.Begin(rel); } - CBDB_WRAP_END; - } - void UpdateIndex(TupleTableSlot *slot) { - CBDB_WRAP_START; - { stub_.UpdateIndex(slot); } - CBDB_WRAP_END; - } - void End() { - CBDB_WRAP_START; - { stub_.End(); } - CBDB_WRAP_END; - } - inline TupleTableSlot *GetSlot() { return stub_.GetSlot(); } - inline bool HasIndex() const { return stub_.HasIndex(); } - - private: - paxc::IndexUpdaterInternal stub_; -}; -} // namespace pax namespace pax { @@ -156,6 +77,11 @@ TableWriter *TableWriter::SetFileSplitStrategy( return this; } +TableWriter *TableWriter::SetEnableStats(bool enable_stats) { + enable_stats_ = enable_stats; + return this; +} + TableWriter::~TableWriter() {} const FileSplitStrategy *TableWriter::GetFileSplitStrategy() const { @@ -202,6 +128,7 @@ std::unique_ptr TableWriter::CreateMicroPartitionWriter( options.file_name = std::move(file_path); options.encoding_opts = GetRelEncodingOptions(); options.storage_format = GetStorageFormat(); + options.enable_stats = enable_stats_; options.enable_min_max_col_idxs = GetMinMaxColumnIndexes(); options.enable_bf_col_idxs = GetBloomFilterColumnIndexes(); @@ -271,8 +198,10 @@ void TableWriter::Open() { if (!mp_stats_) { mp_stats_ = std::make_shared(RelationGetDescr(relation_)); + mp_stats_->Initialize(GetMinMaxColumnIndexes(), GetBloomFilterColumnIndexes()); + } else { mp_stats_->Reset(); } @@ -567,12 +496,16 @@ void TableDeleter::UpdateStatsInAuxTable( std::move(toast_file)); slot = MakeTupleTableSlot(rel_->rd_att, &TTSOpsVirtual); - auto updated_stats = MicroPartitionStatsUpdater(mp_reader.get(), visi_bitmap) + + // if micro partition stats is valid, we need only update the invisible + // tuples, otherwise we need to update all groups + auto updated_stats = MicroPartitionStatsUpdater( + mp_reader.get(), !meta.GetStatsValid(), visi_bitmap) .Update(slot, min_max_col_idxs, bf_col_idxs); // update the statistics in aux table catalog_update.UpdateStatistics(meta.GetMicroPartitionId(), - updated_stats->Serialize()); + updated_stats->Serialize(), true); mp_reader->Close(); @@ -669,15 +602,8 @@ void TableDeleter::DeleteWithVisibilityMap( visimap_file->Close(); } - // TODO: update stats and visimap all in one catalog update - // Update the stats in pax aux table - // Notice that: PAX won't update the stats in group - UpdateStatsInAuxTable( - catalog_update, micro_partition_metadata, - std::make_shared(visi_bitmap->Raw()), min_max_col_idxs, - cbdb::GetBloomFilterColumnIndexes(rel_), stats_updater_projection); - // write pg_pax_blocks_oid + // update visimap file and update stats to invalid catalog_update.UpdateVisimap(block_id, visimap_file_name); } while (iterator->HasNext()); catalog_update.End(); diff --git a/contrib/pax_storage/src/cpp/storage/pax.h b/contrib/pax_storage/src/cpp/storage/pax.h index fec97e613f3..5e4560d5364 100644 --- a/contrib/pax_storage/src/cpp/storage/pax.h +++ b/contrib/pax_storage/src/cpp/storage/pax.h @@ -72,6 +72,8 @@ class TableWriter { TableWriter *SetFileSplitStrategy( std::unique_ptr &&strategy); + TableWriter *SetEnableStats(bool enable_stats); + BlockNumber GetBlockNumber() const { return current_blockno_; } protected: @@ -125,6 +127,7 @@ class TableWriter { BlockNumber current_blockno_ = 0; bool options_cached_; + bool enable_stats_ = false; PaxStorageFormat storage_format_ = PaxStorageFormat::kTypeStoragePorcNonVec; std::vector min_max_col_idx_; std::vector bf_col_idx_; diff --git a/contrib/pax_storage/src/cpp/storage/pax_index_update.cc b/contrib/pax_storage/src/cpp/storage/pax_index_update.cc new file mode 100644 index 00000000000..f47565b5d87 --- /dev/null +++ b/contrib/pax_storage/src/cpp/storage/pax_index_update.cc @@ -0,0 +1,137 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pax_index_update.cc + * + * IDENTIFICATION + * contrib/pax_storage/src/cpp/storage/pax_index_update.cc + * + *------------------------------------------------------------------------- + */ +#include "storage/pax_index_update.h" + +#include "storage/pax_itemptr.h" + +namespace paxc { + +/* + * Callback function to check if an index entry should be reaped during vacuum. + * Returns true if the index entry points to a micro partition that was + * processed (and will be deleted) during compaction. + * + * This has the right signature to be an IndexBulkDeleteCallback. + * Must be defined outside namespace to ensure proper C linkage. + */ +static bool pax_tid_reaped(ItemPointer itemptr, void *state) { + std::unordered_set *processed_mp_ids = + static_cast *>(state); + uint32 block_number = pax::GetBlockNumber(*itemptr); + return processed_mp_ids->find(block_number) != processed_mp_ids->end(); +} + +void IndexUpdaterInternal::Begin(Relation rel) { + Assert(rel); + + rel_ = rel; + slot_ = MakeTupleTableSlot(rel->rd_att, &TTSOpsVirtual); + + if (HasIndex()) { + estate_ = CreateExecutorState(); + + relinfo_ = makeNode(ResultRelInfo); + relinfo_->ri_RelationDesc = rel; + ExecOpenIndices(relinfo_, false); + } +} + +void IndexUpdaterInternal::UpdateIndex(TupleTableSlot *slot) { + Assert(slot == slot_); + Assert(HasIndex()); + /* Ensure tts_nvalid is set correctly for virtual slot before index + * insertion */ + /* This is similar to what aocs_getnext does (setting tts_nvalid = natts) */ + slot->tts_nvalid = slot->tts_tupleDescriptor->natts; + auto recheck_index = + ExecInsertIndexTuples(relinfo_, slot_, estate_, false, false, NULL, NIL); + list_free(recheck_index); +} + +void IndexUpdaterInternal::End() { + if (HasIndex()) { + Assert(relinfo_ && estate_); + + ExecCloseIndices(relinfo_); + pfree(relinfo_); + relinfo_ = nullptr; + + FreeExecutorState(estate_); + estate_ = nullptr; + } + Assert(relinfo_ == nullptr && estate_ == nullptr); + + ExecDropSingleTupleTableSlot(slot_); + slot_ = nullptr; + + rel_ = nullptr; +} + +void IndexUpdaterInternal::CleanIndex( + std::unordered_set &processed_mp_ids) { + Assert(HasIndex()); + int nindexes; + Relation *Irel; + vac_open_indexes(rel_, RowExclusiveLock, &nindexes, &Irel); + + if (Irel != NULL && nindexes > 0) { + IndexVacuumInfo ivinfo = {0}; + ivinfo.analyze_only = false; + ivinfo.message_level = DEBUG2; + ivinfo.num_heap_tuples = rel_->rd_rel->reltuples; + ivinfo.estimated_count = true; + ivinfo.strategy = NULL; // No buffer strategy needed + + // Process all indexes + for (int i = 0; i < nindexes; i++) { + ivinfo.index = Irel[i]; + + // Bulk delete index entries pointing to processed micro partitions + IndexBulkDeleteResult *stats = + index_bulk_delete(&ivinfo, NULL, pax_tid_reaped, + static_cast(&processed_mp_ids)); + + // Always call index_vacuum_cleanup to clean up all dead entries + // This includes entries deleted by index_bulk_delete and entries + // marked as DEAD during index insertion (when old micro partitions + // were found to be invisible) + // Pass stats from index_bulk_delete if available, otherwise pass NULL + // to clean up all remaining dead entries + stats = index_vacuum_cleanup(&ivinfo, stats); + + if (stats) { + // Optionally update index statistics if needed + // Similar to vacuum_appendonly_index(), we could update relstats here + // but for now we just free the stats + pfree(stats); + } + } + } + + vac_close_indexes(nindexes, Irel, NoLock); +} +} // namespace paxc diff --git a/contrib/pax_storage/src/cpp/storage/pax_index_update.h b/contrib/pax_storage/src/cpp/storage/pax_index_update.h new file mode 100644 index 00000000000..58c0db6df55 --- /dev/null +++ b/contrib/pax_storage/src/cpp/storage/pax_index_update.h @@ -0,0 +1,84 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pax_index_update.h + * + * IDENTIFICATION + * contrib/pax_storage/src/cpp/storage/pax_index_update.h + * + *------------------------------------------------------------------------- + */ +#pragma once +#include "comm/cbdb_api.h" + +#include + +#include "comm/cbdb_wrappers.h" + +namespace paxc { +class IndexUpdaterInternal { + public: + void Begin(Relation rel); + void UpdateIndex(TupleTableSlot *slot); + void CleanIndex(std::unordered_set &processed_mp_ids); + void End(); + inline TupleTableSlot *GetSlot() { return slot_; } + inline bool HasIndex() const { return rel_->rd_rel->relhasindex; } + + private: + Relation rel_ = nullptr; + TupleTableSlot *slot_ = nullptr; + EState *estate_ = nullptr; + ResultRelInfo *relinfo_ = nullptr; +}; +} // namespace paxc + +namespace pax { +class IndexUpdater final { + public: + void Begin(Relation rel) { + CBDB_WRAP_START; + { stub_.Begin(rel); } + CBDB_WRAP_END; + } + + void UpdateIndex(TupleTableSlot *slot) { + CBDB_WRAP_START; + { stub_.UpdateIndex(slot); } + CBDB_WRAP_END; + } + + void CleanIndex(std::unordered_set &processed_mp_ids) { + CBDB_WRAP_START; + { stub_.CleanIndex(processed_mp_ids); } + CBDB_WRAP_END; + } + + void End() { + CBDB_WRAP_START; + { stub_.End(); } + CBDB_WRAP_END; + } + inline TupleTableSlot *GetSlot() { return stub_.GetSlot(); } + inline bool HasIndex() const { return stub_.HasIndex(); } + + private: + paxc::IndexUpdaterInternal stub_; +}; +} // namespace pax \ No newline at end of file diff --git a/contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.cc b/contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.cc new file mode 100644 index 00000000000..12fd9a255fc --- /dev/null +++ b/contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.cc @@ -0,0 +1,144 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pax_vacuum_compactor.cc + * + * IDENTIFICATION + * contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.cc + * + *------------------------------------------------------------------------- + */ +#include "storage/pax_vacuum_compactor.h" + +#include + +#include "access/genam.h" +#include "access/pax_visimap.h" +#include "comm/iterator.h" // VectorIterator +#include "comm/pax_memory.h" +#include "commands/vacuum.h" +#include "exceptions/CException.h" +#include "executor/executor.h" +#include "storage/local_file_system.h" +#include "storage/micro_partition_file_factory.h" +#include "storage/pax.h" +#include "storage/pax_index_update.h" +#include "storage/pax_itemptr.h" + +namespace pax { + +static void InsertOrUpdateClusteredMicroPartitionEntry( + const pax::WriteSummary &summary) { + pax::WriteSummary clusterd_summary(summary); + clusterd_summary.is_stats_valid = true; + cbdb::InsertOrUpdateMicroPartitionEntry(clusterd_summary); +} +} // namespace pax + +namespace pax { + +PaxVacuumBatcher::PaxVacuumBatcher(Relation rel) : rel_(rel) {} + +void PaxVacuumBatcher::Process( + std::unique_ptr> &&it, + const std::vector &minmax_col_idxs, + const std::vector &bf_col_idxs) { + pax::IndexUpdater index_updater; + index_updater.Begin(rel_); + Assert(rel_->rd_rel->relhasindex == index_updater.HasIndex()); + auto slot = index_updater.GetSlot(); + slot->tts_tableOid = RelationGetRelid(rel_); + + // Track processed micro partition IDs + std::unordered_set processed_mp_ids; + + // First, collect all micro partition metadata and IDs + // We need to do this before reading tuples because we'll delete micro + // partitions from catalog to make them invisible before inserting new index + // entries + std::vector metadata_list; + while (it->HasNext()) { + auto meta = it->Next(); + metadata_list.push_back(meta); + processed_mp_ids.insert(meta.GetMicroPartitionId()); + } + + // Delete all processed micro partition entries from catalog BEFORE compaction + // This makes them invisible so that when we insert new index entries, + // B-tree's uniqueness check will find old entries pointing to dead micro + // partitions and mark them as DEAD instead of raising an error + Snapshot snapshot = GetActiveSnapshot(); + Oid relid = RelationGetRelid(rel_); + for (auto mp_id : processed_mp_ids) { + cbdb::DeleteMicroPartitionEntry(relid, snapshot, mp_id); + } + + // Clean up old index entries pointing to processed micro partitions + // This is similar to vacuum_appendonly_index() in vacuum_ao.c + if (index_updater.HasIndex() && !processed_mp_ids.empty()) { + index_updater.CleanIndex(processed_mp_ids); + } + + // CommandCounterIncrement is needed to ensure the deletions are visible + // in the current transaction before we start inserting new index entries + CommandCounterIncrement(); + + TableWriter tw(rel_); + tw.SetFileSplitStrategy(std::make_unique()); + tw.SetWriteSummaryCallback(cbdb::InsertOrUpdateMicroPartitionEntry); + tw.SetEnableStats(true); + tw.Open(); + + // Recreate iterator from the collected metadata for reading tuples + auto metadata_iter = std::make_unique>( + std::move(metadata_list)); + + // Create TableReader similar to PaxScanDesc + TableReader::ReaderOptions reader_options{}; + reader_options.filter = std::make_shared(); + auto natts = cbdb::RelationGetAttributesNumber(rel_); + reader_options.filter->SetColumnProjection(std::vector(natts, true)); + reader_options.reused_buffer = nullptr; + reader_options.table_space_id = rel_->rd_rel->reltablespace; + + TableReader reader(std::move(metadata_iter), reader_options); + reader.Open(); + + // Read all tuples using TableReader and insert into new micro partitions + // Note: Old micro partitions are now deleted from catalog, so when we insert + // new index entries, B-tree's uniqueness check will find old entries pointing + // to dead micro partitions and mark them as DEAD instead of raising an error. + // Note: TableReader::ReadTuple already calls ExecStoreVirtualTuple + while (reader.ReadTuple(slot)) { + tw.WriteTuple(slot); + if (index_updater.HasIndex()) { + // Already store the ctid after WriteTuple + Assert(ItemPointerIsValid(&slot->tts_tid)); + // Insert new index entries directly (similar to AO/AOCS compaction) + // The uniqueness check will handle old entries by marking them as DEAD + index_updater.UpdateIndex(slot); + } + } + + index_updater.End(); + reader.Close(); + tw.Close(); +} + +} // namespace pax \ No newline at end of file diff --git a/contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.h b/contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.h new file mode 100644 index 00000000000..837cdc68f60 --- /dev/null +++ b/contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.h @@ -0,0 +1,56 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pax_vacuum_compactor.h +* + * IDENTIFICATION + * contrib/pax_storage/src/cpp/storage/pax_vacuum_compactor.h + * + *------------------------------------------------------------------------- + */ +#pragma once + +#include "comm/cbdb_api.h" + +#include +#include +#include + +#include "comm/bitmap.h" +#include "storage/micro_partition.h" +#include "storage/micro_partition_metadata.h" +#include "comm/iterator.h" + +struct EState; +struct ResultRelInfo; + +namespace pax { + +class PaxVacuumBatcher final { + public: + explicit PaxVacuumBatcher(Relation rel); + void Process(std::unique_ptr> &&it, + const std::vector &minmax_col_idxs, + const std::vector &bf_col_idxs); + + private: + Relation rel_ = nullptr; +}; + +} // namespace pax \ No newline at end of file diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/compaction_utility_insert.out b/contrib/pax_storage/src/test/isolation2/expected/pax/compaction_utility_insert.out index ccffd79ddc9..10bd0d786a2 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/compaction_utility_insert.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/compaction_utility_insert.out @@ -27,6 +27,4 @@ SELECT segment_id, pttupcount FROM get_pax_aux_table_all('foo') where pttupcount segment_id | pttupcount ------------+------------ 0 | 1 - 0 | 1 - 0 | 1 -(3 rows) +(1 row) diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/select_after_vacuum.out b/contrib/pax_storage/src/test/isolation2/expected/pax/select_after_vacuum.out index b0254ab4b26..a4a21c8ab17 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/select_after_vacuum.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/select_after_vacuum.out @@ -149,136 +149,16 @@ INSERT 1 0: SELECT ptblockname, case when pttupcount = 0 then 'zero' when pttupcount = 1 then 'one' when pttupcount <= 5 then 'few' else 'many' end FROM get_pax_aux_table_all('pax_tbl'); ptblockname | case -------------+------ - 0 | many - 1 | many - 2 | many - 3 | many - 4 | many - 5 | many - 6 | many - 7 | many - 8 | many - 9 | many - 10 | many - 11 | many - 12 | many - 13 | many - 14 | many - 15 | many - 16 | many - 17 | many - 18 | many - 19 | many - 20 | many - 21 | one - 0 | many - 1 | many - 2 | many - 3 | many - 4 | many - 5 | many - 6 | many - 7 | many - 8 | many - 9 | many - 10 | many - 11 | many - 12 | many - 13 | many - 14 | many - 15 | many - 16 | many - 17 | many - 18 | many - 19 | many - 20 | many - 0 | many - 1 | many - 2 | many - 3 | many - 4 | many - 5 | many - 6 | many - 7 | many - 8 | many - 9 | many - 10 | many - 11 | many - 12 | many - 13 | many - 14 | many - 15 | many - 16 | many - 17 | many - 18 | many - 19 | many - 20 | many -(64 rows) + 21 | many + 21 | many + 21 | many + 22 | one +(4 rows) 0: SELECT * FROM get_pax_aux_table_all('pax_tbl'); - segment_id | ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered -------------+-------------+------------+---------------------------------+----------------+-----------------+--------------- - 0 | 0 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 1 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 2 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 3 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 4 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 5 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 6 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 7 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 8 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 9 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 10 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 11 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 12 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 13 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 14 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 15 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 16 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 17 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 18 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 19 | 338 | [(false,false),(289),None,None] | t | f | f - 0 | 20 | 338 | [(false,false),(289),None,None] | t | f | f - 1 | 0 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 1 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 2 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 3 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 4 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 5 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 6 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 7 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 8 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 9 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 10 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 11 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 12 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 13 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 14 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 15 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 16 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 17 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 18 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 19 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 20 | 322 | [(false,false),(282),None,None] | t | f | f - 1 | 21 | 1 | [(false,false),(1),None,None] | f | f | f - 2 | 0 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 1 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 2 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 3 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 4 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 5 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 6 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 7 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 8 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 9 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 10 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 11 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 12 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 13 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 14 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 15 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 16 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 17 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 18 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 19 | 340 | [(false,false),(302),None,None] | t | f | f - 2 | 20 | 340 | [(false,false),(302),None,None] | t | f | f -(64 rows) + segment_id | ptblockname | pttupcount | ptstatistics | ptexistvisimap | ptexistexttoast | ptisclustered +------------+-------------+------------+----------------------------------+----------------+-----------------+--------------- + 0 | 21 | 6069 | [(false,false),(6069),None,None] | f | f | f + 1 | 21 | 5922 | [(false,false),(5922),None,None] | f | f | f + 1 | 22 | 1 | [(false,false),(1),None,None] | f | f | f + 2 | 21 | 6342 | [(false,false),(6342),None,None] | f | f | f +(4 rows) diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/select_while_vacuum_serializable2.out b/contrib/pax_storage/src/test/isolation2/expected/pax/select_while_vacuum_serializable2.out index c99c0fd1f73..ecc1be0f109 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/select_while_vacuum_serializable2.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/select_while_vacuum_serializable2.out @@ -138,68 +138,8 @@ INSERT 1 2: SELECT segment_id, ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl'); segment_id | ptblockname | pttupcount ------------+-------------+------------ - 0 | 0 | 338 - 0 | 1 | 338 - 0 | 2 | 338 - 0 | 3 | 338 - 0 | 4 | 338 - 0 | 5 | 338 - 0 | 6 | 338 - 0 | 7 | 338 - 0 | 8 | 338 - 0 | 9 | 338 - 0 | 10 | 338 - 0 | 11 | 338 - 0 | 12 | 338 - 0 | 13 | 338 - 0 | 14 | 338 - 0 | 15 | 338 - 0 | 16 | 338 - 0 | 17 | 338 - 0 | 18 | 338 - 0 | 19 | 338 - 0 | 20 | 338 - 1 | 0 | 322 - 1 | 1 | 322 - 1 | 2 | 322 - 1 | 3 | 322 - 1 | 4 | 322 - 1 | 5 | 322 - 1 | 6 | 322 - 1 | 7 | 322 - 1 | 8 | 322 - 1 | 9 | 322 - 1 | 10 | 322 - 1 | 11 | 322 - 1 | 12 | 322 - 1 | 13 | 322 - 1 | 14 | 322 - 1 | 15 | 322 - 1 | 16 | 322 - 1 | 17 | 322 - 1 | 18 | 322 - 1 | 19 | 322 - 1 | 20 | 322 - 1 | 21 | 1 - 2 | 0 | 340 - 2 | 1 | 340 - 2 | 2 | 340 - 2 | 3 | 340 - 2 | 4 | 340 - 2 | 5 | 340 - 2 | 6 | 340 - 2 | 7 | 340 - 2 | 8 | 340 - 2 | 9 | 340 - 2 | 10 | 340 - 2 | 11 | 340 - 2 | 12 | 340 - 2 | 13 | 340 - 2 | 14 | 340 - 2 | 15 | 340 - 2 | 16 | 340 - 2 | 17 | 340 - 2 | 18 | 340 - 2 | 19 | 340 - 2 | 20 | 340 -(64 rows) + 0 | 21 | 6069 + 1 | 21 | 5922 + 1 | 22 | 1 + 2 | 21 | 6342 +(4 rows) diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_function.out b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_function.out index 55cd4574a32..3210719f058 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_function.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_function.out @@ -34,9 +34,9 @@ VACUUM 2: SELECT ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl'); ptblockname | pttupcount -------------+------------ - 0 | 38 - 0 | 25 - 0 | 37 + 1 | 20 + 1 | 11 + 1 | 19 (3 rows) -- A second VACUUM shouldn't recycle them either. @@ -45,14 +45,14 @@ VACUUM 2: SELECT ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl'); ptblockname | pttupcount -------------+------------ - 0 | 38 - 0 | 37 - 0 | 25 + 1 | 20 + 1 | 19 + 1 | 11 (3 rows) 1<: <... completed> myfunc -------- - 70 + 120 (1 row) -- Now that the first transaction has finished, VACUUM can recycle. @@ -61,7 +61,7 @@ VACUUM 2: SELECT ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl'); ptblockname | pttupcount -------------+------------ - 0 | 38 - 0 | 37 - 0 | 25 + 1 | 20 + 1 | 11 + 1 | 19 (3 rows) diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable.out b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable.out index e5fb6551db3..fbd7b5e441c 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable.out @@ -29,9 +29,9 @@ VACUUM SELECT ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl') where pttupcount > 0; ptblockname | pttupcount -------------+------------ - 0 | 25 - 0 | 38 - 0 | 37 + 1 | 14 + 1 | 30 + 1 | 26 (3 rows) SELECT COUNT(*) FROM pax_tbl; count diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable2.out b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable2.out index ec22aa6880f..eb5fc36aa4b 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable2.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable2.out @@ -39,9 +39,9 @@ VACUUM 2: SELECT ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl'); ptblockname | pttupcount -------------+------------ - 0 | 38 - 0 | 25 - 0 | 37 + 1 | 14 + 1 | 30 + 1 | 26 (3 rows) 1: SELECT COUNT(*) FROM pax_tbl; count diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable3.out b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable3.out index ef575745565..4db14d8db05 100644 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable3.out +++ b/contrib/pax_storage/src/test/isolation2/expected/pax/vacuum_self_serializable3.out @@ -33,9 +33,9 @@ VACUUM SELECT ptblockname, pttupcount FROM get_pax_aux_table_all('pax_tbl') where pttupcount > 0; ptblockname | pttupcount -------------+------------ - 0 | 25 - 0 | 38 - 0 | 37 + 1 | 30 + 1 | 14 + 1 | 26 (3 rows) SELECT COUNT(*) FROM pax_tbl; count