DRAFT - Move Orca to a separate Postgres extension

.abi-check/7.2.0_arenadata7/postgres.symbols.ignore

@@ -1 +1,8 @@
+OptimizerMemoryContext
+optimize_query
+transformGroupedWindows
+optopt@@GLIBC_2.2.5
+optopt
+optimizer_partition_selection_log
+ConfigureNamesBool_gp
 ConfigureNamesInt_gp

.travis.yml

@@ -121,9 +121,9 @@ matrix:
                 --enable-orca \
                 --without-zstd --disable-gpfdist \
                 CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-              bear --append make -s -C src/backend/gpopt
+              bear --append -- make -s -C gpcontrib/orca
               )
-              CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-gpopt ../vpath.debug
+              CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-gpopt ../vpath.debug
             - |
               (
               set -e
@@ -135,13 +135,13 @@ matrix:
                 --enable-orca \
                 --without-zstd --disable-gpfdist \
                 CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-              bear --append make -s -C src/backend/gpopt
+              bear --append -- make -s -C gpcontrib/orca
               )
-              CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-gpopt ../vpath.release
-            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hsrc/backend/gporca -Bbuild.debug
-            - CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-orca build.debug
-            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hsrc/backend/gporca -Bbuild.relwithdebinfo
-            - CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-orca build.relwithdebinfo
+              CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-gpopt ../vpath.release
+            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hgpcontrib/orca/gporca -Bbuild.debug
+            - CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-orca build.debug
+            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hgpcontrib/orca/gporca -Bbuild.relwithdebinfo
+            - CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-orca build.relwithdebinfo
 
         - name: check-format
           stage: lint
@@ -157,9 +157,9 @@ matrix:
           before_script: ~
           after_script: ~
           script:
-            - env CLANG_FORMAT=clang-format-11 src/tools/fmt gen
+            - env CLANG_FORMAT=clang-format-11 gpcontrib/orca/tools/fmt gen
             - git diff --exit-code
-            - env CLANG_FORMAT=clang-format-11 src/tools/fmt chk
+            - env CLANG_FORMAT=clang-format-11 gpcontrib/orca/tools/fmt chk
 
 stages:
   - lint

arenadata/Dockerfile.linter

@@ -9,4 +9,4 @@ COPY . /opt/gpdb_src
 
 ENV CLANG_FORMAT=clang-format-11
 
-ENTRYPOINT src/tools/fmt gen && git diff --exit-code && src/tools/fmt chk
+ENTRYPOINT gpcontrib/orca/tools/fmt gen && git diff --exit-code && gpcontrib/orca/tools/fmt chk

concourse/scripts/unit_tests_gporca.bash

@@ -9,7 +9,7 @@ function test_orca
         return
     fi
     OUTPUT_DIR="../../../../gpAux/ext/${BLD_ARCH}"
-    pushd ${GPDB_SRC_PATH}/src/backend/gporca
+    pushd ${GPDB_SRC_PATH}/gpcontrib/orca/gporca
     concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR}
     concourse/build_and_test.py --build_type=Debug --output_dir=${OUTPUT_DIR}
     popd

concourse/tasks/check_format.yml

@@ -9,7 +9,7 @@ inputs:
   - name: gpdb_src
     path: .
 run:
-  path: src/tools/fmt
+  path: gpcontrib/orca/tools/fmt
   args: [ chk ]
 params:
   CLANG_FORMAT: clang-format-11

concourse/tasks/clang_tidy.yml

@@ -21,7 +21,7 @@ run:
         --enable-orca \
         --without-python --without-readline --without-zlib --without-zstd --without-libbz2 --disable-gpfdist --disable-gpcloud --without-libcurl \
         CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-      bear --append make -s -C src/backend/gpopt
+      bear --append -- make -s -C gpcontrib/orca
     )
     (
       mkdir vpath.release
@@ -31,17 +31,17 @@ run:
         --enable-orca \
         --without-python --without-readline --without-zlib --without-zstd --without-libbz2 --disable-gpfdist --disable-gpcloud --without-libcurl \
         CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-      bear --append make -s -C src/backend/gpopt
+      bear --append -- make -s -C gpcontrib/orca
     )
     (
       cd gpdb_src
-      src/tools/tidy chk-gpopt ../vpath.debug
-      src/tools/tidy chk-gpopt ../vpath.release
+      gpcontrib/orca/tools/tidy chk-gpopt ../vpath.debug
+      gpcontrib/orca/tools/tidy chk-gpopt ../vpath.release
 
-      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hsrc/backend/gporca -Bbuild.debug
-      src/tools/tidy chk-orca build.debug
-      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hsrc/backend/gporca -Bbuild.relwithdebinfo
-      src/tools/tidy chk-orca build.relwithdebinfo
+      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hgpcontrib/orca/gporca -Bbuild.debug
+      gpcontrib/orca/tools/tidy chk-orca build.debug
+      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hgpcontrib/orca/gporca -Bbuild.relwithdebinfo
+      gpcontrib/orca/tools/tidy chk-orca build.relwithdebinfo
     )
 
 params:

configure

@@ -7905,9 +7905,7 @@ if test "${enable_orca+set}" = set; then :
   enableval=$enable_orca;
   case $enableval in
     yes)
-
-$as_echo "#define USE_ORCA 1" >>confdefs.h
-
+      :
       ;;
     no)
       :
@@ -7920,8 +7918,6 @@ $as_echo "#define USE_ORCA 1" >>confdefs.h
 else
   enable_orca=yes
 
-$as_echo "#define USE_ORCA 1" >>confdefs.h
-
 fi
 
 

configure.in

@@ -787,9 +787,7 @@ PGAC_ARG_BOOL(enable, cassert, no, [enable assertion checks (for debugging)],
 #
 # Enable GPORCA optimizer
 #
-PGAC_ARG_BOOL(enable, orca, yes, [disable ORCA optimizer],
-              [AC_DEFINE([USE_ORCA], 1,
-                         [Define to 1 to build with Greenplum ORCA optimizer. (--enable-orca)])])
+PGAC_ARG_BOOL(enable, orca, yes, [disable ORCA optimizer])
 AC_MSG_RESULT([checking whether to build with ORCA... $enable_orca])
 AC_SUBST(enable_orca)
 

contrib/auto_explain/expected/auto_explain.out

@@ -101,7 +101,7 @@ Finalize Aggregate  (cost=35148.64..35148.65 rows=1 width=8) (actual rows=1 loop
                           ->  Broadcast Motion 3:3  (slice2; segments: 3)  (cost=0.00..53.05 rows=1001 width=0) (actual rows=1001 loops=1)
                                 ->  Seq Scan on auto_explain_test.t2  (cost=0.00..13.01 rows=334 width=0) (actual rows=340 loops=1)
 Optimizer: Postgres-based planner
-Settings: enable_nestloop = 'on', optimizer = 'off'
+Settings: enable_nestloop = 'on'
   (slice0)    Executor memory: 131K bytes.
   (slice1)    Executor memory: 152K bytes avg x 3 workers, 152K bytes max (seg0).
   (slice2)    Executor memory: 42K bytes avg x 3 workers, 42K bytes max (seg0).

contrib/auto_explain/expected/auto_explain_optimizer.out

@@ -101,7 +101,7 @@ Finalize Aggregate  (cost=0.00..1326086.34 rows=1 width=8) (actual rows=1 loops=
                           ->  Seq Scan on auto_explain_test.t1  (cost=0.00..431.01 rows=334 width=1) (actual rows=340 loops=1)
                     ->  Seq Scan on auto_explain_test.t2  (cost=0.00..431.01 rows=334 width=1) (actual rows=340 loops=1002)
 Optimizer: GPORCA
-Settings: enable_nestloop = 'on'
+Settings: enable_nestloop = 'on', optimizer = 'on'
   (slice0)    Executor memory: 67K bytes.
   (slice1)    Executor memory: 119K bytes avg x 3 workers, 119K bytes max (seg0).
   (slice2)    Executor memory: 42K bytes avg x 3 workers, 42K bytes max (seg0).

contrib/pg_stat_statements/sql/disable_pg_stat_statements.sql

@@ -1,4 +1,4 @@
 -- start_ignore
-\! gpconfig -r shared_preload_libraries;
+\! gpconfig -c shared_preload_libraries -v "$(psql -At -c "SELECT array_to_string(array_remove(string_to_array(current_setting('shared_preload_libraries'), ','), 'pg_stat_statements'), ',')" postgres)"
 \! gpstop -raq -M fast;
 -- end_ignore

contrib/pg_stat_statements/sql/enable_pg_stat_statements.sql

@@ -1,4 +1,4 @@
 -- start_ignore
-\! gpconfig -c shared_preload_libraries -v 'pg_stat_statements';
+\! gpconfig -c shared_preload_libraries -v "$(psql -At -c "SELECT array_to_string(array_append(string_to_array(current_setting('shared_preload_libraries'), ','), 'pg_stat_statements'), ',')" postgres)"
 \! gpstop -raq -M fast;
 -- end_ignore

gpMgmt/bin/minirepro

@@ -297,7 +297,7 @@ def dump_extstats(cur, oid_str, f_out):
 
 def get_non_default_optimization_gucs(cursor):
     # Ignore 'optimizer' guc as it is turned off by minirepro while creating connection object
-    query = "select name, setting from pg_settings where category like 'Query Tuning%' and setting!=boot_val and name!='optimizer';"
+    query = "select name, setting from pg_settings where (category like 'Query Tuning%' or category='Customized Options') and setting!=boot_val and name!='optimizer';"
     try:
         cursor.execute(query)
     except psycopg2.DatabaseError as e:

gpcontrib/Makefile

@@ -36,6 +36,9 @@ endif
 ifeq "$(with_zstd)" "yes"
 	recurse_targets += zstd
 endif
+ifeq "$(enable_orca)" "yes"
+	recurse_targets += orca
+endif
 $(call recurse,all install clean distclean, $(recurse_targets))
 
 all: gpcloud orafce