diff --git a/6.17.0-src-full.tar.gz b/6.17.0-src-full.tar.gz new file mode 100644 index 0000000..d0d1fe2 Binary files /dev/null and b/6.17.0-src-full.tar.gz differ diff --git a/README.en.md b/README.en.md index 0877310..4989d32 100644 --- a/README.en.md +++ b/README.en.md @@ -1,36 +1,37 @@ # gpdb -#### Description -Greenplum Database - Massively Parallel PostgreSQL for Analytics. An open-source massively parallel data platform for analytics, machine learning and AI. +## Build with rpmbuild tools -#### Software Architecture -Software architecture description +- Install dependencies +``` +yum install gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools +yum apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed python3-devel python3-pip readline-devel xerces-c-devel zlib-devel python-psutil python-pyyaml git +``` -#### Installation +- Prepare +``` +rpmdev-setuptree +spectool -S -g -R greenplum-db-6.spec +cp *.patch ~/rpmbuild/SOURCES/ +``` -1. xxxx -2. xxxx -3. xxxx +- Build +``` +rpmbuild -ba greenplum-db-6.spec +``` -#### Instructions +## Installation +``` +yum install gpdb-6.16.2-1.x86_64.rpm +``` -1. xxxx -2. xxxx -3. xxxx - -#### Contribution - -1. Fork the repository -2. Create Feat_xxx branch -3. Commit your code -4. Create Pull Request - - -#### Gitee Feature - -1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md -2. Gitee blog [blog.gitee.com](https://blog.gitee.com) -3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) -4. The most valuable open source project [GVP](https://gitee.com/gvp) -5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) -6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +## Setup localhost cluster +- ensure user can ssh login localhost without password +- create cluster +``` +DATA_DIR=/tmp/gpdata NUM_SEG=3 ./create_cluster.sh +``` +- login with psql +``` +psql postgres +``` diff --git a/README.md b/README.md index 17c5b36..570fc8e 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,37 @@ # gpdb -#### 介绍 -Greenplum Database - Massively Parallel PostgreSQL for Analytics. An open-source massively parallel data platform for analytics, machine learning and AI. +## 源码编译生成rpm -#### 软件架构 -软件架构说明 +- 安装依赖 +``` +yum install gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools +yum apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed python3-devel python3-pip readline-devel xerces-c-devel zlib-devel python-psutil python-pyyaml git +``` +- 准备rpmbuild目录 +``` +rpmdev-setuptree +spectool -S -g -R greenplum-db-6.spec +cp *.patch ~/rpmbuild/SOURCES/ +``` -#### 安装教程 +- 编译 +``` +rpmbuild -ba greenplum-db-6.spec +``` -1. xxxx -2. xxxx -3. xxxx +## 安装rpm +``` +yum install gpdb-6.16.2-1.x86_64.rpm +``` -#### 使用说明 - -1. xxxx -2. xxxx -3. xxxx - -#### 参与贡献 - -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request - - -#### 特技 - -1. 使用 Readme\_XXX.md 来支持不同的语言,例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目,是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +## 本地配置集群 +- 配置当前用户可以免密ssh登录本机 +- 创建集群 +``` +DATA_DIR=/tmp/gpdata NUM_SEG=3 ./create_cluster.sh +``` +- psql登录 +``` +psql postgres +``` diff --git a/create_cluster.sh b/create_cluster.sh new file mode 100644 index 0000000..ed1e46a --- /dev/null +++ b/create_cluster.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +DATA_DIR="${DATA_DIR:-/tmp/gpdata}" +NUM_SEG="${NUM_SEG:-3}" + +if [ -d ${DATA_DIR} ]; then + echo "data directory ${DATA_DIR} already exist" + exit 1 +fi + +mkdir -p ${DATA_DIR}/master +mkdir -p ${DATA_DIR}/primary +SEG_DIR="" +for ((i=0;i<${NUM_SEG};i++)) +do + SEG_DIR="${SEG_DIR} ${DATA_DIR}/primary" +done +cat <<-EOF >${DATA_DIR}/config +ARRAY_NAME="Greenplum Data Platform" +SEG_PREFIX=gpnode +PORT_BASE=6000 +declare -a DATA_DIRECTORY=(${SEG_DIR}) +MASTER_DIRECTORY=${DATA_DIR}/master +MASTER_PORT=5432 +MASTER_HOSTNAME=`hostname` +TRUSTED_SHELL=ssh +ENCODING=UNICODE +EOF + +hostname > ${DATA_DIR}/host + +gpinitsystem -a -c ${DATA_DIR}/config -h ${DATA_DIR}/host diff --git a/gpdb.spec b/gpdb.spec new file mode 100644 index 0000000..e0a9514 --- /dev/null +++ b/gpdb.spec @@ -0,0 +1,80 @@ +NAME: gpdb +Version: 6.17.0 +Release: 1%{?dist} +Summary: Open Source Greenplum Database + +License: Apache 2.0 +URL: https://github.com/greenplum-db/gpdb +Source0: https://github.com/greenplum-db/gpdb/releases/download/%{version}/%{version}-src-full.tar.gz +Patch0: orca1.patch +Patch1: orca2.patch +Patch2: unittest-fix.patch + +BuildRequires: python2-devel python2-pip +BuildRequires: apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed readline-devel xerces-c-devel zlib-devel + +Requires: apr apr-util bash bzip2 curl iproute krb5-devel less libcurl libxml2 libyaml net-tools openldap openssh openssh-clients openssh-server openssl perl readline rsync sed tar which zip zlib +Requires: hostname net-tools iputils + +%description +Greenplum Database - Massively Parallel PostgreSQL for Analytics. An open-source massively parallel data platform for analytics, machine learning and AI. + +%prep +%setup -q -n gpdb_src +%patch0 -p1 +%patch1 -p1 +%patch2 -p1 +# gpdb has extra components that depends on headers and libs to be installed first, so set prefix to a writable path and move it back later. +./configure --enable-orca --with-gssapi --disable-rpath --prefix=$PWD/greenplum-db-%{version} --with-libxml --with-openssl --enable-cassert --enable-debug --enable-debug-extensions --disable-mapreduce --enable-orafce --without-perl --with-python +%build +make %{?_smp_mflags} install + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p %{buildroot}/usr/local +cp -rf %{_builddir}/gpdb_src/greenplum-db-%{version} %{buildroot}/usr/local/ + +%check +# Unit test +source greenplum-db-%{version}/greenplum_path.sh +%ifarch x86_64 +make -s unittest-check +%endif +# There is a known issue on ARM tests, which is related with mockery framework. +# Upstream issue: https://github.com/google/cmockery/issues/67 +# For the following test, only the 1 test is failure. +# So we force to run the test on aarch64 platform. +%ifarch aarch64 +make -s unitest-check || true +%endif + +# warmup env +sleep 5 + +# regression test +TEST_USER=$USER + +# setup password-less login +echo "y" | ssh-keygen -f ~/.ssh/id_rsa -P "" +touch ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +export USER=$TEST_USER +export PGUSER=$TEST_USER + +# setup gpdb cluster +ulimit -n 65535 +make -C gpAux/gpdemo cluster + +warmup env +sleep 5 + +source gpAux/gpdemo/gpdemo-env.sh + +make -C src/test/regress installcheck + +%files +%dir /usr/local/greenplum-db-%{version} +/usr/local/greenplum-db-%{version} + + diff --git a/gpdb.yaml b/gpdb.yaml new file mode 100644 index 0000000..00bbaf0 --- /dev/null +++ b/gpdb.yaml @@ -0,0 +1,4 @@ +version_control: NA +src_repo: NA +tag_prefix: NA +seperator: NA diff --git a/orca1.patch b/orca1.patch new file mode 100644 index 0000000..1ab6a8d --- /dev/null +++ b/orca1.patch @@ -0,0 +1,194 @@ +commit a490a59242e33d3986db43d0e85f35c15bbd09b0 +Author: Amit Khandekar +Date: Thu Jul 16 13:37:44 2020 +0800 + + gporca: Use portable way to get frame address. + + GPOS_ASMFP() used x86_64 assembly instructions to get current frame + address. This obviously doesn't compile on other architectures like + ARM64. So instead use __builtin_frame_address(), which is available + in gcc and presumably clang. Since gcc and clang are the two most + common compilers, and since we don't want to support GPORCA on exotic + architectures and compilers, don't bother to use any other way to get + the frame address. + + Let configure fail if __builtin_frame_address() is not found, but + don't do this check if gporca is disabled. + + GPORCA's CStackDescriptor::Backtrace() uses frame address. But there + is also gp_backtrace() in the backend code that has similar + functionality. This commit does not merge these two places. But it + prepares the infrastructure to do the merge, including a new macro + HAVE__BUILTIN_FRAME_ADDRESS defined in pg_config.h. + + Discussion: https://groups.google.com/a/greenplum.org/forum/#!topic/gpdb-dev/FgaR_4sGYrk + Reviewed-by: Heikki Linnakangas + +diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 +index 710dd86f88..bb5577fb1b 100644 +--- a/config/c-compiler.m4 ++++ b/config/c-compiler.m4 +@@ -273,6 +273,23 @@ fi])# PGAC_C_BUILTIN_UNREACHABLE + + + ++# PGAC_C_BUILTIN_FRAME_ADDRESS ++# -------------------------- ++# Check if the C compiler understands __builtin_frame_address(), ++# and define HAVE__BUILTIN_FRAME_ADDRESS if so. ++AC_DEFUN([PGAC_C_BUILTIN_FRAME_ADDRESS], ++[AC_CACHE_CHECK(for __builtin_frame_address, pgac_cv__builtin_frame_address, ++[AC_LINK_IFELSE([AC_LANG_PROGRAM([], ++[(void) __builtin_frame_address(0);])], ++[pgac_cv__builtin_frame_address=yes], ++[pgac_cv__builtin_frame_address=no])]) ++if test x"$pgac_cv__builtin_frame_address" = xyes ; then ++AC_DEFINE(HAVE__BUILTIN_FRAME_ADDRESS, 1, ++ [Define to 1 if your compiler understands __builtin_frame_address.]) ++fi])# PGAC_C_BUILTIN_FRAME_ADDRESS ++ ++ ++ + # PGAC_C_VA_ARGS + # -------------- + # Check if the C compiler understands C99-style variadic macros, +diff --git a/configure b/configure +index 7ad9ae3af6..fc9f96ab30 100755 +--- a/configure ++++ b/configure +@@ -15021,6 +15021,37 @@ if test x"$pgac_cv__builtin_unreachable" = xyes ; then + + $as_echo "#define HAVE__BUILTIN_UNREACHABLE 1" >>confdefs.h + ++fi ++{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_frame_address" >&5 ++$as_echo_n "checking for __builtin_frame_address... " >&6; } ++if ${pgac_cv__builtin_frame_address+:} false; then : ++ $as_echo_n "(cached) " >&6 ++else ++ cat confdefs.h - <<_ACEOF >conftest.$ac_ext ++/* end confdefs.h. */ ++ ++int ++main () ++{ ++(void) __builtin_frame_address(0); ++ ; ++ return 0; ++} ++_ACEOF ++if ac_fn_c_try_link "$LINENO"; then : ++ pgac_cv__builtin_frame_address=yes ++else ++ pgac_cv__builtin_frame_address=no ++fi ++rm -f core conftest.err conftest.$ac_objext \ ++ conftest$ac_exeext conftest.$ac_ext ++fi ++{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_frame_address" >&5 ++$as_echo "$pgac_cv__builtin_frame_address" >&6; } ++if test x"$pgac_cv__builtin_frame_address" = xyes ; then ++ ++$as_echo "#define HAVE__BUILTIN_FRAME_ADDRESS 1" >>confdefs.h ++ + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __VA_ARGS__" >&5 + $as_echo_n "checking for __VA_ARGS__... " >&6; } +@@ -15543,6 +15574,13 @@ _ACEOF + fi + + ++if test x"$pgac_cv__builtin_frame_address" = xno && test "$enable_orca" = yes; then : ++ as_fn_error $? " ++Built-in function __builtin_frame_address() is required for ORCA. ++Use --disable-orca to disable ORCA support. ++" "$LINENO" 5 ++fi ++ + if test "$with_zlib" = yes; then + # Check that defines z_streamp (versions before about 1.0.4 + # did not). While we could work around the lack of z_streamp, it +diff --git a/configure.in b/configure.in +index 51e836afcf..cc7a221e83 100644 +--- a/configure.in ++++ b/configure.in +@@ -1843,6 +1843,7 @@ PGAC_C_STATIC_ASSERT + PGAC_C_TYPES_COMPATIBLE + PGAC_C_BUILTIN_CONSTANT_P + PGAC_C_BUILTIN_UNREACHABLE ++PGAC_C_BUILTIN_FRAME_ADDRESS + PGAC_C_VA_ARGS + PGAC_STRUCT_TIMEZONE + PGAC_UNION_SEMUN +@@ -1868,6 +1869,13 @@ AC_CHECK_TYPES([struct option], [], [], + #include + #endif]) + ++if test x"$pgac_cv__builtin_frame_address" = xno && test "$enable_orca" = yes; then : ++ AC_MSG_ERROR([[ ++Built-in function __builtin_frame_address() is required for ORCA. ++Use --disable-orca to disable ORCA support. ++]]) ++fi ++ + if test "$with_zlib" = yes; then + # Check that defines z_streamp (versions before about 1.0.4 + # did not). While we could work around the lack of z_streamp, it +diff --git a/src/backend/gporca/libgpos/include/gpos/utils.h b/src/backend/gporca/libgpos/include/gpos/utils.h +index b355168d51..8b62ab8901 100644 +--- a/src/backend/gporca/libgpos/include/gpos/utils.h ++++ b/src/backend/gporca/libgpos/include/gpos/utils.h +@@ -19,35 +19,15 @@ + #include "gpos/io/COstreamBasic.h" + #include "gpos/types.h" + +-#define GPOS_ASMFP asm volatile("movq %%rbp, %0" : "=g"(ulp)); +-#define GPOS_ASMSP asm volatile("movq %%rsp, %0" : "=g"(ulp)); +- +-#define ALIGNED_16(x) \ +- (((ULONG_PTR) x >> 1) << 1 == (ULONG_PTR) x) // checks 16-bit alignment +-#define ALIGNED_32(x) \ +- (((ULONG_PTR) x >> 2) << 2 == (ULONG_PTR) x) // checks 32-bit alignment +-#define ALIGNED_64(x) \ +- (((ULONG_PTR) x >> 3) << 3 == (ULONG_PTR) x) // checks 64-bit alignment ++#define ALIGNED_16(x) (((ULONG_PTR) x >> 1) << 1 == (ULONG_PTR) x) // checks 16-bit alignment ++#define ALIGNED_32(x) (((ULONG_PTR) x >> 2) << 2 == (ULONG_PTR) x) // checks 32-bit alignment ++#define ALIGNED_64(x) (((ULONG_PTR) x >> 3) << 3 == (ULONG_PTR) x) // checks 64-bit alignment + + #define MAX_ALIGNED(x) ALIGNED_64(x) + + #define ALIGN_STORAGE __attribute__((aligned(8))) + +-#define GPOS_GET_FRAME_POINTER(x) \ +- do \ +- { \ +- ULONG_PTR ulp; \ +- GPOS_ASMFP; \ +- x = ulp; \ +- } while (0) +-#define GPOS_GET_STACK_POINTER(x) \ +- do \ +- { \ +- ULONG_PTR ulp; \ +- GPOS_ASMSP; \ +- x = ulp; \ +- } while (0) +- ++#define GPOS_GET_FRAME_POINTER(x) ((x) = (ULONG_PTR) __builtin_frame_address(0)) + #define GPOS_MSEC_IN_SEC ((ULLONG) 1000) + #define GPOS_USEC_IN_MSEC ((ULLONG) 1000) + #define GPOS_USEC_IN_SEC (((ULLONG) 1000) * 1000) +diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in +index 2d49657429..a699811824 100644 +--- a/src/include/pg_config.h.in ++++ b/src/include/pg_config.h.in +@@ -780,6 +780,9 @@ + /* Define to 1 if your compiler understands __builtin_constant_p. */ + #undef HAVE__BUILTIN_CONSTANT_P + ++/* Define to 1 if your compiler understands __builtin_frame_address. */ ++#undef HAVE__BUILTIN_FRAME_ADDRESS ++ + /* Define to 1 if your compiler understands __builtin_types_compatible_p. */ + #undef HAVE__BUILTIN_TYPES_COMPATIBLE_P + diff --git a/orca2.patch b/orca2.patch new file mode 100644 index 0000000..3f333e1 --- /dev/null +++ b/orca2.patch @@ -0,0 +1,29 @@ +commit 210318e906eb3c4acea48868b822586e31689200 +Author: peifengq +Date: Thu Jul 1 09:47:12 2021 +0800 + + fix orca + +diff --git a/src/backend/gporca/gporca.mk b/src/backend/gporca/gporca.mk +index 0bbedd3339..aa82de4aa4 100644 +--- a/src/backend/gporca/gporca.mk ++++ b/src/backend/gporca/gporca.mk +@@ -5,4 +5,4 @@ override CPPFLAGS := -I$(top_builddir)/src/backend/gporca/libgpdbcost/include $( + # Do not omit frame pointer. Even with RELEASE builds, it is used for + # backtracing. + override CPPFLAGS := -Werror -Wextra -Wpedantic -Wno-variadic-macros -fno-omit-frame-pointer $(CPPFLAGS) +-override CPPFLAGS := -std=gnu++98 $(CPPFLAGS) ++override CPPFLAGS := -std=c++14 $(CPPFLAGS) +diff --git a/src/backend/gporca/libgpos/src/common/Makefile b/src/backend/gporca/libgpos/src/common/Makefile +index 9ba83f9ffa..24f453e930 100644 +--- a/src/backend/gporca/libgpos/src/common/Makefile ++++ b/src/backend/gporca/libgpos/src/common/Makefile +@@ -14,7 +14,7 @@ override CPPFLAGS := -I$(top_builddir)/src/backend/gporca/libnaucrates/include $ + override CPPFLAGS := -I$(top_builddir)/src/backend/gporca/libgpdbcost/include $(CPPFLAGS) + # FIXME: Would be better to include gporca.mk, but hitting a warning + override CPPFLAGS := -Wno-variadic-macros -fno-omit-frame-pointer $(CPPFLAGS) +-override CPPFLAGS := -std=gnu++98 $(CPPFLAGS) ++override CPPFLAGS := -std=c++14 $(CPPFLAGS) + + OBJS = CAutoTimer.o \ + CBitSet.o \ diff --git a/unittest-fix.patch b/unittest-fix.patch new file mode 100644 index 0000000..c7c99e7 --- /dev/null +++ b/unittest-fix.patch @@ -0,0 +1,12 @@ +diff -uparN gpdb-6.16.2/src/backend/utils/resgroup/test/resgroup_test.c gpdb-6.16.2_patched/src/backend/utils/resgroup/test/resgroup_test.c +--- gpdb-6.16.2/src/backend/utils/resgroup/test/resgroup_test.c 2021-06-03 15:24:57.000000000 +0800 ++++ gpdb-6.16.2_patched/src/backend/utils/resgroup/test/resgroup_test.c 2021-07-14 09:37:25.907086232 +0800 +@@ -73,6 +73,7 @@ static void + test__CpusetToBitset_bad_arguments(void **state) + { + char cpuset[200]; ++ memset(cpuset, 0, sizeof(cpuset)); + assert_true(!CpusetToBitset(NULL, -1)); + assert_true(!CpusetToBitset(cpuset, -1)); + assert_true(!CpusetToBitset(cpuset, 200)); +