Add open source gpdb 6.17.0

Add rpmbuild spec, patches, and startup scripts for open source
gpdb 6.17.0.
This commit is contained in:
peifengq 2021-07-21 12:55:09 +00:00
parent 73e2c3ab64
commit 7914017577
9 changed files with 411 additions and 59 deletions

BIN
6.17.0-src-full.tar.gz Normal file

Binary file not shown.

View File

@ -1,36 +1,37 @@
# gpdb
#### Description
Greenplum Database - Massively Parallel PostgreSQL for Analytics. An open-source massively parallel data platform for analytics, machine learning and AI.
## Build with rpmbuild tools
#### Software Architecture
Software architecture description
- Install dependencies
```
yum install gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools
yum apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed python3-devel python3-pip readline-devel xerces-c-devel zlib-devel python-psutil python-pyyaml git
```
#### Installation
- Prepare
```
rpmdev-setuptree
spectool -S -g -R greenplum-db-6.spec
cp *.patch ~/rpmbuild/SOURCES/
```
1. xxxx
2. xxxx
3. xxxx
- Build
```
rpmbuild -ba greenplum-db-6.spec
```
#### Instructions
## Installation
```
yum install gpdb-6.16.2-1.x86_64.rpm
```
1. xxxx
2. xxxx
3. xxxx
#### Contribution
1. Fork the repository
2. Create Feat_xxx branch
3. Commit your code
4. Create Pull Request
#### Gitee Feature
1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md
2. Gitee blog [blog.gitee.com](https://blog.gitee.com)
3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore)
4. The most valuable open source project [GVP](https://gitee.com/gvp)
5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help)
6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)
## Setup localhost cluster
- ensure user can ssh login localhost without password
- create cluster
```
DATA_DIR=/tmp/gpdata NUM_SEG=3 ./create_cluster.sh
```
- login with psql
```
psql postgres
```

View File

@ -1,37 +1,37 @@
# gpdb
#### 介绍
Greenplum Database - Massively Parallel PostgreSQL for Analytics. An open-source massively parallel data platform for analytics, machine learning and AI.
## 源码编译生成rpm
#### 软件架构
软件架构说明
- 安装依赖
```
yum install gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools
yum apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed python3-devel python3-pip readline-devel xerces-c-devel zlib-devel python-psutil python-pyyaml git
```
- 准备rpmbuild目录
```
rpmdev-setuptree
spectool -S -g -R greenplum-db-6.spec
cp *.patch ~/rpmbuild/SOURCES/
```
#### 安装教程
- 编译
```
rpmbuild -ba greenplum-db-6.spec
```
1. xxxx
2. xxxx
3. xxxx
## 安装rpm
```
yum install gpdb-6.16.2-1.x86_64.rpm
```
#### 使用说明
1. xxxx
2. xxxx
3. xxxx
#### 参与贡献
1. Fork 本仓库
2. 新建 Feat_xxx 分支
3. 提交代码
4. 新建 Pull Request
#### 特技
1. 使用 Readme\_XXX.md 来支持不同的语言,例如 Readme\_en.md, Readme\_zh.md
2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com)
3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目
4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目,是综合评定出的优秀开源项目
5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help)
6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)
## 本地配置集群
- 配置当前用户可以免密ssh登录本机
- 创建集群
```
DATA_DIR=/tmp/gpdata NUM_SEG=3 ./create_cluster.sh
```
- psql登录
```
psql postgres
```

32
create_cluster.sh Normal file
View File

@ -0,0 +1,32 @@
#!/bin/bash
DATA_DIR="${DATA_DIR:-/tmp/gpdata}"
NUM_SEG="${NUM_SEG:-3}"
if [ -d ${DATA_DIR} ]; then
echo "data directory ${DATA_DIR} already exist"
exit 1
fi
mkdir -p ${DATA_DIR}/master
mkdir -p ${DATA_DIR}/primary
SEG_DIR=""
for ((i=0;i<${NUM_SEG};i++))
do
SEG_DIR="${SEG_DIR} ${DATA_DIR}/primary"
done
cat <<-EOF >${DATA_DIR}/config
ARRAY_NAME="Greenplum Data Platform"
SEG_PREFIX=gpnode
PORT_BASE=6000
declare -a DATA_DIRECTORY=(${SEG_DIR})
MASTER_DIRECTORY=${DATA_DIR}/master
MASTER_PORT=5432
MASTER_HOSTNAME=`hostname`
TRUSTED_SHELL=ssh
ENCODING=UNICODE
EOF
hostname > ${DATA_DIR}/host
gpinitsystem -a -c ${DATA_DIR}/config -h ${DATA_DIR}/host

80
gpdb.spec Normal file
View File

@ -0,0 +1,80 @@
NAME: gpdb
Version: 6.17.0
Release: 1%{?dist}
Summary: Open Source Greenplum Database
License: Apache 2.0
URL: https://github.com/greenplum-db/gpdb
Source0: https://github.com/greenplum-db/gpdb/releases/download/%{version}/%{version}-src-full.tar.gz
Patch0: orca1.patch
Patch1: orca2.patch
Patch2: unittest-fix.patch
BuildRequires: python2-devel python2-pip
BuildRequires: apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libyaml-devel libxml2-devel libzstd-devel openssl-devel perl-ExtUtils-Embed readline-devel xerces-c-devel zlib-devel
Requires: apr apr-util bash bzip2 curl iproute krb5-devel less libcurl libxml2 libyaml net-tools openldap openssh openssh-clients openssh-server openssl perl readline rsync sed tar which zip zlib
Requires: hostname net-tools iputils
%description
Greenplum Database - Massively Parallel PostgreSQL for Analytics. An open-source massively parallel data platform for analytics, machine learning and AI.
%prep
%setup -q -n gpdb_src
%patch0 -p1
%patch1 -p1
%patch2 -p1
# gpdb has extra components that depends on headers and libs to be installed first, so set prefix to a writable path and move it back later.
./configure --enable-orca --with-gssapi --disable-rpath --prefix=$PWD/greenplum-db-%{version} --with-libxml --with-openssl --enable-cassert --enable-debug --enable-debug-extensions --disable-mapreduce --enable-orafce --without-perl --with-python
%build
make %{?_smp_mflags} install
%install
rm -rf $RPM_BUILD_ROOT
mkdir -p %{buildroot}/usr/local
cp -rf %{_builddir}/gpdb-%{version}/greenplum-db-%{version} %{buildroot}/usr/local/
%check
# Unit test
source greenplum-db-%{version}/greenplum_path.sh
%ifarch x86_64
make -s unittest-check
%endif
# There is a known issue on ARM tests, which is related with mockery framework.
# Upstream issue: https://github.com/google/cmockery/issues/67
# For the following test, only the 1 test is failure.
# So we force to run the test on aarch64 platform.
%ifarch aarch64
make -s unitest-check || true
%endif
# warmup env
sleep 5
# regression test
TEST_USER=$USER
# setup password-less login
echo "y" | ssh-keygen -f ~/.ssh/id_rsa -P ""
touch ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
export USER=$TEST_USER
export PGUSER=$TEST_USER
# setup gpdb cluster
ulimit -n 65535
make -C gpAux/gpdemo cluster
warmup env
sleep 5
source gpAux/gpdemo/gpdemo-env.sh
make -C src/test/regress installcheck
%files
%dir /usr/local/greenplum-db-%{version}
/usr/local/greenplum-db-%{version}

4
gpdb.yaml Normal file
View File

@ -0,0 +1,4 @@
version_control: NA
src_repo: NA
tag_prefix: NA
seperator: NA

194
orca1.patch Normal file
View File

@ -0,0 +1,194 @@
commit a490a59242e33d3986db43d0e85f35c15bbd09b0
Author: Amit Khandekar <amitdkhan.pg@gmail.com>
Date: Thu Jul 16 13:37:44 2020 +0800
gporca: Use portable way to get frame address.
GPOS_ASMFP() used x86_64 assembly instructions to get current frame
address. This obviously doesn't compile on other architectures like
ARM64. So instead use __builtin_frame_address(), which is available
in gcc and presumably clang. Since gcc and clang are the two most
common compilers, and since we don't want to support GPORCA on exotic
architectures and compilers, don't bother to use any other way to get
the frame address.
Let configure fail if __builtin_frame_address() is not found, but
don't do this check if gporca is disabled.
GPORCA's CStackDescriptor::Backtrace() uses frame address. But there
is also gp_backtrace() in the backend code that has similar
functionality. This commit does not merge these two places. But it
prepares the infrastructure to do the merge, including a new macro
HAVE__BUILTIN_FRAME_ADDRESS defined in pg_config.h.
Discussion: https://groups.google.com/a/greenplum.org/forum/#!topic/gpdb-dev/FgaR_4sGYrk
Reviewed-by: Heikki Linnakangas <hlinnakangas@pivotal.io>
diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 710dd86f88..bb5577fb1b 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -273,6 +273,23 @@ fi])# PGAC_C_BUILTIN_UNREACHABLE
+# PGAC_C_BUILTIN_FRAME_ADDRESS
+# --------------------------
+# Check if the C compiler understands __builtin_frame_address(),
+# and define HAVE__BUILTIN_FRAME_ADDRESS if so.
+AC_DEFUN([PGAC_C_BUILTIN_FRAME_ADDRESS],
+[AC_CACHE_CHECK(for __builtin_frame_address, pgac_cv__builtin_frame_address,
+[AC_LINK_IFELSE([AC_LANG_PROGRAM([],
+[(void) __builtin_frame_address(0);])],
+[pgac_cv__builtin_frame_address=yes],
+[pgac_cv__builtin_frame_address=no])])
+if test x"$pgac_cv__builtin_frame_address" = xyes ; then
+AC_DEFINE(HAVE__BUILTIN_FRAME_ADDRESS, 1,
+ [Define to 1 if your compiler understands __builtin_frame_address.])
+fi])# PGAC_C_BUILTIN_FRAME_ADDRESS
+
+
+
# PGAC_C_VA_ARGS
# --------------
# Check if the C compiler understands C99-style variadic macros,
diff --git a/configure b/configure
index 7ad9ae3af6..fc9f96ab30 100755
--- a/configure
+++ b/configure
@@ -15021,6 +15021,37 @@ if test x"$pgac_cv__builtin_unreachable" = xyes ; then
$as_echo "#define HAVE__BUILTIN_UNREACHABLE 1" >>confdefs.h
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_frame_address" >&5
+$as_echo_n "checking for __builtin_frame_address... " >&6; }
+if ${pgac_cv__builtin_frame_address+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+(void) __builtin_frame_address(0);
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv__builtin_frame_address=yes
+else
+ pgac_cv__builtin_frame_address=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_frame_address" >&5
+$as_echo "$pgac_cv__builtin_frame_address" >&6; }
+if test x"$pgac_cv__builtin_frame_address" = xyes ; then
+
+$as_echo "#define HAVE__BUILTIN_FRAME_ADDRESS 1" >>confdefs.h
+
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __VA_ARGS__" >&5
$as_echo_n "checking for __VA_ARGS__... " >&6; }
@@ -15543,6 +15574,13 @@ _ACEOF
fi
+if test x"$pgac_cv__builtin_frame_address" = xno && test "$enable_orca" = yes; then :
+ as_fn_error $? "
+Built-in function __builtin_frame_address() is required for ORCA.
+Use --disable-orca to disable ORCA support.
+" "$LINENO" 5
+fi
+
if test "$with_zlib" = yes; then
# Check that <zlib.h> defines z_streamp (versions before about 1.0.4
# did not). While we could work around the lack of z_streamp, it
diff --git a/configure.in b/configure.in
index 51e836afcf..cc7a221e83 100644
--- a/configure.in
+++ b/configure.in
@@ -1843,6 +1843,7 @@ PGAC_C_STATIC_ASSERT
PGAC_C_TYPES_COMPATIBLE
PGAC_C_BUILTIN_CONSTANT_P
PGAC_C_BUILTIN_UNREACHABLE
+PGAC_C_BUILTIN_FRAME_ADDRESS
PGAC_C_VA_ARGS
PGAC_STRUCT_TIMEZONE
PGAC_UNION_SEMUN
@@ -1868,6 +1869,13 @@ AC_CHECK_TYPES([struct option], [], [],
#include <getopt.h>
#endif])
+if test x"$pgac_cv__builtin_frame_address" = xno && test "$enable_orca" = yes; then :
+ AC_MSG_ERROR([[
+Built-in function __builtin_frame_address() is required for ORCA.
+Use --disable-orca to disable ORCA support.
+]])
+fi
+
if test "$with_zlib" = yes; then
# Check that <zlib.h> defines z_streamp (versions before about 1.0.4
# did not). While we could work around the lack of z_streamp, it
diff --git a/src/backend/gporca/libgpos/include/gpos/utils.h b/src/backend/gporca/libgpos/include/gpos/utils.h
index b355168d51..8b62ab8901 100644
--- a/src/backend/gporca/libgpos/include/gpos/utils.h
+++ b/src/backend/gporca/libgpos/include/gpos/utils.h
@@ -19,35 +19,15 @@
#include "gpos/io/COstreamBasic.h"
#include "gpos/types.h"
-#define GPOS_ASMFP asm volatile("movq %%rbp, %0" : "=g"(ulp));
-#define GPOS_ASMSP asm volatile("movq %%rsp, %0" : "=g"(ulp));
-
-#define ALIGNED_16(x) \
- (((ULONG_PTR) x >> 1) << 1 == (ULONG_PTR) x) // checks 16-bit alignment
-#define ALIGNED_32(x) \
- (((ULONG_PTR) x >> 2) << 2 == (ULONG_PTR) x) // checks 32-bit alignment
-#define ALIGNED_64(x) \
- (((ULONG_PTR) x >> 3) << 3 == (ULONG_PTR) x) // checks 64-bit alignment
+#define ALIGNED_16(x) (((ULONG_PTR) x >> 1) << 1 == (ULONG_PTR) x) // checks 16-bit alignment
+#define ALIGNED_32(x) (((ULONG_PTR) x >> 2) << 2 == (ULONG_PTR) x) // checks 32-bit alignment
+#define ALIGNED_64(x) (((ULONG_PTR) x >> 3) << 3 == (ULONG_PTR) x) // checks 64-bit alignment
#define MAX_ALIGNED(x) ALIGNED_64(x)
#define ALIGN_STORAGE __attribute__((aligned(8)))
-#define GPOS_GET_FRAME_POINTER(x) \
- do \
- { \
- ULONG_PTR ulp; \
- GPOS_ASMFP; \
- x = ulp; \
- } while (0)
-#define GPOS_GET_STACK_POINTER(x) \
- do \
- { \
- ULONG_PTR ulp; \
- GPOS_ASMSP; \
- x = ulp; \
- } while (0)
-
+#define GPOS_GET_FRAME_POINTER(x) ((x) = (ULONG_PTR) __builtin_frame_address(0))
#define GPOS_MSEC_IN_SEC ((ULLONG) 1000)
#define GPOS_USEC_IN_MSEC ((ULLONG) 1000)
#define GPOS_USEC_IN_SEC (((ULLONG) 1000) * 1000)
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 2d49657429..a699811824 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -780,6 +780,9 @@
/* Define to 1 if your compiler understands __builtin_constant_p. */
#undef HAVE__BUILTIN_CONSTANT_P
+/* Define to 1 if your compiler understands __builtin_frame_address. */
+#undef HAVE__BUILTIN_FRAME_ADDRESS
+
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P

29
orca2.patch Normal file
View File

@ -0,0 +1,29 @@
commit 210318e906eb3c4acea48868b822586e31689200
Author: peifengq <qiupf2000@gmail.com>
Date: Thu Jul 1 09:47:12 2021 +0800
fix orca
diff --git a/src/backend/gporca/gporca.mk b/src/backend/gporca/gporca.mk
index 0bbedd3339..aa82de4aa4 100644
--- a/src/backend/gporca/gporca.mk
+++ b/src/backend/gporca/gporca.mk
@@ -5,4 +5,4 @@ override CPPFLAGS := -I$(top_builddir)/src/backend/gporca/libgpdbcost/include $(
# Do not omit frame pointer. Even with RELEASE builds, it is used for
# backtracing.
override CPPFLAGS := -Werror -Wextra -Wpedantic -Wno-variadic-macros -fno-omit-frame-pointer $(CPPFLAGS)
-override CPPFLAGS := -std=gnu++98 $(CPPFLAGS)
+override CPPFLAGS := -std=c++14 $(CPPFLAGS)
diff --git a/src/backend/gporca/libgpos/src/common/Makefile b/src/backend/gporca/libgpos/src/common/Makefile
index 9ba83f9ffa..24f453e930 100644
--- a/src/backend/gporca/libgpos/src/common/Makefile
+++ b/src/backend/gporca/libgpos/src/common/Makefile
@@ -14,7 +14,7 @@ override CPPFLAGS := -I$(top_builddir)/src/backend/gporca/libnaucrates/include $
override CPPFLAGS := -I$(top_builddir)/src/backend/gporca/libgpdbcost/include $(CPPFLAGS)
# FIXME: Would be better to include gporca.mk, but hitting a warning
override CPPFLAGS := -Wno-variadic-macros -fno-omit-frame-pointer $(CPPFLAGS)
-override CPPFLAGS := -std=gnu++98 $(CPPFLAGS)
+override CPPFLAGS := -std=c++14 $(CPPFLAGS)
OBJS = CAutoTimer.o \
CBitSet.o \

12
unittest-fix.patch Normal file
View File

@ -0,0 +1,12 @@
diff -uparN gpdb-6.16.2/src/backend/utils/resgroup/test/resgroup_test.c gpdb-6.16.2_patched/src/backend/utils/resgroup/test/resgroup_test.c
--- gpdb-6.16.2/src/backend/utils/resgroup/test/resgroup_test.c 2021-06-03 15:24:57.000000000 +0800
+++ gpdb-6.16.2_patched/src/backend/utils/resgroup/test/resgroup_test.c 2021-07-14 09:37:25.907086232 +0800
@@ -73,6 +73,7 @@ static void
test__CpusetToBitset_bad_arguments(void **state)
{
char cpuset[200];
+ memset(cpuset, 0, sizeof(cpuset));
assert_true(!CpusetToBitset(NULL, -1));
assert_true(!CpusetToBitset(cpuset, -1));
assert_true(!CpusetToBitset(cpuset, 200));