You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by ji...@apache.org on 2015/10/23 09:53:44 UTC
incubator-hawq git commit: HAWQ-72. Introduce ADDON_DIR macro in
configure/makefile to allow custimizatin
Repository: incubator-hawq
Updated Branches:
refs/heads/master b10897824 -> fdd2ae17b
HAWQ-72. Introduce ADDON_DIR macro in configure/makefile to allow custimizatin
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fdd2ae17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fdd2ae17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fdd2ae17
Branch: refs/heads/master
Commit: fdd2ae17b4e46d1a50fb8c8a0953d681f226611a
Parents: b108978
Author: Lirong Jian <ji...@gmail.com>
Authored: Fri Oct 23 11:04:05 2015 +0800
Committer: Lirong Jian <ji...@gmail.com>
Committed: Fri Oct 23 11:04:26 2015 +0800
----------------------------------------------------------------------
configure | 6 +
configure.in | 5 +
src/Makefile.global.in | 3 +
src/backend/Makefile | 7 +-
src/backend/access/appendonly/appendonlyam.c | 5 -
src/backend/access/common/reloptions.c | 5 +-
src/backend/catalog/Makefile | 7 +-
src/backend/catalog/pg_compression.c | 210 ----------------------
src/backend/catalog/quicklz_compression.c | 58 ++++++
src/include/catalog/pg_compression.h | 2 +
src/include/catalog/pg_proc.h | 20 +++
src/include/storage/gp_compress.h | 5 -
12 files changed, 107 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 8dea6e1..309039f 100755
--- a/configure
+++ b/configure
@@ -714,6 +714,7 @@ ELF_SYS
EGREP
GREP
with_rt
+ADDON_DIR
with_zlib
with_system_tzdata
with_openssl
@@ -5986,6 +5987,11 @@ fi
#
+# addon
+#
+
+
+#
# Realtime library
#
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index 8c7ae10..fbf6552 100644
--- a/configure.in
+++ b/configure.in
@@ -805,6 +805,11 @@ PGAC_ARG_BOOL(with, zlib, yes,
AC_SUBST(with_zlib)
#
+# addon
+#
+AC_SUBST(ADDON_DIR)
+
+#
# Realtime library
#
PGAC_ARG_BOOL(with, rt, yes,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/Makefile.global.in
----------------------------------------------------------------------
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index a619d74..1e569d1 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -37,6 +37,9 @@ GP_MAJORVERSION = @GP_MAJORVERSION@
# HAWQ version number
HQ_MAJORVERSION = @HQ_MAJORVERSION@
+# addon
+ADDON_DIR = @ADDON_DIR@
+
# Support for VPATH builds
vpath_build = @vpath_build@
abs_top_srcdir = @abs_top_srcdir@
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 44f13a2..28bde84 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -14,9 +14,14 @@ subdir = src/backend
top_builddir = ../..
include $(top_builddir)/src/Makefile.global
+ifdef ADDON_DIR
+include $(ADDON_DIR)/src/Makefile
+endif
+
SUBDIRS = access bootstrap catalog parser commands executor foreign \
lib libpq gp_libpq_fe main nodes optimizer port postmaster regex \
- rewrite storage tcop utils resourcemanager $(top_builddir)/src/timezone cdb
+ rewrite storage tcop utils resourcemanager $(top_builddir)/src/timezone cdb \
+ $(ADDON_SUBDIR)
include $(srcdir)/common.mk
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/access/appendonly/appendonlyam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c
index 45a0035..d08e82f 100755
--- a/src/backend/access/appendonly/appendonlyam.c
+++ b/src/backend/access/appendonly/appendonlyam.c
@@ -75,11 +75,6 @@
#include "utils/builtins.h"
#include "utils/guc.h"
-#ifdef HAVE_QUICKLZ
-#include <quicklz1.h>
-#include <quicklz3.h>
-#endif
-
#ifdef HAVE_LIBZ
#include <zlib.h>
#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/access/common/reloptions.c
----------------------------------------------------------------------
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 679cc93..38b404a 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -517,7 +517,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
if (!compresstype_is_valid(compresstype))
{
-#ifndef HAVE_QUICKLZ
if (strcmp(compresstype, "quicklz") == 0)
{
ereport(ERROR,
@@ -526,7 +525,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
errOmitLocation(true)));
}
else
-#endif
{
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_OBJECT),
@@ -615,7 +613,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
compresstype = pstrdup(defaultParquetCompressor);
}
-#ifdef HAVE_QUICKLZ
if (compresstype && (pg_strcasecmp(compresstype, "quicklz") == 0) &&
(compresslevel != 1))
{
@@ -632,7 +629,7 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
compresslevel = setDefaultCompressionLevel(compresstype);
}
}
-#endif
+
if (compresstype && (pg_strcasecmp(compresstype, "rle_type") == 0) &&
(compresslevel > 4))
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/catalog/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index 342a6c6..015a3ab 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -10,13 +10,18 @@ subdir = src/backend/catalog
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
+ifndef ADDON_DIR
+QUICKLZ_COMPRESSION = quicklz_compression.o
+endif
+
OBJS = catalog.o dependency.o heap.o index.o indexing.o namespace.o aclchk.o \
gp_global_sequence.o gp_persistent.o \
pg_aggregate.o pg_appendonly.o pg_constraint.o pg_conversion.o pg_depend.o \
pg_exttable.o pg_extprotocol.o pg_filesystem.o pg_largeobject.o pg_namespace.o \
pg_operator.o pg_proc.o pg_proc_callback.o pg_shdepend.o \
pg_type.o toasting.o aoseg.o aoblkdir.o gp_fastsequence.o \
- pg_attribute_encoding.o pg_compression.o
+ pg_attribute_encoding.o pg_compression.o $(QUICKLZ_COMPRESSION)
+
SUBDIRS = caql core hcatalog
BKIFILES = postgres.bki postgres.description postgres.shdescription
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/catalog/pg_compression.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/pg_compression.c b/src/backend/catalog/pg_compression.c
index c340ee8..e9714db 100644
--- a/src/backend/catalog/pg_compression.c
+++ b/src/backend/catalog/pg_compression.c
@@ -37,33 +37,10 @@
#include "utils/relcache.h"
#include "utils/syscache.h"
-#ifdef HAVE_QUICKLZ
-#include <quicklz1.h>
-#include <quicklz3.h>
-#endif
-
/* names we expect to see in ENCODING clauses */
char *storage_directive_names[] = {"compresstype", "compresslevel",
"blocksize", NULL};
-#ifdef HAVE_QUICKLZ
-/* Internal state for quicklz */
-typedef struct quicklz_state
-{
- void *scratch;
-
- int level;
- bool compress;
-
- /*
- * The actual algorithms. Allows us to handle quicklz1 and quicklz3
- * conveniently.
- */
- size_t (*compress_fn)(int level, const void *, char *, size_t, void *);
- size_t (*decompress_fn)(int level, const char *, void *, void *);
-} quicklz_state;
-#endif
-
/* Internal state for zlib */
typedef struct zlib_state
{
@@ -211,193 +188,6 @@ callCompressionValidator(PGFunction func, char *comptype, int32 complevel,
(void)DirectFunctionCall1(func, PointerGetDatum(&sa));
}
-#ifdef HAVE_QUICKLZ
-/*
- * quicklz helper function.
- */
-static size_t
-quicklz_desired_sz(size_t input)
-{
- /*
- * From the QuickLZ manual:
- *
- * "The destination buffer must be at least size + 400 bytes large because
- * incompressible data may increase in size."
- *
- */
- return input + 400;
-}
-
-/*
- * Wrap up the qlz functions since C's type checking is getting in the
- * way.
- */
-static size_t
-quicklz_compressor(int level, const void *source, char *destination,
- size_t size, void *state)
-{
- if (level == 1)
- {
- return qlz1_compress(source, destination, size,
- (qlz1_state_compress *)state);
- }
- else if (level == 3)
- {
- return qlz3_compress(source, destination, size,
- (qlz3_state_compress *)state);
-
- }
- Insist(false);
-}
-
-static size_t
-quicklz_decompressor(int level, const char *source, void *destination,
- void *state)
-{
- if (level == 1)
- {
- return qlz1_decompress(source, destination,
- (qlz1_state_decompress *)state);
- }
- else if (level == 3)
- {
- return qlz3_decompress(source, destination,
- (qlz3_state_decompress *)state);
- }
- Insist(false);
-}
-
-/* ---------------------------------------------------------------------
- * Quicklz constructor and destructor
- * ---------------------------------------------------------------------
- */
-Datum
-quicklz_constructor(PG_FUNCTION_ARGS)
-{
- TupleDesc td = PG_GETARG_POINTER(0);
- StorageAttributes *sa = PG_GETARG_POINTER(1);
- CompressionState *cs = palloc0(sizeof(CompressionState));
- quicklz_state *state = palloc0(sizeof(quicklz_state));
- bool compress = PG_GETARG_BOOL(2);
- size_t scratchlen = 0;
-
- cs->opaque = (void *)state;
-
- Insist(PointerIsValid(td));
- Insist(PointerIsValid(sa->comptype));
- Insist(strcmp(sa->comptype, "quicklz") == 0);
- Insist(sa->complevel == 1 ||
- sa->complevel == 3);
-
- state->level = sa->complevel;
- state->compress = compress;
- if (sa->complevel == 1)
- {
- state->compress_fn = quicklz_compressor;
- state->decompress_fn = quicklz_decompressor;
- if (compress)
- scratchlen = sizeof(qlz1_state_compress);
- else
- scratchlen = sizeof(qlz1_state_decompress);
- }
- else if (sa->complevel == 3)
- {
- state->compress_fn = quicklz_compressor;
- state->decompress_fn = quicklz_decompressor;
-
- if (compress)
- scratchlen = sizeof(qlz3_state_compress);
- else
- scratchlen = sizeof(qlz3_state_decompress);
- }
- else
- Insist(false); /* shouldn't get here but code defensively */
-
- state->scratch = palloc0(scratchlen);
-
- cs->desired_sz = quicklz_desired_sz;
-
- PG_RETURN_POINTER(cs);
-}
-
-Datum
-quicklz_destructor(PG_FUNCTION_ARGS)
-{
- CompressionState *cs = PG_GETARG_POINTER(0);
- quicklz_state *state = (quicklz_state *) cs->opaque;
-
- Insist(PointerIsValid(cs->opaque));
-
- pfree(state->scratch);
- pfree(cs->opaque);
-
- PG_RETURN_VOID();
-}
-
-/* ---------------------------------------------------------------------
- * SQL invokable compression and decompression routines for built in
- * compression algorithms. All routines have the same SQL signature:
- *
- * void fun(internal, int, internal, int, internal, internal)
- *
- * If we were to think of this as a C function it would be more like:
- *
- * void fun(void *src, size_t src_sz, void *dst, size_t dst_sz,
- * size_t *dst_used, void *opaque)
- *
- * The meaning of each argument is as follows:
- * src - A pointer to data to be compressed/decompressed
- * src_sz - The number of bytes to compress/decompress
- * dst - A pointer to pre-allocated memory. The data compressed or
- * decompressed by the function are written here.
- * dst_sz - The amount of memory in bytes allocated at dst
- * dst_used - The number of bytes written. If dst_sz was too small to
- * store the data, this is set to zero.
- * opaque - Internal to the compression function.
- */
-Datum
-quicklz_compress(PG_FUNCTION_ARGS)
-{
- const void *src = PG_GETARG_POINTER(0);
- int32 src_sz = PG_GETARG_INT32(1);
- void *dst = PG_GETARG_POINTER(2);
- int32 dst_sz = PG_GETARG_INT32(3);
- int32 *dst_used = PG_GETARG_POINTER(4);
- CompressionState *cs = (CompressionState *)PG_GETARG_POINTER(5);
- quicklz_state *state = (quicklz_state *)cs->opaque;
-
- Insist(dst_sz >= quicklz_desired_sz(src_sz));
-
- *dst_used = state->compress_fn(state->level, src, dst, (size_t)src_sz,
- state->scratch);
-
- PG_RETURN_VOID();
-}
-
-Datum
-quicklz_decompress(PG_FUNCTION_ARGS)
-{
- const char *src = PG_GETARG_POINTER(0);
- int32 src_sz = PG_GETARG_INT32(1);
- void *dst = PG_GETARG_POINTER(2);
- int32 dst_sz = PG_GETARG_INT32(3);
- int32 *dst_used = PG_GETARG_POINTER(4);
- CompressionState *cs = (CompressionState *)PG_GETARG_POINTER(5);
- quicklz_state *state = (quicklz_state *)cs->opaque;
-
- Insist(src_sz > 0 && dst_sz > 0);
- *dst_used = state->decompress_fn(state->level, src, dst, state->scratch);
-
- PG_RETURN_VOID();
-}
-
-Datum
-quicklz_validator(PG_FUNCTION_ARGS)
-{
- PG_RETURN_VOID();
-}
-#endif
-
Datum
zlib_constructor(PG_FUNCTION_ARGS)
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/catalog/quicklz_compression.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/quicklz_compression.c b/src/backend/catalog/quicklz_compression.c
new file mode 100644
index 0000000..c6aabe5
--- /dev/null
+++ b/src/backend/catalog/quicklz_compression.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2011 EMC Corporation All Rights Reserved
+ *
+ * This software is protected, without limitation, by copyright law
+ * and international treaties. Use of this software and the intellectual
+ * property contained therein is expressly limited to the terms and
+ * conditions of the License Agreement under which it is provided by
+ * or on behalf of EMC.
+ *
+ * ---------------------------------------------------------------------
+ *
+ * The quicklz implementation is not provided due to licensing issues. The
+ * following stub implementation is built if a proprietary implementation is
+ * not provided.
+ *
+ * Note that other compression algorithms actually work better for new
+ * installations anyway.
+ */
+
+#include "postgres.h"
+#include "utils/builtins.h"
+
+Datum
+quicklz_constructor(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "quicklz compression not supported");
+ PG_RETURN_VOID();
+}
+
+Datum
+quicklz_destructor(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "quicklz compression not supported");
+ PG_RETURN_VOID();
+}
+
+Datum
+quicklz_compress(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "quicklz compression not supported");
+ PG_RETURN_VOID();
+}
+
+Datum
+quicklz_decompress(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "quicklz compression not supported");
+ PG_RETURN_VOID();
+}
+
+Datum
+quicklz_validator(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "quicklz compression not supported");
+ PG_RETURN_VOID();
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/include/catalog/pg_compression.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_compression.h b/src/include/catalog/pg_compression.h
index 6650eba..6c92be7 100644
--- a/src/include/catalog/pg_compression.h
+++ b/src/include/catalog/pg_compression.h
@@ -104,6 +104,8 @@ typedef FormData_pg_compression *Form_pg_compression;
/* Initial contents */
DATA(insert OID = 3060 ( zlib gp_zlib_constructor gp_zlib_destructor gp_zlib_compress gp_zlib_decompress gp_zlib_validator PGUID ));
+DATA(insert OID = 3061 ( quicklz gp_quicklz_constructor gp_quicklz_destructor gp_quicklz_compress gp_quicklz_decompress gp_quicklz_validator PGUID ));
+
DATA(insert OID = 3062 ( rle_type gp_rle_type_constructor gp_rle_type_destructor gp_rle_type_compress gp_rle_type_decompress gp_rle_type_validator PGUID ));
DATA(insert OID = 3063 ( none gp_dummy_compression_constructor gp_dummy_compression_destructor gp_dummy_compression_compress gp_dummy_compression_decompress gp_dummy_compression_validator PGUID ));
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/include/catalog/pg_proc.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index eeb7bd6..64b7661 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -10275,6 +10275,26 @@ DESCR("anytable type serialization input function");
DATA(insert OID = 3055 ( anytable_out PGNSP PGUID 12 f f t f i 1 2275 f "3053" _null_ _null_ _null_ anytable_out - _null_ n ));
DESCR("anytable type serialization output function");
+/* gp_quicklz_constructor(internal, internal, bool) => internal */
+DATA(insert OID = 5076 ( gp_quicklz_constructor PGNSP PGUID 12 f f f f v 3 2281 f "2281 2281 16" _null_ _null_ _null_ quicklz_constructor - _null_ n ));
+DESCR("quicklz constructor");
+
+/* gp_quicklz_destructor(internal) => void */
+DATA(insert OID = 5077 ( gp_quicklz_destructor PGNSP PGUID 12 f f f f v 1 2278 f "2281" _null_ _null_ _null_ quicklz_destructor - _null_ n ));
+DESCR("quicklz destructor");
+
+/* gp_quicklz_compress(internal, int4, internal, int4, internal, internal) => void */
+DATA(insert OID = 5078 ( gp_quicklz_compress PGNSP PGUID 12 f f f f i 6 2278 f "2281 23 2281 23 2281 2281" _null_ _null_ _null_ quicklz_compress - _null_ n ));
+DESCR("quicklz compressor");
+
+/* gp_quicklz_decompress(internal, int4, internal, int4, internal, internal) => void */
+DATA(insert OID = 5079 ( gp_quicklz_decompress PGNSP PGUID 12 f f f f i 6 2278 f "2281 23 2281 23 2281 2281" _null_ _null_ _null_ quicklz_decompress - _null_ n ));
+DESCR("quicklz decompressor");
+
+/* gp_quicklz_validator(internal) => void */
+DATA(insert OID = 9925 ( gp_quicklz_validator PGNSP PGUID 12 f f f f i 1 2278 f "2281" _null_ _null_ _null_ quicklz_validator - _null_ n ));
+DESCR("quicklz compression validator");
+
/* gp_zlib_constructor(internal, internal, bool) => internal */
DATA(insert OID = 9910 ( gp_zlib_constructor PGNSP PGUID 12 f f f f v 3 2281 f "2281 2281 16" _null_ _null_ _null_ zlib_constructor - _null_ n ));
DESCR("zlib constructor");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/include/storage/gp_compress.h
----------------------------------------------------------------------
diff --git a/src/include/storage/gp_compress.h b/src/include/storage/gp_compress.h
index a3df970..08484cf 100644
--- a/src/include/storage/gp_compress.h
+++ b/src/include/storage/gp_compress.h
@@ -12,11 +12,6 @@
#include "catalog/pg_compression.h"
-#ifdef HAVE_QUICKLZ
-#include <quicklz1.h>
-#include <quicklz3.h>
-#endif
-
#ifdef HAVE_LIBZ
#include <zlib.h>
#endif