You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by ml...@apache.org on 2016/02/16 02:33:19 UTC
[1/5] incubator-hawq git commit: HAWQ-396. Fixed crashed at nested
error report calling when close temp file at AbortTransaction()
Repository: incubator-hawq
Updated Branches:
refs/heads/master 111586b0f -> f58baae01
HAWQ-396. Fixed crashed at nested error report calling when close temp file at AbortTransaction()
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/51999827
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/51999827
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/51999827
Branch: refs/heads/master
Commit: 51999827eb643f44897fcf9a4abbbf28deaa8fe7
Parents: 111586b
Author: Ming LI <ml...@pivotal.io>
Authored: Thu Feb 4 11:47:21 2016 +0800
Committer: Ming LI <ml...@pivotal.io>
Committed: Tue Feb 16 09:31:25 2016 +0800
----------------------------------------------------------------------
src/backend/executor/execWorkfile.c | 2 +-
src/backend/storage/file/bfz.c | 6 +++---
src/backend/storage/file/fd.c | 20 ++++++++------------
src/include/storage/bfz.h | 2 +-
src/include/storage/fd.h | 2 +-
5 files changed, 14 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/51999827/src/backend/executor/execWorkfile.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/execWorkfile.c b/src/backend/executor/execWorkfile.c
index 05adea8..ca65c42 100644
--- a/src/backend/executor/execWorkfile.c
+++ b/src/backend/executor/execWorkfile.c
@@ -463,7 +463,7 @@ ExecWorkFile_Close(ExecWorkFile *workfile)
ExecWorkFile_AdjustBFZSize(workfile, file_size);
}
- bfz_close(bfz_file, true);
+ bfz_close(bfz_file, true, true);
break;
default:
insist_log(false, "invalid work file type: %d", workfile->fileType);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/51999827/src/backend/storage/file/bfz.c
----------------------------------------------------------------------
diff --git a/src/backend/storage/file/bfz.c b/src/backend/storage/file/bfz.c
index 3654611..0599896 100644
--- a/src/backend/storage/file/bfz.c
+++ b/src/backend/storage/file/bfz.c
@@ -81,7 +81,7 @@ bfz_string_to_compression(const char *string)
static void
bfz_close_callback(XactEvent event, void *arg)
{
- bfz_close(arg, false);
+ bfz_close(arg, false, (event!=XACT_EVENT_ABORT));
}
#define BFZ_CHECKSUM_EQ(c1, c2) EQ_CRC32(c1, c2)
@@ -438,7 +438,7 @@ bfz_create_internal(bfz_t *bfz_handle, const char *fileName, bool open_existing,
}
void
-bfz_close(bfz_t * thiz, bool unreg)
+bfz_close(bfz_t * thiz, bool unreg, bool canReportError)
{
if (unreg)
UnregisterXactCallbackOnce(bfz_close_callback, thiz);
@@ -461,7 +461,7 @@ bfz_close(bfz_t * thiz, bool unreg)
if (thiz->del_on_close && thiz->filename != NULL)
{
if (unlink(thiz->filename))
- ereport(ERROR,
+ ereport(canReportError?ERROR:WARNING,
(errcode(ERRCODE_IO_ERROR),
errmsg("could not close temporary file %s: %m", thiz->filename)));
pfree(thiz->filename);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/51999827/src/backend/storage/file/fd.c
----------------------------------------------------------------------
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index 5729941..1037a66 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -1209,7 +1209,7 @@ OpenNamedFile(const char *fileName,
* close a file when done with it
*/
void
-LocalFileClose(File file)
+LocalFileClose(File file, bool canReportError)
{
Vfd *vfdP;
@@ -1227,7 +1227,7 @@ LocalFileClose(File file)
/* close the file */
if (gp_retry_close(vfdP->fd))
- elog(ERROR, "could not close file \"%s\": %m",
+ elog(canReportError?ERROR:WARNING, "could not close file \"%s\": %m",
vfdP->fileName);
--nfile;
@@ -2130,7 +2130,7 @@ CleanupTempFiles(bool isProcExit)
{
AssertImply( (fdstate & FD_TEMPORARY), VfdCache[i].fileName != NULL);
if (IsLocalPath(VfdCache[i].fileName))
- LocalFileClose(i);
+ LocalFileClose(i, false);
else
HdfsFileClose(i, false);
}
@@ -2725,14 +2725,10 @@ HdfsFileClose(File file, bool canReportError)
if (retval == -1)
{
- /* do not disconnect. */
- if (canReportError)
- {
- ereport(ERROR,
- (errcode(ERRCODE_IO_ERROR),
- errmsg("could not close file %d : (%s) errno %d", file, fileName, errno),
- errdetail("%s", HdfsGetLastError())));
- }
+ ereport(canReportError?ERROR:WARNING,
+ (errcode(ERRCODE_IO_ERROR),
+ errmsg("could not close file %d : (%s) errno %d", file, fileName, errno),
+ errdetail("%s", HdfsGetLastError())));
}
}
@@ -3160,7 +3156,7 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode) {
void
FileClose(File file) {
if (IsLocalPath(VfdCache[file].fileName))
- LocalFileClose(file);
+ LocalFileClose(file, true);
else
HdfsFileClose(file, true);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/51999827/src/include/storage/bfz.h
----------------------------------------------------------------------
diff --git a/src/include/storage/bfz.h b/src/include/storage/bfz.h
index 27e718a..ca9b85d 100644
--- a/src/include/storage/bfz.h
+++ b/src/include/storage/bfz.h
@@ -97,7 +97,7 @@ extern bfz_t *bfz_create(const char *filePrefix, bool delOnClose, int compress);
extern bfz_t *bfz_open(const char *fileName, bool delOnClose, int compress);
extern int64 bfz_append_end(bfz_t * thiz);
extern void bfz_scan_begin(bfz_t * thiz);
-extern void bfz_close(bfz_t * thiz, bool unreg);
+extern void bfz_close(bfz_t * thiz, bool unreg, bool canReportError);
extern ssize_t readAndRetry(int fd, void *buffer, size_t size);
extern ssize_t writeAndRetry(int fd, const void *buffer, size_t size);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/51999827/src/include/storage/fd.h
----------------------------------------------------------------------
diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h
index 2015a56..d611894 100644
--- a/src/include/storage/fd.h
+++ b/src/include/storage/fd.h
@@ -82,7 +82,7 @@ extern bool enable_secure_filesystem;
/* access local file system */
extern File LocalPathNameOpenFile(FileName fileName, int fileFlags, int fileMode);
-extern void LocalFileClose(File file);
+extern void LocalFileClose(File file, bool canReportError);
extern int LocalFileRead(File file, char *buffer, int amount);
extern int LocalFileWrite(File file, const char *buffer, int amount);
extern int64 LocalFileSeek(File file, int64 offset, int whence);
[5/5] incubator-hawq git commit: HAWQ-49. Remove madlib related codes
and tests.
Posted by ml...@apache.org.
HAWQ-49. Remove madlib related codes and tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f58baae0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f58baae0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f58baae0
Branch: refs/heads/master
Commit: f58baae0194fb94fc83b1dc7b8e3c7f5994ad115
Parents: 5199982
Author: Ming LI <ml...@pivotal.io>
Authored: Mon Feb 15 15:39:07 2016 +0800
Committer: Ming LI <ml...@pivotal.io>
Committed: Tue Feb 16 09:31:57 2016 +0800
----------------------------------------------------------------------
GNUmakefile.in | 4 -
contrib/gp_sparse_vector/Makefile | 38 -
contrib/gp_sparse_vector/README | 132 --
contrib/gp_sparse_vector/README.bitmap | 259 ---
contrib/gp_sparse_vector/README.devel | 63 -
contrib/gp_sparse_vector/README.update | 72 -
contrib/gp_sparse_vector/README_INSTALL | 9 -
contrib/gp_sparse_vector/README_term_proximity | 118 --
contrib/gp_sparse_vector/SparseData.c | 1572 -------------------
contrib/gp_sparse_vector/SparseData.h | 234 ---
contrib/gp_sparse_vector/bugs | 122 --
contrib/gp_sparse_vector/float_specials.h | 171 --
contrib/gp_sparse_vector/operators.c | 939 -----------
contrib/gp_sparse_vector/sfv_test_output | 141 --
contrib/gp_sparse_vector/sparse_vector.c | 411 -----
contrib/gp_sparse_vector/sparse_vector.h | 175 ---
contrib/gp_sparse_vector/test_output | 296 ----
pom.xml | 2 -
src/backend/catalog/Makefile | 2 +-
src/backend/catalog/madlib.sql | 492 ------
src/test/regress/expected/madlib_svec_test.out | 236 ---
src/test/regress/known_good_schedule | 2 -
src/test/regress/sql/madlib_svec_test.sql | 56 -
src/tools/msvc/Mkvcbuild.pm | 2 +-
24 files changed, 2 insertions(+), 5546 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/GNUmakefile.in
----------------------------------------------------------------------
diff --git a/GNUmakefile.in b/GNUmakefile.in
index 40e887b..f20ee2c 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -14,7 +14,6 @@ all:
$(MAKE) -C config all
$(MAKE) -C contrib/formatter_fixedwidth all
$(MAKE) -C contrib/extprotocol all
- $(MAKE) -C contrib/gp_sparse_vector all
$(MAKE) -C contrib/orafce all
$(MAKE) -C contrib/gp_cancel_query all
$(MAKE) -C contrib/hawq-hadoop all
@@ -27,7 +26,6 @@ install:
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
$(MAKE) -C contrib/extprotocol $@
- $(MAKE) -C contrib/gp_sparse_vector $@
$(MAKE) -C contrib/orafce $@
$(MAKE) -C contrib/gp_cancel_query $@
$(MAKE) -C contrib/hawq-hadoop $@
@@ -40,7 +38,6 @@ installdirs uninstall:
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
$(MAKE) -C contrib/extprotocol $@
- $(MAKE) -C contrib/gp_sparse_vector $@
$(MAKE) -C contrib/orafce $@
$(MAKE) -C contrib/gp_cancel_query $@
$(MAKE) -C contrib/hawq-hadoop $@
@@ -60,7 +57,6 @@ clean:
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
$(MAKE) -C contrib/extprotocol $@
- $(MAKE) -C contrib/gp_sparse_vector $@
$(MAKE) -C contrib/orafce $@
$(MAKE) -C contrib/gp_cancel_query $@
$(MAKE) -C contrib/hawq-hadoop $@
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/Makefile
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/Makefile b/contrib/gp_sparse_vector/Makefile
deleted file mode 100644
index 50468be..0000000
--- a/contrib/gp_sparse_vector/Makefile
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#Intel compiler
-ifeq "$(CC)" "icc"
- USE_ICC = 1
-endif
-
-override CFLAGS+=-std=gnu99
-
-MODULE_big = gp_svec
-OBJS = sparse_vector.o operators.o SparseData.o
-ifdef USE_PGXS
- PGXS := $(shell pg_config --pgxs)
- include $(PGXS)
-else
- subdir = contrib/gp_sparse_vector
- top_builddir = ../..
- include $(top_builddir)/src/Makefile.global
- include $(top_srcdir)/contrib/contrib-global.mk
-endif
-ifdef USE_ICC
- override CFLAGS=-O3 -Werror -std=c99 -vec-report2 -vec-threshold0
-endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/README
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/README b/contrib/gp_sparse_vector/README
deleted file mode 100644
index a05985e..0000000
--- a/contrib/gp_sparse_vector/README
+++ /dev/null
@@ -1,132 +0,0 @@
-
- Sparse Vector Datatype
-
-====================
-Background
-====================
-
-When you use arrays of floating point numbers for various calculations, you will often have long runs of zeros. This is common in many kinds of applications such as scientific, retail optimization and text processing. Each floating point number takes 8 bytes of storage in memory and/or disk, so saving those zeros is often impractical. There are also many computations that benefit from skipping over the zeros.
-
-Say for example that we have the following array of doubles in Postgres stored as a "float8[]":
- '{0, 33,...40,000 zeros..., 12, 22 }'::float8[]
-
-This array would occupy slightly more than 320KB of memory/disk, most of it zeros. Also, if we perform various operations on this, we'll be doing work on 40,001 fields that aren't important. This kind of array arises often in text processing, where the dictionary may have 40-100K terms and each vector will store the number of words in a particular document.
-
-We could use the built in "Array" datatype in Postgres/GP, which has a bitmap for null values, but since it is not optimized for float8[], or for long runs of zeros instead of nulls and the bitmap is not RLE compressed, it is a poor choice for this use-case. If we stored the zeros as NULL in the Postgres/GP array, then the bitmap for nulls would have 5KB of bits in it to mark the nulls, which is not nearly as efficient as we'd like.
-
-A simple RLE scheme that is biased toward being efficient for zero value bitmap results in 6 bytes for bitmap storage.
-
-We also want to implement vector operators for our type that take advantage of an efficient sparse storage format to make computations faster.
-
-====================
-What is it?
-====================
-
-The "Sparse Vector Datatype" is named "svec" and implements a vector with compressed storage of zeros. You can take the above vector and cast it into an svec like this:
- ('{0, 33,...40,000 zeros..., 12, 22 }'::float8[])::svec
-
-This is the same vector we input (I only used 5 zeros).
-
-We can use operations with svec type like <, >, *, **, /,=,+,SUM,COUNT_VEC etc, and they have meanings associated with typical vector operations. The plus (+) operator adds each of the terms of two vectors of same dimension together. For instance, if we have a = {0,1,5} and b = {4,3,2}, then a+b = {4,4,7}. We can see this using the svec type like this:
- lukelonergan=# select ('{0,1,5}'::float8[]::svec + '{4,3,2}'::float8[]::svec)::float8[];
- float8
- ---------
- {4,4,7}
-
-A vector dot product (*) between these two will result in a scalar result of type float8. The dot product should be (0*4+1*3+5*2)=13, like this:
- lukelonergan=# select '{0,1,5}'::float8[]::svec * '{4,3,2}'::float8[]::svec;
- ?column?
- ----------
- 13
-
-A scalar product (**) between these two will result in the individual terms being multiplied. The scalar product should be {0*4,1*3,5*2}={0,3,10}, like this:
- lukelonergan=# select ('{0,1,5}'::float8[]::svec ** '{4,3,2}'::float8[]::svec)::float8[];
- float8
- ----------
- {0,3,10}
-
-Special vector aggregate functions are also useful. SUM is self explanatory. COUNT_VEC (as opposed to COUNT) evaluates the count of non-zero terms found in a set of svec and returns an svec with the counts. For instance, if we have {0,1,5},{10,0,3},{0,0,3},{0,1,0}, then COUNT_VEC() would return {1,2,3}:
- lukelonergan=# create table list (a svec);
- CREATE TABLE
- lukelonergan=# insert into list values ('{0,1,5}'::float8[]),('{10,0,3}'::float8[]),('{0,0,3}'::float8[]),('{0,1,0}'::float8[]);
- INSERT 0 4
- lukelonergan=# select COUNT_VEC(a)::float8[] from list;
- count_vec
- -----------
- {1,2,3}
-
-====================
-Example
-====================
-For a text classification example, lets assume we have a dictionary composed of words in a text array:
-create table features (a text[][]) distributed randomly;
-insert into features values ('{am,before,being,bothered,corpus,document,i,in,is,me,never,now,one,really,second,the,third,this,until}');
-We have a set of documents, also defined as arrays of words:
- lukelonergan=# create table documents(a int,b text[]);
- CREATE TABLE
- lukelonergan=# insert into documents values (1,'{this,is,one,document,in,the,corpus}');
- lukelonergan=# insert into documents values (2,'{i,am,the,second,document,in,the,corpus}');
- lukelonergan=# insert into documents values (3,'{being,third,never,really",bothered,me,until,now}');
- lukelonergan=# insert into documents values (4,'{the,document,before,me,is,the,third,document}');
-
-Now we have a dictionary and some documents, we would like to do some document classification using vector arithmetic on word counts and proportions of dictionary words in each document.
-
-To start this process, well need to find the dictionary words in each document. Well prepared what is called a Sparse Feature Vector or SFV for each document. An SFV is a vector of dimension N, where N is the number of dictionary words, and in each SFV is a count of each dictionary word in the document.
-
-Inside the sparse vector toolset, we have a function that will create an SFV from a document, so we can just do this:
- lukelonergan=# select gp_extract_feature_histogram((select a from features limit 1),b)::float8[] from documents;
- gp_extract_feature_histogram
- -----------------------------------------
- {0,0,0,0,1,1,0,1,1,0,0,0,1,0,0,1,0,1,0}
- {0,0,1,1,0,0,0,0,0,1,1,1,0,1,0,0,1,0,1}
- {1,0,0,0,1,1,1,1,0,0,0,0,0,0,1,2,0,0,0}
- {0,1,0,0,0,2,0,0,1,1,0,0,0,0,0,2,1,0,0}
-
-Note that the output of gp_extract_feature_histogram is an svec for each document containing the count of each of the dictionary words in the ordinal positions of the dictionary. This can more easily be understood by doing this:
- lukelonergan=# select gp_extract_feature_histogram((select a from features limit 1),b)::float8[],b from documents;
- gp_extract_feature_histogram | b
- -----------------------------------------+--------------------------------------------------
- {1,0,0,0,1,1,1,1,0,0,0,0,0,0,1,2,0,0,0} | {i,am,the,second,document,in,the,corpus}
- {0,1,0,0,0,2,0,0,1,1,0,0,0,0,0,2,1,0,0} | {the,document,before,me,is,the,third,document}
- {0,0,0,0,1,1,0,1,1,0,0,0,1,0,0,1,0,1,0} | {this,is,one,document,in,the,corpus}
- {0,0,1,1,0,0,0,0,0,1,1,1,0,1,0,0,1,0,1} | {being,third,never,really,bothered,me,until,now}
-
- lukelonergan=# select * from features;
- a
- --------------------------------------------------------------------------------------------------------
- {am,before,being,bothered,corpus,document,i,in,is,me,never,now,one,really,second,the,third,this,until}
-
-Now when we look at the first document "i am the second document in the corpus", its SFV is {1,3*0,1,1,1,1,6*0,1,2}. The word "am" is the first ordinate in the dictionary and there is 1 instance of it in the SFV. "before" has no instances in the document, so its value is "0" and so on.
-
-gp_extract_feature_histogram is very speed optimized - in essence a single routine version of a hash join, and should be able to process large numbers of documents into their SFVs in parallel at the highest possible speeds.
-
-The rest of the classification process is all vector math. As Brian Dolan describes the process:
- The actual count is hardly ever used. Instead, it's turned into a weight. The most common weight is called tf/idf for "Term Frequency / Inverse Document Frequency".
- The calculation for a given term in a given document is {#Times in this document} * log {#Documents / #Documents the term appears in}
- For instance, the term "document" in document A would have weight 1 * log (4/3). In document D it would have weight 2 * log (4/3).
- Terms that appear in every document would have tf/idf weight 0, since log (4/4) = log(1) = 0. (Our example has no term like that.) That usually sends a lot of values to 0.
-
-For this part of the processing, well need to have a sparse vector of the dictionary dimension (19) with the values (log(#documents/#Documents each term appears in). There will be one such vector for the whole list of documents (aka the "corpus"). The #documents is just a count of all of the documents, in this case 4, but there is one divisor for each dictionary word and its value is the count of all the times that word appears in the document. This single vector for the whole corpus can then be scalar product multiplied by each document SFV to produce the Term Frequency/Inverse Document Frequency.
-
-This can be done as follows:
- lukelonergan=# create table corpus as (select a,gp_extract_feature_histogram((select a from features limit 1),b) sfv from documents);
- lukelonergan=# SELECT docnum, (a*logidf) tf_idf FROM (SELECT log(count(a)/count_vec(a)) logidf FROM corpus) foo, corpus ORDER BY docnum;
- docnum | tf_idf
- --------+----------------------------------------------------------------------------------------------------------------------------------------------------------
- 1 | {4,1,1,1,2,3,1,2,1,1,1,1}:{0,0.693147180559945,0.287682072451781,0,0.693147180559945,0,1.38629436111989,0,0.287682072451781,0,1.38629436111989,0}
- 2 | {1,3,1,1,1,1,6,1,1,3}:{1.38629436111989,0,0.693147180559945,0.287682072451781,1.38629436111989,0.693147180559945,0,1.38629436111989,0.575364144903562,0}
- 3 | {2,2,5,1,2,1,1,2,1,1,1}:{0,1.38629436111989,0,0.693147180559945,1.38629436111989,0,1.38629436111989,0,0.693147180559945,0,1.38629436111989}
- 4 | {1,1,3,1,2,2,5,1,1,2}:{0,1.38629436111989,0,0.575364144903562,0,0.693147180559945,0,0.575364144903562,0.693147180559945,0}
-
-We can now get the "angular distance" between one document and the rest of the documents using the ACOS of the dot product of the document vectors:
- lukelonergan=# CREATE TABLE weights AS (SELECT docnum, (a*logidf) tf_idf FROM (SELECT log(count(a)/count_vec(a)) logidf FROM corpus) foo, corpus ORDER BY docnum) DISTRIBUTED RANDOMLY;
- Calculate the angular distance between the first document to each other document
- lukelonergan=# SELECT docnum,180.*(ACOS(dmin(1.,(tf_idf%*%testdoc)/(svec_l2norm(tf_idf)*svec_l2norm(testdoc))))/3.141592654) angular_distance FROM weights,(SELECT tf_idf testdoc FROM weights WHERE docnum = 1 LIMIT 1) foo ORDER BY 1;
- docnum | angular_distance
- --------+------------------
- 1 | 0
- 2 | 78.8235846096986
- 3 | 89.9999999882484
- 4 | 80.0232034288617
-
-We can see that the angular distance between document 1 and itself is 0 degrees and between document 1 and 3 is 90 degrees because they share no features at all.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/README.bitmap
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/README.bitmap b/contrib/gp_sparse_vector/README.bitmap
deleted file mode 100644
index 901a3f5..0000000
--- a/contrib/gp_sparse_vector/README.bitmap
+++ /dev/null
@@ -1,259 +0,0 @@
-Revision: 02/03/10
-
-I’m spending most of my part time on working out the actual mechanics (with examples) of the matrix/vector implementation.
-
-What will help is to get some input on likely use-cases that can drive the work – though I’ve got a good number of examples from scientific and econometric modeling, I could always benefit from something concrete and immediate.
-
-Following is more on mechanics.
-
-Another finding on indexes, RLE compression and matrices:
-input matrices come in various flavors, many of them in non-sorted “not ready for sparse compression” formats
-in many circumstances there is no a-priori choice of whether there are zeros or NVP
-Because of (1), in the input processing and sorting, it is necessary to create an uncompressed index that is identical to the single RLE index I’ve described previously, sans the special values.
-
-I’ve described the input processing here in a program I wrote to process matrices from the Matrix Mart at NIST:
- /* The Matrix Market format does not specify the order in which the
- * successive elements are stored. We need to compress the matrix into
- * one of CRS or CCS format (row or column compression), so we need to
- * sort the matrix into one of those forms.
- *
- * Here we'll use quicksort to sort the matrix elements into row
- * contiguous storage.
- *
- * If we desire row contiguous storage, we'll also need the column order
- * within the row preserved. We'll do this by computing the index into
- * the matrix using a "columns progressive" ordering, which is computed
- * as NJ * I + J. Then we'll sort based on that index. The index value
- * will range from 0 to NJ*NI-1. For any index value, you can compute I
- * and J using:
- * I = index/NJ
- * J = index - NJ * I
- *
- * If we sort the matrix by this index, we'll have things in row/column
- * order, suitable for CRS compression.
- *
- * Note that if we want CCS, we can construct the index using:
- * index = NI * J + I
- */
-
-In the Yale/MKL formats, we deconstruct the single index into a set of two values for each run and that is stored as the compression index.
-
-I’m thinking now that for the RLE index we can do something along these lines:
-Treat all values symmetrically, that is: all runs of the various types we’re discussing (inf,-inf,0,NVP,value) are described by the same kind of word
-Each RLE word is composed of a leading set of descriptor bits, followed by a payload describing how many of the value are found.
-
-So if we pick the descriptor bits = 3, we can represent 8 “things”, of which 5 things must be “what kind of value is this?”, leaving 3 things. A good use of the 3 things can be word length, one of 8, 16 or 32.
-
-So if we go down this path, we’re left with 5, 13 and 29 bits to represent a run and can dynamically choose which one to use based on the data content. That’s 32, 8192 and 536,870,912 values in each run, which is perfect for capturing various kinds of sparsity.
-
-If we expand to one more descriptor bit, then we can represent 8 more things, which could be used to recognize additional types of values to compress out. That would leave us with 16, 4096 and 268,435,456 values in each run, which also works pretty well and leaves us room for the future in case we find there are more values than {inf,-inf,0,NVP,value}.
-
-Some additional results to share (see table below) – I did a computation of the various storage and compute requirements for the MKL/Yale index and the RLE index. I used 5 matrix examples from NIST’s “Matrix Market”, read them in and translated them from Matrix Market format to MKL/RLE which required sorting them into CRS. From this, I calculated the relevant stats on each in terms of storage and number of words in each of the RLE and MKL indices.
-
-Bottom line is that the results corroborated my earlier calculations. The RLE index is a lot smaller than MKL always, and is more computationally efficient for linear scans through the compressed matrix. This means we should use only RLE for most use-cases and calculate MKL only when we need to use the algorithms in the matrix solver libraries. We can optionally store the MKL index if we calculate it – we can provide a marker to indicate the presence/absence of it in the type.
-
-I’ve also thought through the compatibility with all the sparse solver libs – seems like we have easy compatibility with a CRS-based approach, so I think that should be the baseline. We can provide another marker to indicate what is being stored – CRS, CCS, CDS, etc.
-
-I believe I have enough information to produce a draft design of the matrix type that we can review and get some other feedback about. Here is a sketch of how I think it should work:
-Two separate types, one for vectors and one for matrices – but they share the same format and support routines. The separate types allows definition of vector and matrix specific operators.
-The format stores the following metadata about the datum:
- nD, (n(i),i=1,nD): dimension and rank – for a vector, this is {1,N}; for a matrix it’s {2,Nrows,Ncolumns}
- nZ: number of values (non-special values)
- valType: type of value, one of {bit, byte, int4, int8, real4, complex4, real8, complex8}
- storType: type of storage, one of {CRS, CCS, CDS, ... Up to maybe 32 total with some reserved}
- specValList: a list of special values to be used in the compression, up to 8, e.g. {inf,-inf,NVP,0}
- hasMKL: if true, the MKL index is stored
-The format stores the following data:
- values: a dense array of values, size is (nZ*sizeof(valType))
- rleIndex: an RLE index containing location of all values (including specials)
- sizeofRleIndex: the size of the RLE index for allocation purposes
- if (hasMKL) mklIndex: the MKL/Yale index, size is ((nZ+NI)*sizeof(int))
-
-01234567890123456789012345678901234567890123456789012345678901234567890123456789
-
-In order to efficiently compress the storage of multiple data types, we are
-using a bitmap that describes the positions of four special 64-bit floating
-point values: zero, negative infinity, positive infinity, and "no value present"
-or NVP. Of these, the first three are all standard floating point values that
-can be the natural result of performing calculations. NVP is a special case
-often used in statistics and other analytical processing but is not
-representable as a standard floating point number.
-
-Here we're using two patterns for sparse storage and computation:
-1) recognition of compressible types and subsequent bitmap compression
-2) optimized computational usage of previously compressed values
-
-(1) recognizes that we're going to have many situations where special values
-arise as a natural consequence of computation. In these cases, we will
-automatically locate and compress special values into a special bitmap storage
-format.
-
-(2) leverages the pre-existing compressed special value storage for optimized
-computation. For example, if we have a vector of values V and a large fraction
-of the values in V are zero, then computing "W = 1/V" will produce a
-correspondingly large number of positive infinity values in W. With our
-compressed bitmap storage of zeros, we can loop over the compressed zero
-elements and change their storage type from "zero" to "infinity", thereby
-saving the computation, memory and re-compression overheads.
-
-It's important to have a bitmap format that is both storage efficient and
-optimal for computation. There are many existing formats for sparse vector and
-matrix storage, most of them developed for scientific computation. The Intel
-Math Kernel Library (MKL) specifies the most common sparse matrix storage
-format, derived from the Yale Sparse Matrix format:
-
- "The Intel MKL direct sparse solvers use a row-major upper triangular storage
-format: the matrix is compressed row-by-row and for symmetric matrices only non-
-zero elements in the upper triangular half of the matrix are stored."
-
- The Intel MKL sparse matrix storage format for direct sparse solvers is
-specified by three arrays: values, columns, and rowIndex. The following table
-describes the arrays in terms of the values, row, and column positions of the
-non-zero elements in a sparse matrix.
- values - A real or complex array that contains the non-zero elements
- of a sparse matrix. The non-zero elements are mapped into the
- values array using the row-major upper triangular storage
- mapping described above.
- columns - Element I of the integer array columns is the number of the
- column that contains the I-th element in the values array.
- rowIndex - Element j of the integer array rowIndex gives the index of
- the element in the values array that is first non-zero
- element in a row j.
-
- The length of the values and columns arrays is equal to the number of non-zero
-elements in the matrix."
-
-- Intel Math Kernel Library Reference Manual, March 2009, Document Number:
- 630813-031US
-
-The MKL format stores the non-zero values compactly in a sequential array of
-either complex or double precision entries and maintains two index arrays that
-identify the location of the nonzero values in a two dimensional matrix. The
-advantages of the MKL format are it's efficiency for computation and the fact
-it's in widespread usage in numerical libraries.
-
-The uncompressed size of a real matrix with NI rows and NJ columns would be:
- Raw size = (8 bytes) * ( NI*NJ )
-
-The storage required by the MKL format for a real matrix with N nonzero values
-and NI number of rows is:
- Comp Size = (8 bytes) * ( 2N + NI )
-
-For a complex matrix it's:
- Comp Size = (8 bytes) * ( 3N + NI )
-
-Another way of representing the storage is to assess the "overhead" associated
-with the representation of the zero entries, which is the same for real and
-complex:
- Overhead = (8 bytes) * ( N + NI )
-
-Now we can calculate the "savings" associated with using the MLK compressed
-format for a real matrix:
- Savings = (Raw Size) - (Comp Size)
- (8 bytes) ( NI*NJ - 2N - NI )
-
-If we have a dense matrix, then N will be NI*NJ and the savings is equal to the
-negative overhead, which means we'll have a worst case savings of:
- Dense (worst case) savings = -(8 bytes)(NI*NJ + NI)
- = -(8 bytes)*NI*(NJ+1)
-
-For a dense matrix, this is a 100%+ storage penalty.
-
-There is one important shortcoming of the MKL format: it can only represent one
-type of special value - zero. This is a consequence of the implicit nature of
-the approach, the location of the zero values is not stored, it is implied by
-the lack of a non-zero value in that matrix position. There is no place to
-provide an attribute of the special value so there can be only one type of
-special value.
-
-We need an alternative format that can represent different types of special
-values. It will need to refer to the special values explicitly in an efficient
-manner. We can take advantage of the nature of most all sparse storage
-circumstances: the special values occur in relatively large runs compared to
-the non-zero values. We can use a "run length encoding" or RLE approach to
-compose a compressed special value array, in addition to storing a regular
-packed array of the non-special values. The index location of the non-special
-values can be inferred from the RLE array of special values during computation.
-
-Let's represent the array of special values using a 16-bit word to represent
-each run of special values, and an 8 bit word for each run of non-special
-values. Each of the two word types uses the first bit (the sign bit) to
-represent the run as consisting of special values (positive) or non-special
-values (negative). Each special value word uses the next 2 bits to describe what
-kind of special values are in the run, one of zero, inf, -inf and NVP.
-
-A complete array will be represented as a collection of these two types of
-words, terminated by a special "end of array" word consisting of 16 bits of
-zeros (0x0000). For example:
- <special values><non special values><special values><0x0000>
-
-This can be seen in bytes form:
- <2 bytes><1 byte><2 bytes><2 bytes>
-
-And if the special values are zeros (type 0), with runs of 8192 in each of the
-two runs with a run of 128 non-special values, the bits in this array would be:
- <0001111111111111><11111111><0001111111111111><0000000000000000>
-
-Note that if there are typically a large number of zeros followed by only one
-non-zero value, then this representation will waste a lot of space in the
-non-special value words. We could shorten the representation of non-special
-values to say 5 bits, which would allow for runs of 16 to be represented by
-each word and we would save three bits per run. However, the alignment of the
-index computations would be less than 1 byte, which would likely reduce the
-computational efficiency.
-
-If a given run has more than the maximum representable by the word, words are
-concatenated as needed to represent the run.
-
-For matrices we can use the compressed row format, or the compressed column
-format - for purpose of the efficiency discussion let's assumed compressed rows.
-
-The storage efficiency of this approach is more complicated to compute, as the
-storage will depend on the dispersion of values. We can compute best case and
-worst case scenarios however.
-
-A reasonable "best case" scenario will have one non-special value per row. In
-this case the storage required by the RLE format for a real matrix with N nonzero values, NI rows and NJ columns is:
- Comp Size = (9 bytes) * N + (2 bytes) *(NJ*(NI/8192) +1)
- = (9 bytes) *NI + (2 bytes) *(NJ*(NI/8192) +1)
- for the common case of NI < 8192:
- = (9 bytes) *NI + (2 bytes) *(NJ +1)
-
-In this case, the storage is O(NI+NJ). For instance, if we have a matrix of
-size 2,000 by 2,000, the storage is (8 bytes) * 4,000 = 32KB.
-
-For a worst case scenario, we can assume alternating non-special and special
-values, in which case the storage is:
- Comp Size = (9 bytes) * N + (2 bytes) * (NJ*(NI/2) +1)
- = (9 bytes) *NJ*(NI/2) + (2 bytes) * (NJ*(NI/2) +1)
- = (11 bytes) *NJ*(NI/2) + (2 bytes)
-
-What's interesting is that this worst case size is actually smaller (by almost
-half) than storing the full dense matrix. This is due to the more efficient
-representation of the special values in 2 byte format. The overhead is nearly
-equivalent to storing the special values in addition to the non-special values.
-
-If we have a dense matrix, there is almost zero overhead because the special
-value array is blank and we only store the non-special data values.
-
-If we compare the MKL approach savings to the RLE approach in the best case
-scenario, we have an RLE savings of:
- RLE
- Savings = (Raw Size) - (Comp Size)
- = (8 bytes) *NI*NJ - (9 bytes)*NI - (2 bytes)*(NJ +1)
- for NI ~ NJ:
- ~ (8 bytes) *NI*NJ - (11 bytes)*NI
- ~ (8 bytes) *NI*(NJ-1)
-
- MKL
- Savings = (8 bytes) ( NI*NJ - 3NI )
- = (8 bytes) *NI*(NJ-3)
-
-The savings in the best case is nearly the same. In the worst case scenario of
-a dense matrix, the savings of the RLE approach is nearly 0, which is the best
-result possible. For MKL however, the savings in the worst case is more than
-a 100% storage penalty.
-
-Most cases are somewhere between best case and worst case. We can conclude
-that the storage of the RLE scheme is going to be more efficient than MKL in
-all cases.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/README.devel
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/README.devel b/contrib/gp_sparse_vector/README.devel
deleted file mode 100644
index ed39616..0000000
--- a/contrib/gp_sparse_vector/README.devel
+++ /dev/null
@@ -1,63 +0,0 @@
-================================================================================
-Structure of the sparse type support
-================================================================================
-
-There are two parts to the sparse types:
-1) an in-memory data structure and computational library for sparse data called
-SparseData.
-
-2) a Greenplum/Postgres datatype for each case, initially a sparse vector which
-uses only (double) as it's data
-
-Eventually there will be sparse vector support for other base types like float,
-complex double, integer, char and bit. There will also be matrix support,
-which will use the SparseData structure to compose higher dimension structures.
-
-=========================
-Design
-=========================
-
-The Sparse Vector type is a variable length Postgres datatype, which means that
-its data is stored in a "varlena". A varlena is a structure that has an integer
-that identifies the length of it's data area as it's first item.
-
-For the Sparse Vector, the varlena has a leading integer that describes the size
-or "dimension" of the vector it represents, followed by a serialized SparseData
-of type FLOAT8OID.
-
-All of the operators required for the datatype support are wrappers that use the
-SparseData operators. In order to use a SparseData operator, the serialized
-SparseData inside the Sparse Vector is used to create an in-place accessible
-SparseData (no copy required), and then this is passed to the SparseData
-functions.
-
-If you want to implement a new operator or function for Sparse Vector, you should
-try to implement it inside of operators.c using existing SparseData functions or
-building on other Sparse Vector functions first. If something isn't there, you
-can build a base level function inside of the SparseData.h as an inline function
-or inside SparseData.c as a non-inline function, then wrap it in a Sparse Vector
-function in operators.c.
-
-The sparse_vector.c file is for creation and management functions related to the
-type.
-
-=========================
-Code structure
-=========================
-
-SparseData structure and functions:
-------------------------------------------------------------------------
-SparseData.c, SparseData.h
-
-Sparse vector datatype:
-------------------------------------------------------------------------
-sparse_vector.c, sparse_vector.h, operators.c
-
-Functions for text processing (Sparse Feature Vector or SFV):
-------------------------------------------------------------------------
-gp_sfv.c
-
-Definitions for IEEE double and float types including special denormalized
-types defined for use herein as "no value present":
-------------------------------------------------------------------------
-float_specials.h
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/README.update
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/README.update b/contrib/gp_sparse_vector/README.update
deleted file mode 100644
index c7d84d4..0000000
--- a/contrib/gp_sparse_vector/README.update
+++ /dev/null
@@ -1,72 +0,0 @@
-On Dynamic Word Size for the RLE index
-==========================================================================
-I thought we'd use a method where we identify the size of the next word in
-the index by leveraging the top two bits of the word to describe how large
-the following word is.
-
-This idea works if the word is layed out with the "top bits" inside the
-first byte. However, big endian systems (Intel, AMD) store the topmost
-bits of their words as the *last* byte in the word, which means that we
-can't tell what size the word is without reading the word. We can't
-read the word without knowing the size, so this is a non-starter.
-
-We could encode the size information into the bottom bits of the word,
-but then we end up with different code paths for big endian systems and
-little endian systems. This is dangerous, and means we would not have
-a portable binary format between systems with different endian-ness.
-
-An alternative strategy would use the entire first byte of an RLE word
-to encode the following word size. A modification of this scheme would
-encode the word size into the top two bits of the first byte, and if
-the word size is a single byte, then the remainder of the byte would
-represent the encoded length. This preserves the worst-case compression
-of the previous scheme and introduces an additional overhead to the
-middle compression cases equal to one byte per RLE word.
-
-==========================================================================
-A multi-platform friendly dynamic word size RLE scheme
-==========================================================================
-We have a sequence of numbers and we want a scheme that will store them in
-the most compact format possible using binary types of 8, 16, 32 and 64 bit
-widths.
-
-Let's say we have the following sequence: {13, 2000, 2*10^12, 3*10^26}. We
-can see that it would require one byte to minimally store the first number,
-two to store the second, 4 to store the third and 8 for the fourth.
-
-We can store the sequence using these minimum widths and we will have an
-array with sizes (in bytes): {1,2,4,8}. However, to read this array, we
-need to be able to determine the width of each word.
-
-Here we will use a "length word" to describe the length of the storage
-type for each entry. The size of the length word will always be 1 byte,
-though in the case where the stored number is less than 127, we will not
-use a length word, we'll just store a "-127" in a single byte. The value
-of the length word will be one of 1,2,4 or 8 to indicate the size of the
-following storage width.
-
-So our example would be stored in an array with the following word widths
-and contents (in parenthesis):
- {1(-13), 1(2), 2(2000), 1(4), 4(2*10^12), 1(8), 8(3*10^26)}
-
-The total size of this array is 18 bytes, which should be compared to the
-size required if an 8 byte width was used to store all words, in which
-case we would need 32 bytes to store the array. This is a savings of
-nearly half in this simple case. In the worst case where we are storing
-numbers that all require 8 byte widths, we would be adding one byte per
-word, or a 12.5% overhead. If we have lots of 1 byte words interspersed
-with an occasional 8 byte word, the savings move toward an 8:1 compression
-ratio.
-
-The algorithm for reading this array is as follows:
- Read first byte "B"
- if (B < 0)
- N = -(B)
- else
- Copy "B" byte word into N
-
-We have to be careful about alignment when reading the dynamic sized word
-from the array - we can't just cast the next "B" bytes into the appropriate
-sized type and dereference it. This would cause an unaligned access error,
-which will result in an abort on some architectures and poor performance on
-others.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/README_INSTALL
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/README_INSTALL b/contrib/gp_sparse_vector/README_INSTALL
deleted file mode 100644
index 0975541..0000000
--- a/contrib/gp_sparse_vector/README_INSTALL
+++ /dev/null
@@ -1,9 +0,0 @@
-Requirements:
- - Greenplum Database 3.3 or higher
- - A "C" compiler
-
-To install:
- - Make sure that you have the Greenplum binary distribution in your path
- - type 'make install'
- - install the function and type definitions with "psql -d <your_db_name> -f gp_svec.sql"
- - <optional> run tests with "psql -d <your_db_name> -af gp_svec_test.sql | diff - test_output"
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/README_term_proximity
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/README_term_proximity b/contrib/gp_sparse_vector/README_term_proximity
deleted file mode 100644
index 792d070..0000000
--- a/contrib/gp_sparse_vector/README_term_proximity
+++ /dev/null
@@ -1,118 +0,0 @@
-01234567890123456789012345678901234567890123456789012345678901234567890123456789
-In finding the proximity of several terms, we first need to represent the
-positions of the terms in the document. Let's say we use the following to
-represent a set of term occurrences:
- doc1 term1:1 term2:2 term3:3
- doc2 term1:30 term2:10 term3:20
-
-if we performed a search query against these two documents using the query:
-(term1 && term2 && term3), we would like the weight of results to reflect the
-proximity of these terms. In document1, the three terms are clustered together
-without separation and in document2 they are separated by 10 positions, so we
-would expect the weight of search results to reflect this.
-
-One possible weight of the search would take the like of matching terms for each
-document and construct a weight like:
- Matching Weight = MAX(positions) - MIN(positions)
-
-Using this metric for our test case, document1 would have a weight of 2 and
-document2 would have a weight of 20. We can easily construct a method to
-determine the best match.
-
-But what if there are multiple positions at which the terms appear? For
-instance, what if the term occurrences in documents 3 and 4 looked like this:
- doc3 term1:1,40,4 term2:2 term3:3
- doc4 term1:1 term2:2,30,50 term3:3,10
-
-Now the proximity grouping of term1,2,3 is obscured by our previous weight,
-document 3 and 4 evaluate to weights of 39 and 49 for this search.
-
-============================================
-A sparse matrix method for term proximity
-============================================
-If we consider the list of positions for each term as a "position vector" that
-has an entry for each place the term occurs in a document, then we can write our
-terms and frequencies like a matrix for each document where each column is a
-position vector with a 1 where the term exists in the document and zeros where
-it doesn't exist, like this:
- doc1
- term1 term2 term3 term4
-count at pos1 0 1 0 0
-count at pos2 1 0 0 0
-count at pos3 0 0 1 0
-count at pos4 1 0 0 0
-...
-count at posN 0 0 1 0
-
-The dimensions of this matrix are (Nterms x Npositions).
-
-If we were to look for the query (term1 && term3) in the above document, we
-could construct a query vector with a 1 in each of the 1 and 3 term positions
-and zeros elsewhere like this:
- query
- term1 1
- term2 0
- term3 1
- term4 0
-
-Now we can get a histogram of term occurrences by evaluating the matrix x vector
-product like this:
- histogram = |doc1| %*% query
-
-This yields the following vector for this case (assuming we only have 4 positions):
- histogram
- term1 2
- term2 0
- term3 1
- term4 0
-
-We desire a weighting by proximity: now that we have an arithmetic for
-calculating intersections of queries with occurrences perhaps we can amend the
-weighting of the positions to accomplish a proximity weight?
-
-Consider the following two vectors of term positions for terms 1 and 3 in
-document 1:
- term1 term3
-count at pos1 0 0
-count at pos2 0 0
-count at pos3 0 1
-count at pos4 1 0
-...
-count at posN 0 1
-
-In this case, we have two occurrences of term3, but only one is close to term1.
-If we evaluate the histogram, we'll simply get two matches of term3 and one
-match of term1 without any bias or consideration that we have terms close to
-each other.
- histogram
- term1 1
- term2 0
- term3 2
- term4 0
-
-What if we were to amend the term positions, allowing them to "bleed" across the
-position boundaries like this:
- term1 term3
-count at pos1 0 0
-count at pos2 0 .1
-count at pos3 .1 1
-count at pos4 1 .1
-count at pos5 .1 0
-...
-count at posN-1 0 .1
-count at posN 0 1
-
-We can then compose a "blended term vector" that is the scalar multiplication of
-these weighted term position vectors (insert proper matrix multiplication here)
-so that we have a reinforcement of overlapping positions. Where the two terms
-are separated by one position, there will be a value of 1.1.
-
-We can extend this blending to provide a distribution of values across a larger
-range of positions. For example if we use a distribution like:
- 0.2,0.4,0.8,1.0,0.8,0.4,0.2
-
-If we have a value of 1.8 we know that the terms are next to each other and if
-we have 1.2 we know they are 3 characters apart.
-
-With multiple search terms, this becomes a continuous weighting factor that can
-be normalized by the number of search terms.
[4/5] incubator-hawq git commit: HAWQ-49. Remove madlib related codes
and tests.
Posted by ml...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/SparseData.c
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/SparseData.c b/contrib/gp_sparse_vector/SparseData.c
deleted file mode 100644
index 8406f85..0000000
--- a/contrib/gp_sparse_vector/SparseData.c
+++ /dev/null
@@ -1,1572 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * @file
- * \brief This file implements different operations on the SparseData structure.
- *
- * Most functions on sparse vectors are first defined on SparseData, and
- * then packaged up as functions on sparse vectors using wrappers.
- *
- * This is the general procedure for adding functionalities to sparse vectors:
- * 1. Write the function for SparseData.
- * 2. Wrap it up for SvecType in operators.c or sparse_vector.c.
- * 3. Make the function available in gp_svec.sql.
- */
-
-#include <postgres.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <float.h>
-#include "SparseData.h"
-#include "utils/builtins.h"
-#include "utils/syscache.h"
-#include "parser/parse_func.h"
-#include "access/htup.h"
-#include "catalog/pg_proc.h"
-
-#if PG_VERSION_NUM >= 90300
-#include "access/htup_details.h"
-#endif
-
-void* array_pos_ref = NULL;
-
-
-/* -----------------------------------------------------------------------------
- *
- * The following functions used to be defined in SparseData.h. It is unclear to
- * me why (Florian Schoppmann, June 4, 2012).
- *
- * -------------------------------------------------------------------------- */
-
-/* BEGIN Previously in SparseData.h */
-
-/* Returns the size of each basic type
- */
-size_t
-size_of_type(Oid type)
-{
- switch (type)
- {
- case FLOAT4OID: return(4);
- case FLOAT8OID: return(8);
- case CHAROID: return(1);
- case INT2OID: return(2);
- case INT4OID: return(4);
- case INT8OID: return(8);
- }
- return(1);
-}
-
-/* Appends a count to the count array
- * The function appendBinaryStringInfo always make sure to attach a trailing '\0'
- * to the data array of the index StringInfo.
- */
-void append_to_rle_index(StringInfo index, int64 run_len)
-{
- char bytes[]={0,0,0,0,0,0,0,0,0}; /* 9 bytes into which the compressed
- int8 value is written */
- int8_to_compword(run_len,bytes); /* create compressed version of
- int8 value */
- appendBinaryStringInfo(index,bytes,int8compstoragesize(bytes));
-}
-
-/* Adds a new block to a SparseData
- * The function appendBinaryStringInfo always make sure to attach a trailing '\0'
- * to the data array of the vals StringInfo.
- */
-void add_run_to_sdata(char *run_val, int64 run_len, size_t width,
- SparseData sdata)
-{
- StringInfo index = sdata->index;
- StringInfo vals = sdata->vals;
-
- appendBinaryStringInfo(vals,run_val,width);
- append_to_rle_index(index, run_len);
- sdata->unique_value_count++;
- sdata->total_value_count+=run_len;
-}
-
-/*------------------------------------------------------------------------------
- * Each integer count in the RLE index is stored in a number of bytes determined
- * by its size. The larger the integer count, the larger the size of storage.
- * Following is the map of count maximums to storage size:
- * Range Storage
- * --------- -----------------------------------------
- * 0 - 127 signed char stores the negative count
- *
- * All higher than 127 have two parts, the description byte
- * and the count word
- *
- * description byte signed char stores the number of bytes in the
- * count word one of 1,2,4 or 8
- *
- * 128 - 32767 count word is 2 bytes, signed int16_t
- * 32768 - 2147483648 count word is 4 bytes, signed int32_t
- * 2147483648 - max count word is 8 bytes, signed int64
- *------------------------------------------------------------------------------
- */
-/* Transforms an int64 value to an RLE entry. The entry is placed in the
- * provided entry[] array and will have a variable size depending on the value.
- */
-void int8_to_compword(int64 num, char entry[9])
-{
- if (num < 128) {
- /* The reason this is negative is because entry[0] is
- used to record sizes in the other cases. */
- entry[0] = -(char)num;
- return;
- }
-
- entry[1] = (char)(num & 0xFF);
- entry[2] = (char)((num & 0xFF00) >> 8);
-
- if (num < 32768) { entry[0] = 2; return; }
-
- entry[3] = (char)((num & 0xFF0000L) >> 16);
- entry[4] = (char)((num & 0xFF000000L) >> 24);
-
- if (num < 2147483648LL) { entry[0] = 4; return; }
-
- entry[5] = (char)((num & 0xFF00000000LL) >> 32);
- entry[6] = (char)((num & 0xFF0000000000LL) >> 40);
- entry[7] = (char)((num & 0xFF000000000000LL) >> 48);
- entry[8] = (char)((num & 0xFF00000000000000LL) >> 56);
-
- entry[0] = 8;
-}
-
-/* Transforms a count entry into an int64 value when provided with a pointer
- * to an entry.
- */
-int64 compword_to_int8(const char *entry)
-{
- char size = int8compstoragesize(entry);
- int16_t num_2;
- char *numptr2 = (char *)(&num_2);
- int32_t num_4;
- char *numptr4 = (char *)(&num_4);
- int64 num = 0;
- char *numptr8 = (char *)(&num);
-
- switch(size) {
- case 0: /* entry == NULL represents an array of ones; see
- * comment after definition of SparseDataStruct above
- */
- return 1;
- case 1:
- num = -(entry[0]);
- break;
- case 3:
- numptr2[0] = entry[1];
- numptr2[1] = entry[2];
- num = num_2;
- break;
- case 5:
- numptr4[0] = entry[1];
- numptr4[1] = entry[2];
- numptr4[2] = entry[3];
- numptr4[3] = entry[4];
- num = num_4;
- break;
- case 9:
- numptr8[0] = entry[1];
- numptr8[1] = entry[2];
- numptr8[2] = entry[3];
- numptr8[3] = entry[4];
- numptr8[4] = entry[5];
- numptr8[5] = entry[6];
- numptr8[6] = entry[7];
- numptr8[7] = entry[8];
- break;
- }
- return num;
-}
-
-void printout_double(double *vals, int num_values, int stop)
-{
- (void) stop; /* avoid warning about unused parameter */
- char *output_str = (char *)palloc(sizeof(char)*(num_values*(6+18+2))+1);
- char *str = output_str;
- int numout;
- for (int i=0; i<num_values; i++) {
- numout = snprintf(str,26,"%6.2f,%#llX,",vals[i],
- *((long long unsigned int *)(&(vals[i]))));
- str += numout-1;
- }
- *str = '\0';
- elog(NOTICE,"doubles:%s",output_str);
-}
-void printout_index(char *ix, int num_values, int stop)
-{
- (void) stop; /* avoid warning about unused parameter */
- char *output_str = (char *)palloc(sizeof(char)*((num_values*7)+1));
- char *str = output_str;
- int numout;
- elog(NOTICE,"num_values=%d",num_values);
- for (int i=0; i<num_values; i++,ix+=int8compstoragesize(ix)) {
- numout=snprintf(str,7,"%lld,",(long long int)compword_to_int8(ix));
- str+=numout;
- }
- *str = '\0';
- elog(NOTICE,"index:%s",output_str);
-}
-void printout_sdata(SparseData sdata, char *msg, int stop)
-{
- elog(NOTICE,"%s ==> unvct,tvct,ilen,dlen,datatype=%d,%d,%d,%d,%d",
- msg,
- sdata->unique_value_count,sdata->total_value_count,
- sdata->index->len,sdata->vals->len,
- sdata->type_of_data);
- {
- char *ix=sdata->index->data;
- double *ar=(double *)(sdata->vals->data);
- printout_double(ar,sdata->unique_value_count,0);
- printout_index(ix,sdata->unique_value_count,0);
- }
-
- if (stop)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("LAL STOP")));
-}
-
-/*------------------------------------------------------------------------------
- * Multiplication, Addition, Division by scalars
- *------------------------------------------------------------------------------
- */
-#define typref(type,ptr) (*((type *)(ptr)))
-#define valref(type,val,i) (((type *)(val)->vals->data)[(i)])
-
-#define valsquare(val) (val*val)
-#define valcube(val) (val*valsquare(val))
-#define valquad(val) (valsquare(valsquare(val)))
-
-#define apply_const_to_sdata(sdata,i,op,scalar) \
- switch ((sdata)->type_of_data) \
-{ \
- case FLOAT4OID: \
- valref(float,sdata,i) op typref(float,scalar); \
- break; \
- case FLOAT8OID: \
- valref(float8,sdata,i) op typref(float8,scalar); \
- break; \
- case CHAROID: \
- valref(char,sdata,i) op typref(char,scalar); \
- break; \
- case INT2OID: \
- valref(int16,sdata,i) op typref(int16,scalar); \
- break; \
- case INT4OID: \
- valref(int32,sdata,i) op typref(int32,scalar); \
- break; \
- case INT8OID: \
- valref(int64,sdata,i) op typref(int64,scalar); \
- break; \
-}
-
-#define apply_scalar_left_to_sdata(sdata,i,op,scalar) \
- switch ((sdata)->type_of_data) \
-{ \
- case FLOAT4OID: \
- valref(float,sdata,i) = \
- typref(float,scalar) op valref(float,sdata,i); \
- break; \
- case FLOAT8OID: \
- valref(float8,sdata,i) = \
- typref(float8,scalar) op valref(float8,sdata,i); \
- break; \
- case CHAROID: \
- valref(char,sdata,i) = \
- typref(char,scalar) op valref(char,sdata,i); \
- break; \
- case INT2OID: \
- valref(int16,sdata,i) = \
- typref(int16,scalar) op valref(int16_t,sdata,i); \
- break; \
- case INT4OID: \
- valref(int32,sdata,i) = \
- typref(int32,scalar) op valref(int32_t,sdata,i); \
- break; \
- case INT8OID: \
- valref(int64,sdata,i) = \
- typref(int64,scalar) op valref(int64,sdata,i); \
- break; \
-}
-
-#define accum_sdata_result(result,left,i,op,right,j) \
- switch ((left)->type_of_data) \
-{ \
- case FLOAT4OID: \
- typref(float,result) = \
- valref(float,left,i) op \
- valref(float,right,j); \
- break; \
- case FLOAT8OID: \
- typref(float8,result) = \
- valref(float8,left,i) op \
- valref(float8,right,j); \
- break; \
- case CHAROID: \
- typref(char,result) = \
- valref(char,left,i) op \
- valref(char,right,j); \
- break; \
- case INT2OID: \
- typref(int16,result) = \
- valref(int16,left,i) op \
- valref(int16,right,j); \
- break; \
- case INT4OID: \
- typref(int32,result) = \
- valref(int32,left,i) op \
- valref(int32,right,j); \
- break; \
- case INT8OID: \
- typref(int64,result) = \
- valref(int64,left,i) op \
- valref(int64,right,j); \
- break; \
-}
-
-#define apply_function_sdata_scalar(result,func,left,i,scalar) \
- switch ((left)->type_of_data) \
-{ \
- case FLOAT4OID: \
- valref(float,result,i) =\
- (float) func(valref(float,left,i),typref(float,scalar)); \
- break; \
- case FLOAT8OID: \
- valref(float8,result,i) =\
- func(valref(float8,left,i),typref(float8,scalar)); \
- break; \
- case CHAROID: \
- valref(char,result,i) =\
- (char) func(valref(char,left,i),typref(char,scalar)); \
- break; \
- case INT2OID: \
- valref(int16,result,i) =\
- (int16) func(valref(int16,left,i),typref(int16,scalar)); \
- break; \
- case INT4OID: \
- valref(int32,result,i) =\
- (int32) func(valref(int32,left,i),typref(int32,scalar)); \
- break; \
- case INT8OID: \
- valref(int64,result,i) =\
- (int64) func(valref(int64,left,i),typref(int64,scalar)); \
- break; \
-}
-
-#define apply_square_sdata(result,left,i) \
- switch ((left)->type_of_data) \
-{ \
- case FLOAT4OID: \
- valref(float,result,i) = \
- valsquare(valref(float,left,i)); \
- break; \
- case FLOAT8OID: \
- valref(float8,result,i) = \
- valsquare(valref(float8,left,i));\
- break; \
- case CHAROID: \
- valref(char,result,i) = \
- valsquare(valref(char,left,i));\
- break; \
- case INT2OID: \
- valref(int16_t,result,i) = \
- valsquare(valref(int16_t,left,i));\
- break; \
- case INT4OID: \
- valref(int32_t,result,i) = \
- valsquare(valref(int32_t,left,i));\
- break; \
- case INT8OID: \
- valref(int64,result,i) = \
- valsquare(valref(int64,left,i));\
- break; \
-}
-
-#define apply_cube_sdata(result,left,i) \
- switch ((left)->type_of_data) \
-{ \
- case FLOAT4OID: \
- valref(float,result,i) = \
- valcube(valref(float,left,i)); \
- break; \
- case FLOAT8OID: \
- valref(float8,result,i) = \
- valcube(valref(float8,left,i));\
- break; \
- case CHAROID: \
- valref(char,result,i) = \
- valcube(valref(char,left,i));\
- break; \
- case INT2OID: \
- valref(int16_t,result,i) = \
- valcube(valref(int16_t,left,i));\
- break; \
- case INT4OID: \
- valref(int32_t,result,i) = \
- valcube(valref(int32_t,left,i));\
- break; \
- case INT8OID: \
- valref(int64,result,i) = \
- valcube(valref(int64,left,i));\
- break; \
-}
-#define apply_quad_sdata(result,left,i) \
- switch ((left)->type_of_data) \
-{ \
- case FLOAT4OID: \
- valref(float,result,i) = \
- valquad(valref(float,left,i)); \
- break; \
- case FLOAT8OID: \
- valref(float8,result,i) = \
- valquad(valref(float8,left,i));\
- break; \
- case CHAROID: \
- valref(char,result,i) = \
- valquad(valref(char,left,i));\
- break; \
- case INT2OID: \
- valref(int16_t,result,i) = \
- valquad(valref(int16_t,left,i));\
- break; \
- case INT4OID: \
- valref(int32_t,result,i) = \
- valquad(valref(int32_t,left,i));\
- break; \
- case INT8OID: \
- valref(int64,result,i) = \
- valquad(valref(int64,left,i));\
- break; \
-}
-
-/* Checks that two SparseData have the same dimension
- */
-void
-check_sdata_dimensions(SparseData left, SparseData right)
-{
- if (left->total_value_count != right->total_value_count)
- {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("dimensions of vectors must be the same")));
- }
-}
-
-/* Do one of subtract, add, multiply, or divide depending on
- * the value of operation.
- */
-void op_sdata_by_scalar_inplace(enum operation_t operation,
- char *scalar, SparseData sdata, bool scalar_is_right)
-{
- if (scalar_is_right) //scalar is on the right
- {
- for(int i=0; i<sdata->unique_value_count; i++)
- {
- switch(operation)
- {
- case subtract:
- apply_const_to_sdata(sdata,i,-=,scalar)
- break;
- case add:
- apply_const_to_sdata(sdata,i,+=,scalar)
- break;
- case multiply:
- apply_const_to_sdata(sdata,i,*=,scalar)
- break;
- case divide:
- apply_const_to_sdata(sdata,i,/=,scalar)
- break;
- }
- }
- } else { //scalar is on the left
- for(int i=0; i<sdata->unique_value_count; i++)
- {
- switch(operation)
- {
- case subtract:
- apply_scalar_left_to_sdata(sdata,i,-,scalar)
- break;
- case add:
- apply_scalar_left_to_sdata(sdata,i,+,scalar)
- break;
- case multiply:
- apply_scalar_left_to_sdata(sdata,i,*,scalar)
- break;
- case divide:
- apply_scalar_left_to_sdata(sdata,i,/,scalar)
- break;
- }
- }
- }
-
-}
-SparseData op_sdata_by_scalar_copy(enum operation_t operation,
- char *scalar, SparseData source_sdata, bool scalar_is_right)
-{
- SparseData sdata = makeSparseDataCopy(source_sdata);
- op_sdata_by_scalar_inplace(operation,scalar,sdata,scalar_is_right);
- return sdata;
-}
-
-/* Exponentiates an sdata left argument with a right scalar
- */
-SparseData pow_sdata_by_scalar(SparseData sdata,
- char *scalar)
-{
- SparseData result = makeSparseDataCopy(sdata);
- for(int i=0; i<sdata->unique_value_count; i++)
- apply_function_sdata_scalar(result,pow,sdata,i,scalar)
-
- return(result);
-}
-SparseData square_sdata(SparseData sdata)
-{
- SparseData result = makeSparseDataCopy(sdata);
- for(int i=0; i<sdata->unique_value_count; i++)
- apply_square_sdata(result,sdata,i)
-
- return(result);
-}
-SparseData cube_sdata(SparseData sdata)
-{
- SparseData result = makeSparseDataCopy(sdata);
- for(int i=0; i<sdata->unique_value_count; i++)
- apply_cube_sdata(result,sdata,i)
-
- return(result);
-}
-SparseData quad_sdata(SparseData sdata)
-{
- SparseData result = makeSparseDataCopy(sdata);
- for(int i=0; i<sdata->unique_value_count; i++)
- apply_quad_sdata(result,sdata,i)
-
- return(result);
-}
-
-/* Checks the equality of two SparseData. We can't assume that two
- * SparseData are in canonical form.
- *
- * The algorithm is simple: we traverse the left SparseData element by
- * element, and for each such element x, we traverse all the elements of
- * the right SparseData that overlaps with x and check that they are equal.
- *
- * Note: This function only works on SparseData of float8s at present.
- */
-bool sparsedata_eq(SparseData left, SparseData right)
-{
- if (left->total_value_count != right->total_value_count)
- return false;
-
- char * ix = left->index->data;
- double * vals = (double *)left->vals->data;
-
- char * rix = right->index->data;
- double * rvals = (double *)right->vals->data;
-
- int read = 0, rread = 0;
- int rvid = 0;
- int rrun_length, i;
-
- for (i=0; i<left->unique_value_count; i++,ix+=int8compstoragesize(ix)) {
- read += compword_to_int8(ix);
-
- while (true) {
- /*
- * We need to use memcmp to handle NULLs (represented
- * as NaNs) properly
- */
- if (memcmp(&(vals[i]),&(rvals[rvid]),sizeof(float8))!=0)
- return false;
-
- /*
- * We never move the right element pointer beyond
- * the current left element
- */
- rrun_length = compword_to_int8(rix);
- if (rread + rrun_length > read) break;
-
- /*
- * Increase counters if there are more elements in
- * the right SparseData that overlaps with current
- * left element
- */
- rread += rrun_length;
- if (rvid < right->unique_value_count) {
- rix += int8compstoragesize(rix);
- rvid++;
- }
- if (rread == read) break;
- }
- }
- Assert(rread == read);
- return true;
-}
-
-/* Checks the equality of two SparseData. We can't assume that two
- * SparseData are in canonical form.
- *
- * The algorithm is simple: we traverse the left SparseData element by
- * element, and for each such element x, we traverse all the elements of
- * the right SparseData that overlaps with x and check that they are equal.
- *
- * Unlike sparsedata_eq, this function assumes that any zero represents a
- * missing data and hence still implies equality
- *
- * Note: This function only works on SparseData of float8s at present.
- */
-bool sparsedata_eq_zero_is_equal(SparseData left, SparseData right)
-{
- char * ix = left->index->data;
- double* vals = (double *)left->vals->data;
-
- char * rix = right->index->data;
- double* rvals = (double *)right->vals->data;
-
- int read = 0, rread = 0;
- int i=-1, j=-1, minimum = 0;
- minimum = (left->total_value_count > right->total_value_count) ?
- right->total_value_count : left->total_value_count;
-
- for (;(read < minimum)||(rread < minimum);) {
- if (read < rread) {
- read += (int)compword_to_int8(ix);
- ix +=int8compstoragesize(ix);
- i++;
- if ((memcmp(&(vals[i]),&(rvals[j]),sizeof(float8))!=0) &&
- (vals[i]!=0.0)&&(rvals[j]!=0.0)) {
- return false;
- }
- } else if (read > rread){
- rread += (int)compword_to_int8(rix);
- rix+=int8compstoragesize(rix);
- j++;
- if ((memcmp(&(vals[i]),&(rvals[j]),sizeof(float8))!=0) &&
- (vals[i]!=0.0)&&(rvals[j]!=0.0)) {
- return false;
- }
- } else {
- read += (int)compword_to_int8(ix);
- rread += (int)compword_to_int8(rix);
- ix +=int8compstoragesize(ix);
- rix+=int8compstoragesize(rix);
- i++;
- j++;
- if ((memcmp(&(vals[i]),&(rvals[j]),sizeof(float8))!=0) &&
- (vals[i]!=0.0)&&(rvals[j]!=0.0)) {
- return false;
- }
- }
- }
- /*sprintf(result, "result after %d %f", j, rvals[j]);
- ereport(NOTICE,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg(result)));*/
- return true;
-}
-
-/* Checks if one SparseData object contained in another
- *
- * First vector is said to contain second if all non-zero elements
- * of the second data object equal those of the first one
- *
- * Note: This function only works on SparseData of float8s at present.
- */
-bool sparsedata_contains(SparseData left, SparseData right)
-{
- char * ix = left->index->data;
- double* vals = (double *)left->vals->data;
-
- char * rix = right->index->data;
- double* rvals = (double *)right->vals->data;
-
- int read = 0, rread = 0;
- int i=-1, j=-1, minimum = 0;
- int lsize, rsize;
- lsize = left->total_value_count;
- rsize = right->total_value_count;
- if((rsize > lsize)&&(rvals[right->unique_value_count-1]!=0.0)){
- return false;
- }
-
- minimum = (lsize > rsize)?rsize:lsize;
-
- for (;(read < minimum)||(rread < minimum);) {
- if(read < rread){
- read += (int)compword_to_int8(ix);
- ix +=int8compstoragesize(ix);
- i++;
- if ((memcmp(&(vals[i]),&(rvals[j]),sizeof(float8))!=0)&&(rvals[j]!=0.0)){
- return false;
- }
- }else if(read > rread){
- rread += (int)compword_to_int8(rix);
- rix+=int8compstoragesize(rix);
- j++;
- if ((memcmp(&(vals[i]),&(rvals[j]),sizeof(float8))!=0)&&(rvals[j]!=0.0)){
- return false;
- }
- }else{
- read += (int)compword_to_int8(ix);
- rread += (int)compword_to_int8(rix);
- ix +=int8compstoragesize(ix);
- rix+=int8compstoragesize(rix);
- i++;
- j++;
- if ((memcmp(&(vals[i]),&(rvals[j]),sizeof(float8))!=0)&&(rvals[j]!=0.0)){
- return false;
- }
- }
- }
- return true;
-}
-
-
-static inline double id(double x) { return x; }
-static inline double square(double x) { return x*x; }
-static inline double myabs(double x) { return (x < 0) ? -(x) : x ; }
-
-/* This function is introduced to capture a common routine for
- * traversing a SparseData, transforming each element as we go along and
- * summing up the transformed elements. The method is non-destructive to
- * the input SparseData.
- */
-double
-accum_sdata_values_double(SparseData sdata, double (*func)(double))
-{
- double accum=0.;
- char *ix = sdata->index->data;
- double *vals = (double *)sdata->vals->data;
- int64 run_length;
-
- for (int i=0;i<sdata->unique_value_count;i++)
- {
- run_length = compword_to_int8(ix);
- accum += func(vals[i])*run_length;
- ix+=int8compstoragesize(ix);
- }
- return (accum);
-}
-
-/* Computes the running sum of the elements of a SparseData */
-double sum_sdata_values_double(SparseData sdata) {
- return accum_sdata_values_double(sdata, id);
-}
-
-/* Computes the l2 norm of a SparseData */
-double l2norm_sdata_values_double(SparseData sdata) {
- return sqrt(accum_sdata_values_double(sdata, square));
-}
-
-/* Computes the l1 norm of a SparseData */
-double l1norm_sdata_values_double(SparseData sdata) {
- return accum_sdata_values_double(sdata, myabs);
-}
-
-/*
- * Addition, Scalar Product, Division between SparseData arrays
- *
- * There are a few factors to consider:
- * - The dimension of the left and right arguments must be the same
- * - We employ an algorithm that does the computation on the compressed contents
- * which creates a new SparseData array
- *------------------------------------------------------------------------------
- */
-SparseData op_sdata_by_sdata(enum operation_t operation,
- SparseData left, SparseData right)
-{
- SparseData sdata = makeSparseData();
-
- /*
- * Loop over the contents of the left array, operating on elements
- * of the right array and append a new value to the sdata when a
- * new unique value is generated.
- *
- * We will manage two cursors, one for each of left and right arrays
- */
- char *liptr=left->index->data;
- char *riptr=right->index->data;
- int left_run_length, right_run_length;
- char *new_value,*last_new_value;
- int tot_run_length=-1;
- left_run_length = compword_to_int8(liptr);
- right_run_length = compword_to_int8(riptr);
- int left_lst=0,right_lst=0;
- int left_nxt=left_run_length,right_nxt=right_run_length;
- int nextpos = Min(left_nxt,right_nxt),lastpos=0;
- int i=0,j=0;
- size_t width = size_of_type(left->type_of_data);
-
- check_sdata_dimensions(left,right);
-
- new_value = (char *)palloc(width);
- last_new_value = (char *)palloc(width);
-
- while (1)
- {
- switch (operation)
- {
- case subtract:
- accum_sdata_result(new_value,left,i,-,right,j)
- break;
- case add:
- default:
- accum_sdata_result(new_value,left,i,+,right,j)
- break;
- case multiply:
- accum_sdata_result(new_value,left,i,*,right,j)
- break;
- case divide:
- accum_sdata_result(new_value,left,i,/,right,j)
- break;
- }
- /*
- * Potentially add a new run, depending on whether this is a
- * different value from the previous calculation. It may be
- * that this calculation has produced an identical value to
- * the previous, in which case we store it up, waiting for a
- * new value to happen.
- */
- if (tot_run_length==-1)
- {
- memcpy(last_new_value,new_value,width);
- tot_run_length=0;
- }
- if (memcmp(new_value,last_new_value,width))
- {
- add_run_to_sdata(last_new_value,tot_run_length,sizeof(float8),sdata);
- tot_run_length = 0;
- memcpy(last_new_value,new_value,width);
- }
- tot_run_length += (nextpos-lastpos);
-
- if ((left_nxt==right_nxt)&&(left_nxt==(left->total_value_count))) {
- break;
- } else if (left_nxt==right_nxt) {
- i++;j++;
- left_lst=left_nxt;right_lst=right_nxt;
- liptr+=int8compstoragesize(liptr);
- riptr+=int8compstoragesize(riptr);
- } else if (nextpos==left_nxt) {
- i++;
- left_lst=left_nxt;
- liptr+=int8compstoragesize(liptr);
- } else if (nextpos==right_nxt) {
- j++;
- right_lst=right_nxt;
- riptr+=int8compstoragesize(riptr);
- }
- left_run_length = compword_to_int8(liptr);
- right_run_length = compword_to_int8(riptr);
- left_nxt=left_run_length+left_lst;
- right_nxt=right_run_length+right_lst;
- lastpos=nextpos;
- nextpos = Min(left_nxt,right_nxt);
- }
-
- /*
- * Add the last run if we ended with a repeating value
- */
- if (tot_run_length!=0)
- add_run_to_sdata(new_value,tot_run_length,sizeof(float8),sdata);
-
- /*
- * Set the return data type
- */
- sdata->type_of_data = left->type_of_data;
-
- pfree(new_value);
- pfree(last_new_value);
-
- return sdata;
-}
-
-/* END Previously in SparseData.h */
-
-
-
-/**
- * @return A SparseData structure with allocated empty dynamic
- * StringInfo of unknown initial sizes.
- */
-SparseData makeSparseData(void) {
- /* Allocate the struct */
- SparseData sdata = (SparseData)palloc(sizeof(SparseDataStruct));
-
- /* Allocate the included elements */
- sdata->vals = makeStringInfo();
- sdata->index = makeStringInfo();
-
- sdata->unique_value_count=0;
- sdata->total_value_count=0;
- sdata->type_of_data = FLOAT8OID;
- return sdata;
-
- // makeStringInfo ensures vals and index each has a trailing '\0'
-}
-
-/**
- * @return A SparseData with zero storage in its StringInfos.
- */
-SparseData makeEmptySparseData(void) {
- /* Allocate the struct */
- SparseData sdata = (SparseData)palloc(sizeof(SparseDataStruct));
-
- /* Set up the data area */
- sdata->vals = (StringInfo)palloc(sizeof(StringInfoData));
- sdata->vals->data = (char *)palloc(1);
- sdata->vals->maxlen = 1;
- sdata->vals->data[0] = '\0';
- sdata->vals->len = 0;
- sdata->vals->cursor = 0;
-
- sdata->index = (StringInfo)palloc(sizeof(StringInfoData));
- sdata->index->data = (char *)palloc(1);
- sdata->index->maxlen = 1;
- sdata->index->data[0] = '\0';
- sdata->index->len = 0;
- sdata->index->cursor = 0;
-
- sdata->unique_value_count = 0;
- sdata->total_value_count = 0;
- sdata->type_of_data = FLOAT8OID;
-
- return sdata;
-}
-
-/**
- * @param vals An array of data values
- * @param index An array of run-lengths
- * @param datasize The length of the vals array
- * @param indexsize The length of the index array
- * @param datatype The object ID of the elements represented in the vals array
- * @param unique_value_count The number of RLE blocks in the vals array
- * @param total_value_count The total number of individual elements represented in the vals and index arrays
- * @return A SparseData created in place using pointers to the vals and index data
- */
-SparseData makeInplaceSparseData(char *vals, char *index,
- int datasize, int indexsize, Oid datatype,
- int unique_value_count, int total_value_count) {
- SparseData sdata = makeEmptySparseData();
- sdata->unique_value_count = unique_value_count;
- sdata->total_value_count = total_value_count;
-
- /*
- * To be safe, we obey the constraint demanded in lib/stringinfo.h that
- * the data field of StringInfoData is always terminated with a null.
- */
- if (vals != NULL && vals[datasize] != '\0') {
- char * vals_temp = (char *)palloc(datasize+1);
- memcpy(vals_temp, vals, datasize);
- vals_temp[datasize] = '\0';
- vals = vals_temp;
- }
-
- if (index != NULL && index[indexsize] != '\0') {
- char * index_temp = (char *)palloc(indexsize+1);
- memcpy(index_temp, index, indexsize);
- index_temp[indexsize] = '\0';
- index = index_temp;
- }
-
- sdata->vals->data = vals;
- sdata->vals->len = datasize;
- sdata->vals->maxlen = datasize+1;
- sdata->index->data = index;
-
- sdata->index->len = indexsize;
- sdata->index->maxlen = index == NULL ? 0 : indexsize+1;
- // here we are taking care of the special case of null index, which
- // represents uncompressed sparsedata; see SparseDataStruct defn
- sdata->type_of_data = datatype;
-
- return sdata;
-}
-
-/**
- * @return A copy of an existing SparseData.
- */
-SparseData makeSparseDataCopy(SparseData source_sdata) {
- /* Allocate the struct */
- SparseData sdata = (SparseData)palloc(sizeof(SparseDataStruct));
- /* Allocate the included elements */
- sdata->vals = copyStringInfo(source_sdata->vals);
- sdata->index = copyStringInfo(source_sdata->index);
- sdata->type_of_data = source_sdata->type_of_data;
- sdata->unique_value_count = source_sdata->unique_value_count;
- sdata->total_value_count = source_sdata->total_value_count;
- return sdata;
-}
-/**
- * @param constant The value of an RLE block
- * @param dimension The size of an RLE block
- * @return A SparseData with a single RLE block of size dimension having value constant
- */
-SparseData makeSparseDataFromDouble(double constant,int64 dimension) {
- char *bytestore=(char *)palloc(sizeof(char)*9);
- SparseData sdata = float8arr_to_sdata(&constant,1);
- int8_to_compword(dimension,bytestore); /* create compressed version of
- int8 value */
-
- int newlen = int8compstoragesize(bytestore);
- sdata->index->len = 0; // write over existing value
- appendBinaryStringInfo(sdata->index, bytestore, newlen);
-
- sdata->total_value_count = dimension;
-
- return sdata;
-}
-
-/**
- * Frees up the memory occupied by sdata
- */
-void freeSparseData(SparseData sdata) {
- pfree(sdata->vals);
- pfree(sdata->index);
- pfree(sdata);
-}
-
-/**
- * Frees up the memory occupied by sdata, including the data elements of vals and index.
- */
-void freeSparseDataAndData(SparseData sdata) {
- pfree(sdata->vals->data);
- pfree(sdata->index->data);
- freeSparseData(sdata);
-}
-
-/**
- * @param sinfo The StringInfo structure to be copied
- * @return A copy of sinfo
- */
-StringInfo copyStringInfo(StringInfo sinfo) {
- StringInfo result;
- char *data;
- if (sinfo->data == NULL) {
- data = NULL;
- } else {
- data = (char *)palloc(sizeof(char)*(sinfo->len+1));
- memcpy(data,sinfo->data,sinfo->len);
- data[sinfo->len] = '\0';
- }
- result = makeStringInfoFromData(data,sinfo->len);
- return result;
-}
-
-/**
- * @param data Pointer to an array of elements
- * @param len The size of the data array
- * @return A StringInfo from a data pointer and length
- */
-StringInfo makeStringInfoFromData(char *data,int len) {
- StringInfo sinfo;
- sinfo = (StringInfo)palloc(sizeof(StringInfoData));
-
- if (data != NULL && data[len] != '\0') {
- char * data_temp = (char *)palloc(len+1);
- memcpy(data_temp, data, len);
- data_temp[len] = '\0';
- data = data_temp;
- }
-
- sinfo->data = data;
- sinfo->len = len;
- sinfo->maxlen = len+1;
- sinfo->cursor = 0;
- return sinfo;
-}
-
-/**
- * @param array The array of doubles to be converted to a SparseData
- * @param count The size of array
- * @return A SparseData representation of an input array of doubles
- */
-SparseData float8arr_to_sdata(double *array, int count) {
- return arr_to_sdata((char *)array, sizeof(float8), FLOAT8OID, count);
-}
-
-/**
- * @param array The array of elements to be converted to a SparseData
- * @param width The size of the elements in array
- * @param type_of_data The object ID of the elements in array
- * @param count The size of array
- * @return A SparseData representation of an input array
- */
-SparseData arr_to_sdata(char *array, size_t width, Oid type_of_data, int count){
- char *run_val=array;
- int64 run_len=1;
- SparseData sdata = makeSparseData();
-
- sdata->type_of_data=type_of_data;
-
- for (int i=1; i<count; i++) {
- char *curr_val=array+ (i*size_of_type(type_of_data));
-
- /*
- * Note that special double values like denormalized numbers and exceptions
- * like NaN are treated like any other value - if there are duplicates, the
- * value of the special number is preserved and they are counted.
- */
- if (memcmp(curr_val,run_val,width))
- { /*run is interrupted, initiate new run */
- /* package up the finished run */
- add_run_to_sdata(run_val,run_len,width,sdata);
- /* mark beginning of new run */
- run_val=curr_val;
- run_len=1;
- } else
- { /* we're still in the same run */
- run_len++;
- }
- }
- add_run_to_sdata(run_val, run_len, width, sdata); /* package up the last run */
-
- /* Add the final tallies */
- sdata->unique_value_count = sdata->vals->len/width;
- sdata->total_value_count = count;
-
- return sdata;
-}
-
-int compar(const void *i, const void *j){
- return (int)((int64*)array_pos_ref)[*(int*)i] - ((int64*)array_pos_ref)[*(int*)j];
-}
-
-/**
- * @param array_val The array of values to be converted to values in SparseData
- * @param array_pos The array of positions to be converted to runs in SparseData
- * @param type_of_data type of the value element
- * @param count The (common) size of array and array_pos
- * @param end The size of the desired SparseData
- * @param default_val The default value for positions unspecified in array_pos
- * @return A SparseData representation of an input array of doubles
- */
-SparseData position_to_sdata(double *array_val, int64 *array_pos,
- Oid type_of_data,
- int count, int64 end, double default_val) {
-
- char * array = (char *)array_val;
- char * base_val = (char *)&default_val;
- size_t width = size_of_type(type_of_data);
- char *run_val=array;
- int64 run_len;
- SparseData sdata = makeSparseData();
-
- int *index = (int*)palloc(count*sizeof(int));
- for(int i = 0; i < count; ++i)
- index[i] = i;
-
- /* this is a temp solution that will be in place only until qsort_r
- * is a standard library function */
- array_pos_ref = array_pos;
- qsort(index, count, sizeof(int), compar);
-
- sdata->type_of_data=type_of_data;
- if(array_pos[index[0]] > 1){
- run_len = array_pos[index[0]]-1;
- add_run_to_sdata(base_val,run_len,width,sdata);
- }
-
- for (int i = 0; i<count; i++) {
- run_len=1;
- /*
- * Note that special double values like denormalized numbers and exceptions
- * like NaN are treated like any other value - if there are duplicates, the
- * value of the special number is preserved and they are counted.
- */
- while ((i < count-1) &&
- ((array_pos[index[i+1]] - array_pos[index[i]])==1) &&
- (memcmp((array+index[i]*size_of_type(type_of_data)),
- (array+index[i+1]*size_of_type(type_of_data)),width)==0)) {
- run_len++;
- i++;
- }
- while ((i < count-1)&&((array_pos[index[i+1]] - array_pos[index[i]])==0)) {
- if ((memcmp((array+index[i]*size_of_type(type_of_data)),
- (array+index[i+1]*size_of_type(type_of_data)),width)==0)) {
- i++;
- } else {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("posit_to_sdata conflicting values for the same position")));
- }
- }
- run_val = array+index[i]*size_of_type(type_of_data);
- add_run_to_sdata(run_val,run_len,width,sdata);
-
- if(i < count-1){
- run_len = array_pos[index[i+1]]-array_pos[index[i]]-1;
- if (run_len > 0){
- add_run_to_sdata(base_val,run_len,width,sdata);
- }
- }else if(array_pos[index[i]] < end){
- run_len = end-array_pos[index[i]];
- if (run_len > 0){
- add_run_to_sdata(base_val,run_len,width,sdata);
- }
- }
- }
-
- /* Add the final tallies */
- sdata->unique_value_count = sdata->vals->len/width;
- sdata->total_value_count = end;
- pfree(index);
- return sdata;
-}
-
-
-/**
- * @param sdata The SparseData to be converted to an array of float8s
- * @return A float8[] representation of a SparseData
- */
-double *sdata_to_float8arr(SparseData sdata) {
- double *array;
- int j, aptr;
- char *iptr;
-
- if (sdata->type_of_data != FLOAT8OID) {
- ereport(ERROR,(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Data type of SparseData is not FLOAT64\n")));
- }
-
- if ((array=(double *)palloc(sizeof(double)*(sdata->total_value_count)))
- == NULL)
- {
- ereport(ERROR,(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Error allocating memory for array\n")));
- }
-
- iptr = sdata->index->data;
- aptr = 0;
- for (int i=0; i<sdata->unique_value_count; i++) {
- for (j=0;j<compword_to_int8(iptr);j++,aptr++) {
- array[aptr] = ((double *)(sdata->vals->data))[i];
- }
- iptr+=int8compstoragesize(iptr);
- }
-
- if ((aptr) != sdata->total_value_count)
- {
- ereport(ERROR,(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Array size is incorrect, is: %d and should be %d\n",
- aptr,sdata->total_value_count)));
-
- pfree(array);
- return NULL;
- }
-
- return array;
-}
-
-/**
- * @return An array of integers given the (compressed) count array of a SparseData
- */
-int64 *sdata_index_to_int64arr(SparseData sdata) {
- char *iptr;
- int64 *array_ix = (int64 *)palloc(
- sizeof(int64)*(sdata->unique_value_count));
-
- iptr = sdata->index->data;
- for (int i=0; i<sdata->unique_value_count; i++,
- iptr+=int8compstoragesize(iptr)) {
- array_ix[i] = compword_to_int8(iptr);
- }
- return(array_ix);
-}
-
-/**
- * @param target The memory area to store the serialised SparseData
- * @para source The SparseData to be serialised
- * @return The serialisation of a SparseData structure
- */
-void serializeSparseData(char *target, SparseData source)
-{
- /* SparseDataStruct header */
- memcpy(target,source,SIZEOF_SPARSEDATAHDR);
- /* Two StringInfo structures describing the data and index */
- memcpy(SDATA_DATA_SINFO(target), source->vals,sizeof(StringInfoData));
- memcpy(SDATA_INDEX_SINFO(target),source->index,sizeof(StringInfoData));
- /* The unique data values */
- memcpy(SDATA_VALS_PTR(target),source->vals->data,source->vals->maxlen);
- /* The index values */
- memcpy(SDATA_INDEX_PTR(target),source->index->data,source->index->maxlen);
-
- /*
- * Set pointers to the data areas of the serialized structure
- * First the two StringInfo structures contained in the SparseData,
- * then the data areas inside each of the two StringInfos.
- */
- ((SparseData)target)->vals = (StringInfo)SDATA_DATA_SINFO(target);
- ((SparseData)target)->index = (StringInfo)SDATA_INDEX_SINFO(target);
- ((StringInfo)(SDATA_DATA_SINFO(target)))->data = SDATA_VALS_PTR(target);
- if (source->index->data != NULL)
- {
- ((StringInfo)(SDATA_INDEX_SINFO(target)))->data = SDATA_INDEX_PTR(target);
- } else {
- ((StringInfo)(SDATA_INDEX_SINFO(target)))->data = NULL;
- }
-}
-
-/**
- * Prints a SparseData
- */
-void printSparseData(SparseData sdata) {
- int value_count = sdata->unique_value_count;
- {
- char *indexdata = sdata->index->data;
- double *values = (double *)(sdata->vals->data);
- for (int i=0; i<value_count; i++) {
- printf("run_length[%d] = %lld, ",i,(long long int)compword_to_int8(indexdata));
- printf("value[%d] = %f\n",i,values[i]);
- indexdata+=int8compstoragesize(indexdata);
- }
- }
-}
-
-/**
- * @param sdata The SparseData to be projected on
- * @param idx The index to be projected
- * @return The element of a SparseData at location idx.
- */
-double sd_proj(SparseData sdata, int idx) {
- char * ix = sdata->index->data;
- double * vals = (double *)sdata->vals->data;
- int read, i;
-
- /* error checking */
- if (0 >= idx || idx > sdata->total_value_count)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Index out of bounds.")));
-
- /* find desired block; as is normal in SQL, we start counting from one */
- read = compword_to_int8(ix);
- i = 0;
- while (read < idx) {
- ix += int8compstoragesize(ix);
- read += compword_to_int8(ix);
- i++;
- }
- return vals[i];
-}
-
-/**
- * @param sdata The SparseData from which to extract a subarray
- * @param start The start index of the desired subarray
- * @param end The end index of the desired subarray
- * @return The sub-array, indexed by start and end, of a SparseData.
- */
-SparseData subarr(SparseData sdata, int start, int end) {
- char * ix = sdata->index->data;
- double * vals = (double *)sdata->vals->data;
- SparseData ret = makeSparseData();
- size_t wf8 = sizeof(float8);
-
- if (start > end)
- return reverse(subarr(sdata,end,start));
-
- /* error checking */
- if (0 >= start || start > end || end > sdata->total_value_count)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Array index out of bounds.")));
-
- /* find start block */
- int read = compword_to_int8(ix);
- int i = 0;
- while (read < start) {
- ix += int8compstoragesize(ix);
- read += compword_to_int8(ix);
- i++;
- }
- if (end <= read) {
- /* the whole subarray is in the first block, we are done */
- add_run_to_sdata((char *)(&vals[i]), end-start+1, wf8, ret);
- return ret;
- }
- /* else start building subarray */
- add_run_to_sdata((char *)(&vals[i]), read-start+1, wf8, ret);
-
- for (int j=i+1; j<sdata->unique_value_count; j++) {
- ix += int8compstoragesize(ix);
- int esize = compword_to_int8(ix);
- if (read + esize > end) {
- add_run_to_sdata((char *)(&vals[j]), end-read, wf8,ret);
- break;
- }
- add_run_to_sdata((char *)(&vals[j]), esize, wf8, ret);
- read += esize;
- if (read == end) break;
- }
- return ret;
-}
-
-/**
- * @param sdata The SparseData to be reversed
- * @return A copy of the input SparseData, with the order of the elements reversed.
- */
-SparseData reverse(SparseData sdata) {
- char * ix = sdata->index->data;
- double * vals = (double *)sdata->vals->data;
- SparseData ret = makeSparseData();
- size_t w = sizeof(float8);
-
- /* move to the last count array element */
- for (int j=0; j<sdata->unique_value_count-1; j++)
- ix += int8compstoragesize(ix);
-
- /* copy from right to left */
- for (int j=sdata->unique_value_count-1; j!=-1; j--) {
- add_run_to_sdata((char *)(&vals[j]),compword_to_int8(ix),w,ret);
- ix -= int8compstoragesize(ix);
- }
- return ret;
-}
-
-/**
- * @param left The SparseData that comes first in the resulting concatenation
- * @param right The SparseData that comes second in the resulting concatenation
- * @return The concatenation of two input SparseData.
- */
-SparseData concat(SparseData left, SparseData right) {
- if (left == NULL && right == NULL) {
- return NULL;
- } else if (left == NULL && right != NULL) {
- return makeSparseDataCopy(right);
- } else if (left != NULL && right == NULL) {
- return makeSparseDataCopy(left);
- }
- SparseData sdata = makeEmptySparseData();
- char *vals,*index;
- int l_val_len = left->vals->len;
- int r_val_len = right->vals->len;
- int l_ind_len = left->index->len;
- int r_ind_len = right->index->len;
- int val_len = l_val_len + r_val_len;
- int ind_len = l_ind_len + r_ind_len;
-
- vals = (char *)palloc(sizeof(char)*val_len + 1);
- index = (char *)palloc(sizeof(char)*ind_len + 1);
-
- memcpy(vals, left->vals->data,l_val_len);
- memcpy(vals+l_val_len,right->vals->data,r_val_len);
- vals[val_len] = '\0';
-
- memcpy(index, left->index->data,l_ind_len);
- memcpy(index+l_ind_len,right->index->data,r_ind_len);
- index[ind_len] = '\0';
-
- sdata->vals = makeStringInfoFromData(vals,val_len);
- sdata->index = makeStringInfoFromData(index,ind_len);
-
- sdata->type_of_data = left->type_of_data;
- sdata->unique_value_count = left->unique_value_count +
- right->unique_value_count;
- sdata->total_value_count = left->total_value_count +
- right->total_value_count;
- return sdata;
-}
-
-/**
- * @param rep The SparseData to be replicated
- * @param multiplier The number of times to replicate rep
- * @return The input rep SparseData replicated multiplier times.
- */
-SparseData concat_replicate(SparseData rep, int multiplier) {
- if (rep == NULL) return NULL;
-
- SparseData sdata = makeEmptySparseData();
- char *vals,*index;
- int l_val_len = rep->vals->len;
- int l_ind_len = rep->index->len;
- int val_len = l_val_len*multiplier;
- int ind_len = l_ind_len*multiplier;
-
- vals = (char *)palloc(sizeof(char)*val_len + 1);
- index = (char *)palloc(sizeof(char)*ind_len + 1);
-
- for (int i=0;i<multiplier;i++) {
- memcpy(vals+i*l_val_len,rep->vals->data,l_val_len);
- memcpy(index+i*l_ind_len,rep->index->data,l_ind_len);
- }
- vals[val_len] = '\0';
- index[ind_len] = '\0';
-
- sdata->vals = makeStringInfoFromData(vals,val_len);
- sdata->index = makeStringInfoFromData(index,ind_len);
- sdata->type_of_data = rep->type_of_data;
- sdata->unique_value_count = multiplier * rep->unique_value_count;
- sdata->total_value_count = multiplier * rep->total_value_count;
-
- return sdata;
-}
-
-static bool lapply_error_checking(Oid foid, List * funcname);
-
-/**
- * This function applies an input function on all elements of a sparse data.
- * The function is modelled after the corresponding function in R.
- *
- * @param func The name of the function to apply
- * @param sdata The input sparse data to be modified
- * @return A SparseData with the same dimension as sdata but with each element sdata[i] replaced by func(sdata[i])
- */
-SparseData lapply(text * func, SparseData sdata) {
- Oid argtypes[1] = { FLOAT8OID };
- List * funcname = textToQualifiedNameList(func);
- SparseData result = makeSparseDataCopy(sdata);
- Oid foid = LookupFuncName(funcname, 1, argtypes, false);
-
- lapply_error_checking(foid, funcname);
-
- for (int i=0; i<sdata->unique_value_count; i++)
- valref(float8,result,i) =
- DatumGetFloat8(
- OidFunctionCall1(foid,
- Float8GetDatum(valref(float8,sdata,i))));
- return result;
-}
-
-/* This function checks for error conditions in lapply() function calls.
- */
-static bool lapply_error_checking(Oid foid, List * func) {
- /* foid != InvalidOid; otherwise LookupFuncName would raise error.
- Here we check that the return type of foid is float8. */
- HeapTuple ftup = SearchSysCache(PROCOID,
- ObjectIdGetDatum(foid), 0, 0, 0);
- Form_pg_proc pform = (Form_pg_proc) GETSTRUCT(ftup);
-
- if (pform->prorettype != FLOAT8OID)
- ereport(ERROR,
- (errcode(ERRCODE_DATATYPE_MISMATCH),
- errmsg("return type of %s is not double",
- NameListToString(func))));
-
- // check volatility
-
- ReleaseSysCache(ftup);
- return true;
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/SparseData.h
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/SparseData.h b/contrib/gp_sparse_vector/SparseData.h
deleted file mode 100644
index ba0bd61..0000000
--- a/contrib/gp_sparse_vector/SparseData.h
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * @file
- * \brief This file defines the SparseData structure and some basic
- * functions on SparseData.
- *
- * SparseData provides array storage for repetitive data as commonly found
- * in numerical analysis of sparse arrays and matrices.
- * A general run-length encoding scheme is adopted.
- * Sequential duplicate values in the array are represented in an index
- * structure that stores the count of the number of times a given value
- * is duplicated.
- * All storage is allocated with palloc().
- *
- * NOTE:
- * The SparseData structure is an in-memory structure and so must be
- * serialized into a persisted structure like a VARLENA when leaving
- * a GP / Postgres function. This implies a COPY from the SparseData
- * to the VARLENA.
- */
-
-#ifndef SPARSEDATA_H
-#define SPARSEDATA_H
-
-#include <math.h>
-#include <string.h>
-#include "postgres.h"
-#include "lib/stringinfo.h"
-#include "utils/array.h"
-#include "catalog/pg_type.h"
-
-/*!
- * \internal
- * SparseData holds information about a sparse array of values
- * \endinternal
- */
-typedef struct
-{
- Oid type_of_data; /**< The native type of the data entries */
- int unique_value_count; /**< The number of unique values in the data array */
- int total_value_count; /**< The total number of values, including duplicates */
- StringInfo vals; /**< The unique number values are stored here as a stream of bytes */
- StringInfo index; /**< A count of each value is stored in the index */
-} SparseDataStruct;
-
-/*
- * Sometimes we wish to store an uncompressed array inside a SparseDataStruct;
- * instead of storing an array of ones [1,1,..,1,1] in the index field, which
- * is wasteful, we choose to use index->data == NULL to represent this special
- * case.
- */
-
-/**
- * Pointer to a SparseDataStruct
- */
-typedef SparseDataStruct *SparseData;
-
-/*------------------------------------------------------------------------------
- * Serialized SparseData
- *------------------------------------------------------------------------------
- * SparseDataStruct Contents
- * StringInfoData Contents for "vals"
- * StringInfoData Contents for "index"
- * data contents for "vals" (size is vals->maxlen)
- * data contents for "index" (size is index->maxlen)
- *
- * The vals and index fields are serialized as StringInfoData, then the
- * data contents are serialized at the end.
- *
- * Since two StringInfoData structs together are 64-bit aligned, there's
- * no need for padding.
- *
- * For reference, here is the format of the StringInfoData:
- * char * dataptr;
- * -> a placeholder in the serialized version, is filled
- * when the serialized version is used in-place
- * int len;
- * int maxlen;
- * int cursor;
- */
-/**
- * @return The size of a serialized SparseData based on the actual consumed
- * length of the StringInfo data and StringInfoData structures.
- */
-#define SIZEOF_SPARSEDATAHDR MAXALIGN(sizeof(SparseDataStruct))
-/**
- * @param x a SparseData
- * @return The size of x minus the dynamic variables, plus two
- * integers describing the length of the data area and index
- */
-#define SIZEOF_SPARSEDATASERIAL(x) (SIZEOF_SPARSEDATAHDR + \
- (2*sizeof(StringInfoData)) + \
- (x)->vals->maxlen + (x)->index->maxlen)
-
-/*
- * The following take a serialized SparseData as an argument and return
- * pointers to locations inside.
- */
-#define SDATA_DATA_SINFO(x) ((char *)(x)+SIZEOF_SPARSEDATAHDR)
-#define SDATA_INDEX_SINFO(x) (SDATA_DATA_SINFO(x)+sizeof(StringInfoData))
-#define SDATA_DATA_SIZE(x) (((StringInfo)SDATA_DATA_SINFO(x))->maxlen)
-#define SDATA_INDEX_SIZE(x) (((StringInfo)SDATA_INDEX_SINFO(x))->maxlen)
-#define SDATA_VALS_PTR(x) (SDATA_INDEX_SINFO(x)+sizeof(StringInfoData))
-#define SDATA_INDEX_PTR(x) (SDATA_VALS_PTR(x)+SDATA_DATA_SIZE(x))
-
-#define SDATA_UNIQUE_VALCNT(x) (((SparseData)(x))->unique_value_count)
-#define SDATA_TOTAL_VALCNT(x) (((SparseData)(x))->total_value_count)
-
-/**
- * @param x a SparseData
- * @return True if x is a scalar */
-#define SDATA_IS_SCALAR(x) (((((x)->unique_value_count)==((x)->total_value_count))&&((x)->total_value_count==1)) ? 1 : 0)
-
-/**
- * @param ptr Pointer to the start of the count entry of a SparseData
- * @return The size of the integer count in an RLE index pointed to by ptr
- */
-#define int8compstoragesize(ptr) \
- (((ptr) == NULL) ? 0 : (((*((char *)(ptr)) < 0) ? 1 : (1 + *((char *)(ptr))))))
-/* The size of a compressed int8 is stored in the first element of the ptr
- * array; see the explanation at the int8_to_compword() function below.
- *
- * Note that if the ptr is NULL, a zero size is returned
- */
-
-void int8_to_compword(int64 num, char entry[9]);
-int64 compword_to_int8(const char *entry);
-
-/** Serialization function */
-void serializeSparseData(char *target, SparseData source);
-
-/* Constructors and destructors */
-SparseData makeEmptySparseData(void);
-SparseData makeInplaceSparseData(char *vals, char *index,
- int datasize, int indexsize, Oid datatype,
- int unique_value_count, int total_value_count);
-
-SparseData makeSparseDataCopy(SparseData source_sdata);
-SparseData makeSparseDataFromDouble(double scalar,int64 dimension);
-
-SparseData makeSparseData(void);
-
-void freeSparseData(SparseData sdata);
-void freeSparseDataAndData(SparseData sdata);
-
-StringInfo copyStringInfo(StringInfo source_sinfo);
-StringInfo makeStringInfoFromData(char *data,int len);
-
-/* Conversion to and from arrays */
-double *sdata_to_float8arr(SparseData sdata);
-int64 *sdata_index_to_int64arr(SparseData sdata);
-SparseData float8arr_to_sdata(double *array, int count);
-SparseData position_to_sdata(double *array_val, int64 *array_pos, Oid type_of_data, int count, int64 end, double default_val);
-SparseData arr_to_sdata(char *array, size_t width, Oid type_of_data, int count);
-SparseData posit_to_sdata(char *array, int64* array_pos, size_t width, Oid type_of_data, int count, int64 end, char *base_val);
-
-/* Some functions for accessing and changing elements of a SparseData */
-SparseData lapply(text * func, SparseData sdata);
-double sd_proj(SparseData sdata, int idx);
-SparseData subarr(SparseData sdata, int start, int end);
-SparseData reverse(SparseData sdata);
-SparseData concat(SparseData left, SparseData right);
-SparseData concat_replicate(SparseData rep, int multiplier);
-
-
-enum operation_t { subtract, add, multiply, divide };
-
-double sum_sdata_values_double(SparseData sdata);
-SparseData op_sdata_by_sdata(enum operation_t operation, SparseData left,
- SparseData right);
-bool sparsedata_eq(SparseData left, SparseData right);
-bool sparsedata_eq_zero_is_equal(SparseData left, SparseData right);
-bool sparsedata_contains(SparseData left, SparseData right);
-SparseData pow_sdata_by_scalar(SparseData sdata, char *scalar);
-SparseData square_sdata(SparseData sdata);
-SparseData cube_sdata(SparseData sdata);
-SparseData quad_sdata(SparseData sdata);
-void op_sdata_by_scalar_inplace(enum operation_t operation, char *scalar,
- SparseData sdata, bool scalar_is_right);
-void append_to_rle_index(StringInfo index, int64 run_len);
-
-SparseData op_sdata_by_scalar_copy(enum operation_t operation, char *scalar,
- SparseData source_sdata, bool scalar_is_right);
-
-double l2norm_sdata_values_double(SparseData sdata);
-double l1norm_sdata_values_double(SparseData sdata);
-
-size_t size_of_type(Oid type);
-void printout_double(double *vals, int num_values, int stop);
-void printout_index(char *ix, int num_values, int stop);
-void printout_sdata(SparseData sdata, char *msg, int stop);
-
-void add_run_to_sdata(char *run_val, int64 run_len, size_t width,
- SparseData sdata);
-
-/*------------------------------------------------------------------------------
- * macros that will test whether a given double value is in the normal
- * range or is in the special range (denormals, exceptions).
- *------------------------------------------------------------------------------
- */
-/* Anything between LOW and HIGH is a denormal or exception */
-#define SPEC_MASK_HIGH 0xFFF0000000000000
-#define SPEC_MASK_LOW 0x7FF0000000000000
-#define MASKTEST(x,y) (((x)&(y))==x) /* MASKTEST checks for the presence of
- the bits in x in the input y */
-
-/* The input to MASKTEST_double should be an int64 mask and a (double *) to be
- * tested
- */
-#define MASKTEST_double(x,y) MASKTEST((x),*((int64 *)(&(y))))
-
-#define DBL_IS_A_SPECIAL(x) \
- (MASKTEST_double(SPEC_MASK_HIGH,(x)) || MASKTEST_double(SPEC_MASK_LOW,(x)) \
- || ((x) == 0.))
-
-#endif /* SPARSEDATA_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/bugs
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/bugs b/contrib/gp_sparse_vector/bugs
deleted file mode 100644
index dc1c2c4..0000000
--- a/contrib/gp_sparse_vector/bugs
+++ /dev/null
@@ -1,122 +0,0 @@
-===========================================
-04/06/10
-
- Svec / Smat / Dmat
- Mathematics extensions to Greenplum
-
-==========================================================================================
-BUGS
-==========================================================================================
-OPEN
-DONE
-- Direct operations between float8[] and svec
-- Fixed: Constant (Scalar) handling is broken for left operators
-- Fixed: Lack of a binary send/recv function causes interconnect to SEGV
-- Added ^ operator for exponentiation
-- Added >,<,<>,=,==,<=,>= operators (ORDER BY works)
-- Added index support
-- Added dmin and dmax operators for scalars
-- unnest operator that returns setof(double) and takes svec input
-- svec_dimension operator that returns an int and takes svec input
-- concat operator mapped to "||" that returns svec and takes (svec,svec)
-- Serialization format should avoid copies more often
-- array_agg function converts relation to svec
-- vec_median implements partition selection on float8[]
-- Bug: array_agg would fail with incorrect results when there were many duplicate values
-- implement vec_median for svec (using the sparsity to drive the partition selection)
-
-==========================================================================================
-IMPROVEMENTS
-==========================================================================================
-TODO
-- implement an svec_agg_distinct that reorders and distincts to maximize RLE compression
- - implement an in memory median aggregate MEDIAN_INMEMORY that uses svec_agg_distinct
- as a PREFUNC and feeds svecs into svec_concat as the SFUNC, then vec_median as
- FINALFUNC
-- [off topic] implement MEDIAN using only a FINALFUNC that implements an out-of-core
- partition selection algorithm as follows:
- - As the data arrives, fill runs of svec using svec_agg_distinct until
- work_mem size is reached.
- - when a run is filled, partition select it into lower and upper halves
- and output both to disk appending to two files (lower, upper)
- - repeat until end of input with the same pivot value, there are now
- two files pivoted around one value
- - Choose random pivot index, continue partition selection using runs
- of svec, discarding whichever file is not needed
- - There should be no more than 2 writes of relation to disk, all
- sequential IO
-- implement PERCENTILE using the same logic as vec_median
-- phrase match that returns the weight of a match and the locations where the phrases are
- for each document relative to an input phrase (see README_term_proximity)
-- set "contains" operator that returns the intersection of two vectors (Elbit, LinkedIn)
-- slice operator that returns an svec and takes svec,min,max input
-- svec_pivot_count aggregate returns svec and takes (array dictionary,text term)
-- svec_pivot_sum aggregate returns svec and takes (array dictionary, text term, double value)
-- The complete set of scalar operators present in Postgres
-- Complete set of R vector operators
-- Rename or alias svec to _float8_sparse
-- Hash function for the type (for GROUP BY and HASH JOIN)
----- Like hash_numeric()
-- Vector cross product, outer product %o%
-- Matrices
----- "R" matrix functions and notation
----- "R" multi-dimensional arrays
----- Blocked matrix addressing for very large distributed matrices
-
-==========================================================================================
-Reference material about R operators from
- http://cran.r-project.org/doc/manuals/R-lang.html#Operators
-==========================================================================================
-
-- Minus, can be unary or binary
-+ Plus, can be unary or binary
-! Unary not
-~ Tilde, used for model formulae, can be either unary or binary
-? Help
-: Sequence, binary (in model formulae: interaction)
-* Multiplication, binary
-/ Division, binary
-^ Exponentiation, binary
-%x% Special binary operators, x can be replaced by any valid name
-%% Modulus, binary
-%/% Integer divide, binary
-%*% Matrix product, binary
-%o% Outer product, binary
-%x% Kronecker product, binary
-%in% Matching operator, binary (in model formulae: nesting)
-< Less than, binary
-> Greater than, binary
-== Equal to, binary
->= Greater than or equal to, binary
-<= Less than or equal to, binary
-& And, binary, vectorized
-&& And, binary, not vectorized
-| Or, binary, vectorized
-|| Or, binary, not vectorized
-<- Left assignment, binary
--> Right assignment, binary
-$ List subset, binary
-
-R deals with entire vectors of data at a time, and most of the elementary operators and basic mathematical functions like log are vectorized (as indicated in the table above). This means that e.g. adding two vectors of the same length will create a vector containing the element-wise sums, implicitly looping over the vector index. This applies also to other operators like -, *, and / as well as to higher dimensional structures. Notice in particular that multiplying two matrices does not produce the usual matrix product (the %*% operator exists for that purpose). Some finer points relating to vectorized operations will be discussed in Elementary arithmetic operations.
-
-3.4 Indexing
-
-R contains several constructs which allow access to individual elements or subsets through indexing operations. In the case of the basic vector types one can access the i-th element using x[i], but there is also indexing of lists, matrices, and multi-dimensional arrays. There are several forms of indexing in addition to indexing with a single integer. Indexing can be used both to extract part of an object and to replace parts of an object (or to add parts).
-
-R has three basic indexing operators, with syntax displayed by the following examples
-
- x[i]
- x[i, j]
- x[[i]]
- x[[i, j]]
- x$a
- x$"a"
-For vectors and matrices the [[ forms are rarely used, although they have some slight semantic differences from the [ form (e.g. it drops any names or dimnames attribute, and that partial matching is used for character indices). When indexing multi-dimensional structures with a single index, x[[i]] or x[i] will return the ith sequential element of x.
-
-For lists, one generally uses [[ to select any single element, whereas [ returns a list of the selected elements.
-
-The [[ form allows only a single element to be selected using integer or character indices, whereas [ allows indexing by vectors. Note though that for a list or other recursive object, the index can be a vector and each element of the vector is applied in turn to the list, the selected component, the selected component of that component, and so on. The result is still a single element.
-
-The form using $ applies to recursive objects such as lists and pairlists. It allows only a literal character string or a symbol as the index. That is, the index is not computable: for cases where you need to evaluate an expression to find the index, use x[[expr]]. When $ is applied to a non-recursive object the result used to be always NULL: as from R 2.6.0 this is an error.
-
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/float_specials.h
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/float_specials.h b/contrib/gp_sparse_vector/float_specials.h
deleted file mode 100644
index 8ec1d2a..0000000
--- a/contrib/gp_sparse_vector/float_specials.h
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * @file
- * \brief Special floating-point numbers
- */
-#ifndef FLOATSPECIALS_H
-#define FLOATSPECIALS_H
-
-/* Convenience functions */
-#if defined(_MSC_VER)
-#define MKINT(x) (x##UL)
-#define MKINT64(x) (x##Ui64)
-#define BIT(x) (1Ui64 << (x))
-#else
-#define MKINT(x) (x##U)
-#define MKINT64(x) (x##ULL)
-#define BIT(x) (1ULL << (x))
-#endif
-
-/* 32-bit special value ranges */
-
-#define NEG_QUIET_NAN_MIN32 MKINT(0xFFC00001)
-#define NEG_QUIET_NAN_MAX32 MKINT(0xFFFFFFFF)
-
-#define INDETERMINATE_MIN32 MKINT(0xFFC00000)
-#define INDETERMINATE_MAX32 MKINT(0xFFC00000)
-
-#define NEG_SIGNAL_NAN_MIN32 MKINT(0xFF800001)
-#define NEG_SIGNAL_NAN_MAX32 MKINT(0xFFBFFFFF)
-
-#define NEG_INFINITY_MIN32 MKINT(0xFF800000)
-
-#define NEG_NORMALIZED_MIN32 MKINT(0x80800000)
-#define NEG_NORMALIZED_MAX32 MKINT(0xFF7FFFFF)
-
-#define NEG_DENORMALIZED_MIN32 MKINT(0x80000001)
-#define NEG_DENORMALIZED_MAX32 MKINT(0x807FFFFF)
-
-#define NEG_ZERO_MIN32 MKINT(0x80000000)
-#define NEG_ZERO_MAX32 MKINT(0x80000000)
-
-#define POS_ZERO_MIN32 MKINT(0x00000000)
-#define POS_ZERO_MAX32 MKINT(0x00000000)
-
-#define POS_DENORMALIZED_MIN32 MKINT(0x00000001)
-#define POS_DENORMALIZED_MAX32 MKINT(0x007FFFFF)
-
-#define POS_NORMALIZED_MIN32 MKINT(0x00800000)
-#define POS_NORMALIZED_MAX32 MKINT(0x7F7FFFFF)
-
-#define POS_INFINITY_MIN32 MKINT(0x7F800000)
-#define POS_INFINITY_MAX32 MKINT(0x7F800000)
-
-#define POS_SIGNAL_NAN_MIN32 MKINT(0x7F800001)
-#define POS_SIGNAL_NAN_MAX32 MKINT(0x7FBFFFFF)
-
-#define POS_QUIET_NAN_MIN32 MKINT(0x7FC00000)
-#define POS_QUIET_NAN_MAX32 MKINT(0x7FFFFFFF)
-
-/* 64-bit special value ranges */
-
-#define NEG_QUIET_NAN_MIN64 MKINT64(0xFFF8000000000001)
-#define NEG_QUIET_NAN_MAX64 MKINT64(0xFFFFFFFFFFFFFFFF)
-
-#define INDETERMINATE_MIN64 MKINT64(0xFFF8000000000000)
-#define INDETERMINATE_MAX64 MKINT64(0xFFF8000000000000)
-
-#define NEG_SIGNAL_NAN_MIN64 MKINT64(0xFFF7FFFFFFFFFFFF)
-#define NEG_SIGNAL_NAN_MAX64 MKINT64(0xFFF0000000000001)
-
-#define NEG_INFINITY_MIN64 MKINT64(0xFFF0000000000000)
-
-#define NEG_NORMALIZED_MIN64 MKINT64(0xFFEFFFFFFFFFFFFF)
-#define NEG_NORMALIZED_MAX64 MKINT64(0x8010000000000000)
-
-#define NEG_DENORMALIZED_MIN64 MKINT64(0x800FFFFFFFFFFFFF)
-#define NEG_DENORMALIZED_MAX64 MKINT64(0x8000000000000001)
-
-#define NEG_ZERO_MIN64 MKINT64(0x8000000000000000)
-#define NEG_ZERO_MAX64 MKINT64(0x8000000000000000)
-
-#define POS_ZERO_MIN64 MKINT64(0x0000000000000000)
-#define POS_ZERO_MAX64 MKINT64(0x0000000000000000)
-
-#define POS_DENORMALIZED_MIN64 MKINT64(0x0000000000000001)
-#define POS_DENORMALIZED_MAX64 MKINT64(0x000FFFFFFFFFFFFF)
-
-#define POS_NORMALIZED_MIN64 MKINT64(0x0010000000000000)
-#define POS_NORMALIZED_MAX64 MKINT64(0x7FEFFFFFFFFFFFFF)
-
-#define POS_INFINITY_MIN64 MKINT64(0x7FF0000000000000)
-#define POS_INFINITY_MAX64 MKINT64(0x7FF0000000000000)
-
-#define POS_SIGNAL_NAN_MIN64 MKINT64(0x7FF0000000000001)
-#define POS_SIGNAL_NAN_MAX64 MKINT64(0x7FF7FFFFFFFFFFFF)
-
-#define POS_QUIET_NAN_MIN64 MKINT64(0x7FF8000000000000)
-#define POS_QUIET_NAN_MAX64 MKINT64(0x7FFFFFFFFFFFFFFF)
-
-typedef enum
-{
- POS_QNAN_BIT,
- NEG_QNAN_BIT,
- POS_SNAN_BIT,
- NEG_SNAN_BIT,
- POS_INF_BIT,
- NEG_INF_BIT,
- POS_DEN_BIT,
- NEG_DEN_BIT,
- POS_NOR_BIT,
- NEG_NOR_BIT,
- POS_ZERO_BIT,
- NEG_ZERO_BIT,
- INDETERM_BIT,
- BUG_BIT
-} ieee_selects;
-
-#define MSK_POS_QNAN BIT(POS_QNAN_BIT)
-#define MSK_POS_SNAN BIT(POS_SNAN_BIT)
-#define MSK_POS_INF BIT(POS_INF_BIT)
-#define MSK_POS_DEN BIT(POS_DEN_BIT)
-#define MSK_POS_NOR BIT(POS_NOR_BIT)
-#define MSK_POS_ZERO BIT(POS_ZERO_BIT)
-#define MSK_NEG_QNAN BIT(NEG_QNAN_BIT)
-#define MSK_NEG_SNAN BIT(NEG_SNAN_BIT)
-#define MSK_NEG_INF BIT(NEG_INF_BIT)
-#define MSK_NEG_DEN BIT(NEG_DEN_BIT)
-#define MSK_NEG_NOR BIT(NEG_NOR_BIT)
-#define MSK_NEG_ZERO BIT(NEG_ZERO_BIT)
-#define MSK_INDETERM BIT(INDETERM_BIT)
-#define MSK_BUG BIT(BUG_BIT)
-
-// Definitions for IEEE -inf, inf, -0 compressed out of the sparse vector along with zero
-#define ZERO_i MKINT64(0x0)
-#define INF_i POS_INFINITY_MIN64
-#define NEGINF_i NEG_INFINITY_MIN64
-#define NVP_i NEG_QUIET_NAN_MIN64
-
-// Reading a member of a union different from the one written to is undefined in
-// C99, yet it is very common practice.
-static const union {
- uint64_t asInt64;
- double asDouble;
-} COMPVEC[] = { { 5 }, { ZERO_i }, { INF_i }, { NEGINF_i }, { NVP_i } };
-
-static inline double _LAL_for_compiler_unused_warning(void) {return(COMPVEC[0].asDouble);}
-
-#define ZERO COMPVEC[1].asDouble
-#define INF COMPVEC[2].asDouble
-#define NEGINF COMPVEC[3].asDouble
-#define NVP COMPVEC[4].asDouble
-
-#endif /* FLOATSPECIALS_H */
[3/5] incubator-hawq git commit: HAWQ-49. Remove madlib related codes
and tests.
Posted by ml...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/operators.c
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/operators.c b/contrib/gp_sparse_vector/operators.c
deleted file mode 100644
index 879e124..0000000
--- a/contrib/gp_sparse_vector/operators.c
+++ /dev/null
@@ -1,939 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * @file
- * This module defines a collection of operators for svecs. The functions
- * are usually wrappers that call the corresponding operators defined for
- * SparseData.
- */
-
-#include <postgres.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <math.h>
-
-#include "utils/array.h"
-#include "catalog/pg_type.h"
-#include "utils/numeric.h"
-#include "utils/builtins.h"
-#include "utils/memutils.h"
-#include "access/hash.h"
-
-#include "sparse_vector.h"
-
-#ifndef NO_PG_MODULE_MAGIC
-PG_MODULE_MAGIC;
-#endif
-
-/**
- * For many functions defined in this module, the operation has no meaning
- * if the array dimensions aren't the same, unless one of the inputs is a
- * scalar. This routine checks that condition.
- */
-void check_dimension(SvecType *svec1, SvecType *svec2, char *msg) {
- if ((!IS_SCALAR(svec1)) &&
- (!IS_SCALAR(svec2)) &&
- (svec1->dimension != svec2->dimension)) {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("%s: array dimension of inputs are not the same: dim1=%d, dim2=%d\n",
- msg, svec1->dimension, svec2->dimension)));
- }
-}
-
-/**
- * Dot Product of two svec types
- */
-double svec_svec_dot_product(SvecType *svec1, SvecType *svec2) {
- SparseData left = sdata_from_svec(svec1);
- SparseData right = sdata_from_svec(svec2);
-
- check_dimension(svec1,svec2,"svec_svec_dot_product");
- return sum_sdata_values_double( op_sdata_by_sdata(multiply,left,right));
-}
-
-/**
- * svec_concat_replicate - replicates an svec multiple times
- */
-Datum svec_concat_replicate(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_concat_replicate);
-
-Datum svec_concat_replicate(PG_FUNCTION_ARGS)
-{
- int multiplier = PG_GETARG_INT32(0);
- if (multiplier < 0)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("multiplier cannot be negative")));
-
- SvecType *svec = PG_GETARG_SVECTYPE_P(1);
- SparseData rep = sdata_from_svec(svec);
- SparseData sdata = concat_replicate(rep, multiplier);
-
- PG_RETURN_SVECTYPE_P(svec_from_sparsedata(sdata,true));
-}
-
-/**
- * svec_concat - concatenates two svecs
- */
-Datum svec_concat(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_concat );
-Datum svec_concat(PG_FUNCTION_ARGS)
-{
- if (PG_ARGISNULL(0) && (!PG_ARGISNULL(1)))
- PG_RETURN_SVECTYPE_P(PG_GETARG_SVECTYPE_P(1));
- else if (PG_ARGISNULL(0) && PG_ARGISNULL(1))
- PG_RETURN_NULL();
- else if (PG_ARGISNULL(1))
- PG_RETURN_SVECTYPE_P(PG_GETARG_SVECTYPE_P(0));
-
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_from_svec(svec1);
- SparseData right = sdata_from_svec(svec2);
- SparseData sdata = concat(left, right);
-
- PG_RETURN_SVECTYPE_P(svec_from_sparsedata(sdata,true));
-}
-
-/**
- * svec_eq - returns the equality of two svecs
- */
-Datum svec_eq(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_eq );
-Datum svec_eq(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_from_svec(svec1);
- SparseData right = sdata_from_svec(svec2);
- PG_RETURN_BOOL(sparsedata_eq(left,right));
-}
-
-/*
- * Svec comparison functions based on the l2 norm
- */
-static int32_t svec_l2_cmp_internal(SvecType *svec1, SvecType *svec2)
-{
- SparseData left = sdata_from_svec(svec1);
- SparseData right = sdata_from_svec(svec2);
- double magleft = l2norm_sdata_values_double(left);
- double magright = l2norm_sdata_values_double(right);
- int result;
-
- if (IS_NVP(magleft) || IS_NVP(magright)) {
- result = -5;
- PG_RETURN_INT32(result);
- }
-
- if (magleft < magright) result = -1;
- else if (magleft > magright) result = 1;
- else result = 0;
-
- PG_RETURN_INT32(result);
-}
-Datum svec_l2_cmp(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_cmp );
-Datum svec_l2_cmp(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_INT32(result);
-}
-Datum svec_l2_lt(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_lt );
-Datum svec_l2_lt(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_BOOL((result == -1) ? 1 : 0);
-}
-Datum svec_l2_le(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_le );
-Datum svec_l2_le(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_BOOL(((result == -1)||(result == 0)) ? 1 : 0);
-}
-Datum svec_l2_eq(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_eq );
-Datum svec_l2_eq(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_BOOL((result == 0) ? 1 : 0);
-}
-Datum svec_l2_ne(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_ne );
-Datum svec_l2_ne(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_BOOL((result != 0) ? 1 : 0);
-}
-Datum svec_l2_gt(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_gt );
-Datum svec_l2_gt(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_BOOL((result == 1) ? 1 : 0);
-}
-Datum svec_l2_ge(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( svec_l2_ge );
-Datum svec_l2_ge(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- int result = svec_l2_cmp_internal(svec1,svec2);
-
- if (result == -5) PG_RETURN_NULL();
-
- PG_RETURN_BOOL(((result == 0) || (result == 1)) ? 1 : 0);
-}
-
-/**
- * Performs one of subtract, add, multiply, or divide depending on value
- * of operation.
- */
-SvecType * svec_operate_on_sdata_pair(int scalar_args, enum operation_t op,
- SparseData left, SparseData right)
-{
- SparseData sdata = NULL;
- float8 *left_vals = (float8 *)(left->vals->data);
- float8 *right_vals = (float8 *)(right->vals->data);
- float8 data_result;
-
- switch (scalar_args) {
- case 0: //neither arg is scalar
- sdata = op_sdata_by_sdata(op,left,right);
- break;
- case 1: //left arg is scalar
- sdata=op_sdata_by_scalar_copy(op,(char *)left_vals,right,false);
- break;
- case 2: //right arg is scalar
- sdata=op_sdata_by_scalar_copy(op,(char *)right_vals,left,true);
- break;
- case 3: //both args are scalar
- switch (op) {
- case subtract:
- data_result = left_vals[0] - right_vals[0];
- break;
- case add:
- default:
- data_result = left_vals[0] + right_vals[0];
- break;
- case multiply:
- data_result = left_vals[0] * right_vals[0];
- break;
- case divide:
- data_result = left_vals[0] / right_vals[0];
- break;
- }
- return svec_make_scalar(data_result);
- break;
- }
- return svec_from_sparsedata(sdata,true);
-}
-
-
-SvecType * op_svec_by_svec_internal(enum operation_t op, SvecType *svec1, SvecType *svec2)
-{
- SparseData left = sdata_from_svec(svec1);
- SparseData right = sdata_from_svec(svec2);
-
- int scalar_args = check_scalar(IS_SCALAR(svec1),IS_SCALAR(svec2));
-
- return svec_operate_on_sdata_pair(scalar_args,op,left,right);
-}
-
-/*
- * Do exponentiation, only makes sense if the left is a vector and the right
- * is a scalar or if both are scalar
- */
-static SvecType *
-pow_svec_by_scalar_internal(SvecType *svec1, SvecType *svec2)
-{
- SparseData left = sdata_from_svec(svec1);
- SparseData right = sdata_from_svec(svec2);
- SparseData sdata = NULL;
- double *left_vals=(double *)(left->vals->data);
- double *right_vals=(double *)(right->vals->data);
- double data_result;
-
- int scalar_args = check_scalar(IS_SCALAR(svec1),IS_SCALAR(svec2));
-
- switch(scalar_args) {
- case 0: //neither arg is scalar
- case 1: //left arg is scalar
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Svec exponentiation is undefined when the right argument is a vector")));
- break;
- case 2: //right arg is scalar
- if (right_vals[0] == 2.) // the squared case
- {
- sdata = square_sdata(left);
- } else if (right_vals[0] == 3.) // the cubed case
- {
- sdata = cube_sdata(left);
- } else if (right_vals[0] == 4.) // the quad case
- {
- sdata = quad_sdata(left);
- } else {
- sdata = pow_sdata_by_scalar(left,(char *)right_vals);
- }
- break;
- case 3: //both args are scalar
- data_result = pow(left_vals[0],right_vals[0]);
- return svec_make_scalar(data_result);
- break;
- }
- return svec_from_sparsedata(sdata,true);
-}
-
-PG_FUNCTION_INFO_V1( svec_pow );
-Datum svec_pow(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- check_dimension(svec1,svec2,"svec_pow");
- SvecType *result = pow_svec_by_scalar_internal(svec1,svec2);
- PG_RETURN_SVECTYPE_P(result);
-}
-
-PG_FUNCTION_INFO_V1( svec_minus );
-Datum svec_minus(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- check_dimension(svec1,svec2,"svec_minus");
- SvecType *result = op_svec_by_svec_internal(subtract,svec1,svec2);
- PG_RETURN_SVECTYPE_P(result);
-}
-
-PG_FUNCTION_INFO_V1( svec_plus );
-Datum svec_plus(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- check_dimension(svec1,svec2,"svec_plus");
- SvecType *result = op_svec_by_svec_internal(add,svec1,svec2);
- PG_RETURN_SVECTYPE_P(result);
-}
-
-PG_FUNCTION_INFO_V1( svec_mult );
-Datum svec_mult(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- check_dimension(svec1,svec2,"svec_mult");
- SvecType *result = op_svec_by_svec_internal(multiply,svec1,svec2);
- PG_RETURN_SVECTYPE_P(result);
-}
-
-PG_FUNCTION_INFO_V1( svec_div );
-Datum svec_div(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
- check_dimension(svec1,svec2,"svec_div");
- SvecType *result = op_svec_by_svec_internal(divide,svec1,svec2);
- PG_RETURN_SVECTYPE_P(result);
-}
-
-PG_FUNCTION_INFO_V1( svec_dot );
-/**
- * svec_dot - computes the dot product of two svecs
- */
-Datum svec_dot(PG_FUNCTION_ARGS)
-{
- SvecType *svec1 = PG_GETARG_SVECTYPE_P(0);
- SvecType *svec2 = PG_GETARG_SVECTYPE_P(1);
-
- double accum = svec_svec_dot_product( svec1, svec2);
-
- if (IS_NVP(accum)) PG_RETURN_NULL();
-
- PG_RETURN_FLOAT8(accum);
-}
-
-/*
- * Cast from int2,int4,int8,float4,float8 scalar to SvecType
- */
-PG_FUNCTION_INFO_V1( svec_cast_int2 );
-Datum svec_cast_int2(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_INT16(0);
- PG_RETURN_SVECTYPE_P(svec_make_scalar(value));
-}
-PG_FUNCTION_INFO_V1( svec_cast_int4 );
-Datum svec_cast_int4(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_INT32(0);
- PG_RETURN_SVECTYPE_P(svec_make_scalar(value));
-}
-PG_FUNCTION_INFO_V1( svec_cast_int8 );
-Datum svec_cast_int8(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_INT64(0);
- PG_RETURN_SVECTYPE_P(svec_make_scalar(value));
-}
-PG_FUNCTION_INFO_V1( svec_cast_float4 );
-Datum svec_cast_float4(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_FLOAT4(0);
- PG_RETURN_SVECTYPE_P(svec_make_scalar(value));
-}
-PG_FUNCTION_INFO_V1( svec_cast_float8 );
-Datum svec_cast_float8(PG_FUNCTION_ARGS) {
- float8 value=PG_GETARG_FLOAT8(0);
- PG_RETURN_SVECTYPE_P(svec_make_scalar(value));
-}
-PG_FUNCTION_INFO_V1( svec_cast_numeric );
-Datum svec_cast_numeric(PG_FUNCTION_ARGS) {
- Datum num=PG_GETARG_DATUM(0);
- float8 value;
- value = DatumGetFloat8(DirectFunctionCall1(numeric_float8_no_overflow,num));
- PG_RETURN_SVECTYPE_P(svec_make_scalar(value));
-}
-
-/*
- * Cast from int2,int4,int8,float4,float8 scalar to float8[]
- */
-PG_FUNCTION_INFO_V1( float8arr_cast_int2 );
-Datum float8arr_cast_int2(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_INT16(0);
- PG_RETURN_ARRAYTYPE_P(svec_return_array_internal(svec_make_scalar(value)));
-}
-PG_FUNCTION_INFO_V1( float8arr_cast_int4 );
-Datum float8arr_cast_int4(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_INT32(0);
- PG_RETURN_ARRAYTYPE_P(svec_return_array_internal(svec_make_scalar(value)));
-}
-PG_FUNCTION_INFO_V1( float8arr_cast_int8 );
-Datum float8arr_cast_int8(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_INT64(0);
- PG_RETURN_ARRAYTYPE_P(svec_return_array_internal(svec_make_scalar(value)));
-}
-PG_FUNCTION_INFO_V1( float8arr_cast_float4 );
-Datum float8arr_cast_float4(PG_FUNCTION_ARGS) {
- float8 value=(float8 )PG_GETARG_FLOAT4(0);
- PG_RETURN_ARRAYTYPE_P(svec_return_array_internal(svec_make_scalar(value)));
-}
-PG_FUNCTION_INFO_V1( float8arr_cast_float8 );
-Datum float8arr_cast_float8(PG_FUNCTION_ARGS) {
- float8 value=PG_GETARG_FLOAT8(0);
- PG_RETURN_ARRAYTYPE_P(svec_return_array_internal(svec_make_scalar(value)));
-}
-PG_FUNCTION_INFO_V1( float8arr_cast_numeric );
-Datum float8arr_cast_numeric(PG_FUNCTION_ARGS) {
- Datum num=PG_GETARG_DATUM(0);
- float8 value;
- value = DatumGetFloat8(DirectFunctionCall1(numeric_float8_no_overflow,num));
- PG_RETURN_ARRAYTYPE_P(svec_return_array_internal(svec_make_scalar(value)));
-}
-
-/** Constructs an 1-dimensional svec given a float8 */
-SvecType *svec_make_scalar(float8 value) {
- SparseData sdata = float8arr_to_sdata(&value,1);
- SvecType *result = svec_from_sparsedata(sdata,true);
- result->dimension = -1;
- return result;
-}
-
-
-PG_FUNCTION_INFO_V1( svec_cast_float8arr );
-/**
- * svec_cast_float8arr - turns a float8 array into an svec
- */
-Datum svec_cast_float8arr(PG_FUNCTION_ARGS) {
- ArrayType *A_PG = PG_GETARG_ARRAYTYPE_P(0);
- SvecType *output_svec;
- float8 *array_temp;
- bits8 *bitmap;
- int bitmask;
- int i,j;
-
- if (ARR_ELEMTYPE(A_PG) != FLOAT8OID)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_float8arr only defined over float8[]")));
- if (ARR_NDIM(A_PG) != 1)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_float8arr only defined over 1 dimensional arrays")));
-
- /* Extract array */
- int dimension = ARR_DIMS(A_PG)[0];
- float8 *array = (float8 *)ARR_DATA_PTR(A_PG);
-
- /* If the data array has NULLs, then we need to create an array to
- * store the NULL values as NVP values defined in float_specials.h.
- */
- if (ARR_HASNULL(A_PG)) {
- array_temp = array;
- array = (double *)palloc(sizeof(float8) * dimension);
- bitmap = ARR_NULLBITMAP(A_PG);
- bitmask = 1;
- j = 0;
- for (i=0; i<dimension; i++) {
- if (bitmap && (*bitmap & bitmask) == 0) // if NULL
- array[i] = NVP;
- else {
- array[i] = array_temp[j];
- j++;
- }
- if (bitmap) { // advance bitmap pointer
- bitmask <<= 1;
- if (bitmask == 0x100) {
- bitmap++;
- bitmask = 1;
- }
- }
- }
- }
-
- /* Create the output SVEC */
- SparseData sdata = float8arr_to_sdata(array,dimension);
- output_svec = svec_from_sparsedata(sdata,true);
-
- if (ARR_HASNULL(A_PG))
- pfree(array);
-
- PG_RETURN_SVECTYPE_P(output_svec);
-}
-
-PG_FUNCTION_INFO_V1( svec_cast_positions_float8arr );
-/**
- * svec_cast_positions_float8arr - turns a pair of arrays, the first an int4[]
- * denoting positions and the second a float8[] denoting values, into an
- * svec of a given size with a given default value everywhere else.
- */
-Datum svec_cast_positions_float8arr(PG_FUNCTION_ARGS) {
- ArrayType *B_PG = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *A_PG = PG_GETARG_ARRAYTYPE_P(1);
- int64 size = PG_GETARG_INT64(2);
- float8 base_value = PG_GETARG_FLOAT8(3);
- SvecType *output_svec;
- int i = 0;
-
- if (ARR_ELEMTYPE(A_PG) != FLOAT8OID)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr valeus only defined over float8[]")));
- if (ARR_NDIM(A_PG) != 1)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr only defined over 1 dimensional arrays")));
-
- if (ARR_NULLBITMAP(A_PG))
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr does not allow null bitmaps on arrays")));
-
- if (ARR_ELEMTYPE(B_PG) != INT8OID)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr positions only defined over int[]")));
- if (ARR_NDIM(B_PG) != 1)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr only defined over 1 dimensional arrays")));
-
- if (ARR_NULLBITMAP(B_PG))
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr does not allow null bitmaps on arrays")));
-
- /* Extract array */
- int dimension = ARR_DIMS(A_PG)[0];
- int dimension2 = ARR_DIMS(B_PG)[0];
-
- if (dimension != dimension2)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr position and value vectors must be of the same size")));
-
- float8 *array = (float8 *)ARR_DATA_PTR(A_PG);
- int64 *array_pos = (int64 *)ARR_DATA_PTR(B_PG);
-
- if (array_pos[dimension-1] > size)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr some of the position values are larger than maximum array size declared")));
-
- for(i=0;i < dimension;++i){
- if(array_pos[i] <= 0){
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("svec_cast_positions_float8arr only accepts position that are positive integers (x > 0)")));
- }
- }
-
- /* Create the output SVEC */
- SparseData sdata = position_to_sdata(array,array_pos,FLOAT8OID,dimension,size,base_value);
- output_svec = svec_from_sparsedata(sdata,true);
-
- PG_RETURN_SVECTYPE_P(output_svec);
-}
-
-/*
- * Provide some operators for Postgres FLOAT8OID arrays
- */
-/*
- * Equality
- */
-static bool float8arr_equals_internal(ArrayType *left, ArrayType *right)
-{
- /*
- * Note that we are only defined for FLOAT8OID
- */
- int dimleft = ARR_NDIM(left), dimright = ARR_NDIM(right);
- int *dimsleft = ARR_DIMS(left), *dimsright = ARR_DIMS(right);
- int numleft = ArrayGetNItems(dimleft,dimsleft);
- int numright = ArrayGetNItems(dimright,dimsright);
- double *vals_left = (double *)ARR_DATA_PTR(left);
- double *vals_right = (double *)ARR_DATA_PTR(right);
- bits8 *bitmap_left = ARR_NULLBITMAP(left);
- bits8 *bitmap_right = ARR_NULLBITMAP(right);
- int bitmask = 1;
-
- if ((dimsleft!=dimsright) || (numleft!=numright))
- return false;
-
- /*
- * First we'll check to see if the null bitmaps are equivalent
- */
- if (bitmap_left)
- if (! bitmap_right) return false;
- if (bitmap_right)
- if (! bitmap_left) return false;
-
- if (bitmap_left)
- {
- for (int i=0; i<numleft; i++)
- {
- if ((*bitmap_left & bitmask) == 0)
- if ((*bitmap_left & bitmask) != 0)
- return false;
- bitmask <<= 1;
- if (bitmask == 0x100)
- {
- bitmap_left++;
- bitmask = 1;
- }
- }
- }
-
- /*
- * Now we check for equality of all array values
- */
- for (int i=0; i<numleft; i++)
- if (vals_left[i] != vals_right[i]) return false;
-
- return true;
-}
-
-/**
- * float8arr_equals - checks whether two float8 arrays are identical
- */
-Datum float8arr_equals(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( float8arr_equals);
-Datum float8arr_equals(PG_FUNCTION_ARGS) {
- ArrayType *left = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *right = PG_GETARG_ARRAYTYPE_P(1);
-
- PG_RETURN_BOOL(float8arr_equals_internal(left,right));
-}
-
-/*
- * Returns a SparseData formed from a dense float8[] in uncompressed format.
- * This is useful for creating a SparseData without processing that can be
- * used by the SparseData processing routines.
- */
-SparseData sdata_uncompressed_from_float8arr_internal(ArrayType *array)
-{
- int dim = ARR_NDIM(array);
- int *dims = ARR_DIMS(array);
- int num = ArrayGetNItems(dim,dims);
- double *vals =(double *)ARR_DATA_PTR(array);
- bits8 *bitmap = ARR_NULLBITMAP(array);
- int bitmask=1;
-
- /* Convert null items into NVPs */
- if (bitmap)
- {
- int j = 0;
- double *vals_temp = vals;
- vals = (double *)palloc(sizeof(float8) * num);
- for (int i=0; i<num; i++)
- {
- if ((*bitmap & bitmask) == 0) // if NULL
- vals[i] = NVP;
- else {
- vals[i] = vals_temp[j];
- j++;
- }
- bitmask <<= 1;
- if (bitmask == 0x100)
- {
- bitmap++;
- bitmask = 1;
- }
- }
- }
- /* Makes the SparseData; this relies on using NULL to represent a
- * count array of ones, as described in SparseData.h, after definition
- * of SparseDataStruct.
- */
- SparseData result = makeInplaceSparseData(
- (char *)vals,NULL,
- num*sizeof(float8),0,FLOAT8OID,num,num);
-
- return(result);
-}
-
-/**
- * float8arr_dot - computes the dot product of two float8 arrays
- */
-Datum float8arr_dot(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1( float8arr_dot);
-Datum float8arr_dot(PG_FUNCTION_ARGS) {
- ArrayType *arr_left = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *arr_right = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr_left);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr_right);
- SparseData mult_result;
- double accum;
-
- mult_result = op_sdata_by_sdata(multiply,left,right);
- accum = sum_sdata_values_double(mult_result);
- freeSparseData(left);
- freeSparseData(right);
- freeSparseDataAndData(mult_result);
-
- if (IS_NVP(accum)) PG_RETURN_NULL();
-
- PG_RETURN_FLOAT8(accum);
-}
-
-/*
- * Permute the basic operators (minus,plus,mult,div) between SparseData
- * and float8[]
- *
- * For each function, make a version that takes the left and right args as
- * each type (without copies)
- */
-PG_FUNCTION_INFO_V1( float8arr_minus_float8arr );
-Datum
-float8arr_minus_float8arr(PG_FUNCTION_ARGS)
-{
- ArrayType *arr1 = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *arr2 = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr1);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr2);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,subtract,left,right));
-}
-PG_FUNCTION_INFO_V1( svec_minus_float8arr );
-Datum
-svec_minus_float8arr(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_from_svec(svec);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,subtract,left,right));
-}
-PG_FUNCTION_INFO_V1( float8arr_minus_svec );
-Datum
-float8arr_minus_svec(PG_FUNCTION_ARGS)
-{
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0);
- SvecType *svec = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr);
- SparseData right = sdata_from_svec(svec);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,subtract,left,right));
-}
-
-PG_FUNCTION_INFO_V1( float8arr_plus_float8arr );
-Datum
-float8arr_plus_float8arr(PG_FUNCTION_ARGS)
-{
- ArrayType *arr1 = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *arr2 = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr1);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr2);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,add,left,right));
-}
-PG_FUNCTION_INFO_V1( svec_plus_float8arr );
-Datum
-svec_plus_float8arr(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_from_svec(svec);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,add,left,right));
-}
-PG_FUNCTION_INFO_V1( float8arr_plus_svec );
-Datum
-float8arr_plus_svec(PG_FUNCTION_ARGS)
-{
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0);
- SvecType *svec = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr);
- SparseData right = sdata_from_svec(svec);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,add,left,right));
-}
-PG_FUNCTION_INFO_V1( float8arr_mult_float8arr );
-Datum
-float8arr_mult_float8arr(PG_FUNCTION_ARGS)
-{
- ArrayType *arr1 = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *arr2 = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr1);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr2);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- SvecType *svec = svec_operate_on_sdata_pair(scalar_args,multiply,left,right);
- PG_RETURN_SVECTYPE_P(svec);
-}
-PG_FUNCTION_INFO_V1( svec_mult_float8arr );
-Datum
-svec_mult_float8arr(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_from_svec(svec);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- SvecType *result = svec_operate_on_sdata_pair(scalar_args,multiply,left,right);
- PG_RETURN_SVECTYPE_P(result);
-}
-PG_FUNCTION_INFO_V1( float8arr_mult_svec );
-Datum
-float8arr_mult_svec(PG_FUNCTION_ARGS)
-{
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0);
- SvecType *svec = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr);
- SparseData right = sdata_from_svec(svec);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,multiply,left,right));
-}
-PG_FUNCTION_INFO_V1( float8arr_div_float8arr );
-Datum
-float8arr_div_float8arr(PG_FUNCTION_ARGS)
-{
- ArrayType *arr1 = PG_GETARG_ARRAYTYPE_P(0);
- ArrayType *arr2 = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr1);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr2);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,divide,left,right));
-}
-PG_FUNCTION_INFO_V1( svec_div_float8arr );
-Datum
-svec_div_float8arr(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(1);
- SparseData left = sdata_from_svec(svec);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,divide,left,right));
-}
-PG_FUNCTION_INFO_V1( float8arr_div_svec );
-Datum
-float8arr_div_svec(PG_FUNCTION_ARGS)
-{
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0);
- SvecType *svec = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr);
- SparseData right = sdata_from_svec(svec);
- int scalar_args = check_scalar(SDATA_IS_SCALAR(left),SDATA_IS_SCALAR(right));
- PG_RETURN_SVECTYPE_P(svec_operate_on_sdata_pair(scalar_args,divide,left,right));
-}
-PG_FUNCTION_INFO_V1( svec_dot_float8arr );
-Datum
-svec_dot_float8arr(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(1);
- SparseData right = sdata_uncompressed_from_float8arr_internal(arr);
- SparseData left = sdata_from_svec(svec);
- SparseData mult_result;
- double accum;
- mult_result = op_sdata_by_sdata(multiply,left,right);
- accum = sum_sdata_values_double(mult_result);
- freeSparseData(right);
- freeSparseDataAndData(mult_result);
-
- if (IS_NVP(accum)) PG_RETURN_NULL();
-
- PG_RETURN_FLOAT8(accum);
-}
-PG_FUNCTION_INFO_V1( float8arr_dot_svec);
-Datum
-float8arr_dot_svec(PG_FUNCTION_ARGS)
-{
- ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0);
- SvecType *svec = PG_GETARG_SVECTYPE_P(1);
- SparseData left = sdata_uncompressed_from_float8arr_internal(arr);
- SparseData right = sdata_from_svec(svec);
- SparseData mult_result;
- double accum;
- mult_result = op_sdata_by_sdata(multiply,left,right);
- accum = sum_sdata_values_double(mult_result);
- freeSparseData(left);
- freeSparseDataAndData(mult_result);
-
- if (IS_NVP(accum)) PG_RETURN_NULL();
-
- PG_RETURN_FLOAT8(accum);
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/sfv_test_output
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/sfv_test_output b/contrib/gp_sparse_vector/sfv_test_output
deleted file mode 100644
index 7875189..0000000
--- a/contrib/gp_sparse_vector/sfv_test_output
+++ /dev/null
@@ -1,141 +0,0 @@
--- \set FETCH_COUNT 100
-\timing
-Timing is on.
-DROP TABLE IF EXISTS features;
-DROP TABLE
-Time: 33.300 ms
-DROP TABLE IF EXISTS corpus;
-DROP TABLE
-Time: 10.930 ms
-DROP TABLE IF EXISTS documents;
-DROP TABLE
-Time: 11.157 ms
-DROP TABLE IF EXISTS dictionary;
-DROP TABLE
-Time: 0.556 ms
--- Test simple document classIFication routines
-CREATE TABLE features (a text[][]) DISTRIBUTED RANDOMLY;
-CREATE TABLE
-Time: 23.411 ms
-INSERT INTO features values ('{am,before,being,bothered,corpus,document,i,in,is,me,never,now,one,really,second,the,third,this,until}');
-INSERT 0 1
-Time: 58.249 ms
-DROP TABLE IF EXISTS documents;
-DROP TABLE
-Time: 0.581 ms
-CREATE TABLE documents (docnum int, a text[]) DISTRIBUTED RANDOMLY;
-CREATE TABLE
-Time: 18.009 ms
-INSERT INTO documents values (1,'{this,is,one,document,in,the,corpus}');
-INSERT 0 1
-Time: 92.216 ms
-INSERT INTO documents values (2,'{i,am,the,second,document,in,the,corpus}');
-INSERT 0 1
-Time: 5.971 ms
-INSERT INTO documents values (3,'{being,third,never,really,bothered,me,until,now}');
-INSERT 0 1
-Time: 5.199 ms
-INSERT INTO documents values (4,'{the,document,before,me,is,the,third,document}');
-INSERT 0 1
-Time: 6.822 ms
-CREATE TABLE corpus (docnum int, a svec) DISTRIBUTED RANDOMLY;
-CREATE TABLE
-Time: 17.750 ms
-INSERT INTO corpus (SELECT docnum,gp_extract_feature_histogram((SELECT a FROM features LIMIT 1),a) FROM documents);
-INSERT 0 4
-Time: 77.271 ms
-\qecho Show the feature dictionary
-Show the feature dictionary
-SELECT a dictionary FROM features;
- dictionary
---------------------------------------------------------------------------------------------------------
- {am,before,being,bothered,corpus,document,i,in,is,me,never,now,one,really,second,the,third,this,until}
-(1 row)
-
-Time: 2.450 ms
-\qecho Show each document
-Show each document
-SELECT docnum Document_Number, a document FROM documents ORDER BY 1;
- document_number | document
------------------+--------------------------------------------------
- 1 | {this,is,one,document,in,the,corpus}
- 2 | {i,am,the,second,document,in,the,corpus}
- 3 | {being,third,never,really,bothered,me,until,now}
- 4 | {the,document,before,me,is,the,third,document}
-(4 rows)
-
-Time: 1.895 ms
-\qecho The extracted feature vector for each document
-The extracted feature vector for each document
-SELECT docnum Document_Number, a::float8[] feature_vector FROM corpus ORDER BY 1;
- document_number | feature_vector
------------------+-----------------------------------------
- 1 | {0,0,0,0,1,1,0,1,1,0,0,0,1,0,0,1,0,1,0}
- 2 | {1,0,0,0,1,1,1,1,0,0,0,0,0,0,1,2,0,0,0}
- 3 | {0,0,1,1,0,0,0,0,0,1,1,1,0,1,0,0,1,0,1}
- 4 | {0,1,0,0,0,2,0,0,1,1,0,0,0,0,0,2,1,0,0}
-(4 rows)
-
-Time: 2.378 ms
-\qecho Count the number of times each feature occurs at least once in all documents
-Count the number of times each feature occurs at least once in all documents
-SELECT (vec_count_nonzero(a))::float8[] count_in_document FROM corpus;
- count_in_document
------------------------------------------
- {1,1,1,1,2,3,1,2,2,2,1,1,1,1,1,3,2,1,1}
-(1 row)
-
-Time: 2.600 ms
-\qecho Count all occurrences of each term in all documents
-Count all occurrences of each term in all documents
-SELECT (sum(a))::float8[] sum_in_document FROM corpus;
- sum_in_document
------------------------------------------
- {1,1,1,1,2,4,1,2,2,2,1,1,1,1,1,5,2,1,1}
-(1 row)
-
-Time: 2.367 ms
-\qecho Calculate Term Frequency / Inverse Document Frequency
-Calculate Term Frequency / Inverse Document Frequency
-SELECT docnum, (a*logidf)::float8[] tf_idf FROM (SELECT log(count(a)/vec_count_nonzero(a)) logidf FROM corpus) foo, corpus ORDER BY docnum;
- docnum | tf_idf
---------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------
- 1 | {0,0,0,0,0.693147180559945,0.287682072451781,0,0.693147180559945,0.693147180559945,0,0,0,1.38629436111989,0,0,0.287682072451781,0,1.38629436111989,0}
- 2 | {1.38629436111989,0,0,0,0.693147180559945,0.287682072451781,1.38629436111989,0.693147180559945,0,0,0,0,0,0,1.38629436111989,0.575364144903562,0,0,0}
- 3 | {0,0,1.38629436111989,1.38629436111989,0,0,0,0,0,0.693147180559945,1.38629436111989,1.38629436111989,0,1.38629436111989,0,0,0.693147180559945,0,1.38629436111989}
- 4 | {0,1.38629436111989,0,0,0,0.575364144903562,0,0,0.693147180559945,0.693147180559945,0,0,0,0,0,0.575364144903562,0.693147180559945,0,0}
-(4 rows)
-
-Time: 13.013 ms
-\qecho Show the same calculation in compressed vector format
-Show the same calculation in compressed vector format
-SELECT docnum, (a*logidf) tf_idf FROM (SELECT log(count(a)/vec_count_nonzero(a)) logidf FROM corpus) foo, corpus ORDER BY docnum;
- docnum | tf_idf
---------+----------------------------------------------------------------------------------------------------------------------------------------------------------
- 1 | {4,1,1,1,2,3,1,2,1,1,1,1}:{0,0.693147180559945,0.287682072451781,0,0.693147180559945,0,1.38629436111989,0,0.287682072451781,0,1.38629436111989,0}
- 2 | {1,3,1,1,1,1,6,1,1,3}:{1.38629436111989,0,0.693147180559945,0.287682072451781,1.38629436111989,0.693147180559945,0,1.38629436111989,0.575364144903562,0}
- 3 | {2,2,5,1,2,1,1,2,1,1,1}:{0,1.38629436111989,0,0.693147180559945,1.38629436111989,0,1.38629436111989,0,0.693147180559945,0,1.38629436111989}
- 4 | {1,1,3,1,2,2,5,1,1,2}:{0,1.38629436111989,0,0.575364144903562,0,0.693147180559945,0,0.575364144903562,0.693147180559945,0}
-(4 rows)
-
-Time: 5.794 ms
-\qecho Create a table with TF / IDF weighted vectors in it
-Create a table with TF / IDF weighted vectors in it
-DROP TABLE IF EXISTS WEIGHTS;
-DROP TABLE
-Time: 11.575 ms
-CREATE TABLE weights AS (SELECT docnum, (a*logidf) tf_idf FROM (SELECT log(count(a)/vec_count_nonzero(a)) logidf FROM corpus) foo, corpus ORDER BY docnum) DISTRIBUTED RANDOMLY;
-SELECT 4
-Time: 101.241 ms
-\qecho Calculate the angular distance between the first document to each other document
-Calculate the angular distance between the first document to each other document
-SELECT docnum,trunc((180.*(ACOS(dmin(1.,(tf_idf%*%testdoc)/(l2norm(tf_idf)*l2norm(testdoc))))/(4.*ATAN(1.))))::numeric,2) angular_distance FROM weights,(SELECT tf_idf testdoc FROM weights WHERE docnum = 1 LIMIT 1) foo ORDER BY 1;
- docnum | angular_distance
---------+------------------
- 1 | 0.00
- 2 | 78.82
- 3 | 90.00
- 4 | 80.02
-(4 rows)
-
-Time: 8.357 ms
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/sparse_vector.c
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/sparse_vector.c b/contrib/gp_sparse_vector/sparse_vector.c
deleted file mode 100644
index d1fc4e7..0000000
--- a/contrib/gp_sparse_vector/sparse_vector.c
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * @file
- * Sparse Vector Datatype
- * We would like to store sparse arrays in a terse representation that fits
- * in a small amount of memory.
- * We also want to be able to compare the number of instances where the svec
- * of one document intersects another.
- */
-
-#include <postgres.h>
-
-#include <stdio.h>
-#include <string.h>
-#include <search.h>
-#include <stdlib.h>
-#include <math.h>
-
-#include "utils/array.h"
-#include "utils/builtins.h"
-#include "utils/lsyscache.h"
-#include "catalog/pg_type.h"
-#include "libpq/libpq.h"
-#include "libpq/pqformat.h"
-#include "fmgr.h"
-#include "funcapi.h"
-#include "utils/fmgroids.h"
-#include "lib/stringinfo.h"
-#include "utils/memutils.h"
-#include "sparse_vector.h"
-
-/**
- * @return An array of float8s obtained by converting a given sparse vector
- */
-ArrayType *svec_return_array_internal(SvecType *svec)
-{
- SparseData sdata = sdata_from_svec(svec);
- double *array = sdata_to_float8arr(sdata);
-
- ArrayType *pgarray = construct_array((Datum *)array,
- sdata->total_value_count,FLOAT8OID,
- sizeof(float8),true,'d');
-
- pfree(array);
- return(pgarray);
-}
-
-/*
- * Must serialize for binary communication with libpq by
- * creating a StringInfo and sending individual data items like:
- * (from backend/libpq/pqformat.c):
- * pq_beginmessage - initialize StringInfo buffer
- * pq_sendbyte - append a raw byte to a StringInfo buffer
- * pq_sendint - append a binary integer to a StringInfo buffer
- * pq_sendint64 - append a binary 8-byte int to a StringInfo buffer
- * pq_sendfloat4 - append a float4 to a StringInfo buffer
- * pq_sendfloat8 - append a float8 to a StringInfo buffer
- * pq_sendbytes - append raw data to a StringInfo buffer
- * pq_sendcountedtext - append a counted text string (with character set conversion)
- * pq_sendtext - append a text string (with conversion)
- * pq_sendstring - append a null-terminated text string (with conversion)
- * pq_send_ascii_string - append a null-terminated text string (without conversion)
- * pq_endmessage - send the completed message to the frontend
- *
- */
-
-PG_FUNCTION_INFO_V1(svec_send);
-/**
- * svec_send - converts text to binary format
- */
-Datum svec_send(PG_FUNCTION_ARGS)
-{
- StringInfoData buf;
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- SparseData sdata = sdata_from_svec(svec);
-
- pq_begintypsend(&buf);
- pq_sendint(&buf,sdata->type_of_data,sizeof(Oid));
- pq_sendint(&buf,sdata->unique_value_count,sizeof(int));
- pq_sendint(&buf,sdata->total_value_count,sizeof(int));
- pq_sendint(&buf,sdata->vals->len,sizeof(int));
- pq_sendint(&buf,sdata->index->len,sizeof(int));
- pq_sendbytes(&buf,sdata->vals->data,sdata->vals->len);
- pq_sendbytes(&buf,sdata->index->data,sdata->index->len);
-
- PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
-}
-
-PG_FUNCTION_INFO_V1(svec_recv);
-/**
- * svec_recv - converts external binary format to text
- */
-Datum svec_recv(PG_FUNCTION_ARGS)
-{
- StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
- SvecType *svec;
-
- SparseData sdata=makeEmptySparseData();;
- sdata->type_of_data = pq_getmsgint(buf, sizeof(int));
- sdata->unique_value_count = pq_getmsgint(buf, sizeof(int));
- sdata->total_value_count = pq_getmsgint(buf, sizeof(int));
- sdata->vals->len = pq_getmsgint(buf, sizeof(int));
- sdata->index->len = pq_getmsgint(buf, sizeof(int));
- sdata->vals->data = (char *)pq_getmsgbytes(buf,sdata->vals->len);
- sdata->index->data = (char *)pq_getmsgbytes(buf,sdata->index->len);
- svec = svec_from_sparsedata(sdata,true); //Note this copies the data
-// freeSparseDataAndData(sdata);
- pfree(sdata);
-
- PG_RETURN_SVECTYPE_P(svec);
-}
-
-PG_FUNCTION_INFO_V1( svec_return_array );
-/**
- * svec_return_array - returns an uncompressed Array
- */
-Datum svec_return_array(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- ArrayType *pgarray = svec_return_array_internal(svec);
- PG_RETURN_ARRAYTYPE_P(pgarray);
-}
-
-PG_FUNCTION_INFO_V1(svec_out);
-/**
- * svec_out - outputs a sparse vector as a C string
- */
-Datum svec_out(PG_FUNCTION_ARGS)
-{
- SvecType *svec = PG_GETARG_SVECTYPE_P(0);
- char *result = svec_out_internal(svec);
- PG_RETURN_CSTRING(result);
-}
-
-char * svec_out_internal(SvecType *svec)
-{
- char *ix_string,*vals_string,*result;
- int ixlen,vslen;
- SparseData sdata=sdata_from_svec(svec);
- int64 *array_ix =sdata_index_to_int64arr(sdata);
- ArrayType *pgarray_ix,*pgarray_vals;
-
- pgarray_ix = construct_array((Datum *)array_ix,
- sdata->unique_value_count,INT8OID,
- sizeof(int64),true,'d');
-
- ix_string = DatumGetPointer(OidFunctionCall1(F_ARRAY_OUT,
- PointerGetDatum(pgarray_ix)));
- ixlen = strlen(ix_string);
-
- pgarray_vals = construct_array((Datum *)sdata->vals->data,
- sdata->unique_value_count,FLOAT8OID,
- sizeof(float8),true,'d');
-
- vals_string = DatumGetPointer(OidFunctionCall1(F_ARRAY_OUT,
- PointerGetDatum(pgarray_vals)));
- vslen = strlen(vals_string);
-
- result = (char *)palloc(sizeof(char)*(vslen+ixlen+1+1));
-
- /* NULLs are represented as NaN internally; see svec_in();
- * Here we print each NaN as an NVP. */
- for (int i=0; i!=vslen; i++)
- if (vals_string[i] == 'N')
- {
- vals_string[i+1] = 'V';
- vals_string[i+2] = 'P';
- i = i+2;
- }
-
- sprintf(result,"%s:%s",ix_string,vals_string);
- pfree(ix_string);
- pfree(vals_string);
- pfree(array_ix);
-
- return(result);
-}
-
-SvecType * svec_in_internal(char * str);
-
-PG_FUNCTION_INFO_V1(svec_in);
-/**
- * svec_in - reads in a string and convert that to an svec
- */
-Datum svec_in(PG_FUNCTION_ARGS)
-{
- char *str = pstrdup(PG_GETARG_CSTRING(0));
- SvecType *result = svec_in_internal(str);
- PG_RETURN_SVECTYPE_P(result);
-}
-
-SvecType * svec_in_internal(char * str)
-{
- char *values;
- ArrayType *pgarray_vals,*pgarray_ix;
- double *vals, *vals_temp;
- StringInfo index;
- int64 *u_index;
- int32_t num_values,total_value_count;
- SparseData sdata;
- SvecType *result;
- bits8 *bitmap;
- int bitmask;
- int i,j;
-
- /* Read in the two arrays defining the Sparse Vector, first is the array
- * of run lengths (the count array), the second is an array of the
- * unique values (the data array).
- *
- * The input format is a pair of standard Postgres arrays separated by
- * a colon, like this:
- * {1,10,1,5,1}:{4.3,0,0.2,0,7.4}
- *
- * For now, the data array must only have float8 elements.
- */
- if ((values=strchr(str,':')) == NULL) {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Invalid input string for svec")));
- } else {
- *values = '\0';
- values = values+1;
- }
- /* Get the count and data arrays */
- pgarray_ix = DatumGetArrayTypeP(
- OidFunctionCall3(F_ARRAY_IN,CStringGetDatum(str),
- ObjectIdGetDatum(INT8OID),Int32GetDatum(-1)));
-
- pgarray_vals = DatumGetArrayTypeP(
- OidFunctionCall3(F_ARRAY_IN,CStringGetDatum(values),
- ObjectIdGetDatum(FLOAT8OID),Int32GetDatum(-1)));
-
- num_values = *(ARR_DIMS(pgarray_ix));
- u_index = (int64 *)ARR_DATA_PTR(pgarray_ix);
- vals = (double *)ARR_DATA_PTR(pgarray_vals);
-
- /* The count and value arrays must be non-empty */
- int size1 = ARR_NDIM(pgarray_ix);
- int size2 = ARR_NDIM(pgarray_vals);
- if (size1 == 0 || size2 == 0)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("The count and value arrays must be non-empty")));
-
- /* The count and value arrays must have the same dimension */
- if (num_values != *(ARR_DIMS(pgarray_vals)))
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Unique value count not equal to run length count %d != %d", num_values, *(ARR_DIMS(pgarray_vals)))));
-
- /* Count array cannot have NULLs */
- if (ARR_HASNULL(pgarray_ix))
- ereport(ERROR,
- (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
- errmsg("NULL value in the count array.")));
-
- /* If the data array has NULLs, then we need to create an array to
- * store the NULL values as NVP values defined in float_specials.h.
- */
- if (ARR_HASNULL(pgarray_vals)) {
- vals_temp = vals;
- vals = (double *)palloc(sizeof(float8) * num_values);
- bitmap = ARR_NULLBITMAP(pgarray_vals);
- bitmask = 1;
- j = 0;
- for (i=0; i<num_values; i++) {
- if (bitmap && (*bitmap & bitmask) == 0) // if NULL
- vals[i] = NVP;
- else {
- vals[i] = vals_temp[j];
- j++;
- }
- if (bitmap) { // advance bitmap pointer
- bitmask <<= 1;
- if (bitmask == 0x100) {
- bitmap++;
- bitmask = 1;
- }
- }
- }
- }
-
- /* Make an empty StringInfo because we have the data array already */
- index = makeStringInfo();
- total_value_count = 0;
- for (int i=0;i<num_values;i++) {
- if (u_index[i] <= 0)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("Non-positive run length in input")));
-
- total_value_count+=u_index[i];
- append_to_rle_index(index,u_index[i]);
- }
-
- sdata = makeInplaceSparseData((char *)vals,index->data,
- num_values*sizeof(double),index->len,FLOAT8OID,
- num_values,total_value_count);
- sdata->type_of_data = FLOAT8OID;
-
- result = svec_from_sparsedata(sdata,true);
- if (total_value_count == 1) result->dimension = -1; //Scalar
-
- if (ARR_HASNULL(pgarray_vals)) pfree(vals);
- pfree(str); /* str is allocated from a strdup */
- pfree(pgarray_ix);
- pfree(pgarray_vals);
-
- return result;
-}
-
-/**
- * Produces an svec from a SparseData
- */
-SvecType *svec_from_sparsedata(SparseData sdata, bool trim)
-{
- int size;
-
- if (trim)
- {
- /* Trim the extra space off of the StringInfo dynamic strings
- * before serializing the SparseData
- */
- sdata->vals->maxlen=sdata->vals->len;
- sdata->index->maxlen=sdata->index->len;
- }
-
- size = SVECHDRSIZE + SIZEOF_SPARSEDATASERIAL(sdata);
-
- SvecType *result = (SvecType *)palloc(size);
- SET_VARSIZE(result,size);
- serializeSparseData(SVEC_SDATAPTR(result),sdata);
- result->dimension = sdata->total_value_count;
- if (result->dimension == 1) result->dimension=-1; //Scalar
- return (result);
-}
-
-/**
- * Produces an svec from an array
- */
-SvecType *svec_from_float8arr(float8 *array, int dimension)
-{
- SparseData sdata = float8arr_to_sdata(array,dimension);
- SvecType *result = svec_from_sparsedata(sdata,true);
- return result;
-}
-
-/**
- * Makes an empty svec with sufficient memory allocated for the input number
- */
-SvecType *makeEmptySvec(int allocation)
-{
- int val_len = sizeof(float8)*allocation+1;
- int ind_len = 9*allocation+1;
- SvecType *svec;
- SparseData sdata = makeEmptySparseData();
- sdata->vals->data = (char *)palloc(val_len);
- sdata->vals->len = 0;
- sdata->vals->maxlen = val_len;
- sdata->index->data = (char *)palloc(ind_len);
- sdata->index->len = 0;
- sdata->index->maxlen = ind_len;
- svec = svec_from_sparsedata(sdata,false);
- freeSparseDataAndData(sdata);
- return(svec);
-}
-
-/**
- * Allocates more space for the count and data arrays of an svec
- */
-SvecType *reallocSvec(SvecType *source)
-{
- SvecType *svec;
- SparseData sdata = sdata_from_svec(source);
- int val_newmaxlen = Max(2*sizeof(float8)+1, 2 * (Size) sdata->vals->maxlen);
- char *newvals = (char *)palloc(val_newmaxlen);
- int ind_newmaxlen = Max(2*sizeof(int8)+1, 2 * (Size) sdata->index->maxlen);
- char *newindex = (char *)palloc(ind_newmaxlen);
- /*
- * This space was never allocated with palloc, so we can't repalloc it!
- */
- memcpy(newvals ,sdata->vals->data ,sdata->vals->len);
- memcpy(newindex,sdata->index->data,sdata->index->len);
- sdata->vals->data = newvals;
- sdata->vals->maxlen = val_newmaxlen;
- sdata->index->data = newindex;
- sdata->index->maxlen = ind_newmaxlen;
- svec = svec_from_sparsedata(sdata,false);
-// pfree(source);
- return(svec);
-}
-
- //
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/sparse_vector.h
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/sparse_vector.h b/contrib/gp_sparse_vector/sparse_vector.h
deleted file mode 100644
index 4c9f889..0000000
--- a/contrib/gp_sparse_vector/sparse_vector.h
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * @file
- * \brief Persistent storage for the Sparse Vector Datatype
- *
- */
-
-#ifndef SPARSEVECTOR_H
-#define SPARSEVECTOR_H
-
-#include "SparseData.h"
-#include "float_specials.h"
-
-/*!
- * \internal
- * Consists of the dimension of the vector (how many elements) and a SparseData
- * structure that stores the data in a compressed format.
- * \endinternal
- */
-typedef struct {
- int32 vl_len_; /**< This is unused at the moment */
- int32 dimension; /**< Number of elements in this vector, special case is -1 indicates a scalar */
- char data[1]; /**< The serialized SparseData representing the vector here */
-} SvecType;
-
-#define DatumGetSvecTypeP(X) ((SvecType *) PG_DETOAST_DATUM(X))
-#define DatumGetSvecTypePCopy(X) ((SvecType *) PG_DETOAST_DATUM_COPY(X))
-#define PG_GETARG_SVECTYPE_P(n) DatumGetSvecTypeP(PG_GETARG_DATUM(n))
-#define PG_GETARG_SVECTYPE_P_COPY(n) DatumGetSvecTypePCopy(PG_GETARG_DATUM(n))
-#define PG_RETURN_SVECTYPE_P(x) PG_RETURN_POINTER(x)
-
-/* Below are the locations of the SparseData values within the serialized
- * inline SparseData below the Svec header
- *
- * All macros take an (SvecType *) as argument
- */
-#define SVECHDRSIZE (VARHDRSZ + sizeof(int32))
-/* Beginning of the serialized SparseData */
-#define SVEC_SDATAPTR(x) ((char *)(x)+SVECHDRSIZE)
-#define SVEC_SIZEOFSERIAL(x) (SVECHDRSIZE+SIZEOF_SPARSEDATASERIAL((SparseData)SVEC_SDATAPTR(x)))
-#define SVEC_UNIQUE_VALCNT(x) (SDATA_UNIQUE_VALCNT(SVEC_SDATAPTR(x)))
-#define SVEC_TOTAL_VALCNT(x) (SDATA_TOTAL_VALCNT(SVEC_SDATAPTR(x)))
-#define SVEC_DATA_SIZE(x) (SDATA_DATA_SIZE(SVEC_SDATAPTR(x)))
-#define SVEC_VALS_PTR(x) (SDATA_VALS_PTR(SVEC_SDATAPTR(x)))
-/* The size of the index is variable unlike the values, so in the serialized
- * SparseData, we include an int32 that indicates the size of the index.
- */
-#define SVEC_INDEX_SIZE(x) (SDATA_INDEX_SIZE(SVEC_SDATAPTR(x)))
-#define SVEC_INDEX_PTR(x) (SDATA_INDEX_PTR(SVEC_SDATAPTR(x)))
-
-/** @return True if input is a scalar */
-#define IS_SCALAR(x) (((x)->dimension) < 0 ? 1 : 0 )
-
-/** @return True if input is a NULL, represented internally as a NVP */
-#define IS_NVP(x) (memcmp(&(x),&(NVP),sizeof(double)) == 0)
-
-static inline int check_scalar(int i1, int i2)
-{
- if ((!i1) && (!i2)) return(0);
- else if (i1 && i2) return(3);
- else if (i1) return(1);
- else if (i2) return(2);
- return(0);
-}
-
-/*
- * This routine supplies a pointer to a SparseData derived from an SvecType.
- * The SvecType is a serialized structure with fixed memory allocations, so
- * care must be taken not to append to the embedded StringInfo structs
- * without re-serializing the SparseData into the SvecType.
- */
-static inline SparseData sdata_from_svec(SvecType *svec)
-{
- char *sdataptr = SVEC_SDATAPTR(svec);
- SparseData sdata = (SparseData)sdataptr;
- sdata->vals = (StringInfo)SDATA_DATA_SINFO(sdataptr);
- sdata->index = (StringInfo)SDATA_INDEX_SINFO(sdataptr);
- sdata->vals->data = SVEC_VALS_PTR(svec);
- if (sdata->index->maxlen == 0)
- {
- sdata->index->data = NULL;
- } else
- {
- sdata->index->data = SVEC_INDEX_PTR(svec);
- }
- return(sdata);
-}
-
-static inline void printout_svec(SvecType *svec, char *msg, int stop);
-static inline void printout_svec(SvecType *svec, char *msg, int stop)
-{
- printout_sdata((SparseData)SVEC_SDATAPTR(svec), msg, stop);
- elog(NOTICE,"len,dimension=%d,%d",VARSIZE(svec),svec->dimension);
-}
-
-char *svec_out_internal(SvecType *svec);
-SvecType *svec_from_sparsedata(SparseData sdata,bool trim);
-ArrayType *svec_return_array_internal(SvecType *svec);
-char *svec_out_internal(SvecType *svec);
-SvecType *svec_make_scalar(float8 value);
-SvecType *svec_from_float8arr(float8 *array, int dimension);
-SvecType *op_svec_by_svec_internal(enum operation_t operation, SvecType *svec1, SvecType *svec2);
-SvecType *svec_operate_on_sdata_pair(int scalar_args,enum operation_t operation,SparseData left,SparseData right);
-SvecType *makeEmptySvec(int allocation);
-SvecType *reallocSvec(SvecType *source);
-
-Datum svec_in(PG_FUNCTION_ARGS);
-Datum svec_out(PG_FUNCTION_ARGS);
-Datum svec_return_vector(PG_FUNCTION_ARGS);
-Datum svec_return_array(PG_FUNCTION_ARGS);
-Datum svec_send(PG_FUNCTION_ARGS);
-Datum svec_recv(PG_FUNCTION_ARGS);
-
-// Operators
-Datum svec_pow(PG_FUNCTION_ARGS);
-Datum svec_equals(PG_FUNCTION_ARGS);
-Datum svec_minus(PG_FUNCTION_ARGS);
-Datum svec_plus(PG_FUNCTION_ARGS);
-Datum svec_div(PG_FUNCTION_ARGS);
-Datum svec_dot(PG_FUNCTION_ARGS);
-Datum svec_mult(PG_FUNCTION_ARGS);
-Datum svec_summate(PG_FUNCTION_ARGS);
-
-Datum float8arr_minus_float8arr(PG_FUNCTION_ARGS);
-Datum svec_minus_float8arr(PG_FUNCTION_ARGS);
-Datum float8arr_minus_svec(PG_FUNCTION_ARGS);
-Datum float8arr_plus_float8arr(PG_FUNCTION_ARGS);
-Datum svec_plus_float8arr(PG_FUNCTION_ARGS);
-Datum float8arr_plus_svec(PG_FUNCTION_ARGS);
-Datum float8arr_mult_float8arr(PG_FUNCTION_ARGS);
-Datum svec_mult_float8arr(PG_FUNCTION_ARGS);
-Datum float8arr_mult_svec(PG_FUNCTION_ARGS);
-Datum float8arr_div_float8arr(PG_FUNCTION_ARGS);
-Datum svec_div_float8arr(PG_FUNCTION_ARGS);
-Datum float8arr_div_svec(PG_FUNCTION_ARGS);
-Datum svec_dot_float8arr(PG_FUNCTION_ARGS);
-Datum float8arr_dot_svec(PG_FUNCTION_ARGS);
-
-// Casts
-Datum svec_cast_int2(PG_FUNCTION_ARGS);
-Datum svec_cast_int4(PG_FUNCTION_ARGS);
-Datum svec_cast_int8(PG_FUNCTION_ARGS);
-Datum svec_cast_float4(PG_FUNCTION_ARGS);
-Datum svec_cast_float8(PG_FUNCTION_ARGS);
-Datum svec_cast_numeric(PG_FUNCTION_ARGS);
-
-Datum float8arr_cast_int2(PG_FUNCTION_ARGS);
-Datum float8arr_cast_int4(PG_FUNCTION_ARGS);
-Datum float8arr_cast_int8(PG_FUNCTION_ARGS);
-Datum float8arr_cast_float4(PG_FUNCTION_ARGS);
-Datum float8arr_cast_float8(PG_FUNCTION_ARGS);
-Datum float8arr_cast_numeric(PG_FUNCTION_ARGS);
-
-Datum svec_cast_float8arr(PG_FUNCTION_ARGS);
-Datum svec_cast_positions_float8arr(PG_FUNCTION_ARGS);
-
-#endif /* SPARSEVECTOR_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/contrib/gp_sparse_vector/test_output
----------------------------------------------------------------------
diff --git a/contrib/gp_sparse_vector/test_output b/contrib/gp_sparse_vector/test_output
deleted file mode 100644
index e067d70..0000000
--- a/contrib/gp_sparse_vector/test_output
+++ /dev/null
@@ -1,296 +0,0 @@
--- \set FETCH_COUNT 100
-\timing
-Timing is on.
-drop table if exists test;
-DROP TABLE
-Time: 1.163 ms
-create table test (a int, b svec) DISTRIBUTED BY (a);
-CREATE TABLE
-Time: 67.692 ms
-insert into test (select 1,gp_extract_feature_histogram('{"one","two","three","four","five","six"}','{"twe","four","five","six","one","three","two","one"}'));
-INSERT 0 1
-Time: 72.360 ms
-insert into test (select 2,gp_extract_feature_histogram('{"one","two","three","four","five","six"}','{"the","brown","cat","ran","across","three","dogs"}'));
-INSERT 0 1
-Time: 4.543 ms
-insert into test (select 3,gp_extract_feature_histogram('{"one","two","three","four","five","six"}','{"two","four","five","six","one","three","two","one"}'));
-INSERT 0 1
-Time: 3.701 ms
--- Test the equals operator (should be only 3 rows)
-select a,b::float8[] cross_product_equals from (select a,b from test) foo where b = foo.b order by a;
- a | cross_product_equals
----+----------------------
- 1 | {2,1,1,1,1,1}
- 2 | {0,0,1,0,0,0}
- 3 | {2,2,1,1,1,1}
-(3 rows)
-
-Time: 4.044 ms
-drop table if exists test2;
-DROP TABLE
-Time: 0.217 ms
-create table test2 as select * from test DISTRIBUTED BY (a);
-SELECT 3
-Time: 64.780 ms
--- Test the plus operator (should be 9 rows)
-select (t1.b+t2.b)::float8[] cross_product_sum from test t1, test2 t2 order by t1.a;
- cross_product_sum
--------------------
- {4,3,2,2,2,2}
- {4,2,2,2,2,2}
- {2,1,2,1,1,1}
- {0,0,2,0,0,0}
- {2,1,2,1,1,1}
- {2,2,2,1,1,1}
- {4,4,2,2,2,2}
- {4,3,2,2,2,2}
- {2,2,2,1,1,1}
-(9 rows)
-
-Time: 7.121 ms
--- Test ORDER BY
-select (t1.b+t2.b)::float8[] cross_product_sum, l2norm(t1.b+t2.b) l2norm, (t1.b+t2.b) sparse_vector from test t1, test2 t2 order by 3;
- cross_product_sum | l2norm | sparse_vector
--------------------+------------------+---------------------
- {0,0,2,0,0,0} | 2 | {2,1,3}:{0,2,0}
- {2,1,2,1,1,1} | 3.46410161513775 | {1,1,1,3}:{2,1,2,1}
- {2,1,2,1,1,1} | 3.46410161513775 | {1,1,1,3}:{2,1,2,1}
- {2,2,2,1,1,1} | 3.87298334620742 | {3,3}:{2,1}
- {2,2,2,1,1,1} | 3.87298334620742 | {3,3}:{2,1}
- {4,2,2,2,2,2} | 6 | {1,5}:{4,2}
- {4,3,2,2,2,2} | 6.40312423743285 | {1,1,4}:{4,3,2}
- {4,3,2,2,2,2} | 6.40312423743285 | {1,1,4}:{4,3,2}
- {4,4,2,2,2,2} | 6.92820323027551 | {2,4}:{4,2}
-(9 rows)
-
-Time: 5.359 ms
- select (sum(t1.b))::float8[] as features_sum from test t1;
- features_sum
----------------
- {4,3,3,2,2,2}
-(1 row)
-
-Time: 2.844 ms
--- Test the div operator
- select (t1.b/(select sum(b) from test))::float8[] as weights from test t1 order by a;
- weights
--------------------------------------------------------
- {0.5,0.333333333333333,0.333333333333333,0.5,0.5,0.5}
- {0,0,0.333333333333333,0,0,0}
- {0.5,0.666666666666667,0.333333333333333,0.5,0.5,0.5}
-(3 rows)
-
-Time: 4.229 ms
--- Test the * operator
- select t1.b %*% (t1.b/(select sum(b) from test)) as raw_score from test t1 order by a;
- raw_score
--------------------
- 3.16666666666667
- 0.333333333333333
- 4.16666666666667
-(3 rows)
-
-Time: 4.098 ms
--- Test the * and l2norm operators
- select (t1.b %*% (t1.b/(select sum(b) from test))) / (l2norm(t1.b) * l2norm((select sum(b) from test))) as norm_score from test t1 order by a;
- norm_score
--------------------
- 0.15563317594128
- 0.049147318718299
- 0.177345110574739
-(3 rows)
-
-Time: 6.295 ms
--- Test the ^ and l1norm operators
-select ('{1,2}:{20.,10.}'::svec)^('{1}:{3.}'::svec);
- ?column?
--------------------
- {1,2}:{8000,1000}
-(1 row)
-
-Time: 1.060 ms
- select (t1.b %*% (t1.b/(select sum(b) from test))) / (l1norm(t1.b) * l1norm((select sum(b) from test))) as norm_score from test t1 order by a;
- norm_score
---------------------
- 0.0282738095238095
- 0.0208333333333333
- 0.0325520833333333
-(3 rows)
-
-Time: 6.715 ms
--- Test the multi-concatenation and show sizes compared with a normal array
-drop table if exists corpus_proj;
-DROP TABLE
-Time: 0.258 ms
-drop table if exists corpus_proj_array;
-DROP TABLE
-Time: 0.225 ms
-create table corpus_proj as (select 10000 *|| ('{45,2,35,4,15,1}:{0,1,0,1,0,2}'::svec) result ) distributed randomly;
-SELECT 1
-Time: 91.366 ms
-create table corpus_proj_array as (select result::float8[] from corpus_proj) distributed randomly;
-SELECT 1
-Time: 318.754 ms
--- Calculate on-disk size of sparse vector
-select pg_size_pretty(pg_total_relation_size('corpus_proj'));
- pg_size_pretty
-----------------
- 192 kB
-(1 row)
-
-Time: 5.393 ms
--- Calculate on-disk size of normal array
-select pg_size_pretty(pg_total_relation_size('corpus_proj_array'));
- pg_size_pretty
-----------------
- 288 kB
-(1 row)
-
-Time: 3.488 ms
-\timing
-Timing is off.
--- Calculate L1 norm from sparse vector
-select l1norm(result) from corpus_proj;
- l1norm
---------
- 80000
-(1 row)
-
--- Calculate L1 norm from float8[]
-select l1norm(result) from corpus_proj_array;
- l1norm
---------
- 80000
-(1 row)
-
--- Calculate L2 norm from sparse vector
-select l2norm(result) from corpus_proj;
- l2norm
-------------------
- 316.227766016838
-(1 row)
-
--- Calculate L2 norm from float8[]
-select l2norm(result) from corpus_proj_array;
- l2norm
-------------------
- 316.227766016838
-(1 row)
-
-drop table corpus_proj;
-DROP TABLE
-drop table corpus_proj_array;
-DROP TABLE
-drop table test;
-DROP TABLE
-drop table test2;
-DROP TABLE
--- Test operators between svec and float8[]
-select ('{1,2,3,4}:{3,4,5,6}'::svec) %*% ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[];
- ?column?
-----------
- 260
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[] %*% ('{1,2,3,4}:{3,4,5,6}'::svec);
- ?column?
-----------
- 260
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec) / ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[];
- ?column?
-----------
- {10}:{1}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[] / ('{1,2,3,4}:{3,4,5,6}'::svec);
- ?column?
-----------
- {10}:{1}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec) * ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[];
- ?column?
-------------------------
- {1,2,3,4}:{9,16,25,36}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[] * ('{1,2,3,4}:{3,4,5,6}'::svec);
- ?column?
-------------------------
- {1,2,3,4}:{9,16,25,36}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec) + ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[];
- ?column?
------------------------
- {1,2,3,4}:{6,8,10,12}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[] + ('{1,2,3,4}:{3,4,5,6}'::svec);
- ?column?
------------------------
- {1,2,3,4}:{6,8,10,12}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec) - ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[];
- ?column?
-----------
- {10}:{0}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::svec)::float8[] - ('{1,2,3,4}:{3,4,5,6}'::svec);
- ?column?
-----------
- {10}:{0}
-(1 row)
-
--- Test the pivot operator in the presence of NULL values
-drop table if exists pivot_test;
-DROP TABLE
-create table pivot_test(a float8) distributed randomly;
-CREATE TABLE
-insert into pivot_test values (0),(1),(NULL),(2),(3);
-INSERT 0 5
-select array_agg(a) from pivot_test;
- array_agg
----------------------
- {1,1,2,1}:{1,2,0,3}
-(1 row)
-
-select l1norm(array_agg(a)) from pivot_test;
- l1norm
---------
- 6
-(1 row)
-
-drop table if exists pivot_test;
-DROP TABLE
--- Answer should be 5
-select vec_median(array_agg(a)) from (select generate_series(1,9) a) foo;
- vec_median
-------------
- 5
-(1 row)
-
--- Answer should be a 10-wide vector
-select array_agg(a) from (select trunc(random()*10) a,generate_series(1,100000) order by a) foo;
- array_agg
--------------------------------------------------------------------------------
- {9946,10172,10063,9850,9874,10139,9964,9923,10131,9938}:{0,1,2,3,4,5,6,7,8,9}
-(1 row)
-
--- Average is 4.50034, median is 5
-select vec_median('{9960,9926,10053,9993,10080,10050,9938,9941,10030,10029}:{1,9,8,7,6,5,4,3,2,0}'::svec);
- vec_median
-------------
- 5
-(1 row)
-
-select vec_median('{9960,9926,10053,9993,10080,10050,9938,9941,10030,10029}:{1,9,8,7,6,5,4,3,2,0}'::svec::float8[]);
- vec_median
-------------
- 5
-(1 row)
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0a8e600..47f1bbb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,8 +103,6 @@
however it is tricky to maintain headers in them -->
<exclude>depends/libyarn/test/data/*</exclude>
- <exclude>contrib/gp_sparse_vector/bugs</exclude>
- <exclude>contrib/gp_sparse_vector/*output</exclude>
<exclude>contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/resources/dataset</exclude>
<exclude>contrib/hawq-hadoop/**/*.yaml</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/src/backend/catalog/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index be3e5da..08622f1 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -32,7 +32,7 @@ include $(top_srcdir)/src/backend/common.mk
cdb_init.sql: cdb_schema.sql filesystem.sql
- cat $(call vpathsearch,cdb_schema.sql) $(call vpathsearch,cdb_external_extensions.sql) $(call vpathsearch,filesystem.sql) $(call vpathsearch,madlib.sql) > cdb_init.sql
+ cat $(call vpathsearch,cdb_schema.sql) $(call vpathsearch,cdb_external_extensions.sql) $(call vpathsearch,filesystem.sql) > cdb_init.sql
cdb_schema.sql: cdb_schema.in
sed -e 's,@pkglibdir@,$(pkglibdir),g' \
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/src/backend/catalog/madlib.sql
----------------------------------------------------------------------
diff --git a/src/backend/catalog/madlib.sql b/src/backend/catalog/madlib.sql
deleted file mode 100644
index 4c435ad..0000000
--- a/src/backend/catalog/madlib.sql
+++ /dev/null
@@ -1,492 +0,0 @@
-SET statement_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = off;
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-SET escape_string_warning = off;
-
-SET default_with_oids = false;
-
-CREATE SCHEMA madlib;
--- CREATE LANGUAGE plpythonu;
-
-SET search_path = madlib, pg_catalog;
-
-CREATE TYPE bytea8;
-
-CREATE FUNCTION bytea8in(cstring) RETURNS bytea8
- AS $$byteain$$
- LANGUAGE internal IMMUTABLE STRICT;
-
-CREATE FUNCTION bytea8out(bytea8) RETURNS cstring
- AS $$byteaout$$
- LANGUAGE internal IMMUTABLE STRICT;
-
-CREATE FUNCTION bytea8recv(internal) RETURNS bytea8
- AS $$bytearecv$$
- LANGUAGE internal IMMUTABLE STRICT;
-
-CREATE FUNCTION bytea8send(bytea8) RETURNS bytea
- AS $$byteasend$$
- LANGUAGE internal IMMUTABLE STRICT;
-
-CREATE TYPE bytea8 (
- INTERNALLENGTH = variable,
- INPUT = bytea8in,
- OUTPUT = bytea8out,
- RECEIVE = bytea8recv,
- SEND = bytea8send,
- ALIGNMENT = double,
- STORAGE = plain
-);
-
-CREATE TYPE svec;
-
-CREATE FUNCTION svec_in(cstring) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_in'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_out(svec) RETURNS cstring
- AS '$libdir/gp_svec.so', 'svec_out'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_recv(internal) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_recv'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_send(svec) RETURNS bytea
- AS '$libdir/gp_svec.so', 'svec_send'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE TYPE svec (
- INTERNALLENGTH = variable,
- INPUT = svec_in,
- OUTPUT = svec_out,
- RECEIVE = svec_recv,
- SEND = svec_send,
- ALIGNMENT = double,
- STORAGE = extended
-);
-
-CREATE FUNCTION float8arr_cast_float4(real) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'float8arr_cast_float4'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION float8arr_cast_float8(double precision) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'float8arr_cast_float8'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION float8arr_cast_int2(smallint) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'float8arr_cast_int2'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION float8arr_cast_int4(integer) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'float8arr_cast_int4'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION float8arr_cast_int8(bigint) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'float8arr_cast_int8'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION float8arr_cast_numeric(numeric) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'float8arr_cast_numeric'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION float8arr_div_float8arr(double precision[], double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_div_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_div_svec(double precision[], svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_div_svec'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_eq(double precision[], double precision[]) RETURNS boolean
- AS '$libdir/gp_svec.so', 'float8arr_equals'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_minus_float8arr(double precision[], double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_minus_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_minus_svec(double precision[], svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_minus_svec'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_mult_float8arr(double precision[], double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_mult_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_mult_svec(double precision[], svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_mult_svec'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_plus_float8arr(double precision[], double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_plus_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION float8arr_plus_svec(double precision[], svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'float8arr_plus_svec'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_cast_float4(real) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_float4'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_float8(double precision) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_float8'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_float8arr(double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_float8arr'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_int2(smallint) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_int2'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_int4(integer) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_int4'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_int8(bigint) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_int8'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_numeric(numeric) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_numeric'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_cast_positions_float8arr(bigint[], double precision[], bigint, double precision) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_cast_positions_float8arr'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_concat(svec, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_concat'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_concat_replicate(integer, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_concat_replicate'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_div(svec, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_div'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_div_float8arr(svec, double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_div_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_dot(svec, svec) RETURNS double precision
- AS '$libdir/gp_svec.so', 'svec_dot'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_dot(double precision[], double precision[]) RETURNS double precision
- AS '$libdir/gp_svec.so', 'float8arr_dot'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_dot(svec, double precision[]) RETURNS double precision
- AS '$libdir/gp_svec.so', 'svec_dot_float8arr'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_dot(double precision[], svec) RETURNS double precision
- AS '$libdir/gp_svec.so', 'float8arr_dot_svec'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_eq(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_eq'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_l2_cmp(svec, svec) RETURNS integer
- AS '$libdir/gp_svec.so', 'svec_l2_cmp'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_l2_eq(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_l2_eq'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_l2_ge(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_l2_ge'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_l2_gt(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_l2_gt'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_l2_le(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_l2_le'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_l2_lt(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_l2_lt'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_l2_ne(svec, svec) RETURNS boolean
- AS '$libdir/gp_svec.so', 'svec_l2_ne'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_minus(svec, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_minus'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_minus_float8arr(svec, double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_minus_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_mult(svec, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_mult'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_mult_float8arr(svec, double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_mult_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_plus(svec, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_plus'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_plus_float8arr(svec, double precision[]) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_plus_float8arr'
- LANGUAGE c IMMUTABLE;
-
-CREATE FUNCTION svec_pow(svec, svec) RETURNS svec
- AS '$libdir/gp_svec.so', 'svec_pow'
- LANGUAGE c IMMUTABLE STRICT;
-
-CREATE FUNCTION svec_return_array(svec) RETURNS double precision[]
- AS '$libdir/gp_svec.so', 'svec_return_array'
- LANGUAGE c IMMUTABLE;
-
-CREATE OPERATOR %*% (
- PROCEDURE = svec_dot,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR %*% (
- PROCEDURE = svec_dot,
- LEFTARG = double precision[],
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR %*% (
- PROCEDURE = svec_dot,
- LEFTARG = double precision[],
- RIGHTARG = svec
-);
-
-CREATE OPERATOR %*% (
- PROCEDURE = svec_dot,
- LEFTARG = svec,
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR * (
- PROCEDURE = svec_mult,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR * (
- PROCEDURE = float8arr_mult_float8arr,
- LEFTARG = double precision[],
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR * (
- PROCEDURE = float8arr_mult_svec,
- LEFTARG = double precision[],
- RIGHTARG = svec
-);
-
-CREATE OPERATOR * (
- PROCEDURE = svec_mult_float8arr,
- LEFTARG = svec,
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR *|| (
- PROCEDURE = svec_concat_replicate,
- LEFTARG = integer,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR + (
- PROCEDURE = svec_plus,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR + (
- PROCEDURE = float8arr_plus_float8arr,
- LEFTARG = double precision[],
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR + (
- PROCEDURE = float8arr_plus_svec,
- LEFTARG = double precision[],
- RIGHTARG = svec
-);
-
-CREATE OPERATOR + (
- PROCEDURE = svec_plus_float8arr,
- LEFTARG = svec,
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR - (
- PROCEDURE = svec_minus,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR - (
- PROCEDURE = float8arr_minus_float8arr,
- LEFTARG = double precision[],
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR - (
- PROCEDURE = float8arr_minus_svec,
- LEFTARG = double precision[],
- RIGHTARG = svec
-);
-
-CREATE OPERATOR - (
- PROCEDURE = svec_minus_float8arr,
- LEFTARG = svec,
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR / (
- PROCEDURE = svec_div,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR / (
- PROCEDURE = float8arr_div_float8arr,
- LEFTARG = double precision[],
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR / (
- PROCEDURE = float8arr_div_svec,
- LEFTARG = double precision[],
- RIGHTARG = svec
-);
-
-CREATE OPERATOR / (
- PROCEDURE = svec_div_float8arr,
- LEFTARG = svec,
- RIGHTARG = double precision[]
-);
-
-CREATE OPERATOR < (
- PROCEDURE = svec_l2_lt,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = >,
- NEGATOR = >=,
- RESTRICT = scalarltsel,
- JOIN = scalarltjoinsel
-);
-
-CREATE OPERATOR <= (
- PROCEDURE = svec_l2_le,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = >=,
- NEGATOR = >,
- RESTRICT = scalarltsel,
- JOIN = scalarltjoinsel
-);
-
-CREATE OPERATOR <> (
- PROCEDURE = svec_l2_eq,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = <>,
- NEGATOR = =,
- RESTRICT = eqsel,
- JOIN = eqjoinsel
-);
-
-CREATE OPERATOR = (
- PROCEDURE = svec_eq,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = =,
- NEGATOR = <>,
- RESTRICT = eqsel,
- JOIN = eqjoinsel
-);
-
-CREATE OPERATOR == (
- PROCEDURE = svec_l2_eq,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = =,
- NEGATOR = <>,
- RESTRICT = eqsel,
- JOIN = eqjoinsel
-);
-
-CREATE OPERATOR > (
- PROCEDURE = svec_l2_gt,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = <,
- NEGATOR = <=,
- RESTRICT = scalargtsel,
- JOIN = scalargtjoinsel
-);
-
-CREATE OPERATOR >= (
- PROCEDURE = svec_l2_ge,
- LEFTARG = svec,
- RIGHTARG = svec,
- COMMUTATOR = <=,
- NEGATOR = <,
- RESTRICT = scalargtsel,
- JOIN = scalargtjoinsel
-);
-
-CREATE OPERATOR ^ (
- PROCEDURE = svec_pow,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR || (
- PROCEDURE = svec_concat,
- LEFTARG = svec,
- RIGHTARG = svec
-);
-
-CREATE OPERATOR CLASS svec_l2_ops
- DEFAULT FOR TYPE svec USING btree AS
- OPERATOR 1 <(svec,svec) ,
- OPERATOR 2 <=(svec,svec) ,
- OPERATOR 3 ==(svec,svec) ,
- OPERATOR 4 >=(svec,svec) ,
- OPERATOR 5 >(svec,svec) ,
- FUNCTION 1 svec_l2_cmp(svec,svec);
-
-SET search_path = pg_catalog;
-
-CREATE CAST (double precision[] AS madlib.svec) WITH FUNCTION madlib.svec_cast_float8arr(double precision[]);
-
-CREATE CAST (real AS madlib.svec) WITH FUNCTION madlib.svec_cast_float4(real);
-
-CREATE CAST (double precision AS madlib.svec) WITH FUNCTION madlib.svec_cast_float8(double precision);
-
-CREATE CAST (smallint AS madlib.svec) WITH FUNCTION madlib.svec_cast_int2(smallint);
-
-CREATE CAST (integer AS madlib.svec) WITH FUNCTION madlib.svec_cast_int4(integer);
-
-CREATE CAST (bigint AS madlib.svec) WITH FUNCTION madlib.svec_cast_int8(bigint);
-
-CREATE CAST (numeric AS madlib.svec) WITH FUNCTION madlib.svec_cast_numeric(numeric);
-
-CREATE CAST (madlib.svec AS double precision[]) WITH FUNCTION madlib.svec_return_array(madlib.svec);
[2/5] incubator-hawq git commit: HAWQ-49. Remove madlib related codes
and tests.
Posted by ml...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/src/test/regress/expected/madlib_svec_test.out
----------------------------------------------------------------------
diff --git a/src/test/regress/expected/madlib_svec_test.out b/src/test/regress/expected/madlib_svec_test.out
deleted file mode 100644
index 3a7dbd0..0000000
--- a/src/test/regress/expected/madlib_svec_test.out
+++ /dev/null
@@ -1,236 +0,0 @@
-SET client_min_messages TO ERROR;
-DROP SCHEMA IF EXISTS madlib_install_check_gpsql_svec_test CASCADE;
-CREATE SCHEMA madlib_install_check_gpsql_svec_test;
-SET search_path = madlib_install_check_gpsql_svec_test, madlib;
-create table test_pairs( id int, a madlib.svec, b madlib.svec );
-insert into test_pairs values
- (0, '{1,100,1}:{5,0,5}', '{50,50,2}:{1,2,10}'),
- (1, '{1,100,1}:{-5,0,-5}', '{50,50,2}:{-1,-2,-10}');
-insert into test_pairs values
- (11, '{1}:{0}', '{1}:{1}'),
- (12, '{1}:{5}', '{3}:{-8}'),
- (13, '{1}:{0}', '{1}:{NULL}'),
- (14, '{1,2,1}:{2,4,2}', '{2,1,1}:{0,3,5}'),
- (15, '{1,2,1}:{2,4,2}', '{2,1,1}:{NULL,3,5}');
-select id, madlib.svec_plus(a,b) from test_pairs order by id;
- id | svec_plus
-----+----------------------------------
- 0 | {1,49,50,1,1}:{6,1,2,10,15}
- 1 | {1,49,50,1,1}:{-6,-1,-2,-10,-15}
- 11 | {1}:{1}
- 12 | {3}:{-3}
- 13 | {1}:{NVP}
- 14 | {1,1,2}:{2,4,7}
- 15 | {2,2}:{NVP,7}
-(7 rows)
-
-select id, madlib.svec_plus(a,b) = madlib.svec_plus(b,a) from test_pairs order by id;
- id | ?column?
-----+----------
- 0 | t
- 1 | t
- 11 | t
- 12 | t
- 13 | t
- 14 | t
- 15 | t
-(7 rows)
-
-select id, madlib.svec_mult(a,b) from test_pairs order by id;
- id | svec_mult
-----+---------------------
- 0 | {1,100,1}:{5,0,50}
- 1 | {1,100,1}:{5,-0,50}
- 11 | {1}:{0}
- 12 | {3}:{-40}
- 13 | {1}:{NVP}
- 14 | {2,1,1}:{0,12,10}
- 15 | {2,1,1}:{NVP,12,10}
-(7 rows)
-
-select id, madlib.svec_mult(a,b) = madlib.svec_mult(b,a) from test_pairs order by id;
- id | ?column?
-----+----------
- 0 | t
- 1 | t
- 11 | t
- 12 | t
- 13 | t
- 14 | t
- 15 | t
-(7 rows)
-
-select id, madlib.svec_minus(a, b) = madlib.svec_plus(madlib.svec_mult((-1)::svec,b), a) from test_pairs order by id;
- id | ?column?
-----+----------
- 0 | t
- 1 | t
- 11 | t
- 12 | t
- 13 | t
- 14 | t
- 15 | t
-(7 rows)
-
-select id, madlib.svec_pow(a,2::svec) = madlib.svec_mult(a,a) from test_pairs order by id;
- id | ?column?
-----+----------
- 0 | t
- 1 | t
- 11 | t
- 12 | t
- 13 | t
- 14 | t
- 15 | t
-(7 rows)
-
-select madlib.svec_plus('{1,2,3}:{4,5,6}', 5::madlib.svec);
- svec_plus
--------------------
- {1,2,3}:{9,10,11}
-(1 row)
-
-select madlib.svec_plus(5::madlib.svec, '{1,2,3}:{4,5,6}');
- svec_plus
--------------------
- {1,2,3}:{9,10,11}
-(1 row)
-
-select madlib.svec_plus(500::madlib.svec, '{1,2,3}:{4,null,6}');
- svec_plus
------------------------
- {1,2,3}:{504,NVP,506}
-(1 row)
-
-select madlib.svec_div(500::madlib.svec, '{1,2,3}:{4,null,6}');
- svec_div
-------------------------------------
- {1,2,3}:{125,NVP,83.3333333333333}
-(1 row)
-
-select madlib.svec_div('{1,2,3}:{4,null,6}', 500::madlib.svec);
- svec_div
----------------------------
- {1,2,3}:{0.008,NVP,0.012}
-(1 row)
-
--- Test operators between svec and float8[]
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) %*% ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
- ?column?
-----------
- 260
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] %*% ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
- ?column?
-----------
- 260
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) / ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
- ?column?
-----------
- {10}:{1}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] / ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
- ?column?
-----------
- {10}:{1}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) * ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
- ?column?
-------------------------
- {1,2,3,4}:{9,16,25,36}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] * ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
- ?column?
-------------------------
- {1,2,3,4}:{9,16,25,36}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) + ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
- ?column?
------------------------
- {1,2,3,4}:{6,8,10,12}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] + ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
- ?column?
------------------------
- {1,2,3,4}:{6,8,10,12}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) - ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
- ?column?
-----------
- {10}:{0}
-(1 row)
-
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] - ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
- ?column?
-----------
- {10}:{0}
-(1 row)
-
-select id, a, b, madlib.svec_concat(a,b), a, b from test_pairs order by id;
- id | a | b | svec_concat | a | b
-----+---------------------+-----------------------+---------------------------------------+---------------------+-----------------------
- 0 | {1,100,1}:{5,0,5} | {50,50,2}:{1,2,10} | {1,100,1,50,50,2}:{5,0,5,1,2,10} | {1,100,1}:{5,0,5} | {50,50,2}:{1,2,10}
- 1 | {1,100,1}:{-5,0,-5} | {50,50,2}:{-1,-2,-10} | {1,100,1,50,50,2}:{-5,0,-5,-1,-2,-10} | {1,100,1}:{-5,0,-5} | {50,50,2}:{-1,-2,-10}
- 11 | {1}:{0} | {1}:{1} | {1,1}:{0,1} | {1}:{0} | {1}:{1}
- 12 | {1}:{5} | {3}:{-8} | {1,3}:{5,-8} | {1}:{5} | {3}:{-8}
- 13 | {1}:{0} | {1}:{NVP} | {1,1}:{0,NVP} | {1}:{0} | {1}:{NVP}
- 14 | {1,2,1}:{2,4,2} | {2,1,1}:{0,3,5} | {1,2,1,2,1,1}:{2,4,2,0,3,5} | {1,2,1}:{2,4,2} | {2,1,1}:{0,3,5}
- 15 | {1,2,1}:{2,4,2} | {2,1,1}:{NVP,3,5} | {1,2,1,2,1,1}:{2,4,2,NVP,3,5} | {1,2,1}:{2,4,2} | {2,1,1}:{NVP,3,5}
-(7 rows)
-
-select id, madlib.svec_concat_replicate(0, b), b from test_pairs order by id;
- id | svec_concat_replicate | b
-----+-----------------------+-----------------------
- 0 | {}:{} | {50,50,2}:{1,2,10}
- 1 | {}:{} | {50,50,2}:{-1,-2,-10}
- 11 | {}:{} | {1}:{1}
- 12 | {}:{} | {3}:{-8}
- 13 | {}:{} | {1}:{NVP}
- 14 | {}:{} | {2,1,1}:{0,3,5}
- 15 | {}:{} | {2,1,1}:{NVP,3,5}
-(7 rows)
-
-select id, madlib.svec_concat_replicate(1, b) = b from test_pairs order by id;
- id | ?column?
-----+----------
- 0 | t
- 1 | t
- 11 | t
- 12 | t
- 13 | t
- 14 | t
- 15 | t
-(7 rows)
-
-select id, madlib.svec_concat_replicate(3, b), b from test_pairs order by id;
- id | svec_concat_replicate | b
-----+-----------------------------------------------------------+-----------------------
- 0 | {50,50,2,50,50,2,50,50,2}:{1,2,10,1,2,10,1,2,10} | {50,50,2}:{1,2,10}
- 1 | {50,50,2,50,50,2,50,50,2}:{-1,-2,-10,-1,-2,-10,-1,-2,-10} | {50,50,2}:{-1,-2,-10}
- 11 | {1,1,1}:{1,1,1} | {1}:{1}
- 12 | {3,3,3}:{-8,-8,-8} | {3}:{-8}
- 13 | {1,1,1}:{NVP,NVP,NVP} | {1}:{NVP}
- 14 | {2,1,1,2,1,1,2,1,1}:{0,3,5,0,3,5,0,3,5} | {2,1,1}:{0,3,5}
- 15 | {2,1,1,2,1,1,2,1,1}:{NVP,3,5,NVP,3,5,NVP,3,5} | {2,1,1}:{NVP,3,5}
-(7 rows)
-
--- This vfunction test svec creation from position array
-select madlib.svec_cast_positions_float8arr('{1,2,4,6,2,5}'::INT8[], '{.2,.3,.4,.5,.3,.1}'::FLOAT8[], 10000, 0.0);
- svec_cast_positions_float8arr
-----------------------------------------------
- {1,1,1,1,1,1,9994}:{0.2,0.3,0,0.4,0.1,0.5,0}
-(1 row)
-
-DROP SCHEMA madlib_install_check_gpsql_svec_test CASCADE;
--- sanity check; don't delete madlib schema and its objects.
-DROP SCHEMA madlib CASCADE;
-ERROR: cannot drop schema madlib because it is required by the database system
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/src/test/regress/known_good_schedule
----------------------------------------------------------------------
diff --git a/src/test/regress/known_good_schedule b/src/test/regress/known_good_schedule
index cde5a4c..c906071 100755
--- a/src/test/regress/known_good_schedule
+++ b/src/test/regress/known_good_schedule
@@ -202,8 +202,6 @@ ignore: percentile
ignore: resource_queue_function
ignore: gp_optimizer
ignore: co_nestloop_idxscan
-ignore: madlib_array_ops
-test: madlib_svec_test
test: agg_derived_win
ignore: orca_udfs
#
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/src/test/regress/sql/madlib_svec_test.sql
----------------------------------------------------------------------
diff --git a/src/test/regress/sql/madlib_svec_test.sql b/src/test/regress/sql/madlib_svec_test.sql
deleted file mode 100644
index 7e641fb..0000000
--- a/src/test/regress/sql/madlib_svec_test.sql
+++ /dev/null
@@ -1,56 +0,0 @@
-SET client_min_messages TO ERROR;
-DROP SCHEMA IF EXISTS madlib_install_check_gpsql_svec_test CASCADE;
-CREATE SCHEMA madlib_install_check_gpsql_svec_test;
-SET search_path = madlib_install_check_gpsql_svec_test, madlib;
-
-create table test_pairs( id int, a madlib.svec, b madlib.svec );
-insert into test_pairs values
- (0, '{1,100,1}:{5,0,5}', '{50,50,2}:{1,2,10}'),
- (1, '{1,100,1}:{-5,0,-5}', '{50,50,2}:{-1,-2,-10}');
-insert into test_pairs values
- (11, '{1}:{0}', '{1}:{1}'),
- (12, '{1}:{5}', '{3}:{-8}'),
- (13, '{1}:{0}', '{1}:{NULL}'),
- (14, '{1,2,1}:{2,4,2}', '{2,1,1}:{0,3,5}'),
- (15, '{1,2,1}:{2,4,2}', '{2,1,1}:{NULL,3,5}');
-
-
-select id, madlib.svec_plus(a,b) from test_pairs order by id;
-select id, madlib.svec_plus(a,b) = madlib.svec_plus(b,a) from test_pairs order by id;
-select id, madlib.svec_mult(a,b) from test_pairs order by id;
-select id, madlib.svec_mult(a,b) = madlib.svec_mult(b,a) from test_pairs order by id;
-select id, madlib.svec_minus(a, b) = madlib.svec_plus(madlib.svec_mult((-1)::svec,b), a) from test_pairs order by id;
-select id, madlib.svec_pow(a,2::svec) = madlib.svec_mult(a,a) from test_pairs order by id;
-
-select madlib.svec_plus('{1,2,3}:{4,5,6}', 5::madlib.svec);
-select madlib.svec_plus(5::madlib.svec, '{1,2,3}:{4,5,6}');
-select madlib.svec_plus(500::madlib.svec, '{1,2,3}:{4,null,6}');
-select madlib.svec_div(500::madlib.svec, '{1,2,3}:{4,null,6}');
-select madlib.svec_div('{1,2,3}:{4,null,6}', 500::madlib.svec);
-
--- Test operators between svec and float8[]
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) %*% ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] %*% ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) / ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] / ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) * ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] * ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) + ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] + ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec) - ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[];
-select ('{1,2,3,4}:{3,4,5,6}'::madlib.svec)::float8[] - ('{1,2,3,4}:{3,4,5,6}'::madlib.svec);
-
-select id, a, b, madlib.svec_concat(a,b), a, b from test_pairs order by id;
-
-select id, madlib.svec_concat_replicate(0, b), b from test_pairs order by id;
-select id, madlib.svec_concat_replicate(1, b) = b from test_pairs order by id;
-select id, madlib.svec_concat_replicate(3, b), b from test_pairs order by id;
-
-
--- This vfunction test svec creation from position array
-select madlib.svec_cast_positions_float8arr('{1,2,4,6,2,5}'::INT8[], '{.2,.3,.4,.5,.3,.1}'::FLOAT8[], 10000, 0.0);
-
-DROP SCHEMA madlib_install_check_gpsql_svec_test CASCADE;
-
--- sanity check; don't delete madlib schema and its objects.
-DROP SCHEMA madlib CASCADE;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f58baae0/src/tools/msvc/Mkvcbuild.pm
----------------------------------------------------------------------
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 4fbb6c3..259b14e 100755
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -34,7 +34,7 @@ my $contrib_extrasource = {
'cube' => ['cubescan.l','cubeparse.y'],
'seg' => ['segscan.l','segparse.y']
};
-my @contrib_excludes = ('pgcrypto','intagg','uuid-ossp','bufferedtest', 'varblocktest', 'hstore', 'gp_filedump', 'tsearch2', 'dblink', 'xlogdump','xlogviewer', 'changetrackingdump','gp_sparse_vector','xml2','adminpack');
+my @contrib_excludes = ('pgcrypto','intagg','uuid-ossp','bufferedtest', 'varblocktest', 'hstore', 'gp_filedump', 'tsearch2', 'dblink', 'xlogdump','xlogviewer', 'changetrackingdump','xml2','adminpack');
sub mkvcbuild
{