You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/01/29 04:20:14 UTC
[1/6] incubator-hawq git commit: HAWQ-340. Make getVersion API return
JSON format.
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-369 d03cded67 -> 26d9c1a08
HAWQ-340. Make getVersion API return JSON format.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b1d2e90f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b1d2e90f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b1d2e90f
Branch: refs/heads/HAWQ-369
Commit: b1d2e90f1346b3997e060285f39dd9d5957ac65c
Parents: bee0137
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Fri Jan 22 15:58:06 2016 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Mon Jan 25 18:17:30 2016 -0800
----------------------------------------------------------------------
pxf/build.gradle | 44 +++++++++++++++++++-
pxf/gradle.properties | 1 +
.../hawq/pxf/service/rest/VersionResource.java | 26 ++++++++++--
.../pxf/service/rest/VersionResourceTest.java | 7 +++-
4 files changed, 73 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1d2e90f/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 0d260d4..2eb333a 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -17,6 +17,8 @@
* under the License.
*/
+import org.apache.tools.ant.filters.ReplaceTokens
+
buildscript {
repositories {
// mavenCentral without https:
@@ -122,6 +124,47 @@ subprojects { subProject ->
}
project('pxf-service') {
+
+// Copy existing sources and replace any occurrences of @tokenName@ with desired values
+ task generateSources {
+ doFirst {
+ copy {
+ from('src/main/java') {
+ include '**/*.java'
+ filter(ReplaceTokens,
+ tokens:['pxfProtocolVersion': project.pxfProtocolVersion ])}
+ into "tmp/generatedSources"
+ }
+ }
+ }
+
+// Call cleanup taskAfter Java code compilation
+ compileJava.doLast {
+ tasks.cleanGeneratedSources.execute()
+ }
+
+// Delete "tmp" directory under current project directory
+// rm -r pxf-service/tmp
+ task cleanGeneratedSources() {
+ doFirst {
+ delete "tmp"
+ }
+ }
+
+// Call generateSources task before Java compilation
+ gradle.projectsEvaluated {
+ compileJava.dependsOn(generateSources)
+ }
+
+// Use custom sources directory with generated sources
+ sourceSets {
+ main {
+ java {
+ srcDirs = ["tmp/generatedSources"]
+ }
+ }
+ }
+
apply plugin: 'war'
tasks.war {
archiveName = 'pxf.war'
@@ -137,7 +180,6 @@ project('pxf-service') {
}
}
}
-
dependencies {
compile(project(':pxf-api'))
compile 'com.sun.jersey:jersey-core:1.9'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1d2e90f/pxf/gradle.properties
----------------------------------------------------------------------
diff --git a/pxf/gradle.properties b/pxf/gradle.properties
index 6a7b2d6..6827b89 100644
--- a/pxf/gradle.properties
+++ b/pxf/gradle.properties
@@ -23,3 +23,4 @@ hiveVersion=1.2.1
hbaseVersionJar=1.1.2
hbaseVersionRPM=1.1.2
tomcatVersion=7.0.62
+pxfProtocolVersion=v14
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1d2e90f/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
index 6f326d6..db9743e 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
@@ -21,6 +21,7 @@ package org.apache.hawq.pxf.service.rest;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
@@ -33,7 +34,25 @@ import org.apache.commons.logging.LogFactory;
* version e.g. {@code ...pxf/v14/Bridge}
*/
class Version {
- final static String PXF_PROTOCOL_VERSION = "v14";
+ /**
+ * Constant which holds current protocol version. Getting replaced with
+ * actual value on build stage, using pxfProtocolVersion parameter from
+ * gradle.properties
+ */
+ final static String PXF_PROTOCOL_VERSION = "@pxfProtocolVersion@";
+
+ public Version() {
+ }
+
+ public String version;
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
}
/**
@@ -58,11 +77,12 @@ public class VersionResource {
* @return response with the PXF protocol version
*/
@GET
+ @Produces("application/json")
public Response getProtocolVersion() {
ResponseBuilder b = Response.ok();
- b.entity("PXF protocol version " + Version.PXF_PROTOCOL_VERSION);
- b.type(MediaType.TEXT_PLAIN_TYPE);
+ b.entity("{ \"version\": \"" + Version.PXF_PROTOCOL_VERSION + "\"}");
+ b.type(MediaType.APPLICATION_JSON_TYPE);
return b.build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1d2e90f/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/rest/VersionResourceTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/rest/VersionResourceTest.java b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/rest/VersionResourceTest.java
index 58ca4e3..d9f8f36 100644
--- a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/rest/VersionResourceTest.java
+++ b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/rest/VersionResourceTest.java
@@ -22,6 +22,8 @@ package org.apache.hawq.pxf.service.rest;
import static org.junit.Assert.assertEquals;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.junit.Test;
@@ -36,7 +38,10 @@ public class VersionResourceTest {
assertEquals(Response.Status.OK,
Response.Status.fromStatusCode(result.getStatus()));
- assertEquals("PXF protocol version " + Version.PXF_PROTOCOL_VERSION,
+ assertEquals(
+ "{ \"version\": \"" + Version.PXF_PROTOCOL_VERSION + "\"}",
result.getEntity().toString());
+ assertEquals(result.getMetadata().get(HttpHeaders.CONTENT_TYPE).get(0),
+ MediaType.APPLICATION_JSON_TYPE);
}
}
[2/6] incubator-hawq git commit: HAWQ-364. Make resource manager
dynamically adjust minimum YARN container count in each segment
Posted by od...@apache.org.
HAWQ-364. Make resource manager dynamically adjust minimum YARN container count in each segment
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e22956c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e22956c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e22956c6
Branch: refs/heads/HAWQ-369
Commit: e22956c6f84cbeb79ea32615f5f7f9908ffca553
Parents: 898820b
Author: YI JIN <yj...@pivotal.io>
Authored: Thu Jan 28 14:37:07 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Thu Jan 28 14:37:07 2016 +1100
----------------------------------------------------------------------
.../communication/rmcomm_RM2RMSEG.c | 2 +
.../resourcemanager/include/resourcepool.h | 1 +
.../resourcemanager/include/resqueuemanager.h | 6 +-
src/backend/resourcemanager/requesthandler.c | 2 +
.../resourcebroker/resourcebroker_LIBYARN.c | 20 ++++-
src/backend/resourcemanager/resourcemanager.c | 2 +
src/backend/resourcemanager/resqueuemanager.c | 79 +++++++++++++++++++-
7 files changed, 108 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
index 3591ac1..e6b861b 100644
--- a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
+++ b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
@@ -244,6 +244,7 @@ void receivedRUAliveResponse(AsyncCommMessageHandlerContext context,
GET_SEGRESOURCE_HOSTNAME(segres));
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
}
else {
elog(DEBUG3, "Resource manager find host %s is down already.",
@@ -293,6 +294,7 @@ void sentRUAliveError(AsyncCommMessageHandlerContext context)
GET_SEGRESOURCE_HOSTNAME(segres));
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index d63a6cb..b7b25a1 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -390,6 +390,7 @@ struct ResourcePoolData {
*/
ResourceBundleData FTSTotal;
ResourceBundleData GRMTotal;
+ ResourceBundleData GRMTotalHavingNoHAWQNode;
uint64_t LastUpdateTime; /* Last time the GRM cluster report is gotten. */
uint64_t LastRequestTime;/* Last time the GRM cluster report is sent. */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/include/resqueuemanager.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resqueuemanager.h b/src/backend/resourcemanager/include/resqueuemanager.h
index 0b38520..171b399 100644
--- a/src/backend/resourcemanager/include/resqueuemanager.h
+++ b/src/backend/resourcemanager/include/resqueuemanager.h
@@ -329,6 +329,8 @@ struct DynResourceQueueManagerData {
int ForcedReturnGRMContainerCount;
bool toRunQueryDispatch;
bool hasResourceProblem[RESPROBLEM_COUNT];
+
+ int ActualMinGRMContainerPerSeg;
};
typedef struct DynResourceQueueManagerData *DynResourceQueueManager;
typedef struct DynResourceQueueManagerData DynResourceQueueManagerData;
@@ -344,8 +346,10 @@ typedef struct DynResourceQueueManagerData DynResourceQueueManagerData;
void initializeResourceQueueManager(void);
/* collect resource queues' resource usage status from bottom up. */
void refreshMemoryCoreRatioLevelUsage(uint64_t curmicrosec);
-/* Refresh reosurce queue resource capacity and adjusts all queued requests. */
+/* Refresh resource queue resource capacity and adjusts all queued requests. */
void refreshResourceQueueCapacity(bool queuechanged);
+/* Refresh actual minimum GRM container water level. */
+void refreshActualMinGRMContainerPerSeg(void);
/* Dispatch resource to the queuing queries. */
void dispatchResourceToQueries(void);
/* Time out the resource allocated whose QD owner does not have chance to return. */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index c6e9a34..cc2a216 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -783,6 +783,7 @@ bool handleRMSEGRequestIMAlive(void **arg)
{
/* Refresh resource queue capacities. */
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
/* Recalculate all memory/core ratio instances' limits. */
refreshMemoryCoreRatioLimits();
/* Refresh memory/core ratio level water mark. */
@@ -1049,6 +1050,7 @@ bool handleRMRequestSegmentIsDown(void **arg)
}
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
RPCResponseSegmentIsDownData response;
response.Result = res;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index c6d26af..c97e340 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -649,12 +649,21 @@ int handleRB2RM_ClusterReport(void)
setAllSegResourceGRMUnavailable();
/*
- * Start to update resource pool content.
+ * Start to update resource pool content. The YARN cluster total size is
+ * also counted the same time.
*/
+
+ resetResourceBundleData(&(PRESPOOL->GRMTotalHavingNoHAWQNode), 0, 0.0, 0);
+
MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
while( list_length(segstats) > 0 )
{
SegStat segstat = (SegStat)lfirst(list_head(segstats));
+
+ addResourceBundleData(&(PRESPOOL->GRMTotalHavingNoHAWQNode),
+ segstat->GRMTotalMemoryMB,
+ segstat->GRMTotalCore);
+
res = updateHAWQSegWithGRMSegStat(segstat);
if ( res == FUNC_RETURN_OK )
{
@@ -676,6 +685,14 @@ int handleRB2RM_ClusterReport(void)
}
MEMORY_CONTEXT_SWITCH_BACK
+ elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
+ "having (%d MB, %lf CORE) in a YARN cluster of total resource "
+ "(%d MB, %lf CORE).",
+ PRESPOOL->GRMTotal.MemoryMB,
+ PRESPOOL->GRMTotal.Core,
+ PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB,
+ PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
+
/*
* If the segment is not GRM available, RM should return all containers
* located upon them.
@@ -695,6 +712,7 @@ int handleRB2RM_ClusterReport(void)
PQUEMGR->GRMQueueResourceTight = response.ResourceTight > 0 ? true : false;
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
PRESPOOL->LastUpdateTime = gettime_microsec();
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 819da83..b8a7bb5 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -2645,6 +2645,7 @@ void updateStatusOfAllNodes()
if ( changedstatus )
{
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
}
validateResourcePoolStatus(true);
@@ -2808,6 +2809,7 @@ int loadHostInformationIntoResourcePool(void)
/* Refresh resource queue capacities. */
refreshResourceQueueCapacity(false);
+ refreshActualMinGRMContainerPerSeg();
/* Recalculate all memory/core ratio instances' limits. */
refreshMemoryCoreRatioLimits();
/* Refresh memory/core ratio level water mark. */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e22956c6/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index e2a2f43..10a970b 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -258,6 +258,8 @@ void initializeResourceQueueManager(void)
{
PQUEMGR->hasResourceProblem[i] = false;
}
+
+ PQUEMGR->ActualMinGRMContainerPerSeg = rm_min_resource_perseg;
}
/*
@@ -2472,6 +2474,77 @@ int returnResourceToResQueMgr(ConnectionTrack conntrack)
return res;
}
+void refreshActualMinGRMContainerPerSeg(void)
+{
+ /*--------------------------------------------------------------------------
+ * There are 3 limits should be considered, the actual water level is the
+ * least value of the 3 limits : resource queue normal capacity caused mean
+ * GRM container number, minimum value of all segments' maximum GRM container
+ * numbers, user setting saved in guc.
+ *
+ *--------------------------------------------------------------------------
+ */
+
+ /* STEP 1. go through each segment to get segment maximum capacity. */
+ int minctncount = INT32_MAX;
+ int normalctncount = INT32_MAX;
+ if ( DRMGlobalInstance->ImpType != NONE_HAWQ2 )
+ {
+ List *allsegres = NULL;
+ ListCell *cell = NULL;
+ getAllPAIRRefIntoList(&(PRESPOOL->Segments), &allsegres);
+
+ foreach(cell, allsegres)
+ {
+ SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
+ if ( !IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
+ !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+ {
+ continue;
+ }
+
+ if ( segres->Stat->GRMTotalCore < minctncount )
+ {
+ minctncount = segres->Stat->GRMTotalCore;
+ }
+ }
+ freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
+
+ elog(RMLOG, "Resource manager finds minimum global resource manager "
+ "container count can contained by all segments is %d",
+ minctncount);
+
+ /* STEP 2. check the queue normal capacity introduced water level. */
+ if ( PRESPOOL->AvailNodeCount > 0 &&
+ PQUEMGR->GRMQueueCapacity > 0 &&
+ PRESPOOL->GRMTotalHavingNoHAWQNode.Core > 0 )
+ {
+ normalctncount = trunc(PRESPOOL->GRMTotalHavingNoHAWQNode.Core *
+ PQUEMGR->GRMQueueCapacity /
+ PRESPOOL->AvailNodeCount);
+
+ elog(RMLOG, "Resource manager calculates normal global resource "
+ "manager container count based on target queue capacity "
+ "is %d",
+ normalctncount);
+ }
+ }
+
+ /* STEP 3. Get final water level result. */
+ int oldval = PQUEMGR->ActualMinGRMContainerPerSeg;
+ int newval = minctncount < normalctncount ? minctncount : normalctncount;
+ newval = newval < rm_min_resource_perseg ? newval : rm_min_resource_perseg;
+
+ if ( newval != oldval )
+ {
+ elog(WARNING, "Resource manager adjusts minimum global resource manager "
+ "container count in each segment from %d to %d.",
+ oldval,
+ newval);
+ }
+ PQUEMGR->ActualMinGRMContainerPerSeg = newval;
+}
+
void refreshResourceQueueCapacity(bool queuechanged)
{
static char errorbuf[ERRORMESSAGE_SIZE];
@@ -2510,8 +2583,10 @@ void refreshResourceQueuePercentageCapacity(bool queuechanged)
{
if ( DRMGlobalInstance->ImpType == YARN_LIBYARN )
{
- mem = PRESPOOL->GRMTotal.MemoryMB * PQUEMGR->GRMQueueMaxCapacity;
- core = PRESPOOL->GRMTotal.Core * PQUEMGR->GRMQueueMaxCapacity;
+ mem = PRESPOOL->GRMTotalHavingNoHAWQNode.MemoryMB *
+ PQUEMGR->GRMQueueMaxCapacity;
+ core = PRESPOOL->GRMTotalHavingNoHAWQNode.Core *
+ PQUEMGR->GRMQueueMaxCapacity;
}
else if ( DRMGlobalInstance->ImpType == NONE_HAWQ2 )
{
[4/6] incubator-hawq git commit: HAWQ-367. Introduce
seg_max_connections guc to set segment max_connections setting
Posted by od...@apache.org.
HAWQ-367. Introduce seg_max_connections guc to set segment max_connections setting
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ff98f360
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ff98f360
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ff98f360
Branch: refs/heads/HAWQ-369
Commit: ff98f360b2b6c12fb07e7d200c1ab772dcaf80c7
Parents: 1df504f
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Jan 29 11:17:22 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Jan 29 11:17:22 2016 +1100
----------------------------------------------------------------------
src/backend/postmaster/postmaster.c | 15 +++++++++++++++
src/backend/tcop/postgres.c | 15 ++++++++++++++-
src/backend/utils/init/globals.c | 1 +
src/backend/utils/misc/guc.c | 14 ++++++++++++++
src/include/miscadmin.h | 1 +
5 files changed, 45 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ff98f360/src/backend/postmaster/postmaster.c
----------------------------------------------------------------------
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index bc71d97..f15a6eb 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -1071,9 +1071,24 @@ PostmasterMain(int argc, char *argv[])
* postgresql.conf for the first time.
*/
if (!SelectConfigFiles(userDoption, progname))
+
ExitPostmaster(2);
/*
+ * Overwrite MaxBackends in case it is a segment.
+ */
+ if ( !AmIMaster() && !IsUnderPostmaster)
+ {
+ char segmaxconns[32];
+ elog(LOG, "Update segment max_connections to %d", SegMaxBackends);
+ snprintf(segmaxconns, sizeof(segmaxconns), "%d", SegMaxBackends);
+ SetConfigOption("max_connections",
+ segmaxconns,
+ PGC_POSTMASTER,
+ PGC_S_OVERRIDE);
+ }
+
+ /*
* CDB/MPP/GPDB: Set the processor affinity (may be a no-op on
* some platforms). The port number is nice to use because we know
* that different segments on a single host will not have the same
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ff98f360/src/backend/tcop/postgres.c
----------------------------------------------------------------------
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 4d3dfe3..71c4c2a 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -3763,7 +3763,6 @@ PostgresMain(int argc, char *argv[], const char *username)
volatile bool send_ready_for_query = true;
int topErrCode;
-
MemoryAccount *postgresMainMemoryAccount = NULL;
/*
@@ -4145,6 +4144,20 @@ PostgresMain(int argc, char *argv[], const char *username)
PgStartTime = GetCurrentTimestamp();
}
+ /*
+ * Overwrite MaxBackends in case it is a segment.
+ */
+ if ( !AmIMaster() && !IsUnderPostmaster)
+ {
+ char segmaxconns[32];
+ elog(LOG, "Update segment max_connections to %d", SegMaxBackends);
+ snprintf(segmaxconns, sizeof(segmaxconns), "%d", SegMaxBackends);
+ SetConfigOption("max_connections",
+ segmaxconns,
+ PGC_POSTMASTER,
+ PGC_S_OVERRIDE);
+ }
+
if (PostAuthDelay)
pg_usleep(PostAuthDelay * 1000000L);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ff98f360/src/backend/utils/init/globals.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c
index bd069cf..a7202ae 100644
--- a/src/backend/utils/init/globals.c
+++ b/src/backend/utils/init/globals.c
@@ -116,6 +116,7 @@ int maintenance_work_mem = 65536;
/* Primary determinants of sizes of shared-memory structures: */
int NBuffers = 4096;
int MaxBackends = 200;
+int SegMaxBackends = 4800;
int gp_workfile_max_entries = 8192; /* Number of unique entries we can hold in the workfile directory */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ff98f360/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 845bfdc..ceb378c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4617,6 +4617,20 @@ static struct config_int ConfigureNamesInt[] =
200, 10, MAX_MAX_BACKENDS, NULL, NULL
},
+ /*
+ * When HAWQ has one master and one segment deployed together in one physical
+ * machine, we can not separately set different max connection count. Thus,
+ * we introduce this guc for segment setting only.
+ */
+ {
+ {"seg_max_connections", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
+ gettext_noop("Sets the maximum number of concurrent connections in a segment."),
+ NULL
+ },
+ &SegMaxBackends,
+ 4800, 240, MAX_MAX_BACKENDS, NULL, NULL
+ },
+
{
{"superuser_reserved_connections", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
gettext_noop("Sets the number of connection slots reserved for superusers."),
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ff98f360/src/include/miscadmin.h
----------------------------------------------------------------------
diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h
index 23be5f1..b52485c 100644
--- a/src/include/miscadmin.h
+++ b/src/include/miscadmin.h
@@ -194,6 +194,7 @@ extern PGDLLIMPORT char *DataDir;
extern PGDLLIMPORT int NBuffers;
extern int MaxBackends;
+extern int SegMaxBackends;
extern int MaxConnections;
extern int gp_workfile_max_entries;
extern int gp_mdver_max_entries;
[6/6] incubator-hawq git commit: Merge branch 'master' into HAWQ-369
Posted by od...@apache.org.
Merge branch 'master' into HAWQ-369
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/26d9c1a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/26d9c1a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/26d9c1a0
Branch: refs/heads/HAWQ-369
Commit: 26d9c1a0812c9f2c241a65ed4ef0104a25b1310e
Parents: d03cded 226a55c
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Thu Jan 28 19:19:47 2016 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Thu Jan 28 19:19:47 2016 -0800
----------------------------------------------------------------------
pxf/build.gradle | 48 +++++-
pxf/gradle.properties | 1 +
.../hawq/pxf/api/utilities/Utilities.java | 153 ++++++++++++++++++
.../hawq/pxf/api/utilities/UtilitiesTest.java | 117 ++++++++++++++
.../pxf/plugins/hdfs/StringPassResolver.java | 8 +-
.../hawq/pxf/plugins/hdfs/WritableResolver.java | 2 +-
.../plugins/hdfs/utilities/HdfsUtilities.java | 2 +-
.../plugins/hdfs/StringPassResolverTest.java | 105 ++----------
.../plugins/hive/HiveColumnarSerdeResolver.java | 3 +-
.../hawq/pxf/plugins/hive/HiveResolver.java | 3 +-
.../hawq/pxf/service/FragmenterFactory.java | 2 +-
.../org/apache/hawq/pxf/service/ReadBridge.java | 2 +-
.../apache/hawq/pxf/service/WriteBridge.java | 2 +-
.../pxf/service/rest/InvalidPathResource.java | 3 +-
.../hawq/pxf/service/rest/VersionResource.java | 26 ++-
.../hawq/pxf/service/rest/WritableResource.java | 10 +-
.../hawq/pxf/service/utilities/Utilities.java | 154 ------------------
.../pxf/service/BridgeInputBuilderTest.java | 159 +++++++++++++++++++
.../pxf/service/rest/VersionResourceTest.java | 7 +-
.../pxf/service/utilities/UtilitiesTest.java | 116 --------------
src/backend/postmaster/postmaster.c | 15 ++
src/backend/tcop/postgres.c | 15 +-
src/backend/utils/init/globals.c | 1 +
src/backend/utils/misc/guc.c | 14 ++
src/include/miscadmin.h | 1 +
25 files changed, 584 insertions(+), 385 deletions(-)
----------------------------------------------------------------------
[3/6] incubator-hawq git commit: HAWQ-253. Separate pxf-service from
pxf plugins
Posted by od...@apache.org.
HAWQ-253. Separate pxf-service from pxf plugins
1. Move Utilities class from pxf-service to pxf-api package.
2. Split StringPassResolverTest to two unit test files - one testing functionality of StringPassResolver.setFields
and the other testing the functionality of BridgeInputBuilder.makeInput.
3. Close stream in WritableResource using try-with-resources.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1df504f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1df504f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1df504f4
Branch: refs/heads/HAWQ-369
Commit: 1df504f4f27b4559ff32d64b80061a85835440fd
Parents: e22956c
Author: Noa Horn <nh...@pivotal.io>
Authored: Thu Jan 28 14:45:51 2016 -0800
Committer: Noa Horn <nh...@pivotal.io>
Committed: Thu Jan 28 14:45:51 2016 -0800
----------------------------------------------------------------------
pxf/build.gradle | 4 +-
.../hawq/pxf/api/utilities/Utilities.java | 153 ++++++++++++++++++
.../hawq/pxf/api/utilities/UtilitiesTest.java | 117 ++++++++++++++
.../pxf/plugins/hdfs/StringPassResolver.java | 8 +-
.../hawq/pxf/plugins/hdfs/WritableResolver.java | 2 +-
.../plugins/hdfs/utilities/HdfsUtilities.java | 2 +-
.../plugins/hdfs/StringPassResolverTest.java | 105 ++----------
.../plugins/hive/HiveColumnarSerdeResolver.java | 3 +-
.../hawq/pxf/plugins/hive/HiveResolver.java | 3 +-
.../hawq/pxf/service/FragmenterFactory.java | 2 +-
.../org/apache/hawq/pxf/service/ReadBridge.java | 2 +-
.../apache/hawq/pxf/service/WriteBridge.java | 2 +-
.../pxf/service/rest/InvalidPathResource.java | 3 +-
.../hawq/pxf/service/rest/WritableResource.java | 10 +-
.../hawq/pxf/service/utilities/Utilities.java | 154 ------------------
.../pxf/service/BridgeInputBuilderTest.java | 159 +++++++++++++++++++
.../pxf/service/utilities/UtilitiesTest.java | 116 --------------
17 files changed, 466 insertions(+), 379 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 0d260d4..cc913ec 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -237,9 +237,11 @@ project('pxf-service') {
project('pxf-hdfs') {
dependencies {
- compile(project(':pxf-service'))
+ compile(project(':pxf-api'))
compile 'org.apache.avro:avro-mapred:1.7.4'
compile "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
+ compile "org.apache.hadoop:hadoop-common:$hadoopVersion"
+ compile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
}
ospackage {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
new file mode 100644
index 0000000..314583c
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -0,0 +1,153 @@
+package org.apache.hawq.pxf.api.utilities;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Utilities class exposes helper method for PXF classes
+ */
+public class Utilities {
+ private static final Log LOG = LogFactory.getLog(Utilities.class);
+
+ /**
+ * Creates an object using the class name. The class name has to be a class
+ * located in the webapp's CLASSPATH.
+ *
+ * @param confClass the class of the metaData used to initialize the
+ * instance
+ * @param className a class name to be initialized.
+ * @param metaData input data used to initialize the class
+ * @return Initialized instance of given className
+ * @throws Exception throws exception if classname was not found in
+ * classpath, didn't have expected constructor or failed to be
+ * instantiated
+ */
+ public static Object createAnyInstance(Class<?> confClass,
+ String className, InputData metaData)
+ throws Exception {
+
+ Class<?> cls = null;
+ try {
+ cls = Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ // in case the class name uses the old "com.pivotal.pxf" package
+ // name, recommend using the new package "org.apache.hawq.pxf".
+ if (className.startsWith("com.pivotal.pxf")) {
+ throw new Exception(
+ "Class "
+ + className
+ + " does not appear in classpath. "
+ + "Plugins provided by PXF must start with \"org.apache.hawq.pxf\"",
+ e.getCause());
+ } else {
+ throw e;
+ }
+ }
+
+ Constructor<?> con = cls.getConstructor(confClass);
+
+ return instantiate(con, metaData);
+ }
+
+ /**
+ * Creates an object using the class name with its default constructor. The
+ * class name has to be a class located in the webapp's CLASSPATH.
+ *
+ * @param className a class name to be initialized
+ * @return initialized instance of given className
+ * @throws Exception throws exception if classname was not found in
+ * classpath, didn't have expected constructor or failed to be
+ * instantiated
+ */
+ public static Object createAnyInstance(String className) throws Exception {
+ Class<?> cls = Class.forName(className);
+ Constructor<?> con = cls.getConstructor();
+ return instantiate(con);
+ }
+
+ private static Object instantiate(Constructor<?> con, Object... args)
+ throws Exception {
+ try {
+ return con.newInstance(args);
+ } catch (InvocationTargetException e) {
+ /*
+ * We are creating resolvers, accessors, fragmenters, etc. using the
+ * reflection framework. If for example, a resolver, during its
+ * instantiation - in the c'tor, will throw an exception, the
+ * Resolver's exception will reach the Reflection layer and there it
+ * will be wrapped inside an InvocationTargetException. Here we are
+ * above the Reflection layer and we need to unwrap the Resolver's
+ * initial exception and throw it instead of the wrapper
+ * InvocationTargetException so that our initial Exception text will
+ * be displayed in psql instead of the message:
+ * "Internal Server Error"
+ */
+ throw (e.getCause() != null) ? new Exception(e.getCause()) : e;
+ }
+ }
+
+ /**
+ * Transforms a byte array into a string of octal codes in the form
+ * \\xyz\\xyz
+ *
+ * We double escape each char because it is required in postgres bytea for
+ * some bytes. In the minimum all non-printables, backslash, null and single
+ * quote. Easier to just escape everything see
+ * http://www.postgresql.org/docs/9.0/static/datatype-binary.html
+ *
+ * Octal codes must be padded to 3 characters (001, 012)
+ *
+ * @param bytes bytes to escape
+ * @param sb octal codes of given bytes
+ */
+ public static void byteArrayToOctalString(byte[] bytes, StringBuilder sb) {
+ if ((bytes == null) || (sb == null)) {
+ return;
+ }
+
+ sb.ensureCapacity(sb.length()
+ + (bytes.length * 5 /* characters per byte */));
+ for (int b : bytes) {
+ sb.append(String.format("\\\\%03o", b & 0xff));
+ }
+ }
+
+ /**
+ * Replaces any non-alpha-numeric character with a '.'. This measure is used
+ * to prevent cross-site scripting (XSS) when an input string might include
+ * code or script. By removing all special characters and returning a
+ * censured string to the user this threat is avoided.
+ *
+ * @param input string to be masked
+ * @return masked string
+ */
+ public static String maskNonPrintables(String input) {
+ if (StringUtils.isEmpty(input)) {
+ return input;
+ }
+ return input.replaceAll("[^a-zA-Z0-9_:/-]", ".");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
new file mode 100644
index 0000000..355ea42
--- /dev/null
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
@@ -0,0 +1,117 @@
+package org.apache.hawq.pxf.api.utilities;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Utilities;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({Class.class})
+public class UtilitiesTest {
+ @Test
+ public void byteArrayToOctalStringNull() throws Exception {
+ StringBuilder sb = null;
+ byte[] bytes = "nofink".getBytes();
+
+ Utilities.byteArrayToOctalString(bytes, sb);
+
+ assertNull(sb);
+
+ sb = new StringBuilder();
+ bytes = null;
+
+ Utilities.byteArrayToOctalString(bytes, sb);
+
+ assertEquals(0, sb.length());
+ }
+
+ @Test
+ public void byteArrayToOctalString() throws Exception {
+ String orig = "Have Narisha";
+ String octal = "Rash Rash Rash!";
+ String expected = orig + "\\\\122\\\\141\\\\163\\\\150\\\\040"
+ + "\\\\122\\\\141\\\\163\\\\150\\\\040"
+ + "\\\\122\\\\141\\\\163\\\\150\\\\041";
+ StringBuilder sb = new StringBuilder();
+ sb.append(orig);
+
+ Utilities.byteArrayToOctalString(octal.getBytes(), sb);
+
+ assertEquals(orig.length() + (octal.length() * 5), sb.length());
+ assertEquals(expected, sb.toString());
+ }
+
+ @Test
+ public void createAnyInstanceOldPackageName() throws Exception {
+
+ InputData metaData = mock(InputData.class);
+ String className = "com.pivotal.pxf.Lucy";
+ ClassNotFoundException exception = new ClassNotFoundException(className);
+ PowerMockito.mockStatic(Class.class);
+ when(Class.forName(className)).thenThrow(exception);
+
+ try {
+ Utilities.createAnyInstance(InputData.class,
+ className, metaData);
+ fail("creating an instance should fail because the class doesn't exist in classpath");
+ } catch (Exception e) {
+ assertEquals(e.getClass(), Exception.class);
+ assertEquals(
+ e.getMessage(),
+ "Class " + className + " does not appear in classpath. "
+ + "Plugins provided by PXF must start with \"org.apache.hawq.pxf\"");
+ }
+ }
+
+ @Test
+ public void maskNonPrintable() throws Exception {
+ String input = "";
+ String result = Utilities.maskNonPrintables(input);
+ assertEquals("", result);
+
+ input = null;
+ result = Utilities.maskNonPrintables(input);
+ assertEquals(null, result);
+
+ input = "Lucy in the sky";
+ result = Utilities.maskNonPrintables(input);
+ assertEquals("Lucy.in.the.sky", result);
+
+ input = "with <$$$@#$!000diamonds!!?!$#&%/>";
+ result = Utilities.maskNonPrintables(input);
+ assertEquals("with.........000diamonds......../.", result);
+
+ input = "http://www.beatles.com/info?query=whoisthebest";
+ result = Utilities.maskNonPrintables(input);
+ assertEquals("http://www.beatles.com/info.query.whoisthebest", result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolver.java b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolver.java
index a48e263..dc7d1d5 100644
--- a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolver.java
+++ b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolver.java
@@ -8,9 +8,9 @@ package org.apache.hawq.pxf.plugins.hdfs;
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -35,8 +35,8 @@ import static org.apache.hawq.pxf.api.io.DataType.VARCHAR;
/**
* StringPassResolver handles "deserialization" and serialization of
- * String records. StringPassResolver implements IReadResolver and
- * IWriteResolver interfaces. Returns strings as-is.
+ * String records. StringPassResolver implements {@link ReadResolver} and
+ * {@link WriteResolver} interfaces. Returns strings as-is.
*/
public class StringPassResolver extends Plugin implements ReadResolver, WriteResolver {
// for write
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/WritableResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/WritableResolver.java b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/WritableResolver.java
index 89870c6..c758231 100644
--- a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/WritableResolver.java
+++ b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/WritableResolver.java
@@ -24,9 +24,9 @@ import org.apache.hawq.pxf.api.*;
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.api.utilities.Plugin;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.hawq.pxf.plugins.hdfs.utilities.RecordkeyAdapter;
import org.apache.hawq.pxf.plugins.hdfs.utilities.DataSchemaException;
-import org.apache.hawq.pxf.service.utilities.Utilities;
import static org.apache.hawq.pxf.plugins.hdfs.utilities.DataSchemaException.MessageFmt.*;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.java b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.java
index aa8c4b4..59551a9 100644
--- a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.java
+++ b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.java
@@ -20,10 +20,10 @@ package org.apache.hawq.pxf.plugins.hdfs.utilities;
*/
-import org.apache.hawq.pxf.service.utilities.Utilities;
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hawq.pxf.api.OneField;
import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.generic.GenericDatumReader;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-hdfs/src/test/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolverTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/test/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolverTest.java b/pxf/pxf-hdfs/src/test/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolverTest.java
index d03cec8..2e76962 100644
--- a/pxf/pxf-hdfs/src/test/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolverTest.java
+++ b/pxf/pxf-hdfs/src/test/java/org/apache/hawq/pxf/plugins/hdfs/StringPassResolverTest.java
@@ -24,36 +24,22 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
-import org.apache.commons.logging.LogFactory;
import org.apache.hawq.pxf.api.OneField;
import org.apache.hawq.pxf.api.OneRow;
-import org.apache.hawq.pxf.api.OutputFormat;
-import org.apache.hawq.pxf.service.BridgeInputBuilder;
-import org.apache.hawq.pxf.service.io.Text;
-import org.apache.hawq.pxf.service.utilities.ProtocolData;
+import org.apache.hawq.pxf.api.io.DataType;
+import org.apache.hawq.pxf.api.utilities.InputData;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({
- Text.class,
- BridgeInputBuilder.class,
- ProtocolData.class,
- LogFactory.class })
+
public class StringPassResolverTest {
- ProtocolData mockProtocolData;
+ InputData mockInputData;
@Test
/*
- * Test the setFields method: small \n terminated input
+ * Test the setFields method: small input
*/
public void testSetFields() throws Exception {
StringPassResolver resolver = buildResolver();
@@ -68,84 +54,31 @@ public class StringPassResolverTest {
(int) 'o',
(int) '\n' };
- DataInputStream inputStream = new DataInputStream(
- new ByteArrayInputStream(data));
- BridgeInputBuilder inputBuilder = new BridgeInputBuilder(
- mockProtocolData);
- List<OneField> record = inputBuilder.makeInput(inputStream);
+ List<OneField> record = Collections.singletonList(new OneField(DataType.BYTEA.getOID(),
+ Arrays.copyOfRange(data, 0, 5)));
OneRow oneRow = resolver.setFields(record);
verifyOneRow(oneRow, Arrays.copyOfRange(data, 0, 5));
- record = inputBuilder.makeInput(inputStream);
+ record = Collections.singletonList(new OneField(DataType.BYTEA.getOID(),
+ Arrays.copyOfRange(data, 5, 8)));
+
oneRow = resolver.setFields(record);
verifyOneRow(oneRow, Arrays.copyOfRange(data, 5, 8));
}
@Test
/*
- * Test the setFields method: input > buffer size, \n terminated
- */
- public void testSetFieldsBigArray() throws Exception {
-
- StringPassResolver resolver = buildResolver();
-
- byte[] bigArray = new byte[2000];
- for (int i = 0; i < 1999; ++i) {
- bigArray[i] = (byte) (i % 10 + 30);
- }
- bigArray[1999] = (byte) '\n';
-
- DataInputStream inputStream = new DataInputStream(
- new ByteArrayInputStream(bigArray));
- BridgeInputBuilder inputBuilder = new BridgeInputBuilder(
- mockProtocolData);
- List<OneField> record = inputBuilder.makeInput(inputStream);
-
- OneRow oneRow = resolver.setFields(record);
-
- verifyOneRow(oneRow, bigArray);
- }
-
- @Test
- /*
- * Test the setFields method: input > buffer size, no \n
+ * Test the setFields method: empty byte array
*/
- public void testSetFieldsBigArrayNoNewLine() throws Exception {
-
- StringPassResolver resolver = buildResolver();
-
- byte[] bigArray = new byte[2000];
- for (int i = 0; i < 2000; ++i) {
- bigArray[i] = (byte) (i % 10 + 60);
- }
-
- DataInputStream inputStream = new DataInputStream(
- new ByteArrayInputStream(bigArray));
- BridgeInputBuilder inputBuilder = new BridgeInputBuilder(
- mockProtocolData);
- List<OneField> record = inputBuilder.makeInput(inputStream);
-
- OneRow oneRow = resolver.setFields(record);
-
- verifyOneRow(oneRow, bigArray);
- }
-
- @Test
- /*
- * Test the setFields method: empty stream (returns -1)
- */
- public void testSetFieldsEmptyStream() throws Exception {
+ public void testSetFieldsEmptyByteArray() throws Exception {
StringPassResolver resolver = buildResolver();
byte[] empty = new byte[0];
- DataInputStream inputStream = new DataInputStream(
- new ByteArrayInputStream(empty));
- BridgeInputBuilder inputBuilder = new BridgeInputBuilder(
- mockProtocolData);
- List<OneField> record = inputBuilder.makeInput(inputStream);
+ List<OneField> record = Collections.singletonList(new OneField(DataType.BYTEA.getOID(),
+ empty));
OneRow oneRow = resolver.setFields(record);
@@ -156,12 +89,8 @@ public class StringPassResolverTest {
* helpers functions
*/
private StringPassResolver buildResolver() throws Exception {
-
- mockProtocolData = mock(ProtocolData.class);
- PowerMockito.when(mockProtocolData.outputFormat()).thenReturn(
- OutputFormat.TEXT);
-
- return new StringPassResolver(mockProtocolData);
+ mockInputData = mock(InputData.class);
+ return new StringPassResolver(mockInputData);
}
private void verifyOneRow(OneRow oneRow, byte[] expected) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
index 0120d7b..d298bac 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
@@ -27,8 +27,7 @@ import org.apache.hawq.pxf.api.UnsupportedTypeException;
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
import org.apache.hawq.pxf.api.utilities.InputData;
-import org.apache.hawq.pxf.service.utilities.Utilities;
-
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
index 59245d0..2562d3d 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
@@ -23,9 +23,8 @@ import org.apache.hawq.pxf.api.*;
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.api.utilities.Plugin;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities;
-import org.apache.hawq.pxf.service.utilities.Utilities;
-
import org.apache.commons.lang.CharUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/FragmenterFactory.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/FragmenterFactory.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/FragmenterFactory.java
index 0e15093..c516d69 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/FragmenterFactory.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/FragmenterFactory.java
@@ -22,7 +22,7 @@ package org.apache.hawq.pxf.service;
import org.apache.hawq.pxf.api.Fragmenter;
import org.apache.hawq.pxf.api.utilities.InputData;
-import org.apache.hawq.pxf.service.utilities.Utilities;
+import org.apache.hawq.pxf.api.utilities.Utilities;
/**
* Factory class for creation of {@link Fragmenter} objects. The actual {@link Fragmenter} object is "hidden" behind
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ReadBridge.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ReadBridge.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ReadBridge.java
index 0f3c968..01a95ab 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ReadBridge.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ReadBridge.java
@@ -25,9 +25,9 @@ import org.apache.hawq.pxf.api.ReadAccessor;
import org.apache.hawq.pxf.api.ReadResolver;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.api.utilities.Plugin;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.hawq.pxf.service.io.Writable;
import org.apache.hawq.pxf.service.utilities.ProtocolData;
-import org.apache.hawq.pxf.service.utilities.Utilities;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/WriteBridge.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/WriteBridge.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/WriteBridge.java
index 80552af..c3ee731 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/WriteBridge.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/WriteBridge.java
@@ -23,9 +23,9 @@ package org.apache.hawq.pxf.service;
import org.apache.hawq.pxf.api.*;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.api.utilities.Plugin;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.hawq.pxf.service.io.Writable;
import org.apache.hawq.pxf.service.utilities.ProtocolData;
-import org.apache.hawq.pxf.service.utilities.Utilities;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
index 8e987f3..5a9f0d1 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
@@ -22,9 +22,10 @@ package org.apache.hawq.pxf.service.rest;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hawq.pxf.service.utilities.Utilities;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import com.google.common.collect.ImmutableSet;
+
import java.util.Arrays;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java
index 70bec2a..d1dea5e 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java
@@ -37,11 +37,11 @@ import javax.ws.rs.core.Response;
import org.apache.catalina.connector.ClientAbortException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hawq.pxf.api.utilities.Utilities;
import org.apache.hawq.pxf.service.Bridge;
import org.apache.hawq.pxf.service.WriteBridge;
import org.apache.hawq.pxf.service.utilities.ProtocolData;
import org.apache.hawq.pxf.service.utilities.SecuredHDFS;
-import org.apache.hawq.pxf.service.utilities.Utilities;
/*
* Running this resource manually:
@@ -150,11 +150,11 @@ public class WritableResource extends RestResource{
// Open the output file
bridge.beginIteration();
- DataInputStream dataStream = new DataInputStream(inputStream);
-
long totalWritten = 0;
- try {
+ // dataStream will close automatically in the end of the try.
+ // inputStream is closed by dataStream.close().
+ try (DataInputStream dataStream = new DataInputStream(inputStream)) {
while (bridge.setNext(dataStream)) {
++totalWritten;
}
@@ -163,8 +163,6 @@ public class WritableResource extends RestResource{
} catch (Exception ex) {
LOG.debug("totalWritten so far " + totalWritten + " to " + path);
throw ex;
- } finally {
- inputStream.close();
}
String censuredPath = Utilities.maskNonPrintables(path);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/Utilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/Utilities.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/Utilities.java
deleted file mode 100644
index 8467734..0000000
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/Utilities.java
+++ /dev/null
@@ -1,154 +0,0 @@
-package org.apache.hawq.pxf.service.utilities;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hawq.pxf.api.utilities.InputData;
-
-/**
- * Utilities class exposes helper method for PXF classes
- */
-public class Utilities {
- private static final Log LOG = LogFactory.getLog(Utilities.class);
-
- /**
- * Creates an object using the class name. The class name has to be a class
- * located in the webapp's CLASSPATH.
- *
- * @param confClass the class of the metaData used to initialize the
- * instance
- * @param className a class name to be initialized.
- * @param metaData input data used to initialize the class
- * @return Initialized instance of given className
- * @throws Exception throws exception if classname was not found in
- * classpath, didn't have expected constructor or failed to be
- * instantiated
- */
- public static Object createAnyInstance(Class<?> confClass,
- String className, InputData metaData)
- throws Exception {
-
- Class<?> cls = null;
- try {
- cls = Class.forName(className);
- } catch (ClassNotFoundException e) {
- // in case the class name uses the old "com.pivotal.pxf" package
- // name, recommend using the new package "org.apache.hawq.pxf".
- if (className.startsWith("com.pivotal.pxf")) {
- throw new Exception(
- "Class "
- + className
- + " does not appear in classpath. "
- + "Plugins provided by PXF must start with \"org.apache.hawq.pxf\"",
- e.getCause());
- } else {
- throw e;
- }
- }
-
- Constructor<?> con = cls.getConstructor(confClass);
-
- return instantiate(con, metaData);
- }
-
- /**
- * Creates an object using the class name with its default constructor. The
- * class name has to be a class located in the webapp's CLASSPATH.
- *
- * @param className a class name to be initialized
- * @return initialized instance of given className
- * @throws Exception throws exception if classname was not found in
- * classpath, didn't have expected constructor or failed to be
- * instantiated
- */
- public static Object createAnyInstance(String className) throws Exception {
- Class<?> cls = Class.forName(className);
- Constructor<?> con = cls.getConstructor();
- return instantiate(con);
- }
-
- private static Object instantiate(Constructor<?> con, Object... args)
- throws Exception {
- try {
- return con.newInstance(args);
- } catch (InvocationTargetException e) {
- /*
- * We are creating resolvers, accessors, fragmenters, etc. using the
- * reflection framework. If for example, a resolver, during its
- * instantiation - in the c'tor, will throw an exception, the
- * Resolver's exception will reach the Reflection layer and there it
- * will be wrapped inside an InvocationTargetException. Here we are
- * above the Reflection layer and we need to unwrap the Resolver's
- * initial exception and throw it instead of the wrapper
- * InvocationTargetException so that our initial Exception text will
- * be displayed in psql instead of the message:
- * "Internal Server Error"
- */
- throw (e.getCause() != null) ? new Exception(e.getCause()) : e;
- }
- }
-
- /**
- * Transforms a byte array into a string of octal codes in the form
- * \\xyz\\xyz
- *
- * We double escape each char because it is required in postgres bytea for
- * some bytes. In the minimum all non-printables, backslash, null and single
- * quote. Easier to just escape everything see
- * http://www.postgresql.org/docs/9.0/static/datatype-binary.html
- *
- * Octal codes must be padded to 3 characters (001, 012)
- *
- * @param bytes bytes to escape
- * @param sb octal codes of given bytes
- */
- public static void byteArrayToOctalString(byte[] bytes, StringBuilder sb) {
- if ((bytes == null) || (sb == null)) {
- return;
- }
-
- sb.ensureCapacity(sb.length()
- + (bytes.length * 5 /* characters per byte */));
- for (int b : bytes) {
- sb.append(String.format("\\\\%03o", b & 0xff));
- }
- }
-
- /**
- * Replaces any non-alpha-numeric character with a '.'. This measure is used
- * to prevent cross-site scripting (XSS) when an input string might include
- * code or script. By removing all special characters and returning a
- * censured string to the user this threat is avoided.
- *
- * @param input string to be masked
- * @return masked string
- */
- public static String maskNonPrintables(String input) {
- if (StringUtils.isEmpty(input)) {
- return input;
- }
- return input.replaceAll("[^a-zA-Z0-9_:/-]", ".");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeInputBuilderTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeInputBuilderTest.java b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeInputBuilderTest.java
new file mode 100644
index 0000000..2668ef3
--- /dev/null
+++ b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeInputBuilderTest.java
@@ -0,0 +1,159 @@
+package org.apache.hawq.pxf.service;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hawq.pxf.api.OneField;
+import org.apache.hawq.pxf.api.OutputFormat;
+import org.apache.hawq.pxf.api.io.DataType;
+import org.apache.hawq.pxf.service.utilities.ProtocolData;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+public class BridgeInputBuilderTest {
+ ProtocolData mockProtocolData;
+ BridgeInputBuilder inputBuilder;
+ DataInputStream inputStream;
+
+ @Test
+ /*
+ * Test makeInput method: small \n terminated input
+ */
+ public void makeInput() throws Exception {
+
+ byte[] data = new byte[] {
+ (int) 'a',
+ (int) 'b',
+ (int) 'c',
+ (int) 'd',
+ (int) '\n',
+ (int) 'n',
+ (int) 'o',
+ (int) '\n' };
+
+ prepareInput(data);
+
+ List<OneField> record = inputBuilder.makeInput(inputStream);
+
+ verifyRecord(record, Arrays.copyOfRange(data, 0, 5));
+
+ record = inputBuilder.makeInput(inputStream);
+ verifyRecord(record, Arrays.copyOfRange(data, 5, 8));
+ }
+
+ @Test
+ /*
+ * Test the makeInput method: input > buffer size, \n terminated
+ */
+ public void makeInputBigArray() throws Exception {
+
+ byte[] bigArray = new byte[2000];
+ for (int i = 0; i < 1999; ++i) {
+ bigArray[i] = (byte) (i % 10 + 30);
+ }
+ bigArray[1999] = (byte) '\n';
+
+ prepareInput(bigArray);
+
+ List<OneField> record = inputBuilder.makeInput(inputStream);
+
+ verifyRecord(record, bigArray);
+ }
+
+ @Test
+ /*
+ * Test the makeInput method: input > buffer size, no \n
+ */
+ public void makeInputBigArrayNoNewLine() throws Exception {
+
+ byte[] bigArray = new byte[2000];
+ for (int i = 0; i < 2000; ++i) {
+ bigArray[i] = (byte) (i % 10 + 60);
+ }
+
+ prepareInput(bigArray);
+
+ List<OneField> record = inputBuilder.makeInput(inputStream);
+
+ verifyRecord(record, bigArray);
+ }
+
+ @Test
+ /*
+ * Test the makeInput method: empty stream (returns -1)
+ */
+ public void makeInputEmptyStream() throws Exception {
+
+ byte[] empty = new byte[0];
+
+ prepareInput(empty);
+
+ List<OneField> record = inputBuilder.makeInput(inputStream);
+
+ verifyRecord(record, empty);
+ }
+
+ /*
+ * helpers functions
+ */
+
+ @After
+ public void cleanUp() throws IOException {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
+
+ private void prepareInput(byte[] data) throws Exception {
+ mockProtocolData = mock(ProtocolData.class);
+ PowerMockito.when(mockProtocolData.outputFormat()).thenReturn(
+ OutputFormat.TEXT);
+ inputBuilder = new BridgeInputBuilder(
+ mockProtocolData);
+ inputStream = new DataInputStream(
+ new ByteArrayInputStream(data));
+ }
+
+ private void verifyRecord(List<OneField> record, byte[] expected) {
+ assertEquals(record.size(), 1);
+
+ OneField field = record.get(0);
+ assertEquals(field.type, DataType.BYTEA.getOID());
+
+ byte[] bytes = (byte[]) field.val;
+ byte[] result = Arrays.copyOfRange(bytes, 0, bytes.length);
+ assertEquals(result.length, expected.length);
+ assertTrue(Arrays.equals(result, expected));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1df504f4/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/UtilitiesTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/UtilitiesTest.java b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/UtilitiesTest.java
deleted file mode 100644
index 0afb4e2..0000000
--- a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/UtilitiesTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-package org.apache.hawq.pxf.service.utilities;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hawq.pxf.api.utilities.InputData;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({Class.class})
-public class UtilitiesTest {
- @Test
- public void byteArrayToOctalStringNull() throws Exception {
- StringBuilder sb = null;
- byte[] bytes = "nofink".getBytes();
-
- Utilities.byteArrayToOctalString(bytes, sb);
-
- assertNull(sb);
-
- sb = new StringBuilder();
- bytes = null;
-
- Utilities.byteArrayToOctalString(bytes, sb);
-
- assertEquals(0, sb.length());
- }
-
- @Test
- public void byteArrayToOctalString() throws Exception {
- String orig = "Have Narisha";
- String octal = "Rash Rash Rash!";
- String expected = orig + "\\\\122\\\\141\\\\163\\\\150\\\\040"
- + "\\\\122\\\\141\\\\163\\\\150\\\\040"
- + "\\\\122\\\\141\\\\163\\\\150\\\\041";
- StringBuilder sb = new StringBuilder();
- sb.append(orig);
-
- Utilities.byteArrayToOctalString(octal.getBytes(), sb);
-
- assertEquals(orig.length() + (octal.length() * 5), sb.length());
- assertEquals(expected, sb.toString());
- }
-
- @Test
- public void createAnyInstanceOldPackageName() throws Exception {
-
- InputData metaData = mock(InputData.class);
- String className = "com.pivotal.pxf.Lucy";
- ClassNotFoundException exception = new ClassNotFoundException(className);
- PowerMockito.mockStatic(Class.class);
- when(Class.forName(className)).thenThrow(exception);
-
- try {
- Utilities.createAnyInstance(InputData.class,
- className, metaData);
- fail("creating an instance should fail because the class doesn't exist in classpath");
- } catch (Exception e) {
- assertEquals(e.getClass(), Exception.class);
- assertEquals(
- e.getMessage(),
- "Class " + className + " does not appear in classpath. "
- + "Plugins provided by PXF must start with \"org.apache.hawq.pxf\"");
- }
- }
-
- @Test
- public void maskNonPrintable() throws Exception {
- String input = "";
- String result = Utilities.maskNonPrintables(input);
- assertEquals("", result);
-
- input = null;
- result = Utilities.maskNonPrintables(input);
- assertEquals(null, result);
-
- input = "Lucy in the sky";
- result = Utilities.maskNonPrintables(input);
- assertEquals("Lucy.in.the.sky", result);
-
- input = "with <$$$@#$!000diamonds!!?!$#&%/>";
- result = Utilities.maskNonPrintables(input);
- assertEquals("with.........000diamonds......../.", result);
-
- input = "http://www.beatles.com/info?query=whoisthebest";
- result = Utilities.maskNonPrintables(input);
- assertEquals("http://www.beatles.com/info.query.whoisthebest", result);
- }
-}
[5/6] incubator-hawq git commit: HAWQ-340. Make getVersion API return
JSON format.
Posted by od...@apache.org.
HAWQ-340. Make getVersion API return JSON format.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/226a55c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/226a55c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/226a55c3
Branch: refs/heads/HAWQ-369
Commit: 226a55c3855a8273b3b850e2b98ea8d2273083df
Parents: ff98f36 b1d2e90
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Thu Jan 28 18:06:17 2016 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Thu Jan 28 18:06:17 2016 -0800
----------------------------------------------------------------------
pxf/build.gradle | 44 +++++++++++++++++++-
pxf/gradle.properties | 1 +
.../hawq/pxf/service/rest/VersionResource.java | 26 ++++++++++--
.../pxf/service/rest/VersionResourceTest.java | 7 +++-
4 files changed, 73 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/226a55c3/pxf/build.gradle
----------------------------------------------------------------------