You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by sh...@apache.org on 2017/07/12 18:08:29 UTC
incubator-hawq git commit: HAWQ-1490. Added Demo PXF classes for
standalone testing
Repository: incubator-hawq
Updated Branches:
refs/heads/master 80e8322c2 -> 2222f4b4a
HAWQ-1490. Added Demo PXF classes for standalone testing
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2222f4b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2222f4b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2222f4b4
Branch: refs/heads/master
Commit: 2222f4b4afa2fa8f3f862fb22aabbacc77e990f5
Parents: 80e8322
Author: John Gaskin <jo...@gmail.com>
Authored: Wed Jul 12 11:08:16 2017 -0700
Committer: shivzone <sh...@gmail.com>
Committed: Wed Jul 12 11:08:16 2017 -0700
----------------------------------------------------------------------
.../hawq/pxf/api/examples/DemoAccessor.java | 98 ++++++++++++++++++++
.../hawq/pxf/api/examples/DemoFragmenter.java | 59 ++++++++++++
.../hawq/pxf/api/examples/DemoResolver.java | 67 +++++++++++++
.../hawq/pxf/api/examples/DemoTextResolver.java | 63 +++++++++++++
.../apache/hawq/pxf/api/DemoAccessorTest.java | 79 ++++++++++++++++
.../apache/hawq/pxf/api/DemoResolverTest.java | 70 ++++++++++++++
6 files changed, 436 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
new file mode 100644
index 0000000..bec3146
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
@@ -0,0 +1,98 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.OneRow;
+import org.apache.hawq.pxf.api.ReadAccessor;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Internal interface that would defined the access to a file on HDFS, but in
+ * this case contains the data required.
+ *
+ * Demo implementation
+ */
+public class DemoAccessor extends Plugin implements ReadAccessor {
+
+ private static final Log LOG = LogFactory.getLog(DemoAccessor.class);
+ private int rowNumber;
+ private int fragmentNumber;
+ private static int NUM_ROWS = 2;
+
+ /**
+ * Constructs a DemoAccessor
+ *
+ * @param metaData the InputData
+ */
+ public DemoAccessor(InputData metaData) {
+ super(metaData);
+ }
+ @Override
+ public boolean openForRead() throws Exception {
+ /* no-op, because this plugin doesn't read a file. */
+ return true;
+ }
+
+ /**
+ * Read the next record
+ * The record contains as many fields as defined by the DDL schema.
+ *
+ * @return one row which corresponds to one record
+ */
+ @Override
+ public OneRow readNextObject() throws Exception {
+ /* return next row , <key=fragmentNo.rowNo, val=rowNo,text,fragmentNo>*/
+ /* check for EOF */
+ if (fragmentNumber > 0)
+ return null; /* signal EOF, close will be called */
+ int fragment = inputData.getDataFragment();
+ String fragmentMetadata = new String(inputData.getFragmentMetadata());
+ int colCount = inputData.getColumns();
+
+ /* generate row with (colCount) columns */
+ StringBuilder colValue = new StringBuilder(fragmentMetadata + " row" + (rowNumber+1));
+ for(int colIndex=1; colIndex<colCount; colIndex++) {
+ colValue.append(",").append("value" + colIndex);
+ }
+ OneRow row = new OneRow(fragment + "." + rowNumber, colValue.toString());
+
+ /* advance */
+ rowNumber += 1;
+ if (rowNumber == NUM_ROWS) {
+ rowNumber = 0;
+ fragmentNumber += 1;
+ }
+
+ /* return data */
+ return row;
+ }
+
+ /**
+ * close the reader. no action here
+ *
+ */
+ @Override
+ public void closeForRead() throws Exception {
+ /* Demo close doesn't do anything */
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
new file mode 100644
index 0000000..685fcf2
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
@@ -0,0 +1,59 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.Fragmenter;
+import org.apache.hawq.pxf.api.Fragment;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import java.util.List;
+
+/**
+ * Class that defines the splitting of a data resource into fragments that can be processed in parallel
+ * getFragments() returns the fragments information of a given path (source name and location of each fragment).
+ *
+ * Demo implementation
+ */
+
+public class DemoFragmenter extends Fragmenter{
+ /**
+ * Constructs the DemoFragmenter
+ *
+ * @param metaData
+ */
+ public DemoFragmenter(InputData metaData) {
+ super(metaData);
+ }
+
+ /**
+ * Provide metadata for each data partition of the given datasource
+ *
+ * @return list of fragments
+ */
+ @Override
+ public List<Fragment> getFragments() throws Exception {
+ String localhostname = java.net.InetAddress.getLocalHost().getHostName();
+ String[] localHosts = new String[]{localhostname, localhostname};
+ fragments.add(new Fragment(inputData.getDataSource() + ".1", localHosts, "fragment1".getBytes()));
+ fragments.add(new Fragment(inputData.getDataSource() + ".2", localHosts, "fragment2".getBytes()));
+ fragments.add(new Fragment(inputData.getDataSource() + ".3", localHosts, "fragment3".getBytes()));
+ return fragments;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
new file mode 100644
index 0000000..8780a19
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
@@ -0,0 +1,67 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.OneField;
+import org.apache.hawq.pxf.api.OneRow;
+import org.apache.hawq.pxf.api.ReadResolver;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+import java.util.LinkedList;
+import java.util.List;
+import static org.apache.hawq.pxf.api.io.DataType.INTEGER;
+import static org.apache.hawq.pxf.api.io.DataType.VARCHAR;
+
+/**
+ * Class that defines the deserializtion of one record brought from the external input data.
+ *
+ * Demo implementation that returns record custom format
+ */
+public class DemoResolver extends Plugin implements ReadResolver {
+ /**
+ * Constructs the DemoResolver
+ *
+ * @param metaData
+ */
+ public DemoResolver(InputData metaData) {
+ super(metaData);
+ }
+
+ /**
+ * Read the next record
+ * The record contains as many fields as defined by the DDL schema.
+ *
+ * @param row one record
+ * @return list of fields or columns
+ */
+ @Override
+ public List<OneField> getFields(OneRow row) throws Exception {
+ List<OneField> output = new LinkedList<OneField>();
+ Object data = row.getData();
+
+ /* break up the row into fields */
+ String[] fields = ((String) data).split(",");
+ for(int colIndex=0; colIndex<fields.length; colIndex++) {
+ output.add(new OneField(VARCHAR.getOID(), fields[colIndex]));
+ }
+
+ return output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
new file mode 100644
index 0000000..772550e
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
@@ -0,0 +1,63 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.OneField;
+import org.apache.hawq.pxf.api.OneRow;
+import org.apache.hawq.pxf.api.ReadResolver;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.apache.hawq.pxf.api.io.DataType.INTEGER;
+import static org.apache.hawq.pxf.api.io.DataType.VARCHAR;
+
+/**
+ * Class that defines the deserializtion of one record brought from the external input data.
+ *
+ * Demo implementation of resolver that returns text format
+ */
+public class DemoTextResolver extends Plugin implements ReadResolver {
+ /**
+ * Constructs the DemoResolver
+ *
+ * @param metaData
+ */
+ public DemoTextResolver(InputData metaData) {
+ super(metaData);
+ }
+
+ /**
+ * Read the next record
+ * The record contains as many fields as defined by the DDL schema.
+ *
+ * @param row one record
+ * @return the first column contains the entire text data
+ */
+ @Override
+ public List<OneField> getFields(OneRow row) throws Exception {
+ List<OneField> output = new LinkedList<OneField>();
+ Object data = row.getData();
+ output.add(new OneField(VARCHAR.getOID(), data));
+ return output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java
new file mode 100755
index 0000000..36cbf21
--- /dev/null
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java
@@ -0,0 +1,79 @@
+package org.apache.hawq.pxf.api;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+
+
+import org.apache.hawq.pxf.api.examples.DemoAccessor;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({DemoAccessor.class}) // Enables mocking 'new' calls
+
+public class DemoAccessorTest {
+
+ @Mock InputData inputData;
+ DemoAccessor accessor;
+
+ @Before
+ public void setup() throws Exception {
+ accessor = new DemoAccessor(inputData);
+ }
+
+ @Test
+ public void testRowsWithSingleColumn() throws Exception {
+
+ when(inputData.getDataFragment()).thenReturn(0);
+ when(inputData.getFragmentMetadata()).thenReturn("fragment1".getBytes(), "fragment1".getBytes());
+ when(inputData.getColumns()).thenReturn(1);
+
+ int numRows = 2;
+ for (int i = 0; i < numRows; i++) {
+ OneRow row = accessor.readNextObject();
+ assertEquals(row.toString(), "OneRow:0." + i + "->fragment1 row" + (i+1));
+ }
+ assertNull(accessor.readNextObject());
+ }
+
+ @Test
+ public void testRowsWithMultipleColumns() throws Exception {
+
+ when(inputData.getDataFragment()).thenReturn(0);
+ when(inputData.getFragmentMetadata()).thenReturn("fragment1".getBytes(), "fragment1".getBytes());
+ when(inputData.getColumns()).thenReturn(3);
+
+ int numRows = 2;
+ for (int i = 0; i < numRows; i++) {
+ OneRow row = accessor.readNextObject();
+ assertEquals(row.toString(), "OneRow:0." + i + "->fragment1 row" + (i+1) + ",value1,value2");
+ }
+ assertNull(accessor.readNextObject());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java
new file mode 100755
index 0000000..dd22c87
--- /dev/null
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java
@@ -0,0 +1,70 @@
+package org.apache.hawq.pxf.api;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.hawq.pxf.api.examples.DemoAccessor;
+import org.apache.hawq.pxf.api.examples.DemoResolver;
+import org.apache.hawq.pxf.api.examples.DemoTextResolver;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({DemoAccessor.class}) // Enables mocking 'new' calls
+
+public class DemoResolverTest {
+
+ @Mock InputData inputData;
+ DemoResolver customResolver;
+ DemoTextResolver textResolver;
+ OneRow row;
+
+ @Before
+ public void setup() throws Exception {
+ customResolver = new DemoResolver(inputData);
+ textResolver = new DemoTextResolver(inputData);
+ row = new OneRow("0.0","value1,value2");
+ }
+
+ @Test
+ public void testCustomData() throws Exception {
+
+ List<OneField> output = customResolver.getFields(row);
+ assertEquals("value1", output.get(0).toString());
+ assertEquals("value2", output.get(1).toString());
+ }
+
+ @Test
+ public void testTextData() throws Exception {
+
+ List<OneField> output = textResolver.getFields(row);
+ assertEquals("value1,value2", output.get(0).toString());
+
+ }
+}