You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2015/04/28 01:12:18 UTC
[21/51] [partial] parquet-mr git commit: PARQUET-23: Rename to
org.apache.parquet.
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateEvaluator.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateEvaluator.java b/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateEvaluator.java
deleted file mode 100644
index 6ff296d..0000000
--- a/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateEvaluator.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2.recordlevel;
-
-import org.junit.Test;
-
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And;
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.Or;
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicateEvaluator.evaluate;
-
-public class TestIncrementallyUpdatedFilterPredicateEvaluator {
-
- public static class ShortCircuitException extends RuntimeException {
- public ShortCircuitException() {
- super("this was supposed to short circuit and never get here!");
- }
- }
-
- public static ValueInspector intIsNull() {
- return new ValueInspector() {
- @Override
- public void updateNull() {
- setResult(true);
- }
-
- @Override
- public void update(int value) {
- setResult(false);
- }
- };
- }
-
- public static ValueInspector intIsEven() {
- return new ValueInspector() {
- @Override
- public void updateNull() {
- setResult(false);
- }
-
- @Override
- public void update(int value) {
- setResult(value % 2 == 0);
- }
- };
- }
-
- public static ValueInspector doubleMoreThan10() {
- return new ValueInspector() {
- @Override
- public void updateNull() {
- setResult(false);
- }
-
- @Override
- public void update(double value) {
- setResult(value > 10.0);
- }
- };
- }
-
- @Test
- public void testValueInspector() {
- // known, and set to false criteria, null considered false
- ValueInspector v = intIsEven();
- v.update(11);
- assertFalse(evaluate(v));
- v.reset();
-
- // known and set to true criteria, null considered false
- v.update(12);
- assertTrue(evaluate(v));
- v.reset();
-
- // known and set to null, null considered false
- v.updateNull();
- assertFalse(evaluate(v));
- v.reset();
-
- // known, and set to false criteria, null considered true
- ValueInspector intIsNull = intIsNull();
- intIsNull.update(10);
- assertFalse(evaluate(intIsNull));
- intIsNull.reset();
-
- // known, and set to false criteria, null considered true
- intIsNull.updateNull();
- assertTrue(evaluate(intIsNull));
- intIsNull.reset();
-
- // unknown, null considered false
- v.reset();
- assertFalse(evaluate(v));
-
- // unknown, null considered true
- intIsNull.reset();
- assertTrue(evaluate(intIsNull));
- }
-
- private void doOrTest(ValueInspector v1, ValueInspector v2, int v1Value, int v2Value, boolean expected) {
- v1.update(v1Value);
- v2.update(v2Value);
- IncrementallyUpdatedFilterPredicate or = new Or(v1, v2);
- assertEquals(expected, evaluate(or));
- v1.reset();
- v2.reset();
- }
-
- private void doAndTest(ValueInspector v1, ValueInspector v2, int v1Value, int v2Value, boolean expected) {
- v1.update(v1Value);
- v2.update(v2Value);
- IncrementallyUpdatedFilterPredicate and = new And(v1, v2);
- assertEquals(expected, evaluate(and));
- v1.reset();
- v2.reset();
- }
-
-
- @Test
- public void testOr() {
- ValueInspector v1 = intIsEven();
- ValueInspector v2 = intIsEven();
-
- int F = 11;
- int T = 12;
-
- // F || F == F
- doOrTest(v1, v2, F, F, false);
- // F || T == T
- doOrTest(v1, v2, F, T, true);
- // T || F == T
- doOrTest(v1, v2, T, F, true);
- // T || T == T
- doOrTest(v1, v2, T, T, true);
-
- }
-
- @Test
- public void testAnd() {
- ValueInspector v1 = intIsEven();
- ValueInspector v2 = intIsEven();
-
- int F = 11;
- int T = 12;
-
- // F && F == F
- doAndTest(v1, v2, F, F, false);
- // F && T == F
- doAndTest(v1, v2, F, T, false);
- // T && F == F
- doAndTest(v1, v2, T, F, false);
- // T && T == T
- doAndTest(v1, v2, T, T, true);
-
- }
-
- @Test
- public void testShortCircuit() {
- ValueInspector neverCalled = new ValueInspector() {
- @Override
- public boolean accept(Visitor visitor) {
- throw new ShortCircuitException();
- }
- };
-
- try {
- evaluate(neverCalled);
- fail("this should throw");
- } catch (ShortCircuitException e) {
- //
- }
-
- // T || X should evaluate to true without inspecting X
- ValueInspector v = intIsEven();
- v.update(10);
- IncrementallyUpdatedFilterPredicate or = new Or(v, neverCalled);
- assertTrue(evaluate(or));
- v.reset();
-
- // F && X should evaluate to false without inspecting X
- v.update(11);
- IncrementallyUpdatedFilterPredicate and = new And(v, neverCalled);
- assertFalse(evaluate(and));
- v.reset();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateResetter.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateResetter.java b/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateResetter.java
deleted file mode 100644
index 8d10549..0000000
--- a/parquet-column/src/test/java/parquet/filter2/recordlevel/TestIncrementallyUpdatedFilterPredicateResetter.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2.recordlevel;
-
-
-import org.junit.Test;
-
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.And;
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.Or;
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static parquet.filter2.recordlevel.TestIncrementallyUpdatedFilterPredicateEvaluator.doubleMoreThan10;
-import static parquet.filter2.recordlevel.TestIncrementallyUpdatedFilterPredicateEvaluator.intIsEven;
-import static parquet.filter2.recordlevel.TestIncrementallyUpdatedFilterPredicateEvaluator.intIsNull;
-
-public class TestIncrementallyUpdatedFilterPredicateResetter {
- @Test
- public void testReset() {
-
- ValueInspector intIsNull = intIsNull();
- ValueInspector intIsEven = intIsEven();
- ValueInspector doubleMoreThan10 = doubleMoreThan10();
-
- IncrementallyUpdatedFilterPredicate pred = new Or(intIsNull, new And(intIsEven, doubleMoreThan10));
-
- intIsNull.updateNull();
- intIsEven.update(11);
- doubleMoreThan10.update(20.0D);
-
- assertTrue(intIsNull.isKnown());
- assertTrue(intIsEven.isKnown());
- assertTrue(doubleMoreThan10.isKnown());
-
- IncrementallyUpdatedFilterPredicateResetter.reset(pred);
-
- assertFalse(intIsNull.isKnown());
- assertFalse(intIsEven.isKnown());
- assertFalse(doubleMoreThan10.isKnown());
-
- intIsNull.updateNull();
- assertTrue(intIsNull.isKnown());
- assertFalse(intIsEven.isKnown());
- assertFalse(doubleMoreThan10.isKnown());
-
- IncrementallyUpdatedFilterPredicateResetter.reset(pred);
- assertFalse(intIsNull.isKnown());
- assertFalse(intIsEven.isKnown());
- assertFalse(doubleMoreThan10.isKnown());
-
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/filter2/recordlevel/TestValueInspector.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/filter2/recordlevel/TestValueInspector.java b/parquet-column/src/test/java/parquet/filter2/recordlevel/TestValueInspector.java
deleted file mode 100644
index 0bd401b..0000000
--- a/parquet-column/src/test/java/parquet/filter2/recordlevel/TestValueInspector.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2.recordlevel;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.junit.Test;
-
-import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static parquet.filter2.recordlevel.TestIncrementallyUpdatedFilterPredicateEvaluator.intIsEven;
-
-public class TestValueInspector {
-
- @Test
- public void testLifeCycle() {
- ValueInspector v = intIsEven();
-
- // begins in unknown state
- assertFalse(v.isKnown());
- // calling getResult in unknown state throws
- try {
- v.getResult();
- fail("this should throw");
- } catch (IllegalStateException e) {
- assertEquals("getResult() called on a ValueInspector whose result is not yet known!", e.getMessage());
- }
-
- // update state to known
- v.update(10);
-
- // v was updated with value 10, so result is known and should be true
- assertTrue(v.isKnown());
- assertTrue(v.getResult());
-
- // calling update w/o resetting should throw
- try {
- v.update(11);
- fail("this should throw");
- } catch (IllegalStateException e) {
- assertEquals("setResult() called on a ValueInspector whose result is already known!"
- + " Did you forget to call reset()?", e.getMessage());
- }
-
- // back to unknown state
- v.reset();
-
- assertFalse(v.isKnown());
- // calling getResult in unknown state throws
- try {
- v.getResult();
- fail("this should throw");
- } catch (IllegalStateException e) {
- assertEquals("getResult() called on a ValueInspector whose result is not yet known!", e.getMessage());
- }
-
- // v was updated with value 11, so result is known and should be false
- v.update(11);
- assertTrue(v.isKnown());
- assertFalse(v.getResult());
-
- }
-
- @Test
- public void testReusable() {
- List<Integer> values = Arrays.asList(2, 4, 7, 3, 8, 8, 11, 200);
- ValueInspector v = intIsEven();
-
- for (Integer x : values) {
- v.update(x);
- assertEquals(x % 2 == 0, v.getResult());
- v.reset();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/io/ConverterConsumer.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/io/ConverterConsumer.java b/parquet-column/src/test/java/parquet/io/ConverterConsumer.java
deleted file mode 100644
index d3de0a4..0000000
--- a/parquet-column/src/test/java/parquet/io/ConverterConsumer.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.io;
-
-import java.util.ArrayDeque;
-import java.util.Deque;
-
-import parquet.io.api.Binary;
-import parquet.io.api.GroupConverter;
-import parquet.io.api.PrimitiveConverter;
-import parquet.io.api.RecordConsumer;
-import parquet.schema.MessageType;
-import parquet.schema.Type;
-
-public class ConverterConsumer extends RecordConsumer {
-
- private final GroupConverter root;
- private final MessageType schema;
-
- private Deque<GroupConverter> path = new ArrayDeque<GroupConverter>();
- private Deque<Type> typePath = new ArrayDeque<Type>();
- private GroupConverter current;
- private PrimitiveConverter currentPrimitive;
- private Type currentType;
-
- public ConverterConsumer(GroupConverter recordConsumer, MessageType schema) {
- this.root = recordConsumer;
- this.schema = schema;
- }
-
- @Override
- public void startMessage() {
- root.start();
- this.currentType = schema;
- this.current = root;
- }
-
- @Override
- public void endMessage() {
- root.end();
- }
-
- @Override
- public void startField(String field, int index) {
- path.push(current);
- typePath.push(currentType);
- currentType = currentType.asGroupType().getType(index);
- if (currentType.isPrimitive()) {
- currentPrimitive = current.getConverter(index).asPrimitiveConverter();
- } else {
- current = current.getConverter(index).asGroupConverter();
- }
- }
-
- @Override
- public void endField(String field, int index) {
- currentType = typePath.pop();
- current = path.pop();
- }
-
- @Override
- public void startGroup() {
- current.start();
- }
-
- @Override
- public void endGroup() {
- current.end();
- }
-
- @Override
- public void addInteger(int value) {
- currentPrimitive.addInt(value);
- }
-
- @Override
- public void addLong(long value) {
- currentPrimitive.addLong(value);
- }
-
- @Override
- public void addBoolean(boolean value) {
- currentPrimitive.addBoolean(value);
- }
-
- @Override
- public void addBinary(Binary value) {
- currentPrimitive.addBinary(value);
- }
-
- @Override
- public void addFloat(float value) {
- currentPrimitive.addFloat(value);
- }
-
- @Override
- public void addDouble(double value) {
- currentPrimitive.addDouble(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/io/ExpectationValidatingConverter.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/io/ExpectationValidatingConverter.java b/parquet-column/src/test/java/parquet/io/ExpectationValidatingConverter.java
deleted file mode 100644
index e06b466..0000000
--- a/parquet-column/src/test/java/parquet/io/ExpectationValidatingConverter.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.io;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Deque;
-import java.util.List;
-
-import parquet.io.api.Binary;
-import parquet.io.api.Converter;
-import parquet.io.api.GroupConverter;
-import parquet.io.api.PrimitiveConverter;
-import parquet.io.api.RecordMaterializer;
-import parquet.schema.GroupType;
-import parquet.schema.MessageType;
-import parquet.schema.PrimitiveType;
-import parquet.schema.Type;
-import parquet.schema.TypeConverter;
-
-public class ExpectationValidatingConverter extends RecordMaterializer<Void> {
-
- private GroupConverter root;
-
- private final Deque<String> expectations;
- int count = 0;
-
- public void validate(String got) {
- assertEquals("event #"+count, expectations.pop(), got);
- ++count;
- }
-
- public ExpectationValidatingConverter(String[] expectations, MessageType schema) {
- this(new ArrayDeque<String>(Arrays.asList(expectations)), schema);
- }
-
- public ExpectationValidatingConverter(Deque<String> expectations, MessageType schema) {
- this.expectations = expectations;
- this.root = (GroupConverter)schema.convertWith(new TypeConverter<Converter>() {
-
- @Override
- public Converter convertPrimitiveType(final List<GroupType> path, final PrimitiveType primitiveType) {
- return new PrimitiveConverter() {
-
- private void validate(String message) {
- ExpectationValidatingConverter.this.validate(path(path, primitiveType) + message);
- }
-
- @Override
- public void addBinary(Binary value) {
- validate("addBinary("+value.toStringUsingUTF8()+")");
- }
-
- @Override
- public void addBoolean(boolean value) {
- validate("addBoolean("+value+")");
- }
-
- @Override
- public void addDouble(double value) {
- validate("addDouble("+value+")");
- }
-
- @Override
- public void addFloat(float value) {
- validate("addFloat("+value+")");
- }
-
- @Override
- public void addInt(int value) {
- validate("addInt("+value+")");
- }
-
- @Override
- public void addLong(long value) {
- validate("addLong("+value+")");
- }
- };
- }
-
- @Override
- public Converter convertGroupType(final List<GroupType> path, final GroupType groupType, final List<Converter> children) {
- return new GroupConverter() {
-
- private void validate(String message) {
- ExpectationValidatingConverter.this.validate(path(path, groupType) + message);
- }
-
- @Override
- public void start() {
- validate("start()");
- }
-
- @Override
- public void end() {
- validate("end()");
- }
-
- @Override
- public Converter getConverter(int fieldIndex) {
- return children.get(fieldIndex);
- }
-
- };
- }
-
- @Override
- public Converter convertMessageType(MessageType messageType, final List<Converter> children) {
- return new GroupConverter() {
-
- @Override
- public Converter getConverter(int fieldIndex) {
- return children.get(fieldIndex);
- }
-
- @Override
- public void start() {
- validate("startMessage()");
- }
-
- @Override
- public void end() {
- validate("endMessage()");
- }
- };
- }
- });
- }
-
- @Override
- public Void getCurrentRecord() {
- return null;
- }
-
- private String path(List<GroupType> path, Type type) {
- String pathString = "";
- if (path.size() > 0) {
- for (int i = 1; i < path.size(); i++) {
- pathString += path.get(i).getName() + ".";
- }
- }
- pathString += type.getName() + ".";
- return pathString;
- }
-
- @Override
- public GroupConverter getRootConverter() {
- return root;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/io/ExpectationValidatingRecordConsumer.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/io/ExpectationValidatingRecordConsumer.java b/parquet-column/src/test/java/parquet/io/ExpectationValidatingRecordConsumer.java
deleted file mode 100644
index 6ec1474..0000000
--- a/parquet-column/src/test/java/parquet/io/ExpectationValidatingRecordConsumer.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.io;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.Deque;
-
-import parquet.io.api.Binary;
-import parquet.io.api.RecordConsumer;
-
-final public class ExpectationValidatingRecordConsumer extends
- RecordConsumer {
- private final Deque<String> expectations;
- int count = 0;
-
- public ExpectationValidatingRecordConsumer(Deque<String> expectations) {
- this.expectations = expectations;
- }
-
- private void validate(String got) {
-// System.out.println(" \"" + got + "\";");
- assertEquals("event #"+count, expectations.pop(), got);
- ++count;
- }
-
- @Override
- public void startMessage() {
- validate("startMessage()");
- }
-
- @Override
- public void startGroup() {
- validate("startGroup()");
- }
-
- @Override
- public void startField(String field, int index) {
- validate("startField("+field+", "+index+")");
- }
-
- @Override
- public void endMessage() {
- validate("endMessage()");
- }
-
- @Override
- public void endGroup() {
- validate("endGroup()");
- }
-
- @Override
- public void endField(String field, int index) {
- validate("endField("+field+", "+index+")");
- }
-
- @Override
- public void addInteger(int value) {
- validate("addInt("+value+")");
- }
-
- @Override
- public void addLong(long value) {
- validate("addLong("+value+")");
- }
-
- @Override
- public void addBoolean(boolean value) {
- validate("addBoolean("+value+")");
- }
-
- @Override
- public void addBinary(Binary value) {
- validate("addBinary("+value.toStringUsingUTF8()+")");
- }
-
- @Override
- public void addFloat(float value) {
- validate("addFloat("+value+")");
- }
-
- @Override
- public void addDouble(double value) {
- validate("addDouble("+value+")");
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/io/PerfTest.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/io/PerfTest.java b/parquet-column/src/test/java/parquet/io/PerfTest.java
deleted file mode 100644
index 4281897..0000000
--- a/parquet-column/src/test/java/parquet/io/PerfTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.io;
-
-import static parquet.example.Paper.r1;
-import static parquet.example.Paper.r2;
-import static parquet.example.Paper.schema;
-import static parquet.example.Paper.schema2;
-import static parquet.example.Paper.schema3;
-
-import java.util.logging.Level;
-
-import parquet.Log;
-import parquet.column.ParquetProperties.WriterVersion;
-import parquet.column.impl.ColumnWriteStoreV1;
-import parquet.column.page.mem.MemPageStore;
-import parquet.example.DummyRecordConverter;
-import parquet.example.data.GroupWriter;
-import parquet.io.api.RecordMaterializer;
-import parquet.schema.MessageType;
-
-
-/**
- * make sure {@link Log#LEVEL} is set to {@link Level#OFF}
- *
- * @author Julien Le Dem
- *
- */
-public class PerfTest {
-
- public static void main(String[] args) {
- MemPageStore memPageStore = new MemPageStore(0);
- write(memPageStore);
- read(memPageStore);
- }
-
- private static void read(MemPageStore memPageStore) {
- read(memPageStore, schema, "read all");
- read(memPageStore, schema, "read all");
- read(memPageStore, schema2, "read projected");
- read(memPageStore, schema3, "read projected no Strings");
- }
-
- private static void read(MemPageStore memPageStore, MessageType myschema,
- String message) {
- MessageColumnIO columnIO = newColumnFactory(myschema);
- System.out.println(message);
- RecordMaterializer<Object> recordConsumer = new DummyRecordConverter(myschema);
- RecordReader<Object> recordReader = columnIO.getRecordReader(memPageStore, recordConsumer);
-
- read(recordReader, 2, myschema);
- read(recordReader, 10000, myschema);
- read(recordReader, 10000, myschema);
- read(recordReader, 10000, myschema);
- read(recordReader, 10000, myschema);
- read(recordReader, 10000, myschema);
- read(recordReader, 100000, myschema);
- read(recordReader, 1000000, myschema);
- System.out.println();
- }
-
-
- private static void write(MemPageStore memPageStore) {
- ColumnWriteStoreV1 columns = new ColumnWriteStoreV1(memPageStore, 50*1024*1024, 50*1024*1024, false, WriterVersion.PARQUET_1_0);
- MessageColumnIO columnIO = newColumnFactory(schema);
-
- GroupWriter groupWriter = new GroupWriter(columnIO.getRecordWriter(columns), schema);
- groupWriter.write(r1);
- groupWriter.write(r2);
-
- write(memPageStore, groupWriter, 10000);
- write(memPageStore, groupWriter, 10000);
- write(memPageStore, groupWriter, 10000);
- write(memPageStore, groupWriter, 10000);
- write(memPageStore, groupWriter, 10000);
- write(memPageStore, groupWriter, 100000);
- write(memPageStore, groupWriter, 1000000);
- columns.flush();
- System.out.println();
- System.out.println(columns.getBufferedSize() + " bytes used total");
- System.out.println("max col size: "+columns.maxColMemSize()+" bytes");
- }
-
- private static MessageColumnIO newColumnFactory(MessageType schema) {
- return new ColumnIOFactory().getColumnIO(schema);
- }
- private static void read(RecordReader<Object> recordReader, int count, MessageType schema) {
- Object[] records = new Object[count];
- System.gc();
- System.out.print("no gc <");
- long t0 = System.currentTimeMillis();
- for (int i = 0; i < records.length; i++) {
- records[i] = recordReader.read();
- }
- long t1 = System.currentTimeMillis();
- System.out.print("> ");
- long t = t1-t0;
- float err = (float)100 * 2 / t; // (+/- 1 ms)
- System.out.printf(" read %,9d recs in %,5d ms at %,9d rec/s err: %3.2f%%\n", count , t, t == 0 ? 0 : count * 1000 / t, err);
- if (!records[0].equals("end()")) {
- throw new RuntimeException(""+records[0]);
- }
- }
-
- private static void write(MemPageStore memPageStore, GroupWriter groupWriter, int count) {
- long t0 = System.currentTimeMillis();
- for (int i = 0; i < count; i++) {
- groupWriter.write(r1);
- }
- long t1 = System.currentTimeMillis();
- long t = t1-t0;
- memPageStore.addRowCount(count);
- System.out.printf("written %,9d recs in %,5d ms at %,9d rec/s\n", count, t, t == 0 ? 0 : count * 1000 / t );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/io/TestColumnIO.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/io/TestColumnIO.java b/parquet-column/src/test/java/parquet/io/TestColumnIO.java
deleted file mode 100644
index f351eaf..0000000
--- a/parquet-column/src/test/java/parquet/io/TestColumnIO.java
+++ /dev/null
@@ -1,674 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.io;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static parquet.example.Paper.pr1;
-import static parquet.example.Paper.pr2;
-import static parquet.example.Paper.r1;
-import static parquet.example.Paper.r2;
-import static parquet.example.Paper.schema;
-import static parquet.example.Paper.schema2;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
-import static parquet.schema.Type.Repetition.OPTIONAL;
-import static parquet.schema.Type.Repetition.REQUIRED;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import parquet.Log;
-import parquet.column.ColumnDescriptor;
-import parquet.column.ColumnWriteStore;
-import parquet.column.ColumnWriter;
-import parquet.column.ParquetProperties.WriterVersion;
-import parquet.column.impl.ColumnWriteStoreV1;
-import parquet.column.page.PageReadStore;
-import parquet.column.page.mem.MemPageStore;
-import parquet.example.data.Group;
-import parquet.example.data.GroupFactory;
-import parquet.example.data.GroupWriter;
-import parquet.example.data.simple.NanoTime;
-import parquet.example.data.simple.SimpleGroupFactory;
-import parquet.example.data.simple.convert.GroupRecordConverter;
-import parquet.io.api.Binary;
-import parquet.io.api.RecordConsumer;
-import parquet.io.api.RecordMaterializer;
-import parquet.schema.GroupType;
-import parquet.schema.MessageType;
-import parquet.schema.MessageTypeParser;
-import parquet.schema.PrimitiveType;
-import parquet.schema.PrimitiveType.PrimitiveTypeName;
-import parquet.schema.Type;
-import parquet.schema.Type.Repetition;
-
-@RunWith(Parameterized.class)
-public class TestColumnIO {
- private static final Log LOG = Log.getLog(TestColumnIO.class);
-
- private static final String oneOfEach =
- "message Document {\n"
- + " required int64 a;\n"
- + " required int32 b;\n"
- + " required float c;\n"
- + " required double d;\n"
- + " required boolean e;\n"
- + " required binary f;\n"
- + " required int96 g;\n"
- + " required fixed_len_byte_array(3) h;\n"
- + "}\n";
-
- private static final String schemaString =
- "message Document {\n"
- + " required int64 DocId;\n"
- + " optional group Links {\n"
- + " repeated int64 Backward;\n"
- + " repeated int64 Forward;\n"
- + " }\n"
- + " repeated group Name {\n"
- + " repeated group Language {\n"
- + " required binary Code;\n"
- + " optional binary Country;\n"
- + " }\n"
- + " optional binary Url;\n"
- + " }\n"
- + "}\n";
-
- int[][] expectedFSA = new int[][] {
- { 1 }, // 0: DocId
- { 2, 1 }, // 1: Links.Backward
- { 3, 2 }, // 2: Links.Forward
- { 4, 4, 4 },// 3: Name.Language.Code
- { 5, 5, 3 },// 4: Name.Language.Country
- { 6, 3 } // 5: Name.Url
- };
-
- int[][] expectedFSA2 = new int[][] {
- { 1 }, // 0: DocId
- { 2, 1, 1 },// 1: Name.Language.Country
- };
-
- public static final String[] expectedEventsForR1 = {
- "startMessage()",
- "DocId.addLong(10)",
- "Links.start()",
- "Links.Forward.addLong(20)",
- "Links.Forward.addLong(40)",
- "Links.Forward.addLong(60)",
- "Links.end()",
- "Name.start()",
- "Name.Language.start()",
- "Name.Language.Code.addBinary(en-us)",
- "Name.Language.Country.addBinary(us)",
- "Name.Language.end()",
- "Name.Language.start()",
- "Name.Language.Code.addBinary(en)",
- "Name.Language.end()",
- "Name.Url.addBinary(http://A)",
- "Name.end()",
- "Name.start()",
- "Name.Url.addBinary(http://B)",
- "Name.end()",
- "Name.start()",
- "Name.Language.start()",
- "Name.Language.Code.addBinary(en-gb)",
- "Name.Language.Country.addBinary(gb)",
- "Name.Language.end()",
- "Name.end()",
- "endMessage()"
- };
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() throws IOException {
- Object[][] data = {
- { true },
- { false } };
- return Arrays.asList(data);
- }
-
- private boolean useDictionary;
-
- public TestColumnIO(boolean useDictionary) {
- this.useDictionary = useDictionary;
- }
-
- @Test
- public void testSchema() {
- assertEquals(schemaString, schema.toString());
- }
-
- @Test
- public void testReadUsingRequestedSchemaWithExtraFields(){
- MessageType orginalSchema = new MessageType("schema",
- new PrimitiveType(REQUIRED, INT32, "a"),
- new PrimitiveType(OPTIONAL, INT32, "b")
- );
- MessageType schemaWithExtraField = new MessageType("schema",
- new PrimitiveType(OPTIONAL, INT32, "b"),
- new PrimitiveType(OPTIONAL, INT32, "a"),
- new PrimitiveType(OPTIONAL, INT32, "c")
- );
- MemPageStore memPageStoreForOriginalSchema = new MemPageStore(1);
- MemPageStore memPageStoreForSchemaWithExtraField = new MemPageStore(1);
- SimpleGroupFactory groupFactory = new SimpleGroupFactory(orginalSchema);
- writeGroups(orginalSchema, memPageStoreForOriginalSchema, groupFactory.newGroup().append("a", 1).append("b", 2));
-
- SimpleGroupFactory groupFactory2 = new SimpleGroupFactory(schemaWithExtraField);
- writeGroups(schemaWithExtraField, memPageStoreForSchemaWithExtraField, groupFactory2.newGroup().append("a", 1).append("b", 2).append("c",3));
-
- {
- List<Group> groups = new ArrayList<Group>();
- groups.addAll(readGroups(memPageStoreForOriginalSchema, orginalSchema, schemaWithExtraField, 1));
- groups.addAll(readGroups(memPageStoreForSchemaWithExtraField, schemaWithExtraField, schemaWithExtraField, 1));
- // TODO: add once we have the support for empty projection
-// groups1.addAll(readGroups(memPageStore3, schema3, schema2, 1));
- Object[][] expected = {
- { 2, 1, null},
- { 2, 1, 3},
-// { null, null}
- };
- validateGroups(groups, expected);
- }
- }
-
- @Test
- public void testReadUsingRequestedSchemaWithIncompatibleField(){
- MessageType originalSchema = new MessageType("schema",
- new PrimitiveType(OPTIONAL, INT32, "e"));
- MemPageStore store = new MemPageStore(1);
- SimpleGroupFactory groupFactory = new SimpleGroupFactory(originalSchema);
- writeGroups(originalSchema, store, groupFactory.newGroup().append("e", 4));
-
- try {
- MessageType schemaWithIncompatibleField = new MessageType("schema",
- new PrimitiveType(OPTIONAL, BINARY, "e")); // Incompatible schema: different type
- readGroups(store, originalSchema, schemaWithIncompatibleField, 1);
- fail("should have thrown an incompatible schema exception");
- } catch (ParquetDecodingException e) {
- assertEquals("The requested schema is not compatible with the file schema. incompatible types: optional binary e != optional int32 e", e.getMessage());
- }
- }
-
- @Test
- public void testReadUsingSchemaWithRequiredFieldThatWasOptional(){
- MessageType originalSchema = new MessageType("schema",
- new PrimitiveType(OPTIONAL, INT32, "e"));
- MemPageStore store = new MemPageStore(1);
- SimpleGroupFactory groupFactory = new SimpleGroupFactory(originalSchema);
- writeGroups(originalSchema, store, groupFactory.newGroup().append("e", 4));
-
- try {
- MessageType schemaWithRequiredFieldThatWasOptional = new MessageType("schema",
- new PrimitiveType(REQUIRED, INT32, "e")); // Incompatible schema: required when it was optional
- readGroups(store, originalSchema, schemaWithRequiredFieldThatWasOptional, 1);
- fail("should have thrown an incompatible schema exception");
- } catch (ParquetDecodingException e) {
- assertEquals("The requested schema is not compatible with the file schema. incompatible types: required int32 e != optional int32 e", e.getMessage());
- }
- }
-
- @Test
- public void testReadUsingProjectedSchema(){
- MessageType orginalSchema = new MessageType("schema",
- new PrimitiveType(REQUIRED, INT32, "a"),
- new PrimitiveType(REQUIRED, INT32, "b")
- );
- MessageType projectedSchema = new MessageType("schema",
- new PrimitiveType(OPTIONAL, INT32, "b")
- );
- MemPageStore store = new MemPageStore(1);
- SimpleGroupFactory groupFactory = new SimpleGroupFactory(orginalSchema);
- writeGroups(orginalSchema, store, groupFactory.newGroup().append("a", 1).append("b", 2));
-
- {
- List<Group> groups = new ArrayList<Group>();
- groups.addAll(readGroups(store, orginalSchema, projectedSchema, 1));
- Object[][] expected = {
- {2},
- };
- validateGroups(groups, expected);
- }
- }
-
- private void validateGroups(List<Group> groups1, Object[][] e1) {
- Iterator<Group> i1 = groups1.iterator();
- for (int i = 0; i < e1.length; i++) {
- Object[] objects = e1[i];
- Group next = i1.next();
- for (int j = 0; j < objects.length; j++) {
- Object object = objects[j];
- if (object == null) {
- assertEquals(0, next.getFieldRepetitionCount(j));
- } else {
- assertEquals("looking for r[" + i + "][" + j + "][0]=" + object, 1, next.getFieldRepetitionCount(j));
- assertEquals(object, next.getInteger(j, 0));
- }
- }
- }
- }
-
- private List<Group> readGroups(MemPageStore memPageStore, MessageType fileSchema, MessageType requestedSchema, int n) {
- ColumnIOFactory columnIOFactory = new ColumnIOFactory(true);
- MessageColumnIO columnIO = columnIOFactory.getColumnIO(requestedSchema, fileSchema);
- RecordReaderImplementation<Group> recordReader = getRecordReader(columnIO, requestedSchema, memPageStore);
- List<Group> groups = new ArrayList<Group>();
- for (int i = 0; i < n; i++) {
- groups.add(recordReader.read());
- }
- return groups;
- }
-
- private void writeGroups(MessageType writtenSchema, MemPageStore memPageStore, Group... groups) {
- ColumnIOFactory columnIOFactory = new ColumnIOFactory(true);
- ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
- MessageColumnIO columnIO = columnIOFactory.getColumnIO(writtenSchema);
- GroupWriter groupWriter = new GroupWriter(columnIO.getRecordWriter(columns), writtenSchema);
- for (Group group : groups) {
- groupWriter.write(group);
- }
- columns.flush();
- }
-
- @Test
- public void testColumnIO() {
- log(schema);
- log("r1");
- log(r1);
- log("r2");
- log(r2);
-
- MemPageStore memPageStore = new MemPageStore(2);
- ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
-
- ColumnIOFactory columnIOFactory = new ColumnIOFactory(true);
- {
- MessageColumnIO columnIO = columnIOFactory.getColumnIO(schema);
- log(columnIO);
- GroupWriter groupWriter = new GroupWriter(columnIO.getRecordWriter(columns), schema);
- groupWriter.write(r1);
- groupWriter.write(r2);
- columns.flush();
- log(columns);
- log("=========");
-
- RecordReaderImplementation<Group> recordReader = getRecordReader(columnIO, schema, memPageStore);
-
- validateFSA(expectedFSA, columnIO, recordReader);
-
- List<Group> records = new ArrayList<Group>();
- records.add(recordReader.read());
- records.add(recordReader.read());
-
- int i = 0;
- for (Group record : records) {
- log("r" + (++i));
- log(record);
- }
- assertEquals("deserialization does not display the same result", r1.toString(), records.get(0).toString());
- assertEquals("deserialization does not display the same result", r2.toString(), records.get(1).toString());
- }
- {
- MessageColumnIO columnIO2 = columnIOFactory.getColumnIO(schema2);
-
- List<Group> records = new ArrayList<Group>();
- RecordReaderImplementation<Group> recordReader = getRecordReader(columnIO2, schema2, memPageStore);
-
- validateFSA(expectedFSA2, columnIO2, recordReader);
-
- records.add(recordReader.read());
- records.add(recordReader.read());
-
- int i = 0;
- for (Group record : records) {
- log("r" + (++i));
- log(record);
- }
- assertEquals("deserialization does not display the expected result", pr1.toString(), records.get(0).toString());
- assertEquals("deserialization does not display the expected result", pr2.toString(), records.get(1).toString());
- }
- }
-
- @Test
- public void testOneOfEach() {
- MessageType oneOfEachSchema = MessageTypeParser.parseMessageType(oneOfEach);
- GroupFactory gf = new SimpleGroupFactory(oneOfEachSchema);
- Group g1 = gf.newGroup()
- .append("a", 1l)
- .append("b", 2)
- .append("c", 3.0f)
- .append("d", 4.0d)
- .append("e", true)
- .append("f", Binary.fromString("6"))
- .append("g", new NanoTime(1234, System.currentTimeMillis() * 1000))
- .append("h", Binary.fromString("abc"));
-
- testSchema(oneOfEachSchema, Arrays.asList(g1));
- }
-
- @Test
- public void testRequiredOfRequired() {
- MessageType reqreqSchema = MessageTypeParser.parseMessageType(
- "message Document {\n"
- + " required group foo {\n"
- + " required int64 bar;\n"
- + " }\n"
- + "}\n");
-
- GroupFactory gf = new SimpleGroupFactory(reqreqSchema);
- Group g1 = gf.newGroup();
- g1.addGroup("foo").append("bar", 2l);
-
- testSchema(reqreqSchema, Arrays.asList(g1));
- }
-
- @Test
- public void testOptionalRequiredInteraction() {
- for (int i = 0; i < 6; i++) {
- Type current = new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "primitive");
- for (int j = 0; j < i; j++) {
- current = new GroupType(Repetition.REQUIRED, "req" + j, current);
- }
- MessageType groupSchema = new MessageType("schema"+i, current);
- GroupFactory gf = new SimpleGroupFactory(groupSchema);
- List<Group> groups = new ArrayList<Group>();
- Group root = gf.newGroup();
- Group currentGroup = root;
- for (int j = 0; j < i; j++) {
- currentGroup = currentGroup.addGroup(0);
- }
- currentGroup.add(0, Binary.fromString("foo"));
- groups.add(root);
- testSchema(groupSchema, groups);
- }
- for (int i = 0; i < 6; i++) {
- Type current = new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, "primitive");
- for (int j = 0; j < i; j++) {
- current = new GroupType(Repetition.REQUIRED, "req" + j, current);
- }
- MessageType groupSchema = new MessageType("schema"+(i+6), current);
- GroupFactory gf = new SimpleGroupFactory(groupSchema);
- List<Group> groups = new ArrayList<Group>();
- Group rootDefined = gf.newGroup();
- Group rootUndefined = gf.newGroup();
- Group currentDefinedGroup = rootDefined;
- Group currentUndefinedGroup = rootUndefined;
- for (int j = 0; j < i; j++) {
- currentDefinedGroup = currentDefinedGroup.addGroup(0);
- currentUndefinedGroup = currentUndefinedGroup.addGroup(0);
- }
- currentDefinedGroup.add(0, Binary.fromString("foo"));
- groups.add(rootDefined);
- groups.add(rootUndefined);
- testSchema(groupSchema, groups);
- }
- for (int i = 0; i < 6; i++) {
- Type current = new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, "primitive");
- for (int j = 0; j < 6; j++) {
- current = new GroupType(i==j ? Repetition.OPTIONAL : Repetition.REQUIRED, "req" + j, current);
- }
- MessageType groupSchema = new MessageType("schema"+(i+12), current);
- GroupFactory gf = new SimpleGroupFactory(groupSchema);
- List<Group> groups = new ArrayList<Group>();
- Group rootDefined = gf.newGroup();
- Group rootUndefined = gf.newGroup();
- Group currentDefinedGroup = rootDefined;
- Group currentUndefinedGroup = rootUndefined;
- for (int j = 0; j < 6; j++) {
- currentDefinedGroup = currentDefinedGroup.addGroup(0);
- if (i < j) {
- currentUndefinedGroup = currentUndefinedGroup.addGroup(0);
- }
- }
- currentDefinedGroup.add(0, Binary.fromString("foo"));
- groups.add(rootDefined);
- groups.add(rootUndefined);
- testSchema(groupSchema, groups);
- }
- }
-
- private void testSchema(MessageType messageSchema, List<Group> groups) {
- MemPageStore memPageStore = new MemPageStore(groups.size());
- ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
-
- ColumnIOFactory columnIOFactory = new ColumnIOFactory(true);
- MessageColumnIO columnIO = columnIOFactory.getColumnIO(messageSchema);
- log(columnIO);
-
- // Write groups.
- GroupWriter groupWriter =
- new GroupWriter(columnIO.getRecordWriter(columns), messageSchema);
- for (Group group : groups) {
- groupWriter.write(group);
- }
- columns.flush();
-
- // Read groups and verify.
- RecordReaderImplementation<Group> recordReader =
- getRecordReader(columnIO, messageSchema, memPageStore);
- for (Group group : groups) {
- final Group got = recordReader.read();
- assertEquals("deserialization does not display the same result",
- group.toString(), got.toString());
- }
- }
-
- private RecordReaderImplementation<Group> getRecordReader(MessageColumnIO columnIO, MessageType schema, PageReadStore pageReadStore) {
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
-
- return (RecordReaderImplementation<Group>)columnIO.getRecordReader(pageReadStore, recordConverter);
- }
-
- private void log(Object o) {
- LOG.info(o);
- }
-
- private void validateFSA(int[][] expectedFSA, MessageColumnIO columnIO, RecordReaderImplementation<?> recordReader) {
- log("FSA: ----");
- List<PrimitiveColumnIO> leaves = columnIO.getLeaves();
- for (int i = 0; i < leaves.size(); ++i) {
- PrimitiveColumnIO primitiveColumnIO = leaves.get(i);
- log(Arrays.toString(primitiveColumnIO.getFieldPath()));
- for (int r = 0; r < expectedFSA[i].length; r++) {
- int next = expectedFSA[i][r];
- log(" "+r+" -> "+ (next==leaves.size() ? "end" : Arrays.toString(leaves.get(next).getFieldPath()))+": "+recordReader.getNextLevel(i, r));
- assertEquals(Arrays.toString(primitiveColumnIO.getFieldPath())+": "+r+" -> ", next, recordReader.getNextReader(i, r));
- }
- }
- log("----");
- }
-
- @Test
- public void testPushParser() {
- MemPageStore memPageStore = new MemPageStore(1);
- ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
- MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
- new GroupWriter(columnIO.getRecordWriter(columns), schema).write(r1);
- columns.flush();
-
- RecordReader<Void> recordReader = columnIO.getRecordReader(memPageStore, new ExpectationValidatingConverter(expectedEventsForR1, schema));
- recordReader.read();
-
- }
-
- private ColumnWriteStoreV1 newColumnWriteStore(MemPageStore memPageStore) {
- return new ColumnWriteStoreV1(memPageStore, 800, 800, useDictionary, WriterVersion.PARQUET_1_0);
- }
-
- @Test
- public void testEmptyField() {
- MemPageStore memPageStore = new MemPageStore(1);
- ColumnWriteStoreV1 columns = newColumnWriteStore(memPageStore);
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- final RecordConsumer recordWriter = columnIO.getRecordWriter(columns);
- recordWriter.startMessage();
- recordWriter.startField("DocId", 0);
- recordWriter.addLong(0);
- recordWriter.endField("DocId", 0);
- recordWriter.startField("Links", 1);
- try {
- recordWriter.endField("Links", 1);
- Assert.fail("expected exception because of empty field");
- } catch (ParquetEncodingException e) {
- Assert.assertEquals("empty fields are illegal, the field should be ommited completely instead", e.getMessage());
- }
- }
-
- @Test
- public void testGroupWriter() {
- List<Group> result = new ArrayList<Group>();
- final GroupRecordConverter groupRecordConverter = new GroupRecordConverter(schema);
- RecordConsumer groupConsumer = new ConverterConsumer(groupRecordConverter.getRootConverter(), schema);
- GroupWriter groupWriter = new GroupWriter(new RecordConsumerLoggingWrapper(groupConsumer), schema);
- groupWriter.write(r1);
- result.add(groupRecordConverter.getCurrentRecord());
- groupWriter.write(r2);
- result.add(groupRecordConverter.getCurrentRecord());
- assertEquals("deserialization does not display the expected result", result.get(0).toString(), r1.toString());
- assertEquals("deserialization does not display the expected result", result.get(1).toString(), r2.toString());
- }
-
- @Test
- public void testWriteWithGroupWriter() {
-
- final String[] expected = {
- "[DocId]: 10, r:0, d:0",
- "[Links, Forward]: 20, r:0, d:2",
- "[Links, Forward]: 40, r:1, d:2",
- "[Links, Forward]: 60, r:1, d:2",
- "[Links, Backward]: null, r:0, d:1",
- "[Name, Language, Code]: en-us, r:0, d:2",
- "[Name, Language, Country]: us, r:0, d:3",
- "[Name, Language, Code]: en, r:2, d:2",
- "[Name, Language, Country]: null, r:2, d:2",
- "[Name, Url]: http://A, r:0, d:2",
- "[Name, Url]: http://B, r:1, d:2",
- "[Name, Language, Code]: null, r:1, d:1",
- "[Name, Language, Country]: null, r:1, d:1",
- "[Name, Language, Code]: en-gb, r:1, d:2",
- "[Name, Language, Country]: gb, r:1, d:3",
- "[Name, Url]: null, r:1, d:1",
- "[DocId]: 20, r:0, d:0",
- "[Links, Backward]: 10, r:0, d:2",
- "[Links, Backward]: 30, r:1, d:2",
- "[Links, Forward]: 80, r:0, d:2",
- "[Name, Url]: http://C, r:0, d:2",
- "[Name, Language, Code]: null, r:0, d:1",
- "[Name, Language, Country]: null, r:0, d:1"
- };
-
- ValidatingColumnWriteStore columns = new ValidatingColumnWriteStore(expected);
- MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
- GroupWriter groupWriter = new GroupWriter(columnIO.getRecordWriter(columns), schema);
- groupWriter.write(r1);
- groupWriter.write(r2);
- columns.validate();
- }
-}
-final class ValidatingColumnWriteStore implements ColumnWriteStore {
- private final String[] expected;
- int counter = 0;
-
- ValidatingColumnWriteStore(String[] expected) {
- this.expected = expected;
- }
-
- @Override
- public ColumnWriter getColumnWriter(final ColumnDescriptor path) {
- return new ColumnWriter() {
- private void validate(Object value, int repetitionLevel,
- int definitionLevel) {
- String actual = Arrays.toString(path.getPath())+": "+value+", r:"+repetitionLevel+", d:"+definitionLevel;
- assertEquals("event #" + counter, expected[counter], actual);
- ++ counter;
- }
-
- @Override
- public void writeNull(int repetitionLevel, int definitionLevel) {
- validate(null, repetitionLevel, definitionLevel);
- }
-
- @Override
- public void write(Binary value, int repetitionLevel, int definitionLevel) {
- validate(value.toStringUsingUTF8(), repetitionLevel, definitionLevel);
- }
-
- @Override
- public void write(boolean value, int repetitionLevel, int definitionLevel) {
- validate(value, repetitionLevel, definitionLevel);
- }
-
- @Override
- public void write(int value, int repetitionLevel, int definitionLevel) {
- validate(value, repetitionLevel, definitionLevel);
- }
-
- @Override
- public void write(long value, int repetitionLevel, int definitionLevel) {
- validate(value, repetitionLevel, definitionLevel);
- }
-
- @Override
- public void write(float value, int repetitionLevel, int definitionLevel) {
- validate(value, repetitionLevel, definitionLevel);
- }
-
- @Override
- public void write(double value, int repetitionLevel, int definitionLevel) {
- validate(value, repetitionLevel, definitionLevel);
- }
- };
- }
-
- public void validate() {
- assertEquals("read all events", expected.length, counter);
- }
-
- @Override
- public void endRecord() {
- }
-
- @Override
- public void flush() {
- }
-
- @Override
- public long getAllocatedSize() {
- return 0;
- }
-
- @Override
- public long getBufferedSize() {
- return 0;
- }
-
- @Override
- public String memUsageString() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/io/TestFiltered.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/io/TestFiltered.java b/parquet-column/src/test/java/parquet/io/TestFiltered.java
deleted file mode 100644
index c1d538f..0000000
--- a/parquet-column/src/test/java/parquet/io/TestFiltered.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.io;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-
-import parquet.column.ParquetProperties.WriterVersion;
-import parquet.column.impl.ColumnWriteStoreV1;
-import parquet.column.page.mem.MemPageStore;
-import parquet.example.data.Group;
-import parquet.example.data.GroupWriter;
-import parquet.example.data.simple.convert.GroupRecordConverter;
-import parquet.filter.ColumnPredicates.LongPredicateFunction;
-import parquet.filter.ColumnPredicates.PredicateFunction;
-import parquet.filter2.compat.FilterCompat;
-import parquet.io.api.RecordMaterializer;
-
-import static org.junit.Assert.assertEquals;
-import static parquet.example.Paper.r1;
-import static parquet.example.Paper.r2;
-import static parquet.example.Paper.schema;
-import static parquet.filter.AndRecordFilter.and;
-import static parquet.filter.ColumnPredicates.applyFunctionToLong;
-import static parquet.filter.ColumnPredicates.applyFunctionToString;
-import static parquet.filter.ColumnPredicates.equalTo;
-import static parquet.filter.ColumnRecordFilter.column;
-import static parquet.filter.NotRecordFilter.not;
-import static parquet.filter.OrRecordFilter.or;
-import static parquet.filter.PagedRecordFilter.page;
-
-public class TestFiltered {
-
- /* Class that implements applyFunction filter for long. Checks for long greater than 15. */
- public class LongGreaterThan15Predicate implements LongPredicateFunction {
- @Override
- public boolean functionToApply(long input) {
- return input > 15;
- }
- };
-
- /* Class that implements applyFunction filter for string. Checks for string ending in 'A'. */
- public class StringEndsWithAPredicate implements PredicateFunction<String> {
- @Override
- public boolean functionToApply(String input) {
- return input.endsWith("A");
- }
- };
-
- private List<Group> readAll(RecordReader<Group> reader) {
- List<Group> result = new ArrayList<Group>();
- Group g;
- while ((g = reader.read()) != null) {
- result.add(g);
- }
- return result;
- }
-
- private void readOne(RecordReader<Group> reader, String message, Group expected) {
- List<Group> result = readAll(reader);
- assertEquals(message + ": " + result, 1, result.size());
- assertEquals("filtering did not return the correct record", expected.toString(), result.get(0).toString());
- }
-
- @Test
- public void testFilterOnInteger() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 1);
-
- // Get first record
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter, FilterCompat.get(column("DocId", equalTo(10l))));
-
- readOne(recordReader, "r2 filtered out", r1);
-
- // Get second record
- recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("DocId", equalTo(20l))));
-
- readOne(recordReader, "r1 filtered out", r2);
-
- }
-
- @Test
- public void testApplyFunctionFilterOnLong() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 1);
-
- // Get first record
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("DocId", equalTo(10l))));
-
- readOne(recordReader, "r2 filtered out", r1);
-
- // Get second record
- recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("DocId", applyFunctionToLong(new LongGreaterThan15Predicate()))));
-
- readOne(recordReader, "r1 filtered out", r2);
- }
-
- @Test
- public void testFilterOnString() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 1);
-
- // First try matching against the A url in record 1
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("Name.Url", equalTo("http://A"))));
-
- readOne(recordReader, "r2 filtered out", r1);
-
- // Second try matching against the B url in record 1 - it should fail as we only match
- // against the first instance of a
- recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("Name.Url", equalTo("http://B"))));
-
- List<Group> all = readAll(recordReader);
- assertEquals("There should be no matching records: " + all , 0, all.size());
-
- // Finally try matching against the C url in record 2
- recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("Name.Url", equalTo("http://C"))));
-
- readOne(recordReader, "r1 filtered out", r2);
-
- }
-
- @Test
- public void testApplyFunctionFilterOnString() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 1);
-
- // First try matching against the A url in record 1
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("Name.Url", applyFunctionToString(new StringEndsWithAPredicate()))));
-
- readOne(recordReader, "r2 filtered out", r1);
-
- // Second try matching against the B url in record 1 - it should fail as we only match
- // against the first instance of a
- recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("Name.Url", equalTo("http://B"))));
-
- List<Group> all = readAll(recordReader);
- assertEquals("There should be no matching records: " + all , 0, all.size());
-
- // Finally try matching against the C url in record 2
- recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(column("Name.Url", equalTo("http://C"))));
-
- readOne(recordReader, "r1 filtered out", r2);
-
- }
-
- @Test
- public void testPaged() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 6);
-
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(page(4, 4)));
-
- List<Group> all = readAll(recordReader);
- assertEquals("expecting records " + all, 4, all.size());
- for (int i = 0; i < all.size(); i++) {
- assertEquals("expecting record", (i%2 == 0 ? r2 : r1).toString(), all.get(i).toString());
- }
- }
-
- @Test
- public void testFilteredAndPaged() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 8);
-
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(and(column("DocId", equalTo(10l)), page(2, 4))));
-
- List<Group> all = readAll(recordReader);
- assertEquals("expecting 4 records " + all, 4, all.size());
- for (int i = 0; i < all.size(); i++) {
- assertEquals("expecting record1", r1.toString(), all.get(i).toString());
- }
-
- }
-
- @Test
- public void testFilteredOrPaged() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 8);
-
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(or(column("DocId", equalTo(10l)),
- column("DocId", equalTo(20l)))));
-
- List<Group> all = readAll(recordReader);
- assertEquals("expecting 8 records " + all, 16, all.size());
- for (int i = 0; i < all.size () / 2; i++) {
- assertEquals("expecting record1", r1.toString(), all.get(2 * i).toString());
- assertEquals("expecting record2", r2.toString(), all.get(2 * i + 1).toString());
- }
- }
-
- @Test
- public void testFilteredNotPaged() {
- MessageColumnIO columnIO = new ColumnIOFactory(true).getColumnIO(schema);
- MemPageStore memPageStore = writeTestRecords(columnIO, 8);
-
- RecordMaterializer<Group> recordConverter = new GroupRecordConverter(schema);
- RecordReaderImplementation<Group> recordReader = (RecordReaderImplementation<Group>)
- columnIO.getRecordReader(memPageStore, recordConverter,
- FilterCompat.get(not(column("DocId", equalTo(10l)))));
-
- List<Group> all = readAll(recordReader);
- assertEquals("expecting 8 records " + all, 8, all.size());
- for (int i = 0; i < all.size(); i++) {
- assertEquals("expecting record2", r2.toString(), all.get(i).toString());
- }
- }
-
- private MemPageStore writeTestRecords(MessageColumnIO columnIO, int number) {
- MemPageStore memPageStore = new MemPageStore(number * 2);
- ColumnWriteStoreV1 columns = new ColumnWriteStoreV1(memPageStore, 800, 800, false, WriterVersion.PARQUET_1_0);
-
- GroupWriter groupWriter = new GroupWriter(columnIO.getRecordWriter(columns), schema);
- for ( int i = 0; i < number; i++ ) {
- groupWriter.write(r1);
- groupWriter.write(r2);
- }
- columns.flush();
- return memPageStore;
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/parser/TestParquetParser.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/parser/TestParquetParser.java b/parquet-column/src/test/java/parquet/parser/TestParquetParser.java
deleted file mode 100644
index 8d2e4a3..0000000
--- a/parquet-column/src/test/java/parquet/parser/TestParquetParser.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.parser;
-
-import static org.junit.Assert.assertEquals;
-import static parquet.schema.MessageTypeParser.parseMessageType;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
-import static parquet.schema.Type.Repetition.OPTIONAL;
-import static parquet.schema.Type.Repetition.REPEATED;
-import static parquet.schema.Type.Repetition.REQUIRED;
-import static parquet.schema.OriginalType.*;
-import static parquet.schema.Types.buildMessage;
-
-import org.junit.Test;
-
-import parquet.schema.GroupType;
-import parquet.schema.MessageType;
-import parquet.schema.OriginalType;
-import parquet.schema.MessageTypeParser;
-import parquet.schema.PrimitiveType;
-import parquet.schema.PrimitiveType.PrimitiveTypeName;
-import parquet.schema.Types;
-import parquet.schema.Types.MessageTypeBuilder;
-
-public class TestParquetParser {
- @Test
- public void testPaperExample() throws Exception {
- String example =
- "message Document {\n" +
- " required int64 DocId;\n" +
- " optional group Links {\n" +
- " repeated int64 Backward;\n" +
- " repeated int64 Forward; }\n" +
- " repeated group Name {\n" +
- " repeated group Language {\n" +
- " required binary Code;\n" +
- " required binary Country; }\n" +
- " optional binary Url; }}";
- MessageType parsed = parseMessageType(example);
- MessageType manuallyMade =
- new MessageType("Document",
- new PrimitiveType(REQUIRED, INT64, "DocId"),
- new GroupType(OPTIONAL, "Links",
- new PrimitiveType(REPEATED, INT64, "Backward"),
- new PrimitiveType(REPEATED, INT64, "Forward")),
- new GroupType(REPEATED, "Name",
- new GroupType(REPEATED, "Language",
- new PrimitiveType(REQUIRED, BINARY, "Code"),
- new PrimitiveType(REQUIRED, BINARY, "Country")),
- new PrimitiveType(OPTIONAL, BINARY, "Url")));
- assertEquals(manuallyMade, parsed);
-
- MessageType parsedThenReparsed = parseMessageType(parsed.toString());
-
- assertEquals(manuallyMade, parsedThenReparsed);
-
- parsed = parseMessageType("message m { required group a {required binary b;} required group c { required int64 d; }}");
- manuallyMade =
- new MessageType("m",
- new GroupType(REQUIRED, "a",
- new PrimitiveType(REQUIRED, BINARY, "b")),
- new GroupType(REQUIRED, "c",
- new PrimitiveType(REQUIRED, INT64, "d")));
-
- assertEquals(manuallyMade, parsed);
-
- parsedThenReparsed = parseMessageType(parsed.toString());
-
- assertEquals(manuallyMade, parsedThenReparsed);
- }
-
- @Test
- public void testEachPrimitiveType() {
- MessageTypeBuilder builder = buildMessage();
- StringBuilder schema = new StringBuilder();
- schema.append("message EachType {\n");
- for (PrimitiveTypeName type : PrimitiveTypeName.values()) {
- // add a schema entry, e.g., " required int32 int32_;\n"
- if (type == FIXED_LEN_BYTE_ARRAY) {
- schema.append(" required fixed_len_byte_array(3) fixed_;");
- builder.required(FIXED_LEN_BYTE_ARRAY).length(3).named("fixed_");
- } else {
- schema.append(" required ").append(type)
- .append(" ").append(type).append("_;\n");
- builder.required(type).named(type.toString() + "_");
- }
- }
- schema.append("}\n");
- MessageType expected = builder.named("EachType");
-
- MessageType parsed = parseMessageType(schema.toString());
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testUTF8Annotation() {
- String message =
- "message StringMessage {\n" +
- " required binary string (UTF8);\n" +
- "}\n";
-
- MessageType parsed = parseMessageType(message);
- MessageType expected = buildMessage()
- .required(BINARY).as(UTF8).named("string")
- .named("StringMessage");
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testIDs() {
- String message =
- "message Message {\n" +
- " required binary string (UTF8) = 6;\n" +
- " required int32 i=1;\n" +
- " required binary s2= 3;\n" +
- " required binary s3 =4;\n" +
- "}\n";
-
- MessageType parsed = parseMessageType(message);
- MessageType expected = buildMessage()
- .required(BINARY).as(OriginalType.UTF8).id(6).named("string")
- .required(INT32).id(1).named("i")
- .required(BINARY).id(3).named("s2")
- .required(BINARY).id(4).named("s3")
- .named("Message");
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testMAPAnnotations() {
- // this is primarily to test group annotations
- String message =
- "message Message {\n" +
- " optional group aMap (MAP) {\n" +
- " repeated group map (MAP_KEY_VALUE) {\n" +
- " required binary key (UTF8);\n" +
- " required int32 value;\n" +
- " }\n" +
- " }\n" +
- "}\n";
-
- MessageType parsed = parseMessageType(message);
- MessageType expected = buildMessage()
- .optionalGroup()
- .repeatedGroup()
- .required(BINARY).as(UTF8).named("key")
- .required(INT32).named("value")
- .named("map")
- .named("aMap")
- .named("Message");
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testLISTAnnotation() {
- // this is primarily to test group annotations
- String message =
- "message Message {\n" +
- " required group aList (LIST) {\n" +
- " repeated binary string (UTF8);\n" +
- " }\n" +
- "}\n";
-
- MessageType parsed = parseMessageType(message);
- MessageType expected = buildMessage()
- .requiredGroup()
- .repeated(BINARY).as(UTF8).named("string")
- .named("aList")
- .named("Message");
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testDecimalFixedAnnotation() {
- String message =
- "message DecimalMessage {\n" +
- " required FIXED_LEN_BYTE_ARRAY(4) aDecimal (DECIMAL(9,2));\n" +
- "}\n";
-
- MessageType parsed = parseMessageType(message);
- MessageType expected = buildMessage()
- .required(FIXED_LEN_BYTE_ARRAY).length(4)
- .as(DECIMAL).precision(9).scale(2)
- .named("aDecimal")
- .named("DecimalMessage");
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testDecimalBinaryAnnotation() {
- String message =
- "message DecimalMessage {\n" +
- " required binary aDecimal (DECIMAL(9,2));\n" +
- "}\n";
-
- MessageType parsed = parseMessageType(message);
- MessageType expected = buildMessage()
- .required(BINARY).as(DECIMAL).precision(9).scale(2)
- .named("aDecimal")
- .named("DecimalMessage");
-
- assertEquals(expected, parsed);
- MessageType reparsed = parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testTimeAnnotations() {
- String message = "message TimeMessage {" +
- " required int32 date (DATE);" +
- " required int32 time (TIME_MILLIS);" +
- " required int64 timestamp (TIMESTAMP_MILLIS);" +
- " required FIXED_LEN_BYTE_ARRAY(12) interval (INTERVAL);" +
- "}\n";
-
- MessageType parsed = MessageTypeParser.parseMessageType(message);
- MessageType expected = Types.buildMessage()
- .required(INT32).as(DATE).named("date")
- .required(INT32).as(TIME_MILLIS).named("time")
- .required(INT64).as(TIMESTAMP_MILLIS).named("timestamp")
- .required(FIXED_LEN_BYTE_ARRAY).length(12).as(INTERVAL).named("interval")
- .named("TimeMessage");
-
- assertEquals(expected, parsed);
- MessageType reparsed = MessageTypeParser.parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testIntAnnotations() {
- String message = "message IntMessage {" +
- " required int32 i8 (INT_8);" +
- " required int32 i16 (INT_16);" +
- " required int32 i32 (INT_32);" +
- " required int64 i64 (INT_64);" +
- " required int32 u8 (UINT_8);" +
- " required int32 u16 (UINT_16);" +
- " required int32 u32 (UINT_32);" +
- " required int64 u64 (UINT_64);" +
- "}\n";
-
- MessageType parsed = MessageTypeParser.parseMessageType(message);
- MessageType expected = Types.buildMessage()
- .required(INT32).as(INT_8).named("i8")
- .required(INT32).as(INT_16).named("i16")
- .required(INT32).as(INT_32).named("i32")
- .required(INT64).as(INT_64).named("i64")
- .required(INT32).as(UINT_8).named("u8")
- .required(INT32).as(UINT_16).named("u16")
- .required(INT32).as(UINT_32).named("u32")
- .required(INT64).as(UINT_64).named("u64")
- .named("IntMessage");
-
- assertEquals(expected, parsed);
- MessageType reparsed = MessageTypeParser.parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-
- @Test
- public void testEmbeddedAnnotations() {
- String message = "message EmbeddedMessage {" +
- " required binary json (JSON);" +
- " required binary bson (BSON);" +
- "}\n";
-
- MessageType parsed = MessageTypeParser.parseMessageType(message);
- MessageType expected = Types.buildMessage()
- .required(BINARY).as(JSON).named("json")
- .required(BINARY).as(BSON).named("bson")
- .named("EmbeddedMessage");
-
- assertEquals(expected, parsed);
- MessageType reparsed = MessageTypeParser.parseMessageType(parsed.toString());
- assertEquals(expected, reparsed);
- }
-}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-column/src/test/java/parquet/schema/TestMessageType.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/parquet/schema/TestMessageType.java b/parquet-column/src/test/java/parquet/schema/TestMessageType.java
deleted file mode 100644
index 2374f1f..0000000
--- a/parquet-column/src/test/java/parquet/schema/TestMessageType.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.schema;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
-import static parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
-import static parquet.schema.Type.Repetition.OPTIONAL;
-import static parquet.schema.Type.Repetition.REPEATED;
-import static parquet.schema.Type.Repetition.REQUIRED;
-
-import org.junit.Test;
-
-import parquet.example.Paper;
-import parquet.schema.PrimitiveType.PrimitiveTypeName;
-
-
-public class TestMessageType {
- @Test
- public void test() throws Exception {
- MessageType schema = MessageTypeParser.parseMessageType(Paper.schema.toString());
- assertEquals(Paper.schema, schema);
- assertEquals(schema.toString(), Paper.schema.toString());
- }
-
- @Test
- public void testNestedTypes() {
- MessageType schema = MessageTypeParser.parseMessageType(Paper.schema.toString());
- Type type = schema.getType("Links", "Backward");
- assertEquals(PrimitiveTypeName.INT64,
- type.asPrimitiveType().getPrimitiveTypeName());
- assertEquals(0, schema.getMaxRepetitionLevel("DocId"));
- assertEquals(1, schema.getMaxRepetitionLevel("Name"));
- assertEquals(2, schema.getMaxRepetitionLevel("Name", "Language"));
- assertEquals(0, schema.getMaxDefinitionLevel("DocId"));
- assertEquals(1, schema.getMaxDefinitionLevel("Links"));
- assertEquals(2, schema.getMaxDefinitionLevel("Links", "Backward"));
- }
-
-
- @Test
- public void testMergeSchema() {
- MessageType t1 = new MessageType("root1",
- new PrimitiveType(REPEATED, BINARY, "a"),
- new PrimitiveType(OPTIONAL, BINARY, "b"));
- MessageType t2 = new MessageType("root2",
- new PrimitiveType(REQUIRED, BINARY, "c"));
-
- assertEquals(
- t1.union(t2),
- new MessageType("root1",
- new PrimitiveType(REPEATED, BINARY, "a"),
- new PrimitiveType(OPTIONAL, BINARY, "b"),
- new PrimitiveType(REQUIRED, BINARY, "c"))
- );
-
- assertEquals(
- t2.union(t1),
- new MessageType("root2",
- new PrimitiveType(REQUIRED, BINARY, "c"),
- new PrimitiveType(REPEATED, BINARY, "a"),
- new PrimitiveType(OPTIONAL, BINARY, "b"))
- );
-
- MessageType t3 = new MessageType("root1",
- new PrimitiveType(OPTIONAL, BINARY, "a"));
- MessageType t4 = new MessageType("root2",
- new PrimitiveType(REQUIRED, BINARY, "a"));
-
- try {
- t3.union(t4);
- fail("moving from optional to required");
- } catch (IncompatibleSchemaModificationException e) {
- assertEquals("repetition constraint is more restrictive: can not merge type required binary a into optional binary a", e.getMessage());
- }
-
- assertEquals(
- t4.union(t3),
- new MessageType("root2",
- new PrimitiveType(OPTIONAL, BINARY, "a"))
- );
-
- MessageType t5 = new MessageType("root1",
- new GroupType(REQUIRED, "g1",
- new PrimitiveType(OPTIONAL, BINARY, "a")),
- new GroupType(REQUIRED, "g2",
- new PrimitiveType(OPTIONAL, BINARY, "b")));
- MessageType t6 = new MessageType("root1",
- new GroupType(REQUIRED, "g1",
- new PrimitiveType(OPTIONAL, BINARY, "a")),
- new GroupType(REQUIRED, "g2",
- new GroupType(REQUIRED, "g3",
- new PrimitiveType(OPTIONAL, BINARY, "c")),
- new PrimitiveType(OPTIONAL, BINARY, "b")));
-
- assertEquals(
- t5.union(t6),
- new MessageType("root1",
- new GroupType(REQUIRED, "g1",
- new PrimitiveType(OPTIONAL, BINARY, "a")),
- new GroupType(REQUIRED, "g2",
- new PrimitiveType(OPTIONAL, BINARY, "b"),
- new GroupType(REQUIRED, "g3",
- new PrimitiveType(OPTIONAL, BINARY, "c"))))
- );
-
- MessageType t7 = new MessageType("root1",
- new PrimitiveType(OPTIONAL, BINARY, "a"));
- MessageType t8 = new MessageType("root2",
- new PrimitiveType(OPTIONAL, INT32, "a"));
- try {
- t7.union(t8);
- fail("moving from BINARY to INT32");
- } catch (IncompatibleSchemaModificationException e) {
- assertEquals("can not merge type optional int32 a into optional binary a", e.getMessage());
- }
- }
-
- @Test
- public void testIDs() throws Exception {
- MessageType schema = new MessageType("test",
- new PrimitiveType(REQUIRED, BINARY, "foo").withId(4),
- new GroupType(REQUIRED, "bar",
- new PrimitiveType(REQUIRED, BINARY, "baz").withId(3)
- ).withId(8)
- );
- MessageType schema2 = MessageTypeParser.parseMessageType(schema.toString());
- assertEquals(schema, schema2);
- assertEquals(schema.toString(), schema2.toString());
- }
-}