You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2017/01/24 18:41:22 UTC
orc git commit: ORC-100. Build should fail if there are findbug
errors in modules (Tarun Kumar and omalley via omalley)
Repository: orc
Updated Branches:
refs/heads/master c523b97c1 -> 8bfa18982
ORC-100. Build should fail if there are findbug errors in modules (Tarun Kumar and omalley
via omalley)
Fixes #85
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/8bfa1898
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/8bfa1898
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/8bfa1898
Branch: refs/heads/master
Commit: 8bfa189827803209ebc3bd8c1cee786ee140950d
Parents: c523b97
Author: Tarun Kumar <ag...@gmail.com>
Authored: Mon Jan 23 19:02:53 2017 +0530
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Jan 24 10:40:51 2017 -0800
----------------------------------------------------------------------
java/core/src/findbugs/exclude.xml | 15 ++++++++
.../src/java/org/apache/orc/DataReader.java | 2 +-
java/core/src/java/org/apache/orc/Reader.java | 23 ++++++-------
.../java/org/apache/orc/TypeDescription.java | 2 +-
.../java/org/apache/orc/impl/BufferChunk.java | 14 ++++++++
.../apache/orc/impl/ColumnStatisticsImpl.java | 4 +--
.../orc/impl/ConvertTreeReaderFactory.java | 5 ---
.../org/apache/orc/impl/DynamicByteArray.java | 2 +-
.../org/apache/orc/impl/HadoopShimsCurrent.java | 2 +-
.../org/apache/orc/impl/HadoopShims_2_2.java | 2 +-
.../org/apache/orc/impl/PhysicalFsWriter.java | 2 +-
.../java/org/apache/orc/impl/ReaderImpl.java | 13 ++++---
.../org/apache/orc/impl/RecordReaderImpl.java | 11 ++----
.../org/apache/orc/impl/RecordReaderUtils.java | 16 +++------
.../org/apache/orc/impl/SchemaEvolution.java | 2 +-
.../java/org/apache/orc/impl/StreamName.java | 2 +-
.../org/apache/orc/impl/TreeReaderFactory.java | 7 ----
.../java/org/apache/orc/impl/WriterImpl.java | 2 +-
.../java/org/apache/orc/util/BloomFilter.java | 25 +++++++++-----
java/pom.xml | 36 +++++++++++---------
java/tools/src/findbugs/exclude.xml | 19 +++++++++++
.../src/java/org/apache/orc/tools/FileDump.java | 7 ++--
.../java/org/apache/orc/tools/JsonFileDump.java | 2 +-
23 files changed, 127 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/findbugs/exclude.xml
----------------------------------------------------------------------
diff --git a/java/core/src/findbugs/exclude.xml b/java/core/src/findbugs/exclude.xml
index 2f1148d..1f079cb 100644
--- a/java/core/src/findbugs/exclude.xml
+++ b/java/core/src/findbugs/exclude.xml
@@ -16,4 +16,19 @@
<Match>
<Class name="~org\.apache\.orc\.OrcProto.*"/>
</Match>
+ <Match>
+ <Bug pattern="EI_EXPOSE_REP,EI_EXPOSE_REP2"/>
+ </Match>
+ <Match>
+ <Bug pattern="CN_IDIOM_NO_SUPER_CALL"/>
+ <Class name="org.apache.orc.TypeDescription"/>
+ </Match>
+ <Match>
+ <Bug pattern="SF_SWITCH_FALLTHROUGH,SF_SWITCH_NO_DEFAULT"/>
+ <Class name="org.apache.orc.util.Murmur3"/>
+ </Match>
+ <Match>
+ <Bug pattern="MS_PKGPROTECT"/>
+ <Class name="org.apache.orc.impl.RecordReaderImpl$SargApplier"/>
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/DataReader.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/DataReader.java b/java/core/src/java/org/apache/orc/DataReader.java
index b3f91f2..ae11bf3 100644
--- a/java/core/src/java/org/apache/orc/DataReader.java
+++ b/java/core/src/java/org/apache/orc/DataReader.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.common.io.DiskRangeList;
import org.apache.orc.impl.OrcIndex;
/** An abstract data reader that IO formats can use to read bytes from underlying storage. */
-public interface DataReader extends AutoCloseable {
+public interface DataReader extends AutoCloseable, Cloneable {
/** Opens the DataReader, making it ready to use. */
void open() throws IOException;
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/Reader.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java
index 3c9681c..2ef64d7 100644
--- a/java/core/src/java/org/apache/orc/Reader.java
+++ b/java/core/src/java/org/apache/orc/Reader.java
@@ -148,7 +148,7 @@ public interface Reader {
/**
* Options for creating a RecordReader.
*/
- public static class Options {
+ public static class Options implements Cloneable {
private boolean[] include;
private long offset = 0;
private long length = Long.MAX_VALUE;
@@ -311,18 +311,15 @@ public interface Reader {
}
public Options clone() {
- Options result = new Options();
- result.include = include;
- result.offset = offset;
- result.length = length;
- result.sarg = sarg;
- result.schema = schema;
- result.columnNames = columnNames;
- result.useZeroCopy = useZeroCopy;
- result.skipCorruptRecords = skipCorruptRecords;
- result.dataReader = dataReader == null ? null : dataReader.clone();
- result.tolerateMissingSchema = tolerateMissingSchema;
- return result;
+ try {
+ Options result = (Options) super.clone();
+ if (dataReader != null) {
+ result.dataReader = dataReader.clone();
+ }
+ return result;
+ } catch (CloneNotSupportedException e) {
+ throw new UnsupportedOperationException("uncloneable", e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/TypeDescription.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/TypeDescription.java b/java/core/src/java/org/apache/orc/TypeDescription.java
index 69d46b0..2086f3a 100644
--- a/java/core/src/java/org/apache/orc/TypeDescription.java
+++ b/java/core/src/java/org/apache/orc/TypeDescription.java
@@ -40,7 +40,7 @@ import java.util.regex.Pattern;
* This is the description of the types in an ORC file.
*/
public class TypeDescription
- implements Comparable<TypeDescription>, Serializable {
+ implements Comparable<TypeDescription>, Serializable, Cloneable {
private static final int MAX_PRECISION = 38;
private static final int MAX_SCALE = 38;
private static final int DEFAULT_PRECISION = 38;
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/BufferChunk.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/BufferChunk.java b/java/core/src/java/org/apache/orc/impl/BufferChunk.java
index da43b96..afde82f 100644
--- a/java/core/src/java/org/apache/orc/impl/BufferChunk.java
+++ b/java/core/src/java/org/apache/orc/impl/BufferChunk.java
@@ -79,6 +79,20 @@ public class BufferChunk extends DiskRangeList {
}
@Override
+ public boolean equals(Object other) {
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+ BufferChunk ob = (BufferChunk) other;
+ return chunk.equals(ob.chunk);
+ }
+
+ @Override
+ public int hashCode() {
+ return chunk.hashCode();
+ }
+
+ @Override
public ByteBuffer getData() {
return chunk;
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java b/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
index cdf9c46..a8596db 100644
--- a/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
@@ -569,7 +569,7 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
maximum = new Text();
maximum.set(bytes, offset, length);
}
- sum += length * repetitions;
+ sum += (long)length * repetitions;
}
@Override
@@ -711,7 +711,7 @@ public class ColumnStatisticsImpl implements ColumnStatistics {
@Override
public void updateBinary(byte[] bytes, int offset, int length,
int repetitions) {
- sum += length * repetitions;
+ sum += (long)length * repetitions;
}
@Override
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index e1415d3..ae43824 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -1343,8 +1343,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private DecimalColumnVector fileDecimalColVector;
private int filePrecision;
private int fileScale;
- private int readerPrecision;
- private int readerScale;
private DecimalColumnVector decimalColVector;
DecimalFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, Context context)
@@ -1352,8 +1350,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
super(columnId);
filePrecision = fileType.getPrecision();
fileScale = fileType.getScale();
- readerPrecision = readerType.getPrecision();
- readerScale = readerType.getScale();
decimalTreeReader = new DecimalTreeReader(columnId, context);
setConvertTreeReader(decimalTreeReader);
}
@@ -1524,7 +1520,6 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
private final TypeDescription readerType;
private DecimalColumnVector decimalColVector;
private BytesColumnVector bytesColVector;
- private byte[] scratchBuffer;
StringGroupFromDecimalTreeReader(int columnId, TypeDescription fileType,
TypeDescription readerType, Context context) throws IOException {
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/DynamicByteArray.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/DynamicByteArray.java b/java/core/src/java/org/apache/orc/impl/DynamicByteArray.java
index 986c2ac..96ea1ea 100644
--- a/java/core/src/java/org/apache/orc/impl/DynamicByteArray.java
+++ b/java/core/src/java/org/apache/orc/impl/DynamicByteArray.java
@@ -298,6 +298,6 @@ public final class DynamicByteArray {
* Get the size of the buffers.
*/
public long getSizeInBytes() {
- return initializedChunks * chunkSize;
+ return (long) initializedChunks * chunkSize;
}
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/HadoopShimsCurrent.java b/java/core/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
index 5c53f74..54a4824 100644
--- a/java/core/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
+++ b/java/core/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
@@ -71,7 +71,7 @@ public class HadoopShimsCurrent implements HadoopShims {
return ZeroCopyShims.getZeroCopyReader(in, pool);
}
- private final class FastTextReaderShim implements TextReaderShim {
+ private static final class FastTextReaderShim implements TextReaderShim {
private final DataInputStream din;
public FastTextReaderShim(InputStream in) {
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/HadoopShims_2_2.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/HadoopShims_2_2.java b/java/core/src/java/org/apache/orc/impl/HadoopShims_2_2.java
index 3f65e74..501101f 100644
--- a/java/core/src/java/org/apache/orc/impl/HadoopShims_2_2.java
+++ b/java/core/src/java/org/apache/orc/impl/HadoopShims_2_2.java
@@ -70,7 +70,7 @@ public class HadoopShims_2_2 implements HadoopShims {
return null;
}
- private final class BasicTextReaderShim implements TextReaderShim {
+ private static final class BasicTextReaderShim implements TextReaderShim {
private final InputStream in;
public BasicTextReaderShim(InputStream in) {
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
index 17c73ff..dd4e23b 100644
--- a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
+++ b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java
@@ -147,7 +147,7 @@ public class PhysicalFsWriter implements PhysicalWriter {
* An output receiver that writes the ByteBuffers to the output stream
* as they are received.
*/
- private class DirectStream implements OutputReceiver {
+ private static class DirectStream implements OutputReceiver {
private final FSDataOutputStream output;
DirectStream(FSDataOutputStream output) {
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
index 9c8c06b..ac5cfb2 100644
--- a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -690,11 +690,14 @@ public class ReaderImpl implements Reader {
if (fieldNames.contains(colName)) {
fieldIdx = fieldNames.indexOf(colName);
} else {
- String s = "Cannot find field for: " + colName + " in ";
+ StringBuilder s = new StringBuilder("Cannot find field for: ");
+ s.append(colName);
+ s.append(" in ");
for (String fn : fieldNames) {
- s += fn + ", ";
+ s.append(fn);
+ s.append(", ");
}
- LOG.warn(s);
+ LOG.warn(s.toString());
continue;
}
@@ -747,8 +750,10 @@ public class ReaderImpl implements Reader {
@Override
public List<StripeStatistics> getStripeStatistics() throws IOException {
- if (stripeStats == null && metadata == null) {
+ if (metadata == null) {
metadata = extractMetadata(tail.getSerializedTail(), 0, metadataSize, codec, bufferSize);
+ }
+ if (stripeStats == null) {
stripeStats = metadata.getStripeStatsList();
}
List<StripeStatistics> result = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
index bb6dd2d..2624cc7 100644
--- a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
@@ -411,9 +411,7 @@ public class RecordReaderImpl implements RecordReader {
// in case failed conversion, return the default YES_NO_NULL truth value
} catch (Exception e) {
if (LOG.isWarnEnabled()) {
- final String statsType = min == null ?
- (max == null ? "null" : max.getClass().getSimpleName()) :
- min.getClass().getSimpleName();
+ final String statsType = min.getClass().getSimpleName();
final String predicateType = baseObj == null ? "null" : baseObj.getClass().getSimpleName();
final String reason = e.getClass().getSimpleName() + " when evaluating predicate." +
" Skipping ORC PPD." +
@@ -682,10 +680,7 @@ public class RecordReaderImpl implements RecordReader {
}
break;
case STRING:
- if (obj != null) {
- return (obj.toString());
- }
- break;
+ return (obj.toString());
case TIMESTAMP:
if (obj instanceof Timestamp) {
return obj;
@@ -712,7 +707,7 @@ public class RecordReaderImpl implements RecordReader {
}
throw new IllegalArgumentException(String.format(
- "ORC SARGS could not convert from %s to %s", obj == null ? "(null)" : obj.getClass()
+ "ORC SARGS could not convert from %s to %s", obj.getClass()
.getSimpleName(), type));
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java b/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
index cadee35..1b8db99 100644
--- a/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
+++ b/java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java
@@ -149,16 +149,6 @@ public class RecordReaderUtils {
private final int bufferSize;
private final int typeCount;
- private DefaultDataReader(DefaultDataReader other) {
- this.pool = other.pool;
- this.bufferSize = other.bufferSize;
- this.typeCount = other.typeCount;
- this.fs = other.fs;
- this.path = other.path;
- this.useZeroCopy = other.useZeroCopy;
- this.codec = other.codec;
- }
-
private DefaultDataReader(DataReaderProperties properties) {
this.fs = properties.getFileSystem();
this.path = properties.getPath();
@@ -308,7 +298,11 @@ public class RecordReaderUtils {
@Override
public DataReader clone() {
- return new DefaultDataReader(this);
+ try {
+ return (DataReader) super.clone();
+ } catch (CloneNotSupportedException e) {
+ throw new UnsupportedOperationException("uncloneable", e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index f6f3ac7..a438db3 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -490,7 +490,7 @@ public class SchemaEvolution {
return typeDescription.getChildren().get(ACID_ROW_OFFSET);
}
- public static final List<String> acidEventFieldNames=
+ private static final List<String> acidEventFieldNames=
new ArrayList<String>();
static {
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/StreamName.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/StreamName.java b/java/core/src/java/org/apache/orc/impl/StreamName.java
index e3561bf..1e87788 100644
--- a/java/core/src/java/org/apache/orc/impl/StreamName.java
+++ b/java/core/src/java/org/apache/orc/impl/StreamName.java
@@ -53,7 +53,7 @@ public class StreamName implements Comparable<StreamName> {
Area area = getArea(kind);
Area otherArea = streamName.getArea(streamName.kind);
if (area != otherArea) {
- return -area.compareTo(otherArea);
+ return otherArea.compareTo(area);
}
if (column != streamName.column) {
return column < streamName.column ? -1 : 1;
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
index ccf2f29..0308393 100644
--- a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
@@ -96,7 +96,6 @@ public class TreeReaderFactory {
public abstract static class TreeReader {
protected final int columnId;
protected BitFieldReader present = null;
- protected boolean valuePresent = false;
protected int vectorColumnCount;
protected final Context context;
@@ -109,7 +108,6 @@ public class TreeReaderFactory {
this.context = context;
if (in == null) {
present = null;
- valuePresent = true;
} else {
present = new BitFieldReader(in, 1);
}
@@ -151,7 +149,6 @@ public class TreeReaderFactory {
OrcProto.Stream.Kind.PRESENT));
if (in == null) {
present = null;
- valuePresent = true;
} else {
present = new BitFieldReader(in, 1);
}
@@ -1369,7 +1366,6 @@ public class TreeReaderFactory {
public static class StringDirectTreeReader extends TreeReader {
private static final HadoopShims SHIMS = HadoopShims.Factory.get();
protected InStream stream;
- protected HadoopShims.TextReaderShim data;
protected IntegerReader lengths;
private final LongColumnVector scratchlcv;
@@ -1384,7 +1380,6 @@ public class TreeReaderFactory {
this.stream = data;
if (length != null && encoding != null) {
this.lengths = createIntegerReader(encoding, length, false, context);
- this.data = SHIMS.getTextReaderShim(this.stream);
}
}
@@ -1405,7 +1400,6 @@ public class TreeReaderFactory {
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
stream = streams.get(name);
- data = SHIMS.getTextReaderShim(this.stream);
lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new StreamName(columnId, OrcProto.Stream.Kind.LENGTH)),
false, context);
@@ -2146,7 +2140,6 @@ public class TreeReaderFactory {
Context context
) throws IOException {
final SchemaEvolution evolution = context.getSchemaEvolution();
- final boolean[] included = evolution.getReaderIncluded();
TypeDescription fileType = evolution.getFileType(readerType);
if (fileType == null || !evolution.includeReaderColumn(readerType.getId())){
return new NullTreeReader(0);
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/impl/WriterImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index 2779212..7766408 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -208,7 +208,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback {
// we want to guarantee that each stream gets ~10 buffers.
// This keeps buffers small enough that we don't get really small stripe
// sizes.
- int estBufferSize = (int) (stripeSize / (20 * numColumns));
+ int estBufferSize = (int) (stripeSize / (20L * numColumns));
estBufferSize = getClosestBufferSize(estBufferSize);
return estBufferSize > bs ? bs : estBufferSize;
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/core/src/java/org/apache/orc/util/BloomFilter.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/util/BloomFilter.java b/java/core/src/java/org/apache/orc/util/BloomFilter.java
index 0d9db24..d609d56 100644
--- a/java/core/src/java/org/apache/orc/util/BloomFilter.java
+++ b/java/core/src/java/org/apache/orc/util/BloomFilter.java
@@ -99,12 +99,13 @@ public class BloomFilter {
bitSet.equals(((BloomFilter) other).bitSet);
}
+ @Override
+ public int hashCode() {
+ return bitSet.hashCode() + numHashFunctions * 5;
+ }
+
public void add(byte[] val) {
- if (val == null) {
- addBytes(val, -1, -1);
- } else {
- addBytes(val, 0, val.length);
- }
+ addBytes(val, 0, val == null ? 0 : val.length);
}
public void addBytes(byte[] val, int offset, int length) {
@@ -151,10 +152,7 @@ public class BloomFilter {
}
public boolean test(byte[] val) {
- if (val == null) {
- return testBytes(val, -1, -1);
- }
- return testBytes(val, 0, val.length);
+ return testBytes(val, 0, val == null ? 0 : val.length);
}
public boolean testBytes(byte[] val, int offset, int length) {
@@ -324,5 +322,14 @@ public class BloomFilter {
other.getClass() == getClass() &&
Arrays.equals(data, ((BitSet) other).data);
}
+
+ @Override
+ public int hashCode() {
+ int result = 0;
+ for(long l: data) {
+ result = (int) (result * 13 + l);
+ }
+ return result;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 62310aa..2fd3008 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -145,6 +145,26 @@
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>2.9</version>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <version>3.0.3</version>
+ <configuration>
+ <effort>Max</effort>
+ <excludeFilterFile>${basedir}/src/findbugs/exclude.xml</excludeFilterFile>
+ <xmlOutput>true</xmlOutput>
+ <findbugsXmlOutputDirectory>${project.build.directory}/findbugs</findbugsXmlOutputDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <id>analyze-compile</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<pluginManagement>
<plugins>
@@ -203,22 +223,6 @@
</pluginManagement>
</build>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <version>3.0.3</version>
- <configuration>
- <effort>Max</effort>
- <excludeFilterFile>${basedir}/src/findbugs/exclude.xml</excludeFilterFile>
- <xmlOutput>true</xmlOutput>
- <findbugsXmlOutputDirectory>${project.build.directory}/findbugs</findbugsXmlOutputDirectory>
- </configuration>
- </plugin>
- </plugins>
- </reporting>
-
<profiles>
<profile>
<id>cmake</id>
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/tools/src/findbugs/exclude.xml
----------------------------------------------------------------------
diff --git a/java/tools/src/findbugs/exclude.xml b/java/tools/src/findbugs/exclude.xml
new file mode 100644
index 0000000..d6441f1
--- /dev/null
+++ b/java/tools/src/findbugs/exclude.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Bug pattern="EI_EXPOSE_REP,EI_EXPOSE_REP2"/>
+ </Match>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/tools/src/java/org/apache/orc/tools/FileDump.java
----------------------------------------------------------------------
diff --git a/java/tools/src/java/org/apache/orc/tools/FileDump.java b/java/tools/src/java/org/apache/orc/tools/FileDump.java
index 7206503..7b79ce5 100644
--- a/java/tools/src/java/org/apache/orc/tools/FileDump.java
+++ b/java/tools/src/java/org/apache/orc/tools/FileDump.java
@@ -18,6 +18,7 @@
package org.apache.orc.tools;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -432,11 +433,11 @@ public final class FileDump {
// find all MAGIC string and see if the file is readable from there
int index = 0;
long nextFooterOffset;
-
+ byte[] magicBytes = OrcFile.MAGIC.getBytes(StandardCharsets.UTF_8);
while (index != -1) {
- index = indexOf(data, OrcFile.MAGIC.getBytes(), index + 1);
+ index = indexOf(data, magicBytes, index + 1);
if (index != -1) {
- nextFooterOffset = startPos + index + OrcFile.MAGIC.length() + 1;
+ nextFooterOffset = startPos + index + magicBytes.length + 1;
if (isReadable(corruptPath, conf, nextFooterOffset)) {
footerOffsets.add(nextFooterOffset);
}
http://git-wip-us.apache.org/repos/asf/orc/blob/8bfa1898/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
----------------------------------------------------------------------
diff --git a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
index aa3072c..0de07ad 100644
--- a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
+++ b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
@@ -220,7 +220,7 @@ public class JsonFileDump {
rows.close();
writer.endObject();
- } catch (Exception e) {
+ } catch (Throwable e) {
writer.key("status").value("FAILED");
throw e;
}