You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by ta...@apache.org on 2018/02/20 21:06:06 UTC
incubator-hivemall git commit: Fix typo: Statefull -> Stateful
Repository: incubator-hivemall
Updated Branches:
refs/heads/master 3a718713a -> d902dc257
Fix typo: Statefull -> Stateful
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/d902dc25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/d902dc25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/d902dc25
Branch: refs/heads/master
Commit: d902dc25777ef5be37d7394bf57acf84d6f2836d
Parents: 3a71871
Author: Takuya Kitazawa <ta...@apache.org>
Authored: Wed Feb 21 06:05:44 2018 +0900
Committer: Takuya Kitazawa <ta...@apache.org>
Committed: Wed Feb 21 06:05:44 2018 +0900
----------------------------------------------------------------------
.../java/hivemall/GeneralLearnerBaseUDTF.java | 12 +--
.../hivemall/fm/FactorizationMachineUDTF.java | 14 ++--
.../main/java/hivemall/recommend/SlimUDTF.java | 12 +--
.../ProbabilisticTopicModelBaseUDTF.java | 12 +--
.../hivemall/utils/io/NioStatefulSegment.java | 80 ++++++++++++++++++++
.../hivemall/utils/io/NioStatefullSegment.java | 80 --------------------
6 files changed, 105 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
index f1bc045..2651da2 100644
--- a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
+++ b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java
@@ -34,7 +34,7 @@ import hivemall.utils.collections.IMapIterator;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.io.FileUtils;
import hivemall.utils.io.NIOUtils;
-import hivemall.utils.io.NioStatefullSegment;
+import hivemall.utils.io.NioStatefulSegment;
import hivemall.utils.lang.FloatAccumulator;
import hivemall.utils.lang.NumberUtils;
import hivemall.utils.lang.Primitives;
@@ -106,7 +106,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
// for iterations
@Nullable
- protected transient NioStatefullSegment fileIO;
+ protected transient NioStatefulSegment fileIO;
@Nullable
protected transient ByteBuffer inputBuf;
private int iterations;
@@ -304,7 +304,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
}
ByteBuffer buf = inputBuf;
- NioStatefullSegment dst = fileIO;
+ NioStatefulSegment dst = fileIO;
if (buf == null) {
final File file;
@@ -322,7 +322,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
throw new UDFArgumentException(e);
}
this.inputBuf = buf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MB
- this.fileIO = dst = new NioStatefullSegment(file, false);
+ this.fileIO = dst = new NioStatefulSegment(file, false);
}
int featureVectorBytes = 0;
@@ -416,7 +416,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
return featureVector;
}
- private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefullSegment dst)
+ private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefulSegment dst)
throws HiveException {
srcBuf.flip();
try {
@@ -536,7 +536,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF {
protected final void runIterativeTraining(@Nonnegative final int iterations)
throws HiveException {
final ByteBuffer buf = this.inputBuf;
- final NioStatefullSegment dst = this.fileIO;
+ final NioStatefulSegment dst = this.fileIO;
assert (buf != null);
assert (dst != null);
final long numTrainingExamples = count;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java b/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
index e1982a8..cb72d0f 100644
--- a/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
+++ b/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java
@@ -29,7 +29,7 @@ import hivemall.optimizer.LossFunctions.LossType;
import hivemall.utils.collections.Fastutil;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.io.FileUtils;
-import hivemall.utils.io.NioStatefullSegment;
+import hivemall.utils.io.NioStatefulSegment;
import hivemall.utils.lang.NumberUtils;
import hivemall.utils.lang.SizeOf;
import hivemall.utils.math.MathUtils;
@@ -111,7 +111,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions {
// file IO
private ByteBuffer _inputBuf;
- private NioStatefullSegment _fileIO;
+ private NioStatefulSegment _fileIO;
@Override
protected Options getOptions() {
@@ -304,7 +304,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions {
}
ByteBuffer inputBuf = _inputBuf;
- NioStatefullSegment dst = _fileIO;
+ NioStatefulSegment dst = _fileIO;
if (inputBuf == null) {
final File file;
try {
@@ -322,7 +322,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions {
}
this._inputBuf = inputBuf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MiB
- this._fileIO = dst = new NioStatefullSegment(file, false);
+ this._fileIO = dst = new NioStatefulSegment(file, false);
}
int xBytes = Feature.requiredBytes(x);
@@ -341,7 +341,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions {
inputBuf.putDouble(y);
}
- private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefullSegment dst)
+ private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefulSegment dst)
throws HiveException {
srcBuf.flip();
try {
@@ -404,7 +404,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions {
/**
* Update regularization parameters `lambda` as follows:
- *
+ *
* <pre>
* grad_lambdaw0 = (grad l(p,y)) * (-2 * alpha * w_0)
* grad_lambdawg = (grad l(p,y)) * (-2 * alpha * (\sum_{l \in group(g)} x_l * w_l))
@@ -536,7 +536,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions {
protected void runTrainingIteration(int iterations) throws HiveException {
final ByteBuffer inputBuf = this._inputBuf;
- final NioStatefullSegment fileIO = this._fileIO;
+ final NioStatefulSegment fileIO = this._fileIO;
assert (inputBuf != null);
assert (fileIO != null);
final long numTrainingExamples = _t;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/recommend/SlimUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/recommend/SlimUDTF.java b/core/src/main/java/hivemall/recommend/SlimUDTF.java
index 1689744..7977a38 100644
--- a/core/src/main/java/hivemall/recommend/SlimUDTF.java
+++ b/core/src/main/java/hivemall/recommend/SlimUDTF.java
@@ -27,7 +27,7 @@ import hivemall.math.vector.VectorProcedure;
import hivemall.utils.collections.Fastutil;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.io.FileUtils;
-import hivemall.utils.io.NioStatefullSegment;
+import hivemall.utils.io.NioStatefulSegment;
import hivemall.utils.lang.NumberUtils;
import hivemall.utils.lang.Primitives;
import hivemall.utils.lang.SizeOf;
@@ -139,7 +139,7 @@ public class SlimUDTF extends UDTFWithOptions {
private transient FloatMatrix _dataMatrix;
// used to store KNN data into temporary file for iterative training
- private transient NioStatefullSegment _fileIO;
+ private transient NioStatefulSegment _fileIO;
private transient ByteBuffer _inputBuf;
private ConversionState _cvState;
@@ -302,7 +302,7 @@ public class SlimUDTF extends UDTFWithOptions {
@Nonnull final Int2ObjectMap<Int2FloatMap> knnItems, final int numKNNItems)
throws HiveException {
ByteBuffer buf = this._inputBuf;
- NioStatefullSegment dst = this._fileIO;
+ NioStatefulSegment dst = this._fileIO;
if (buf == null) {
// invoke only at task node (initialize is also invoked in compilation)
@@ -319,7 +319,7 @@ public class SlimUDTF extends UDTFWithOptions {
}
this._inputBuf = buf = ByteBuffer.allocateDirect(8 * 1024 * 1024); // 8MB
- this._fileIO = dst = new NioStatefullSegment(file, false);
+ this._fileIO = dst = new NioStatefulSegment(file, false);
}
int recordBytes = SizeOf.INT + SizeOf.INT + SizeOf.INT * 2 * knnItems.size()
@@ -349,7 +349,7 @@ public class SlimUDTF extends UDTFWithOptions {
}
private static void writeBuffer(@Nonnull final ByteBuffer srcBuf,
- @Nonnull final NioStatefullSegment dst) throws HiveException {
+ @Nonnull final NioStatefulSegment dst) throws HiveException {
srcBuf.flip();
try {
dst.write(srcBuf);
@@ -488,7 +488,7 @@ public class SlimUDTF extends UDTFWithOptions {
private void runIterativeTraining() throws HiveException {
final ByteBuffer buf = this._inputBuf;
- final NioStatefullSegment dst = this._fileIO;
+ final NioStatefulSegment dst = this._fileIO;
assert (buf != null);
assert (dst != null);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java b/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java
index c3dab89..b740d47 100644
--- a/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java
+++ b/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java
@@ -23,7 +23,7 @@ import hivemall.annotations.VisibleForTesting;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.io.FileUtils;
import hivemall.utils.io.NIOUtils;
-import hivemall.utils.io.NioStatefullSegment;
+import hivemall.utils.io.NioStatefulSegment;
import hivemall.utils.lang.NumberUtils;
import hivemall.utils.lang.Primitives;
import hivemall.utils.lang.SizeOf;
@@ -76,7 +76,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions {
protected ListObjectInspector wordCountsOI;
// for iterations
- protected NioStatefullSegment fileIO;
+ protected NioStatefulSegment fileIO;
protected ByteBuffer inputBuf;
private float cumPerplexity;
@@ -188,7 +188,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions {
}
ByteBuffer buf = inputBuf;
- NioStatefullSegment dst = fileIO;
+ NioStatefulSegment dst = fileIO;
if (buf == null) {
final File file;
@@ -206,7 +206,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions {
throw new UDFArgumentException(e);
}
this.inputBuf = buf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MB
- this.fileIO = dst = new NioStatefullSegment(file, false);
+ this.fileIO = dst = new NioStatefulSegment(file, false);
}
// wordCounts length, wc1 length, wc1 string, wc2 length, wc2 string, ...
@@ -254,7 +254,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions {
miniBatchCount = 0;
}
- private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefullSegment dst)
+ private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefulSegment dst)
throws HiveException {
srcBuf.flip();
try {
@@ -289,7 +289,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions {
protected final void runIterativeTraining(@Nonnegative final int iterations)
throws HiveException {
final ByteBuffer buf = this.inputBuf;
- final NioStatefullSegment dst = this.fileIO;
+ final NioStatefulSegment dst = this.fileIO;
assert (buf != null);
assert (dst != null);
final long numTrainingExamples = model.getDocCount();
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java b/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java
new file mode 100644
index 0000000..95111c5
--- /dev/null
+++ b/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.utils.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.NotThreadSafe;
+
+@NotThreadSafe
+public final class NioStatefulSegment extends NioSegment {
+
+ private long curPos;
+
+ public NioStatefulSegment(File file) {
+ this(file, false);
+ }
+
+ public NioStatefulSegment(File file, boolean readOnly) {
+ super(file, readOnly);
+ this.curPos = 0L;
+ }
+
+ public void resetPosition() {
+ this.curPos = 0L;
+ }
+
+ public long getPosition() {
+ return curPos;
+ }
+
+ public void setPosition(long pos) {
+ this.curPos = pos;
+ }
+
+ public int read(@Nonnull ByteBuffer buf) throws IOException {
+ int bytes = super.read(curPos, buf);
+ this.curPos += bytes;
+ return bytes;
+ }
+
+ public int write(@Nonnull ByteBuffer buf) throws IOException {
+ int bytes = super.write(curPos, buf);
+ this.curPos += bytes;
+ return bytes;
+ }
+
+ @Override
+ public int read(long filePos, @Nonnull ByteBuffer buf) throws IOException {
+ int bytes = super.read(filePos, buf);
+ this.curPos = filePos + bytes;
+ return bytes;
+ }
+
+ @Override
+ public int write(long filePos, @Nonnull ByteBuffer buf) throws IOException {
+ int bytes = super.write(filePos, buf);
+ this.curPos = filePos + bytes;
+ return bytes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java b/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java
deleted file mode 100644
index 5074dca..0000000
--- a/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package hivemall.utils.io;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.NotThreadSafe;
-
-@NotThreadSafe
-public final class NioStatefullSegment extends NioSegment {
-
- private long curPos;
-
- public NioStatefullSegment(File file) {
- this(file, false);
- }
-
- public NioStatefullSegment(File file, boolean readOnly) {
- super(file, readOnly);
- this.curPos = 0L;
- }
-
- public void resetPosition() {
- this.curPos = 0L;
- }
-
- public long getPosition() {
- return curPos;
- }
-
- public void setPosition(long pos) {
- this.curPos = pos;
- }
-
- public int read(@Nonnull ByteBuffer buf) throws IOException {
- int bytes = super.read(curPos, buf);
- this.curPos += bytes;
- return bytes;
- }
-
- public int write(@Nonnull ByteBuffer buf) throws IOException {
- int bytes = super.write(curPos, buf);
- this.curPos += bytes;
- return bytes;
- }
-
- @Override
- public int read(long filePos, @Nonnull ByteBuffer buf) throws IOException {
- int bytes = super.read(filePos, buf);
- this.curPos = filePos + bytes;
- return bytes;
- }
-
- @Override
- public int write(long filePos, @Nonnull ByteBuffer buf) throws IOException {
- int bytes = super.write(filePos, buf);
- this.curPos = filePos + bytes;
- return bytes;
- }
-
-}