[GitHub] incubator-hivemall pull request #111: [HIVEMALL-17] Support SLIM

[nzw0301 <gi...@git.apache.org>]

Github user nzw0301 commented on a diff in the pull request:

    https://github.com/apache/incubator-hivemall/pull/111#discussion_r138523391
  
    --- Diff: core/src/main/java/hivemall/recommend/SlimUDTF.java ---
    @@ -0,0 +1,636 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *   http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing,
    + * software distributed under the License is distributed on an
    + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    + * KIND, either express or implied.  See the License for the
    + * specific language governing permissions and limitations
    + * under the License.
    + */
    +package hivemall.recommend;
    +
    +
    +import hivemall.UDTFWithOptions;
    +import hivemall.annotations.VisibleForTesting;
    +import hivemall.common.ConversionState;
    +import hivemall.math.matrix.sparse.DoKMatrix;
    +import hivemall.math.vector.VectorProcedure;
    +import hivemall.utils.collections.maps.Int2FloatOpenHashTable;
    +import hivemall.utils.hadoop.HiveUtils;
    +import hivemall.utils.io.FileUtils;
    +import hivemall.utils.io.NioStatefullSegment;
    +import hivemall.utils.lang.NumberUtils;
    +import hivemall.utils.lang.Primitives;
    +import hivemall.utils.lang.SizeOf;
    +import hivemall.utils.lang.mutable.MutableDouble;
    +import org.apache.commons.cli.CommandLine;
    +import org.apache.commons.cli.Options;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hive.ql.exec.Description;
    +import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
    +import org.apache.hadoop.hive.ql.metadata.Hive;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.serde2.objectinspector.*;
    +import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
    +import org.apache.hadoop.io.DoubleWritable;
    +import org.apache.hadoop.io.IntWritable;
    +import org.apache.hadoop.mapred.Counters;
    +import org.apache.hadoop.mapred.Reporter;
    +
    +import javax.annotation.Nonnull;
    +import java.io.File;
    +import java.io.IOException;
    +import java.nio.ByteBuffer;
    +import java.util.*;
    +
    +
    +@Description(
    +        name = "train_slim",
    +        value = "_FUNC_( int i, map<int, double> r_i, map<int, map<int, double>> topKRatesOfI, int j, map<int, double> r_j [, constant string options]) " +
    +                "- Returns row index, column index and non-zero weight value of prediction model")
    +public class SlimUDTF extends UDTFWithOptions {
    +    private static final Log logger = LogFactory.getLog(SlimUDTF.class);
    +
    +    private double l1;
    +    private double l2;
    +    private int numIterations;
    +    private int previousItemId;
    +
    +    private transient DoKMatrix weightMatrix; // item-item weight matrix
    +    private transient DoKMatrix dataMatrix; // item-user matrix to get the number of nnz values in column
    +
    +    private PrimitiveObjectInspector itemIOI;
    +    private PrimitiveObjectInspector itemJOI;
    +    private MapObjectInspector riOI;
    +    private MapObjectInspector rjOI;
    +
    +    private MapObjectInspector knnItemsOI;
    +    private PrimitiveObjectInspector knnItemsKeyOI;
    +    private MapObjectInspector knnItemsValueOI;
    +    private PrimitiveObjectInspector knnItemsValueKeyOI;
    +    private PrimitiveObjectInspector knnItemsValueValueOI;
    +
    +    private PrimitiveObjectInspector riKeyOI;
    +    private PrimitiveObjectInspector riValueOI;
    +
    +    private PrimitiveObjectInspector rjKeyOI;
    +    private PrimitiveObjectInspector rjValueOI;
    +
    +    // used to store KNN data into temporary file for iterative training
    +    private NioStatefullSegment fileIO;
    +    private ByteBuffer inputBuf;
    +
    +    private ConversionState cvState;
    +    private long observedTrainingExamples;
    +
    +    public SlimUDTF() {}
    +
    +    @Override
    +    public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    +        final int numArgs = argOIs.length;
    +
    +        if (numArgs == 1 && HiveUtils.isStringOI(argOIs[0])) {
    --- End diff --
    
    I add this a few line to show the slim explanation without other arguments. But it may not be conventional way to show `help` for hivemall.


---