You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "Feng Yuan (JIRA)" <ji...@apache.org> on 2017/09/11 06:07:01 UTC
[jira] [Updated] (HIVE-17498) Does hive have mr-nativetask support
refer to MAPREDUCE-2841
[ https://issues.apache.org/jira/browse/HIVE-17498?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Feng Yuan updated HIVE-17498:
-----------------------------
Description:
I try to implement a HivePlatform extends org.apache.hadoop.mapred.nativetask.Platform.
{code}
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred.nativetask;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class HivePlatform extends Platform {
private static final Logger LOG = Logger.getLogger(HivePlatform.class);
public HivePlatform() {
}
@Override
public void init() throws IOException {
registerKey("org.apache.hadoop.hive.ql.io.HiveKey", HiveKeySerializer.class);
LOG.info("Hive platform inited");
}
@Override
public String name() {
return "Hive";
}
@Override
public boolean support(String keyClassName, INativeSerializer serializer, JobConf job) {
if (keyClassNames.contains(keyClassName) && serializer instanceof INativeComparable) {
String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "." + keyClassName;
job.set(nativeComparator, "HivePlatform.HivePlatform::HiveKeyComparator");
if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN, "HivePlatform=libnativetask.so");
}
return true;
} else {
return false;
}
}
@Override
public boolean define(Class comparatorClass) {
return false;
}
public static class HiveKeySerializer implements INativeComparable, INativeSerializer<HiveKey> {
public HiveKeySerializer() throws ClassNotFoundException, SecurityException, NoSuchMethodException {
}
@Override
public int getLength(HiveKey w) throws IOException {
return 4 + w.getLength();
}
@Override
public void serialize(HiveKey w, DataOutput out) throws IOException {
w.write(out);
}
@Override
public void deserialize(DataInput in, int length, HiveKey w ) throws IOException {
w.readFields(in);
}
}
}
{code}
and throws exceptions:
{code}
Error: java.io.IOException: Initialization of all the collectors failed. Error in last collector was :Native output collector cannot be loaded; at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.io.IOException: Native output collector cannot be loaded; at org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165) at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402) ... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is NULL, or comparator is not set /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76) [0x7ffdcbba6436] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector4initEjjPFiPKcjS2_jEPNS_14ICombineRunnerE+0x36a) [0x7ffdcbb9ad6a] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector9configureEPNS_6ConfigE+0x24a) [0x7ffdcbb9b37a] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask23MCollectorOutputHandler9configureEPNS_6ConfigE+0x80) [0x7ffdcbb91b40] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask12BatchHandler7onSetupEPNS_6ConfigEPcjS3_j+0xe9) [0x7ffdcbb90b29] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_setupHandler+0x1fe) [0x7ffdcbb90e0e] [0x7ffde9017a34] at org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.setupHandler(Native Method) at org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.init(NativeBatchProcessor.java:122) at org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.create(NativeBatchProcessor.java:91) at org.apache.hadoop.mapred.nativetask.handlers.NativeCollectorOnlyHandler.create(NativeCollectorOnlyHandler.java:85) at org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:161) ... 8 more
{code}
My environments are:
hadoop 2.7.3 compile base on:
{code}
Linux x.x.x.x 2.6.32-220.el6.x86_64 #1 SMP Tue Dec 6 19:48:22 GMT 2011 x86_64 x86_64 x86_64 GNU/Linux
{code}
compile command:
{code}
mvn package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist,native -Dtar -Drequire.snappy -Dbundle.snappy -Dsnappy.lib=/usr/local/lib
{code}
hive running parameter:
{code}
set mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;
set mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
{code}
was:
I try to implement a HivePlatform extends org.apache.hadoop.mapred.nativetask.Platform.
{code}
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
@Public
@Evolving
public abstract class Platform {
private final NativeSerialization serialization = NativeSerialization.getInstance();
protected Set<String> keyClassNames = new HashSet();
public Platform() {
}
public abstract void init() throws IOException;
public abstract String name();
protected void registerKey(String keyClassName, Class<?> key) throws IOException {
this.serialization.register(keyClassName, key);
this.keyClassNames.add(keyClassName);
}
protected abstract boolean support(String var1, INativeSerializer<?> var2, JobConf var3);
protected abstract boolean define(Class<?> var1);
}
{code}
> Does hive have mr-nativetask support refer to MAPREDUCE-2841
> ------------------------------------------------------------
>
> Key: HIVE-17498
> URL: https://issues.apache.org/jira/browse/HIVE-17498
> Project: Hive
> Issue Type: Bug
> Components: Hive
> Reporter: Feng Yuan
>
> I try to implement a HivePlatform extends org.apache.hadoop.mapred.nativetask.Platform.
> {code}
> /**
> * Licensed to the Apache Software Foundation (ASF) under one
> * or more contributor license agreements. See the NOTICE file
> * distributed with this work for additional information
> * regarding copyright ownership. The ASF licenses this file
> * to you under the Apache License, Version 2.0 (the
> * "License"); you may not use this file except in compliance
> * with the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.hadoop.mapred.nativetask;
> import org.apache.hadoop.hive.ql.io.HiveKey;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
> import org.apache.log4j.Logger;
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> public class HivePlatform extends Platform {
> private static final Logger LOG = Logger.getLogger(HivePlatform.class);
> public HivePlatform() {
> }
> @Override
> public void init() throws IOException {
> registerKey("org.apache.hadoop.hive.ql.io.HiveKey", HiveKeySerializer.class);
> LOG.info("Hive platform inited");
> }
> @Override
> public String name() {
> return "Hive";
> }
> @Override
> public boolean support(String keyClassName, INativeSerializer serializer, JobConf job) {
> if (keyClassNames.contains(keyClassName) && serializer instanceof INativeComparable) {
> String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "." + keyClassName;
> job.set(nativeComparator, "HivePlatform.HivePlatform::HiveKeyComparator");
> if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
> job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN, "HivePlatform=libnativetask.so");
> }
> return true;
> } else {
> return false;
> }
> }
> @Override
> public boolean define(Class comparatorClass) {
> return false;
> }
> public static class HiveKeySerializer implements INativeComparable, INativeSerializer<HiveKey> {
> public HiveKeySerializer() throws ClassNotFoundException, SecurityException, NoSuchMethodException {
> }
> @Override
> public int getLength(HiveKey w) throws IOException {
> return 4 + w.getLength();
> }
> @Override
> public void serialize(HiveKey w, DataOutput out) throws IOException {
> w.write(out);
> }
> @Override
> public void deserialize(DataInput in, int length, HiveKey w ) throws IOException {
> w.readFields(in);
> }
> }
> }
> {code}
> and throws exceptions:
> {code}
> Error: java.io.IOException: Initialization of all the collectors failed. Error in last collector was :Native output collector cannot be loaded; at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.io.IOException: Native output collector cannot be loaded; at org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165) at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402) ... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is NULL, or comparator is not set /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76) [0x7ffdcbba6436] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector4initEjjPFiPKcjS2_jEPNS_14ICombineRunnerE+0x36a) [0x7ffdcbb9ad6a] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask18MapOutputCollector9configureEPNS_6ConfigE+0x24a) [0x7ffdcbb9b37a] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask23MCollectorOutputHandler9configureEPNS_6ConfigE+0x80) [0x7ffdcbb91b40] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask12BatchHandler7onSetupEPNS_6ConfigEPcjS3_j+0xe9) [0x7ffdcbb90b29] /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(Java_org_apache_hadoop_mapred_nativetask_NativeBatchProcessor_setupHandler+0x1fe) [0x7ffdcbb90e0e] [0x7ffde9017a34] at org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.setupHandler(Native Method) at org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.init(NativeBatchProcessor.java:122) at org.apache.hadoop.mapred.nativetask.NativeBatchProcessor.create(NativeBatchProcessor.java:91) at org.apache.hadoop.mapred.nativetask.handlers.NativeCollectorOnlyHandler.create(NativeCollectorOnlyHandler.java:85) at org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:161) ... 8 more
> {code}
> My environments are:
> hadoop 2.7.3 compile base on:
> {code}
> Linux x.x.x.x 2.6.32-220.el6.x86_64 #1 SMP Tue Dec 6 19:48:22 GMT 2011 x86_64 x86_64 x86_64 GNU/Linux
> {code}
> compile command:
> {code}
> mvn package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist,native -Dtar -Drequire.snappy -Dbundle.snappy -Dsnappy.lib=/usr/local/lib
> {code}
> hive running parameter:
> {code}
> set mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;
> set mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)