You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@oozie.apache.org by munna hadoop <mu...@gmail.com> on 2013/12/24 18:14:50 UTC
Fwd: Sqoop action mapper issue with Oozie
Hi All,
I am using Hadoop 2.0.0+YARN+Sqoop-1.4.3+Oozie-3.3.2. I am facing a issues
with Sqoop action in Oozie.
while invoking sqoop action from oozie it is taking only one Mapper. We
have set this as 3 mapper.
Same job when I executed from linux shell(sqoop job –exec <jobname>), it is
taken actual number of mappers.
please help us to resolve this issue.
Please find the below configuration details:
*Workflow.xml*
<?xml version="1.0" encoding="UTF-8"?>
<workflow-app xmlns='uri:oozie:workflow:0.4' name='sqoop-workflow'>
<start to='sqoop-load' />
<action name="sqoop-load">
<sqoop xmlns="uri:oozie:sqoop-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<job-xml>sqoop.xml</job-xml>
<configuration>
<property>
<name>mapreduce.job.queuename</name>
<value>${queueName}</value>
</property>
</configuration>
<arg>job</arg>
<arg>--exec</arg>
<arg>emp.tag.rep</arg>
<arg>--</arg>
<arg>--num-mappers</arg>
<arg>3</arg>
</sqoop>
<ok to="end"/>
<error to="sqoop-load-fail"/>
</action>
<kill name="sqoop-load-fail">
<message>Sqoop export failed, error
message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<end name='end' />
</workflow-app>
*Job.properties*
nameNode=hdfs://hostname:8020
jobTracker=hostname.com:8032
queueName=default
oozie.use.system.libpath=true
oozie.libpath=/user/oozie/share/lib
oozie.wf.application.path=${nameNode}/user/shaik/jobs/sqoop_poc
*hive-site.xml*
<property>
<name>sqoop.metastore.client.record.password</name>
<value>true</value>
<description>If true, allow saved passwords in the metastore.
</description>
</property>
<property>
<name>sqoop.metastore.server.port</name>
<value>16000</value>
<description>Port that this metastore should listen on.
</description>
</property>
<property>
<name>sqoop.metastore.client.autoconnect.url</name>
<value>jdbc:hsqldb:hsql://hostname:16000/sqoop</value>
<description>The connect string to use when connecting to a
job-management metastore. If unspecified, uses ~/.sqoop/.
You can specify a different path here.
</description>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://hostname/hive</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
<description>Driver class name for a JDBC
metastore</description>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>hive</value>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>hive</value>
</property>
<property>
<name>datanucleus.autoCreateSchema</name>
<value>false</value>
</property>
<property>
<name>datanucleus.fixedDatastore</name>
<value>true</value>
</property>
<property>
<name>hive.metastore.uris</name>
<value>thrift://hostname:9083</value>
</property>
Thank you,
Munna
Re: Sqoop action mapper issue with Oozie
Posted by munna hadoop <mu...@gmail.com>.
Please help on below issue....
On 24 December 2013 22:44, munna hadoop <mu...@gmail.com> wrote:
> Hi All,
>
>
>
> I am using Hadoop 2.0.0+YARN+Sqoop-1.4.3+Oozie-3.3.2. I am facing a issues
> with Sqoop action in Oozie.
>
>
>
> while invoking sqoop action from oozie it is taking only one Mapper. We
> have set this as 3 mapper.
>
>
>
> Same job when I executed from linux shell(sqoop job –exec <jobname>), it
> is taken actual number of mappers.
>
>
> please help us to resolve this issue.
>
>
>
> Please find the below configuration details:
>
>
>
> *Workflow.xml*
>
>
>
> <?xml version="1.0" encoding="UTF-8"?>
>
> <workflow-app xmlns='uri:oozie:workflow:0.4' name='sqoop-workflow'>
>
> <start to='sqoop-load' />
>
>
>
> <action name="sqoop-load">
>
> <sqoop xmlns="uri:oozie:sqoop-action:0.2">
>
> <job-tracker>${jobTracker}</job-tracker>
>
> <name-node>${nameNode}</name-node>
>
> <job-xml>sqoop.xml</job-xml>
>
> <configuration>
>
> <property>
>
>
> <name>mapreduce.job.queuename</name>
>
> <value>${queueName}</value>
>
> </property>
>
> </configuration>
>
> <arg>job</arg>
>
> <arg>--exec</arg>
>
> <arg>emp.tag.rep</arg>
>
> <arg>--</arg>
>
> <arg>--num-mappers</arg>
>
> <arg>3</arg>
>
> </sqoop>
>
> <ok to="end"/>
>
> <error to="sqoop-load-fail"/>
>
> </action>
>
> <kill name="sqoop-load-fail">
>
> <message>Sqoop export failed, error
> message[${wf:errorMessage(wf:lastErrorNode())}]</message>
>
> </kill>
>
> <end name='end' />
>
> </workflow-app>
>
>
>
> *Job.properties*
>
>
>
> nameNode=hdfs://hostname:8020
>
> jobTracker=hostname.com:8032
>
> queueName=default
>
>
>
> oozie.use.system.libpath=true
>
> oozie.libpath=/user/oozie/share/lib
>
>
>
> oozie.wf.application.path=${nameNode}/user/shaik/jobs/sqoop_poc
>
>
>
> *hive-site.xml*
>
>
>
> <property>
>
> <name>sqoop.metastore.client.record.password</name>
>
> <value>true</value>
>
> <description>If true, allow saved passwords in the metastore.
>
> </description>
>
> </property>
>
>
>
> <property>
>
> <name>sqoop.metastore.server.port</name>
>
> <value>16000</value>
>
> <description>Port that this metastore should listen on.
>
> </description>
>
> </property>
>
>
>
> <property>
>
> <name>sqoop.metastore.client.autoconnect.url</name>
>
> <value>jdbc:hsqldb:hsql://hostname:16000/sqoop</value>
>
> <description>The connect string to use when connecting to a
>
> job-management metastore. If unspecified, uses ~/.sqoop/.
>
> You can specify a different path here.
>
> </description>
>
> </property>
>
>
>
> <property>
>
> <name>javax.jdo.option.ConnectionURL</name>
>
> <value>jdbc:mysql://hostname/hive</value>
>
> </property>
>
>
>
> <property>
>
> <name>javax.jdo.option.ConnectionDriverName</name>
>
> <value>com.mysql.jdbc.Driver</value>
>
> <description>Driver class name for a JDBC
> metastore</description>
>
> </property>
>
>
>
> <property>
>
> <name>javax.jdo.option.ConnectionUserName</name>
>
> <value>hive</value>
>
> </property>
>
>
>
> <property>
>
> <name>javax.jdo.option.ConnectionPassword</name>
>
> <value>hive</value>
>
> </property>
>
>
>
> <property>
>
> <name>datanucleus.autoCreateSchema</name>
>
> <value>false</value>
>
> </property>
>
>
>
> <property>
>
> <name>datanucleus.fixedDatastore</name>
>
> <value>true</value>
>
> </property>
>
>
>
> <property>
>
> <name>hive.metastore.uris</name>
>
> <value>thrift://hostname:9083</value>
>
> </property>
>
>
>
>
> Thank you,
>
> Munna
>