You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@sqoop.apache.org by Matthieu Labour <ma...@actionx.com> on 2013/03/15 18:36:50 UTC
sqoop-1.4.2 --export-dir
Hi
I am using sqoop-1.4.2 to export to Sql. Thank you for creating this too.
My question concerns the use of --export-dir. It seems that --export-dir
does not work when the dir being passed is the root of subdirectories.
Please can you advise? Thank you.
Example:
The following command exports the data to Sql
~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:XXXX/xxxxxxxxxxxxx
--username xxxxxxxxxx --password xxxxxxxxxx --table
ml_ys_log_gmt_daily_experiment_2 --export-dir
=hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
--input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
--batch
hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
Found 1 items
-rw-r--r-- 1 hadoop supergroup 15931406 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01/part-r-00001
The following command does not export the data to Sql
~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:XXXX/xxxxxxxxxxxxx
--username xxxxxxxxxx --password xxxxxxxxxx --table
ml_ys_log_gmt_daily_experiment_2 --export-dir
=hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized
--input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
--batch
hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/
Found 44 items
-rw-r--r-- 1 hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/_SUCCESS
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-02
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-03
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-04
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-05
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-06
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-07
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-08
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-09
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-10
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-11
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-12
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-13
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-14
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-15
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-16
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-17
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-18
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-19
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-20
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-21
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-22
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-23
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-24
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-25
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-26
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-27
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-28
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-01
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-02
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-03
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-04
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-05
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-06
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-07
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-08
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-09
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-10
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-11
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-12
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-13
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-14
drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
/mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-15
Re: sqoop-1.4.2 --export-dir
Posted by Jarek Jarcec Cecho <ja...@apache.org>.
Please do not hesitate and file a JIRA [1] for this feature request!
Jarcec
Links:
1: https://issues.apache.org/jira/browse/SQOOP
On Fri, Mar 15, 2013 at 02:38:48PM -0400, Matthieu Labour wrote:
> Jarcec,
> Thank you for your answer.
> Is it something you plan to support in the future?
> Cheers
> -matt
>
> On Fri, Mar 15, 2013 at 2:05 PM, Jarek Jarcec Cecho <ja...@apache.org>wrote:
>
> > Hi Matthieu,
> > You're right, parameter --export-dir is not doing any recursive lookup for
> > files. It expects directory with files that you want export.
> >
> > Jarcec
> >
> > On Fri, Mar 15, 2013 at 01:36:50PM -0400, Matthieu Labour wrote:
> > > Hi
> > >
> > > I am using sqoop-1.4.2 to export to Sql. Thank you for creating this too.
> > >
> > > My question concerns the use of --export-dir. It seems that --export-dir
> > > does not work when the dir being passed is the root of subdirectories.
> > > Please can you advise? Thank you.
> > >
> > > Example:
> > >
> > > The following command exports the data to Sql
> > >
> > > ~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
> > > jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:
> > XXXX/xxxxxxxxxxxxx
> > > --username xxxxxxxxxx --password xxxxxxxxxx --table
> > > ml_ys_log_gmt_daily_experiment_2 --export-dir
> > > =hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> > > --input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
> > > --batch
> > >
> > >
> > > hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
> > > hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> > > Found 1 items
> > > -rw-r--r-- 1 hadoop supergroup 15931406 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01/part-r-00001
> > >
> > > The following command does not export the data to Sql
> > >
> > > ~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
> > > jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:
> > XXXX/xxxxxxxxxxxxx
> > > --username xxxxxxxxxx --password xxxxxxxxxx --table
> > > ml_ys_log_gmt_daily_experiment_2 --export-dir
> > > =hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized
> > > --input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
> > > --batch
> > >
> > > hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
> > > hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/
> > > Found 44 items
> > > -rw-r--r-- 1 hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/_SUCCESS
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-02
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-03
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-04
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-05
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-06
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-07
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-08
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-09
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-10
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-11
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-12
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-13
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-14
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-15
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-16
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-17
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-18
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-19
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-20
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-21
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-22
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-23
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-24
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-25
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-26
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-27
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-28
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-01
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-02
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-03
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-04
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-05
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-06
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-07
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-08
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-09
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-10
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-11
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-12
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-13
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-14
> > > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-15
> >
>
>
>
> --
> Matthieu Labour, Engineering | *Action**X* |
> 584 Broadway, Suite 1002 – NY, NY 10012
> 415-994-3480 (m)
Re: sqoop-1.4.2 --export-dir
Posted by Matthieu Labour <ma...@actionx.com>.
Jarcec,
Thank you for your answer.
Is it something you plan to support in the future?
Cheers
-matt
On Fri, Mar 15, 2013 at 2:05 PM, Jarek Jarcec Cecho <ja...@apache.org>wrote:
> Hi Matthieu,
> You're right, parameter --export-dir is not doing any recursive lookup for
> files. It expects directory with files that you want export.
>
> Jarcec
>
> On Fri, Mar 15, 2013 at 01:36:50PM -0400, Matthieu Labour wrote:
> > Hi
> >
> > I am using sqoop-1.4.2 to export to Sql. Thank you for creating this too.
> >
> > My question concerns the use of --export-dir. It seems that --export-dir
> > does not work when the dir being passed is the root of subdirectories.
> > Please can you advise? Thank you.
> >
> > Example:
> >
> > The following command exports the data to Sql
> >
> > ~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
> > jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:
> XXXX/xxxxxxxxxxxxx
> > --username xxxxxxxxxx --password xxxxxxxxxx --table
> > ml_ys_log_gmt_daily_experiment_2 --export-dir
> > =hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> > --input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
> > --batch
> >
> >
> > hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
> > hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> > Found 1 items
> > -rw-r--r-- 1 hadoop supergroup 15931406 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01/part-r-00001
> >
> > The following command does not export the data to Sql
> >
> > ~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
> > jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:
> XXXX/xxxxxxxxxxxxx
> > --username xxxxxxxxxx --password xxxxxxxxxx --table
> > ml_ys_log_gmt_daily_experiment_2 --export-dir
> > =hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized
> > --input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
> > --batch
> >
> > hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
> > hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/
> > Found 44 items
> > -rw-r--r-- 1 hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/_SUCCESS
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-02
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-03
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-04
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-05
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-06
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-07
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-08
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-09
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-10
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-11
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-12
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-13
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-14
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-15
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-16
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-17
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-18
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-19
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-20
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-21
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-22
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-23
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-24
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-25
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-26
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-27
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-28
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-01
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-02
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-03
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-04
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-05
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-06
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-07
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-08
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-09
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-10
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-11
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-12
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-13
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-14
> > drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> > /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-15
>
--
Matthieu Labour, Engineering | *Action**X* |
584 Broadway, Suite 1002 – NY, NY 10012
415-994-3480 (m)
Re: sqoop-1.4.2 --export-dir
Posted by Jarek Jarcec Cecho <ja...@apache.org>.
Hi Matthieu,
You're right, parameter --export-dir is not doing any recursive lookup for files. It expects directory with files that you want export.
Jarcec
On Fri, Mar 15, 2013 at 01:36:50PM -0400, Matthieu Labour wrote:
> Hi
>
> I am using sqoop-1.4.2 to export to Sql. Thank you for creating this too.
>
> My question concerns the use of --export-dir. It seems that --export-dir
> does not work when the dir being passed is the root of subdirectories.
> Please can you advise? Thank you.
>
> Example:
>
> The following command exports the data to Sql
>
> ~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
> jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:XXXX/xxxxxxxxxxxxx
> --username xxxxxxxxxx --password xxxxxxxxxx --table
> ml_ys_log_gmt_daily_experiment_2 --export-dir
> =hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> --input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
> --batch
>
>
> hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
> hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> Found 1 items
> -rw-r--r-- 1 hadoop supergroup 15931406 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01/part-r-00001
>
> The following command does not export the data to Sql
>
> ~/sqoop-1.4.2.bin__hadoop-1.0.0/bin/sqoop export --connect
> jdbc:postgresql://ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com:XXXX/xxxxxxxxxxxxx
> --username xxxxxxxxxx --password xxxxxxxxxx --table
> ml_ys_log_gmt_daily_experiment_2 --export-dir
> =hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized
> --input-fields-terminated-by='\t' --lines-terminated-by='\n' --verbose
> --batch
>
> hadoop@domU-XX-XX-XX-XX-XX-XX:/mnt/var/lib/hadoop/steps/2$ hadoop fs -ls
> hdfs:///mnt/var/lib/hadoop/dfs/logs_daily_sanitized/
> Found 44 items
> -rw-r--r-- 1 hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/_SUCCESS
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-01
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-02
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-03
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-04
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-05
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-06
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-07
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-08
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-09
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-10
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-11
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-12
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-13
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-14
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-15
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-16
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-17
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-18
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-19
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-20
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-21
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:04
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-22
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-23
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-24
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-25
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-26
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-27
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-02-28
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-01
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-02
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-03
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-04
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-05
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-06
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-07
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-08
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-09
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-10
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-11
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-12
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-13
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-14
> drwxr-xr-x - hadoop supergroup 0 2013-03-15 17:03
> /mnt/var/lib/hadoop/dfs/logs_daily_sanitized/dt=2013-03-15