You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/07/23 07:41:54 UTC
svn commit: r796953 -
/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java
Author: stack
Date: Thu Jul 23 05:41:54 2009
New Revision: 796953
URL: http://svn.apache.org/viewvc?rev=796953&view=rev
Log:
HBASE-1215 fix for fact that we only run one map
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java?rev=796953&r1=796952&r2=796953&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java Thu Jul 23 05:41:54 2009
@@ -69,72 +69,85 @@
if (fs.exists(dir)) {
throw new IOException("Input exists -- please specify empty input dir");
}
- FSDataOutputStream out = fs.create(new Path(dir, "mapfiles"));
- try {
- gathermapfiles(conf, fs, out);
- } finally {
- out.close();
- }
+ gathermapfiles(conf, fs, dir);
}
private static void gathermapfiles(final HBaseConfiguration conf,
- final FileSystem fs, final FSDataOutputStream out)
+ final FileSystem fs, final Path dir)
throws IOException {
- // Presumes any directory under hbase.rootdir is a table.
- FileStatus [] tableDirs =
- fs.listStatus(FSUtils.getRootDir(conf), new DirFilter(fs));
- for (int i = 0; i < tableDirs.length; i++) {
- // Inside a table, there are compaction.dir directories to skip.
- // Otherwise, all else should be regions. Then in each region, should
- // only be family directories. Under each of these, should be a mapfile
- // and info directory and in these only one file.
- Path d = tableDirs[i].getPath();
- if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
- FileStatus [] regionDirs = fs.listStatus(d, new DirFilter(fs));
- for (int j = 0; j < regionDirs.length; j++) {
- Path dd = regionDirs[j].getPath();
- if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
- // Else its a region name. Now look in region for families.
- FileStatus [] familyDirs = fs.listStatus(dd, new DirFilter(fs));
- for (int k = 0; k < familyDirs.length; k++) {
- Path family = familyDirs[k].getPath();
- FileStatus [] infoAndMapfile = fs.listStatus(family);
- // Assert that only info and mapfile in family dir.
- if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
- LOG.warn(family.toString() +
- " has more than just info and mapfile: " + infoAndMapfile.length + ". Continuing...");
- continue;
- }
- // Make sure directory named info or mapfile.
- for (int ll = 0; ll < 2; ll++) {
- if (infoAndMapfile[ll].getPath().getName().equals("info") ||
- infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
+ int index = 0;
+ FSDataOutputStream out = getOut(fs, dir, index, null);
+ try {
+ // Presumes any directory under hbase.rootdir is a table.
+ FileStatus [] tableDirs =
+ fs.listStatus(FSUtils.getRootDir(conf), new DirFilter(fs));
+ for (int i = 0; i < tableDirs.length; i++) {
+ // Inside a table, there are compaction.dir directories to skip.
+ // Otherwise, all else should be regions. Then in each region, should
+ // only be family directories. Under each of these, should be a mapfile
+ // and info directory and in these only one file.
+ Path d = tableDirs[i].getPath();
+ if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+ FileStatus [] regionDirs = fs.listStatus(d, new DirFilter(fs));
+ for (int j = 0; j < regionDirs.length; j++) {
+ Path dd = regionDirs[j].getPath();
+ if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+ // Else its a region name. Now look in region for families.
+ FileStatus [] familyDirs = fs.listStatus(dd, new DirFilter(fs));
+ for (int k = 0; k < familyDirs.length; k++) {
+ Path family = familyDirs[k].getPath();
+ FileStatus [] infoAndMapfile = fs.listStatus(family);
+ // Assert that only info and mapfile in family dir.
+ if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
+ LOG.warn(family.toString() +
+ " has more than just info and mapfile: " + infoAndMapfile.length + ". Continuing...");
continue;
- LOG.warn("Unexpected directory name: " +
- infoAndMapfile[ll].getPath() + ". Continuing...");
- continue;
- }
- // Now in family, there are 'mapfile' and 'info' subdirs. Just
- // look in the 'mapfile' subdir.
- FileStatus [] familyStatus =
- fs.listStatus(new Path(family, "mapfiles"));
- if (familyStatus.length > 1) {
- LOG.warn(family.toString() + " has " + familyStatus.length +
+ }
+ // Make sure directory named info or mapfile.
+ for (int ll = 0; ll < 2; ll++) {
+ if (infoAndMapfile[ll].getPath().getName().equals("info") ||
+ infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
+ continue;
+ LOG.warn("Unexpected directory name: " +
+ infoAndMapfile[ll].getPath() + ". Continuing...");
+ continue;
+ }
+ // Now in family, there are 'mapfile' and 'info' subdirs. Just
+ // look in the 'mapfile' subdir.
+ FileStatus [] familyStatus =
+ fs.listStatus(new Path(family, "mapfiles"));
+ if (familyStatus.length > 1) {
+ LOG.warn(family.toString() + " has " + familyStatus.length +
" files. Continuing...");
- continue;
- }
- if (familyStatus.length == 1) {
- // If we got here, then this is good. Add the mapfile to out
- String str = familyStatus[0].getPath().makeQualified(fs).toString();
- LOG.info(str);
- out.write(Bytes.toBytes(str + "\n"));
- } else {
- LOG.warn("Empty store " + family.toString());
+ continue;
+ }
+ if (familyStatus.length == 1) {
+ // If we got here, then this is good. Add the mapfile to out
+ String str = familyStatus[0].getPath().makeQualified(fs).toString();
+ LOG.info(str);
+ out.write(Bytes.toBytes(str + "\n"));
+ if (index++ % 100 == 0) {
+ if (index != 0) {
+ out = getOut(fs, dir, index, out);
+ }
+ }
+ } else {
+ LOG.warn("Empty store " + family.toString());
+ }
}
}
}
+ } finally {
+ out.close();
}
}
+
+ private static FSDataOutputStream getOut(final FileSystem fs, final Path dir,
+ final int index, FSDataOutputStream out)
+ throws IOException {
+ if (out == null) out.close();
+ return fs.create(new Path(dir, "mapfiles-" + index));
+ }
public static void main(String[] args) throws Exception {
HBaseConfiguration conf = new HBaseConfiguration();
@@ -158,4 +171,4 @@
FileOutputFormat.setOutputPath(job, output);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
-}
+}
\ No newline at end of file