You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2009/01/07 05:17:51 UTC
svn commit: r732218 - in /incubator/hama/trunk/src/java/org/apache/hama:
AbstractMatrix.java DenseMatrix.java HamaAdminImpl.java
Author: edwardyoon
Date: Tue Jan 6 20:17:51 2009
New Revision: 732218
URL: http://svn.apache.org/viewvc?rev=732218&view=rev
Log:
Added "while (admin.isTableEnabled(matrixPath)) { disable table }" to perpectly delete, Changed that table autoflush is true.
Modified:
incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java
Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=732218&r1=732217&r2=732218&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Tue Jan 6 20:17:51 2009
@@ -22,9 +22,11 @@
import java.io.IOException;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.RegionException;
+import org.apache.hadoop.hbase.HColumnDescriptor.CompressionType;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.BatchUpdate;
@@ -33,8 +35,6 @@
import org.apache.hama.io.VectorUpdate;
import org.apache.hama.util.BytesUtil;
import org.apache.log4j.Logger;
-import org.apache.hadoop.hbase.HColumnDescriptor.CompressionType;
-import org.apache.hadoop.hbase.HConstants;
/**
* Methods of the matrix classes
@@ -76,10 +76,9 @@
this.tableDesc.addFamily(new HColumnDescriptor(Constants.COLUMN));
this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
this.tableDesc.addFamily(new HColumnDescriptor(Constants.ALIASEFAMILY));
- this.tableDesc.addFamily(new HColumnDescriptor(
- Bytes.toBytes(Constants.BLOCK), 1, CompressionType.NONE,
- false, false, Integer.MAX_VALUE, HConstants.FOREVER, false
- ));
+ this.tableDesc.addFamily(new HColumnDescriptor(Bytes
+ .toBytes(Constants.BLOCK), 1, CompressionType.NONE, false, false,
+ Integer.MAX_VALUE, HConstants.FOREVER, false));
LOG.info("Initializing the matrix storage.");
this.admin.createTable(this.tableDesc);
@@ -87,6 +86,8 @@
// connect to the table.
table = new HTable(config, matrixPath);
+ table.setAutoFlush(true);
+
// Record the matrix type in METADATA_TYPE
BatchUpdate update = new BatchUpdate(Constants.METADATA);
update.put(Constants.METADATA_TYPE, Bytes.toBytes(this.getClass()
@@ -102,7 +103,7 @@
public HTable getHTable() {
return this.table;
}
-
+
/** {@inheritDoc} */
public int getRows() throws IOException {
Cell rows = null;
@@ -121,7 +122,7 @@
VectorUpdate update = new VectorUpdate(i);
update.put(j, value);
table.commit(update.getBatchUpdate());
- table.flushCommits();
+
}
/** {@inheritDoc} */
@@ -129,7 +130,7 @@
VectorUpdate update = new VectorUpdate(i);
update.put(j, value + this.get(i, j));
table.commit(update.getBatchUpdate());
- table.flushCommits();
+
}
/** {@inheritDoc} */
@@ -139,7 +140,6 @@
update.put(Constants.METADATA_COLUMNS, columns);
table.commit(update.getBatchUpdate());
- table.flushCommits();
}
public String getRowLabel(int row) throws IOException {
@@ -154,7 +154,7 @@
VectorUpdate update = new VectorUpdate(row);
update.put(Constants.ATTRIBUTE + "string", name);
table.commit(update.getBatchUpdate());
- table.flushCommits();
+
}
public String getColumnLabel(int column) throws IOException {
@@ -178,7 +178,7 @@
BatchUpdate update = new BatchUpdate(Constants.METADATA);
update.put(Constants.METADATA_REFERENCE, Bytes.toBytes(reference));
table.commit(update);
- table.flushCommits();
+
}
protected int incrementAndGetRef() throws IOException {
@@ -220,12 +220,15 @@
if (!hasAliaseName()) { // the table has not been aliased, we delete the
// table.
if (admin.isTableEnabled(matrixPath)) {
- try {
- admin.disableTable(matrixPath);
- admin.deleteTable(matrixPath);
- } catch (RegionException e) {
- LOG.warn(e);
- }
+ while (admin.isTableEnabled(matrixPath)) {
+ try {
+ admin.disableTable(matrixPath);
+ } catch (RegionException e) {
+ LOG.warn(e);
+ }
+ }
+
+ admin.deleteTable(matrixPath);
}
}
}
@@ -238,7 +241,7 @@
BatchUpdate update = new BatchUpdate(Constants.METADATA);
update.put(Constants.ALIASENAME, Bytes.toBytes(aliasename));
table.commit(update);
- table.flushCommits();
+
return hamaAdmin.save(this, aliasename);
}
}
Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=732218&r1=732217&r2=732218&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Jan 6 20:17:51 2009
@@ -434,7 +434,6 @@
VectorUpdate update = new VectorUpdate(row);
update.putAll(((DenseVector) vector).getEntries().entrySet());
table.commit(update.getBatchUpdate());
- table.flushCommits();
}
public void setColumn(int column, Vector vector) throws IOException {
@@ -442,7 +441,6 @@
VectorUpdate update = new VectorUpdate(i);
update.put(column, vector.get(i));
table.commit(update.getBatchUpdate());
- table.flushCommits();
}
}
@@ -490,7 +488,6 @@
BatchUpdate update = new BatchUpdate(new BlockID(i, j).getBytes());
update.put(Bytes.toBytes(Constants.BLOCK), matrix.getBytes());
table.commit(update);
- table.flushCommits();
}
/**
@@ -530,7 +527,6 @@
update.put(Constants.BLOCK_PATH, Bytes.toBytes(path));
update.put(Constants.BLOCK_SIZE, Bytes.toBytes(size));
table.commit(update);
- table.flushCommits();
}
public int getBlockedMatrixSize() throws IOException {
Modified: incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java?rev=732218&r1=732217&r2=732218&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/HamaAdminImpl.java Tue Jan 6 20:17:51 2009
@@ -33,8 +33,8 @@
import org.apache.log4j.Logger;
/**
- * An Implementation of {@link org.apache.hama.HamaAdmin} to manage
- * the matrix's namespace, and table allocation & garbage collection.
+ * An Implementation of {@link org.apache.hama.HamaAdmin} to manage the matrix's
+ * namespace, and table allocation & garbage collection.
*/
public class HamaAdminImpl implements HamaAdmin {
static final Logger LOG = Logger.getLogger(HamaAdminImpl.class);
@@ -67,7 +67,7 @@
}
/**
- * Initializing the admin.
+ * Initializing the admin.
*/
private void initialJob() {
try {
@@ -78,6 +78,7 @@
}
table = new HTable(conf, Constants.ADMINTABLE);
+ table.setAutoFlush(true);
} catch (Exception e) {
e.printStackTrace();
}
@@ -111,13 +112,14 @@
boolean result = false;
// we just store the name -> path(tablename) here.
- // the matrix type is stored in its hbase table. we don't need to store again.
+ // the matrix type is stored in its hbase table. we don't need to store
+ // again.
BatchUpdate update = new BatchUpdate(aliaseName);
update.put(Constants.PATHCOLUMN, Bytes.toBytes(mat.getPath()));
-
+
try {
table.commit(update);
- table.flushCommits();
+
result = true;
} catch (IOException e) {
e.printStackTrace();
@@ -126,60 +128,64 @@
return result;
}
- /** remove the entry of 'matrixName' in admin table. **/
+ /** remove the entry of 'matrixName' in admin table. * */
private void removeEntry(String matrixName) throws IOException {
table.deleteAll(matrixName);
}
-
+
private int getReference(String tableName) throws IOException {
HTable matrix = new HTable(conf, tableName);
-
+
Cell rows = null;
rows = matrix.get(Constants.METADATA, Constants.METADATA_REFERENCE);
-
- return ( rows == null ) ? 0 : Bytes.toInt(rows.getValue());
+
+ return (rows == null) ? 0 : Bytes.toInt(rows.getValue());
}
-
+
private void clearAliaseInfo(String tableName) throws IOException {
HTable matrix = new HTable(conf, tableName);
-
+
matrix.deleteAll(Constants.METADATA, Constants.ALIASENAME);
}
-
+
public void delete(String matrixName) throws IOException {
// we remove the aliase entry store in Admin table, and
// clear the aliase info store in matrix table.
// And check the reference of the matrix table:
// 1) if the reference of the matrix table is zero:
- // we delete the table.
+ // we delete the table.
// 2) if the reference of the matrix table is not zero:
- // we let the matrix who still reference the table to
- // do the garbage collection.
+ // we let the matrix who still reference the table to
+ // do the garbage collection.
if (matrixExists(matrixName)) {
String tablename = getPath(matrixName);
-
+
// i) remove the aliase entry first.
removeEntry(matrixName);
-
- if(tablename == null) { // a matrixName point to a null table. we delete the entry.
- return ;
+
+ if (tablename == null) { // a matrixName point to a null table. we delete
+ // the entry.
+ return;
}
-
- if(!admin.tableExists(tablename)) { // have not specified table.
+
+ if (!admin.tableExists(tablename)) { // have not specified table.
return;
}
-
+
// ii) clear the aliase info store in matrix table.
clearAliaseInfo(tablename);
-
- if(getReference(tablename) <= 0) { // no reference, do gc!!
+
+ if (getReference(tablename) <= 0) { // no reference, do gc!!
if (admin.isTableEnabled(tablename)) {
- try {
- admin.disableTable(tablename);
- admin.deleteTable(tablename);
- } catch (RegionException e) {
- LOG.warn(e);
- }
+ while (admin.isTableEnabled(tablename)) {
+ try {
+ admin.disableTable(tablename);
+ } catch (RegionException e) {
+ LOG.warn(e);
+ }
+ }
+
+ admin.deleteTable(tablename);
}
}
}