svn commit: r1556216 - in /gora/branches/GORA_94: CHANGES.txt gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java

[le...@apache.org]

Author: lewismc
Date: Tue Jan  7 13:41:25 2014
New Revision: 1556216

URL: http://svn.apache.org/r1556216
Log:
port of GORA-117 gora hbase does not have a mechanism to set the caching on a scanner, which makes for poor performance on map/reduce jobs

Modified:
    gora/branches/GORA_94/CHANGES.txt
    gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
    gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java

Modified: gora/branches/GORA_94/CHANGES.txt
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/CHANGES.txt?rev=1556216&r1=1556215&r2=1556216&view=diff
==============================================================================
--- gora/branches/GORA_94/CHANGES.txt (original)
+++ gora/branches/GORA_94/CHANGES.txt Tue Jan  7 13:41:25 2014
@@ -6,6 +6,8 @@ Gora Change Log
 
 * GORA-94 Upgrade to Apache Avro 1.7.x  ==1st Attempt== (Ed Kohlwey via lewismc)
 
+* GORA-117 gora hbase does not have a mechanism to set the caching on a scanner, which makes for poor performance on map/reduce jobs (alfonsonishikawa)
+
 * GORA-281 More flexible file locations for cassandra's config and log4j (Nate McCall via hsaputra)
 
 * GORA-275 Update Gora stores to pass conf when creating instance of PartitionQueryImpl (Damien Raude-Morvan via hsaputra)

Modified: gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1556216&r1=1556215&r2=1556216&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original)
+++ gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Tue Jan  7 13:41:25 2014
@@ -48,6 +48,7 @@ import org.apache.gora.persistency.impl.
 import org.apache.gora.query.PartitionQuery;
 import org.apache.gora.query.Query;
 import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.store.DataStoreFactory;
 import org.apache.gora.store.impl.DataStoreBase;
 
 import org.apache.hadoop.conf.Configurable;
@@ -87,6 +88,9 @@ implements Configurable {
   private static final String DEPRECATED_MAPPING_FILE = "hbase-mapping.xml";
   public static final String DEFAULT_MAPPING_FILE = "gora-hbase-mapping.xml";
 
+  private static final String SCANNER_CACHING_PROPERTIES_KEY = "scanner.caching" ;
+  private static final int SCANNER_CACHING_PROPERTIES_DEFAULT = 0 ;
+  
   private volatile HBaseAdmin admin;
 
   private volatile HBaseTableConnection table;
@@ -95,6 +99,8 @@ implements Configurable {
 
   private volatile HBaseMapping mapping;
 
+  private int scannerCaching = SCANNER_CACHING_PROPERTIES_DEFAULT ;
+  
   public HBaseStore()  {
   }
 
@@ -126,6 +132,17 @@ implements Configurable {
       throw new RuntimeException(e);
     }
     
+    // Set scanner caching option
+    try {
+      this.setScannerCaching(
+          Integer.valueOf(DataStoreFactory.findProperty(this.properties, this,
+              SCANNER_CACHING_PROPERTIES_KEY,
+              String.valueOf(SCANNER_CACHING_PROPERTIES_DEFAULT)))) ;
+    }catch(Exception e){
+      LOG.error("Can not load " + SCANNER_CACHING_PROPERTIES_KEY + " from gora.properties. Setting to default value: " + SCANNER_CACHING_PROPERTIES_DEFAULT, e) ;
+      this.setScannerCaching(SCANNER_CACHING_PROPERTIES_DEFAULT) ; // Default value if something is wrong
+    }
+    
     if(autoCreateSchema) {
       createSchema();
     }
@@ -436,6 +453,9 @@ implements Configurable {
 
   public ResultScanner createScanner(Query<K, T> query) throws IOException {
     final Scan scan = new Scan();
+    
+    scan.setCaching(this.getScannerCaching()) ; 
+    
     if (query.getStartKey() != null) {
       scan.setStartRow(toBytes(query.getStartKey()));
     }
@@ -699,4 +719,29 @@ implements Configurable {
     this.conf = conf;
   }
 
+  /**
+   * Gets the Scanner Caching optimization value
+   * @return The value used internally in {@link Scan#setCaching(int)}
+   */
+  public int getScannerCaching() {
+    return this.scannerCaching ;
+  }
+  
+  /**
+   * Sets the value for Scanner Caching optimization
+   * 
+   * @see Scan#setCaching(int)
+   * 
+   * @param numRows the number of rows for caching >= 0
+   * @return &lt;&lt;Fluent interface&gt;&gt;
+   */
+  public HBaseStore<K, T> setScannerCaching(int numRows) {
+    if (numRows < 0) {
+      LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: " + numRows + ".") ;
+      return this ;
+    }
+    this.scannerCaching = numRows ;
+    return this ;
+  }
+  
 }

Modified: gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java?rev=1556216&r1=1556215&r2=1556216&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java (original)
+++ gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java Tue Jan  7 13:41:25 2014
@@ -229,6 +229,11 @@ public class TestHBaseStore extends Data
     table.close();
   }
 
+  @Test
+  public void assertScannerCachingValue() {
+    assertEquals(1000, ((HBaseStore<String,WebPage>)this.webPageStore).getScannerCaching()) ;
+    assertEquals(1000, ((HBaseStore<String,Employee>)this.employeeStore).getScannerCaching()) ;
+  }
 
   @Ignore("We need to skip this test since gora considers endRow inclusive, while its exclusive for HBase.")
   @Override