You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2014/10/16 18:19:43 UTC

[1/3] git commit: Fix hint replay with many accumulated expired hints

Repository: cassandra
Updated Branches:
  refs/heads/trunk bb6b0d64e -> 96f40e66b


Fix hint replay with many accumulated expired hints

patch by Aleksey Yeschenko; reviewed by Jonathan Ellis for
CASSANDRA-6998


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/98bcf402
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/98bcf402
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/98bcf402

Branch: refs/heads/trunk
Commit: 98bcf40226db7c8dbafa6eacee9ce5ef3feaeb1b
Parents: 19c2f54
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Thu Oct 16 19:04:33 2014 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Thu Oct 16 19:05:59 2014 +0300

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../cassandra/db/HintedHandOffManager.java      | 56 +++++++++++---------
 .../cassandra/service/StorageService.java       |  2 +-
 .../apache/cassandra/db/HintedHandOffTest.java  |  7 +--
 4 files changed, 33 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/98bcf402/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 967b90c..158a48b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.0.11:
+ * Fix hint replay with many accumulated expired hints (CASSANDRA-6998)
  * Fix duplicate results in DISTINCT queries on static columns with query
    paging (CASSANDRA-8108)
  * Properly validate ascii and utf8 string literals in CQL queries (CASSANDRA-8101)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/98bcf402/src/java/org/apache/cassandra/db/HintedHandOffManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
index f6bb033..a6b6d4c 100644
--- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java
+++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
@@ -216,6 +216,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
                 {
                     logger.info("Deleting any stored hints for {}", endpoint);
                     rm.apply();
+                    hintStore.forceBlockingFlush();
                     compact();
                 }
                 catch (Exception e)
@@ -250,13 +251,20 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
     }
 
     @VisibleForTesting
-    protected Future<?> compact()
+    protected void compact()
     {
-        hintStore.forceBlockingFlush();
-        ArrayList<Descriptor> descriptors = new ArrayList<Descriptor>();
+        ArrayList<Descriptor> descriptors = new ArrayList<>();
         for (SSTable sstable : hintStore.getDataTracker().getUncompactingSSTables())
             descriptors.add(sstable.descriptor);
-        return CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000));
+
+        try
+        {
+            CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get();
+        }
+        catch (InterruptedException | ExecutionException e)
+        {
+            throw new RuntimeException(e);
+        }
     }
 
     private static boolean pagingFinished(ColumnFamily hintColumnFamily, ByteBuffer startColumn)
@@ -333,9 +341,8 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
     /*
      * 1. Get the key of the endpoint we need to handoff
      * 2. For each column, deserialize the mutation and send it to the endpoint
-     * 3. Delete the subcolumn if the write was successful
+     * 3. Delete the column if the write was successful
      * 4. Force a flush
-     * 5. Do major compaction to clean up all deletes etc.
      */
     private void doDeliverHintsToEndpoint(InetAddress endpoint)
     {
@@ -357,7 +364,6 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
                            / (StorageService.instance.getTokenMetadata().getAllEndpoints().size() - 1);
         RateLimiter rateLimiter = RateLimiter.create(throttleInKB == 0 ? Double.MAX_VALUE : throttleInKB * 1024);
 
-        boolean finished = false;
         delivery:
         while (true)
         {
@@ -375,7 +381,6 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
             if (pagingFinished(hintsPage, startColumn))
             {
                 logger.info("Finished hinted handoff of {} rows to endpoint {}", rowsReplayed, endpoint);
-                finished = true;
                 break;
             }
 
@@ -469,17 +474,8 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
             }
         }
 
-        if (finished || rowsReplayed.get() >= DatabaseDescriptor.getTombstoneWarnThreshold())
-        {
-            try
-            {
-                compact().get();
-            }
-            catch (Exception e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
+        // Flush all the tombstones to disk
+        hintStore.forceBlockingFlush();
     }
 
     // read less columns (mutations) per page if they are very large
@@ -503,8 +499,12 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
      */
     private void scheduleAllDeliveries()
     {
-        if (logger.isDebugEnabled())
-          logger.debug("Started scheduleAllDeliveries");
+        logger.debug("Started scheduleAllDeliveries");
+
+        // Force a major compaction to get rid of the tombstones and expired hints. Do it once, before we schedule any
+        // individual replay, to avoid N - 1 redundant individual compactions (when N is the number of nodes with hints
+        // to deliver to).
+        compact();
 
         IPartitioner p = StorageService.getPartitioner();
         RowPosition minPos = p.getMinimumToken().minKeyBound();
@@ -517,11 +517,10 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
             InetAddress target = StorageService.instance.getTokenMetadata().getEndpointForHostId(hostId);
             // token may have since been removed (in which case we have just read back a tombstone)
             if (target != null)
-                scheduleHintDelivery(target);
+                scheduleHintDelivery(target, false);
         }
 
-        if (logger.isDebugEnabled())
-          logger.debug("Finished scheduleAllDeliveries");
+        logger.debug("Finished scheduleAllDeliveries");
     }
 
     /*
@@ -529,7 +528,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
      * When we learn that some endpoint is back up we deliver the data
      * to him via an event driven mechanism.
     */
-    public void scheduleHintDelivery(final InetAddress to)
+    public void scheduleHintDelivery(final InetAddress to, final boolean precompact)
     {
         // We should not deliver hints to the same host in 2 different threads
         if (!queuedDeliveries.add(to))
@@ -543,6 +542,11 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
             {
                 try
                 {
+                    // If it's an individual node hint replay (triggered by Gossip or via JMX), and not the global scheduled replay
+                    // (every 10 minutes), force a major compaction to get rid of the tombstones and expired hints.
+                    if (precompact)
+                        compact();
+
                     deliverHintsToEndpoint(to);
                 }
                 finally
@@ -555,7 +559,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
 
     public void scheduleHintDelivery(String to) throws UnknownHostException
     {
-        scheduleHintDelivery(InetAddress.getByName(to));
+        scheduleHintDelivery(InetAddress.getByName(to), true);
     }
 
     public void pauseHintsDelivery(boolean b)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/98bcf402/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 0cd4be5..cca6f79 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1977,7 +1977,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
         if (tokenMetadata.isMember(endpoint))
         {
-            HintedHandOffManager.instance.scheduleHintDelivery(endpoint);
+            HintedHandOffManager.instance.scheduleHintDelivery(endpoint, true);
             for (IEndpointLifecycleSubscriber subscriber : lifecycleSubscribers)
                 subscriber.onUp(endpoint);
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/98bcf402/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
index 9ffd702..302a1e7 100644
--- a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
+++ b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
@@ -23,7 +23,6 @@ package org.apache.cassandra.db;
 import java.net.InetAddress;
 import java.util.Map;
 import java.util.UUID;
-import java.util.concurrent.TimeUnit;
 
 import org.junit.Test;
 
@@ -31,12 +30,10 @@ import com.google.common.collect.Iterators;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
 
 import static org.junit.Assert.assertEquals;
 import static org.apache.cassandra.cql3.QueryProcessor.processInternal;
@@ -75,9 +72,7 @@ public class HintedHandOffTest extends SchemaLoader
         assertEquals(1, hintStore.getSSTables().size());
 
         // submit compaction
-        FBUtilities.waitOnFuture(HintedHandOffManager.instance.compact());
-        while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0)
-            TimeUnit.SECONDS.sleep(1);
+        HintedHandOffManager.instance.compact();
 
         // single row should not be removed because of gc_grace_seconds
         // is 10 hours and there are no any tombstones in sstable


[2/3] git commit: Merge branch 'cassandra-2.0' into cassandra-2.1

Posted by al...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1

Conflicts:
	CHANGES.txt
	src/java/org/apache/cassandra/db/HintedHandOffManager.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2966b61c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2966b61c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2966b61c

Branch: refs/heads/trunk
Commit: 2966b61cf1a37c466b5cf8e6dcf59354252aa1af
Parents: ed7eed8 98bcf40
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Thu Oct 16 19:16:12 2014 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Thu Oct 16 19:16:12 2014 +0300

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../cassandra/db/HintedHandOffManager.java      | 56 +++++++++++---------
 .../cassandra/service/StorageService.java       |  2 +-
 .../apache/cassandra/db/HintedHandOffTest.java  |  7 +--
 4 files changed, 33 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/2966b61c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 67b25dc,158a48b..aa14be9
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,86 -1,5 +1,87 @@@
 -2.0.11:
 +2.1.1
 + * Fix ClassCastException in DISTINCT query on static columns with
 +   query paging (CASSANDRA-8108)
 + * Fix NPE on null nested UDT inside a set (CASSANDRA-8105)
 + * Fix exception when querying secondary index on set items or map keys
 +   when some clustering columns are specified (CASSANDRA-8073)
 + * Send proper error response when there is an error during native
 +   protocol message decode (CASSANDRA-8118)
 + * Gossip should ignore generation numbers too far in the future (CASSANDRA-8113)
 + * Fix NPE when creating a table with frozen sets, lists (CASSANDRA-8104)
 + * Fix high memory use due to tracking reads on incrementally opened sstable
 +   readers (CASSANDRA-8066)
 + * Fix EXECUTE request with skipMetadata=false returning no metadata
 +   (CASSANDRA-8054)
 + * Allow concurrent use of CQLBulkOutputFormat (CASSANDRA-7776)
 + * Shutdown JVM on OOM (CASSANDRA-7507)
 + * Upgrade netty version and enable epoll event loop (CASSANDRA-7761)
 + * Don't duplicate sstables smaller than split size when using
 +   the sstablesplitter tool (CASSANDRA-7616)
 + * Avoid re-parsing already prepared statements (CASSANDRA-7923)
 + * Fix some Thrift slice deletions and updates of COMPACT STORAGE
 +   tables with some clustering columns omitted (CASSANDRA-7990)
 + * Fix filtering for CONTAINS on sets (CASSANDRA-8033)
 + * Properly track added size (CASSANDRA-7239)
 + * Allow compilation in java 8 (CASSANDRA-7208)
 + * Fix Assertion error on RangeTombstoneList diff (CASSANDRA-8013)
 + * Release references to overlapping sstables during compaction (CASSANDRA-7819)
 + * Send notification when opening compaction results early (CASSANDRA-8034)
 + * Make native server start block until properly bound (CASSANDRA-7885)
 + * (cqlsh) Fix IPv6 support (CASSANDRA-7988)
 + * Ignore fat clients when checking for endpoint collision (CASSANDRA-7939)
 + * Make sstablerepairedset take a list of files (CASSANDRA-7995)
 + * (cqlsh) Tab completeion for indexes on map keys (CASSANDRA-7972)
 + * (cqlsh) Fix UDT field selection in select clause (CASSANDRA-7891)
 + * Fix resource leak in event of corrupt sstable
 + * (cqlsh) Add command line option for cqlshrc file path (CASSANDRA-7131)
 + * Provide visibility into prepared statements churn (CASSANDRA-7921, CASSANDRA-7930)
 + * Invalidate prepared statements when their keyspace or table is
 +   dropped (CASSANDRA-7566)
 + * cassandra-stress: fix support for NetworkTopologyStrategy (CASSANDRA-7945)
 + * Fix saving caches when a table is dropped (CASSANDRA-7784)
 + * Add better error checking of new stress profile (CASSANDRA-7716)
 + * Use ThreadLocalRandom and remove FBUtilities.threadLocalRandom (CASSANDRA-7934)
 + * Prevent operator mistakes due to simultaneous bootstrap (CASSANDRA-7069)
 + * cassandra-stress supports whitelist mode for node config (CASSANDRA-7658)
 + * GCInspector more closely tracks GC; cassandra-stress and nodetool report it (CASSANDRA-7916)
 + * nodetool won't output bogus ownership info without a keyspace (CASSANDRA-7173)
 + * Add human readable option to nodetool commands (CASSANDRA-5433)
 + * Don't try to set repairedAt on old sstables (CASSANDRA-7913)
 + * Add metrics for tracking PreparedStatement use (CASSANDRA-7719)
 + * (cqlsh) tab-completion for triggers (CASSANDRA-7824)
 + * (cqlsh) Support for query paging (CASSANDRA-7514)
 + * (cqlsh) Show progress of COPY operations (CASSANDRA-7789)
 + * Add syntax to remove multiple elements from a map (CASSANDRA-6599)
 + * Support non-equals conditions in lightweight transactions (CASSANDRA-6839)
 + * Add IF [NOT] EXISTS to create/drop triggers (CASSANDRA-7606)
 + * (cqlsh) Display the current logged-in user (CASSANDRA-7785)
 + * (cqlsh) Don't ignore CTRL-C during COPY FROM execution (CASSANDRA-7815)
 + * (cqlsh) Order UDTs according to cross-type dependencies in DESCRIBE
 +   output (CASSANDRA-7659)
 + * (cqlsh) Fix handling of CAS statement results (CASSANDRA-7671)
 + * (cqlsh) COPY TO/FROM improvements (CASSANDRA-7405)
 + * Support list index operations with conditions (CASSANDRA-7499)
 + * Add max live/tombstoned cells to nodetool cfstats output (CASSANDRA-7731)
 + * Validate IPv6 wildcard addresses properly (CASSANDRA-7680)
 + * (cqlsh) Error when tracing query (CASSANDRA-7613)
 + * Avoid IOOBE when building SyntaxError message snippet (CASSANDRA-7569)
 + * SSTableExport uses correct validator to create string representation of partition
 +   keys (CASSANDRA-7498)
 + * Avoid NPEs when receiving type changes for an unknown keyspace (CASSANDRA-7689)
 + * Add support for custom 2i validation (CASSANDRA-7575)
 + * Pig support for hadoop CqlInputFormat (CASSANDRA-6454)
 + * Add listen_interface and rpc_interface options (CASSANDRA-7417)
 + * Improve schema merge performance (CASSANDRA-7444)
 + * Adjust MT depth based on # of partition validating (CASSANDRA-5263)
 + * Optimise NativeCell comparisons (CASSANDRA-6755)
 + * Configurable client timeout for cqlsh (CASSANDRA-7516)
 + * Include snippet of CQL query near syntax error in messages (CASSANDRA-7111)
 + * Make repair -pr work with -local (CASSANDRA-7450)
 + * Fix error in sstableloader with -cph > 1 (CASSANDRA-8007)
 + * Fix snapshot repair error on indexed tables (CASSANDRA-8020)
 + * Do not exit nodetool repair when receiving JMX NOTIF_LOST (CASSANDRA-7909)
 +Merged from 2.0:
+  * Fix hint replay with many accumulated expired hints (CASSANDRA-6998)
   * Fix duplicate results in DISTINCT queries on static columns with query
     paging (CASSANDRA-8108)
   * Properly validate ascii and utf8 string literals in CQL queries (CASSANDRA-8101)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2966b61c/src/java/org/apache/cassandra/db/HintedHandOffManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/HintedHandOffManager.java
index 76c0b99,a6b6d4c..2eb49c7
--- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java
+++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
@@@ -216,7 -215,8 +216,8 @@@ public class HintedHandOffManager imple
                  try
                  {
                      logger.info("Deleting any stored hints for {}", endpoint);
 -                    rm.apply();
 +                    mutation.apply();
+                     hintStore.forceBlockingFlush();
                      compact();
                  }
                  catch (Exception e)
@@@ -251,16 -251,23 +252,23 @@@
      }
  
      @VisibleForTesting
-     protected Future<?> compact()
+     protected void compact()
      {
-         hintStore.forceBlockingFlush();
-         ArrayList<Descriptor> descriptors = new ArrayList<Descriptor>();
+         ArrayList<Descriptor> descriptors = new ArrayList<>();
          for (SSTable sstable : hintStore.getDataTracker().getUncompactingSSTables())
              descriptors.add(sstable.descriptor);
-         return CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000));
+ 
+         try
+         {
+             CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get();
+         }
+         catch (InterruptedException | ExecutionException e)
+         {
+             throw new RuntimeException(e);
+         }
      }
  
 -    private static boolean pagingFinished(ColumnFamily hintColumnFamily, ByteBuffer startColumn)
 +    private static boolean pagingFinished(ColumnFamily hintColumnFamily, Composite startColumn)
      {
          // done if no hints found or the start column (same as last column processed in previous iteration) is the only one
          return hintColumnFamily == null

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2966b61c/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2966b61c/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/HintedHandOffTest.java
index 9bc0724,302a1e7..c29c08e
--- a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
+++ b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
@@@ -30,17 -29,14 +29,15 @@@ import org.junit.Test
  import com.google.common.collect.Iterators;
  
  import org.apache.cassandra.SchemaLoader;
 +import org.apache.cassandra.Util;
  import org.apache.cassandra.cql3.UntypedResultSet;
- import org.apache.cassandra.db.compaction.CompactionManager;
  import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
  import org.apache.cassandra.db.marshal.Int32Type;
  import org.apache.cassandra.db.marshal.UUIDType;
  import org.apache.cassandra.utils.ByteBufferUtil;
- import org.apache.cassandra.utils.FBUtilities;
  
  import static org.junit.Assert.assertEquals;
 -import static org.apache.cassandra.cql3.QueryProcessor.processInternal;
 +import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
  
  public class HintedHandOffTest extends SchemaLoader
  {


[3/3] git commit: Merge branch 'cassandra-2.1' into trunk

Posted by al...@apache.org.
Merge branch 'cassandra-2.1' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/96f40e66
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/96f40e66
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/96f40e66

Branch: refs/heads/trunk
Commit: 96f40e66b0eb3a41df0a116fa7da9db2f2b40e3f
Parents: bb6b0d6 2966b61
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Thu Oct 16 19:16:54 2014 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Thu Oct 16 19:16:54 2014 +0300

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../cassandra/db/HintedHandOffManager.java      | 56 +++++++++++---------
 .../cassandra/service/StorageService.java       |  2 +-
 .../apache/cassandra/db/HintedHandOffTest.java  |  7 +--
 4 files changed, 33 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/96f40e66/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/96f40e66/src/java/org/apache/cassandra/db/HintedHandOffManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/96f40e66/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageService.java
index b341276,46248cb..dac1705
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@@ -1963,9 -2020,12 +1963,9 @@@ public class StorageService extends Not
      {
          MigrationManager.instance.scheduleSchemaPull(endpoint, state);
  
 -        if (isClientMode)
 -            return;
 -
          if (tokenMetadata.isMember(endpoint))
          {
-             HintedHandOffManager.instance.scheduleHintDelivery(endpoint);
+             HintedHandOffManager.instance.scheduleHintDelivery(endpoint, true);
              for (IEndpointLifecycleSubscriber subscriber : lifecycleSubscribers)
                  subscriber.onUp(endpoint);
          }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/96f40e66/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/HintedHandOffTest.java
index c49cc7a,c29c08e..13f9b2d
--- a/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
+++ b/test/unit/org/apache/cassandra/db/HintedHandOffTest.java
@@@ -23,25 -23,18 +23,22 @@@ package org.apache.cassandra.db
  import java.net.InetAddress;
  import java.util.Map;
  import java.util.UUID;
- import java.util.concurrent.TimeUnit;
  
 +import org.junit.BeforeClass;
  import org.junit.Test;
  
  import com.google.common.collect.Iterators;
  
  import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.Util;
 +import org.apache.cassandra.config.KSMetaData;
  import org.apache.cassandra.cql3.UntypedResultSet;
  import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
  import org.apache.cassandra.db.marshal.Int32Type;
  import org.apache.cassandra.db.marshal.UUIDType;
 +import org.apache.cassandra.exceptions.ConfigurationException;
 +import org.apache.cassandra.locator.SimpleStrategy;
  import org.apache.cassandra.utils.ByteBufferUtil;
- import org.apache.cassandra.utils.FBUtilities;
  
  import static org.junit.Assert.assertEquals;
  import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;