You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2009/03/02 07:13:17 UTC
svn commit: r749207 [12/12] - in
/incubator/cassandra/src/org/apache/cassandra: loader/ locator/ net/
net/http/ net/io/ net/sink/ procedures/ service/ test/ tools/
Added: incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleaner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleaner.java?rev=749207&view=auto
==============================================================================
--- incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleaner.java (added)
+++ incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleaner.java Mon Mar 2 06:13:14 2009
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.cassandra.io.ICompactSerializer;
+import org.apache.cassandra.net.EndPoint;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.io.*;
+import org.apache.cassandra.utils.*;
+
+public class MembershipCleaner
+{
+ private static final int port_ = 7000;
+ private static final long waitTime_ = 10000;
+
+ public static void main(String[] args) throws Throwable
+ {
+ if ( args.length != 3 )
+ {
+ System.out.println("Usage : java com.facebook.infrastructure.tools.MembershipCleaner " +
+ "<ip:port to send the message> " +
+ "<node which needs to be removed> " +
+ "<file containing all nodes in the cluster>");
+ System.exit(1);
+ }
+
+ String ipPort = args[0];
+ String node = args[1];
+ String file = args[2];
+
+ String[] ipPortPair = ipPort.split(":");
+ EndPoint target = new EndPoint(ipPortPair[0], Integer.valueOf(ipPortPair[1]));
+ MembershipCleanerMessage mcMessage = new MembershipCleanerMessage(node);
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(bos);
+ MembershipCleanerMessage.serializer().serialize(mcMessage, dos);
+ /* Construct the token update message to be sent */
+ Message mbrshipCleanerMessage = new Message( new EndPoint(FBUtilities.getHostName(), port_), "", StorageService.mbrshipCleanerVerbHandler_, new Object[]{bos.toByteArray()} );
+
+ BufferedReader bufReader = new BufferedReader( new InputStreamReader( new FileInputStream(file) ) );
+ String line = null;
+
+ while ( ( line = bufReader.readLine() ) != null )
+ {
+ mbrshipCleanerMessage.addHeader(line, line.getBytes());
+ }
+
+ System.out.println("Sending a membership clean message to " + target);
+ MessagingService.getMessagingInstance().sendOneWay(mbrshipCleanerMessage, target);
+ Thread.sleep(MembershipCleaner.waitTime_);
+ System.out.println("Done sending the update message");
+ }
+
+ public static class MembershipCleanerMessage implements Serializable
+ {
+ private static ICompactSerializer<MembershipCleanerMessage> serializer_;
+ private static AtomicInteger idGen_ = new AtomicInteger(0);
+
+ static
+ {
+ serializer_ = new MembershipCleanerMessageSerializer();
+ }
+
+ static ICompactSerializer<MembershipCleanerMessage> serializer()
+ {
+ return serializer_;
+ }
+
+ private String target_;
+
+ MembershipCleanerMessage(String target)
+ {
+ target_ = target;
+ }
+
+ String getTarget()
+ {
+ return target_;
+ }
+ }
+
+ public static class MembershipCleanerMessageSerializer implements ICompactSerializer<MembershipCleanerMessage>
+ {
+ public void serialize(MembershipCleanerMessage mcMessage, DataOutputStream dos) throws IOException
+ {
+ dos.writeUTF(mcMessage.getTarget() );
+ }
+
+ public MembershipCleanerMessage deserialize(DataInputStream dis) throws IOException
+ {
+ return new MembershipCleanerMessage(dis.readUTF());
+ }
+ }
+}
Added: incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleanerVerbHandler.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleanerVerbHandler.java?rev=749207&view=auto
==============================================================================
--- incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleanerVerbHandler.java (added)
+++ incubator/cassandra/src/org/apache/cassandra/tools/MembershipCleanerVerbHandler.java Mon Mar 2 06:13:14 2009
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.util.*;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.DataInputBuffer;
+import org.apache.cassandra.net.EndPoint;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tools.TokenUpdater.TokenInfoMessage;
+import org.apache.cassandra.utils.LogUtil;
+import org.apache.log4j.Logger;
+import org.apache.cassandra.io.*;
+import org.apache.cassandra.config.*;
+
+/**
+ * Author : Avinash Lakshman ( alakshman@facebook.com) & Prashant Malik ( pmalik@facebook.com )
+ */
+
+public class MembershipCleanerVerbHandler implements IVerbHandler
+{
+ private static Logger logger_ = Logger.getLogger(MembershipCleanerVerbHandler.class);
+
+ public void doVerb(Message message)
+ {
+ byte[] body = (byte[])message.getMessageBody()[0];
+
+ try
+ {
+ DataInputBuffer bufIn = new DataInputBuffer();
+ bufIn.reset(body, body.length);
+ /* Deserialize to get the token for this endpoint. */
+ MembershipCleaner.MembershipCleanerMessage mcMessage = MembershipCleaner.MembershipCleanerMessage.serializer().deserialize(bufIn);
+
+ String target = mcMessage.getTarget();
+ logger_.info("Removing the node [" + target + "] from membership");
+ EndPoint targetEndPoint = new EndPoint(target, DatabaseDescriptor.getControlPort());
+ /* Remove the token related information for this endpoint */
+ StorageService.instance().removeTokenState(targetEndPoint);
+
+ /* Get the headers for this message */
+ Map<String, byte[]> headers = message.getHeaders();
+ headers.remove( StorageService.getLocalStorageEndPoint().getHost() );
+ logger_.debug("Number of nodes in the header " + headers.size());
+ Set<String> nodes = headers.keySet();
+
+ for ( String node : nodes )
+ {
+ logger_.debug("Processing node " + node);
+ byte[] bytes = headers.remove(node);
+ /* Send a message to this node to alter its membership state. */
+ EndPoint targetNode = new EndPoint(node, DatabaseDescriptor.getStoragePort());
+
+ logger_.debug("Sending a membership clean message to " + targetNode);
+ MessagingService.getMessagingInstance().sendOneWay(message, targetNode);
+ break;
+ }
+ }
+ catch( IOException ex )
+ {
+ logger_.debug(LogUtil.throwableToString(ex));
+ }
+ }
+
+}
Added: incubator/cassandra/src/org/apache/cassandra/tools/ThreadListBuilder.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/src/org/apache/cassandra/tools/ThreadListBuilder.java?rev=749207&view=auto
==============================================================================
--- incubator/cassandra/src/org/apache/cassandra/tools/ThreadListBuilder.java (added)
+++ incubator/cassandra/src/org/apache/cassandra/tools/ThreadListBuilder.java Mon Mar 2 06:13:14 2009
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.io.DataOutputBuffer;
+import org.apache.cassandra.utils.BloomFilter;
+
+
+public class ThreadListBuilder
+{
+ private final static int bufSize_ = 64*1024*1024;
+ private final static int count_ = 128*1024*1024;
+
+ public static void main(String[] args) throws Throwable
+ {
+ if ( args.length != 2 )
+ {
+ System.out.println("Usage : java com.facebook.infrastructure.tools.ThreadListBuilder <directory containing files to be processed> <directory to dump the bloom filter in.>");
+ System.exit(1);
+ }
+
+ File directory = new File(args[0]);
+ File[] files = directory.listFiles();
+ List<DataOutputBuffer> buffers = new ArrayList<DataOutputBuffer>();
+ BloomFilter bf = new BloomFilter(count_, 8);
+ int keyCount = 0;
+
+ /* Process the list of files. */
+ for ( File file : files )
+ {
+ System.out.println("Processing file " + file);
+ BufferedReader bufReader = new BufferedReader( new InputStreamReader( new FileInputStream(file) ), ThreadListBuilder.bufSize_ );
+ String line = null;
+
+ while ( (line = bufReader.readLine()) != null )
+ {
+ /* After accumulating count_ keys reset the bloom filter. */
+ if ( keyCount > 0 && keyCount % count_ == 0 )
+ {
+ DataOutputBuffer bufOut = new DataOutputBuffer();
+ BloomFilter.serializer().serialize(bf, bufOut);
+ System.out.println("Finished serializing the bloom filter");
+ buffers.add(bufOut);
+ bf = new BloomFilter(count_, 8);
+ }
+ line = line.trim();
+ bf.fill(line);
+ ++keyCount;
+ }
+ }
+
+ /* Add the bloom filter assuming the last one was left out */
+ DataOutputBuffer bufOut = new DataOutputBuffer();
+ BloomFilter.serializer().serialize(bf, bufOut);
+ buffers.add(bufOut);
+
+
+ int size = buffers.size();
+ for ( int i = 0; i < size; ++i )
+ {
+ DataOutputBuffer buffer = buffers.get(i);
+ String file = args[1] + System.getProperty("file.separator") + "Bloom-Filter-" + i + ".dat";
+ RandomAccessFile raf = new RandomAccessFile(file, "rw");
+ raf.write(buffer.getData(), 0, buffer.getLength());
+ raf.close();
+ buffer.close();
+ }
+ System.out.println("Done writing the bloom filter to disk");
+ }
+}
Added: incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdateVerbHandler.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdateVerbHandler.java?rev=749207&view=auto
==============================================================================
--- incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdateVerbHandler.java (added)
+++ incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdateVerbHandler.java Mon Mar 2 06:13:14 2009
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.util.*;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.DataInputBuffer;
+import org.apache.cassandra.net.EndPoint;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tools.TokenUpdater.TokenInfoMessage;
+import org.apache.cassandra.utils.LogUtil;
+import org.apache.log4j.Logger;
+import org.apache.cassandra.io.*;
+import org.apache.cassandra.config.*;
+
+/**
+ * Author : Avinash Lakshman ( alakshman@facebook.com) & Prashant Malik ( pmalik@facebook.com )
+ */
+
+public class TokenUpdateVerbHandler implements IVerbHandler
+{
+ private static Logger logger_ = Logger.getLogger(TokenUpdateVerbHandler.class);
+
+ public void doVerb(Message message)
+ {
+ byte[] body = (byte[])message.getMessageBody()[0];
+
+ try
+ {
+ DataInputBuffer bufIn = new DataInputBuffer();
+ bufIn.reset(body, body.length);
+ /* Deserialize to get the token for this endpoint. */
+ TokenUpdater.TokenInfoMessage tiMessage = TokenUpdater.TokenInfoMessage.serializer().deserialize(bufIn);
+
+ BigInteger token = tiMessage.getToken();
+ logger_.info("Updating the token to [" + token + "]");
+ StorageService.instance().updateToken(token);
+
+ /* Get the headers for this message */
+ Map<String, byte[]> headers = message.getHeaders();
+ headers.remove( StorageService.getLocalStorageEndPoint().getHost() );
+ logger_.debug("Number of nodes in the header " + headers.size());
+ Set<String> nodes = headers.keySet();
+
+ for ( String node : nodes )
+ {
+ logger_.debug("Processing node " + node);
+ byte[] bytes = headers.remove(node);
+ /* Send a message to this node to update its token to the one retreived. */
+ EndPoint target = new EndPoint(node, DatabaseDescriptor.getStoragePort());
+ token = new BigInteger(bytes);
+
+ /* Reset the new TokenInfoMessage */
+ tiMessage = new TokenUpdater.TokenInfoMessage(target, token );
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(bos);
+ TokenInfoMessage.serializer().serialize(tiMessage, dos);
+ message.setMessageBody(new Object[]{bos.toByteArray()});
+
+ logger_.debug("Sending a token update message to " + target + " to update it to " + token);
+ MessagingService.getMessagingInstance().sendOneWay(message, target);
+ break;
+ }
+ }
+ catch( IOException ex )
+ {
+ logger_.debug(LogUtil.throwableToString(ex));
+ }
+ }
+
+}
Added: incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdater.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdater.java?rev=749207&view=auto
==============================================================================
--- incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdater.java (added)
+++ incubator/cassandra/src/org/apache/cassandra/tools/TokenUpdater.java Mon Mar 2 06:13:14 2009
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.cassandra.io.ICompactSerializer;
+import org.apache.cassandra.net.EndPoint;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.io.*;
+import org.apache.cassandra.utils.*;
+
+public class TokenUpdater
+{
+ private static final int port_ = 7000;
+ private static final long waitTime_ = 10000;
+
+ public static void main(String[] args) throws Throwable
+ {
+ if ( args.length != 3 )
+ {
+ System.out.println("Usage : java com.facebook.infrastructure.tools.TokenUpdater <ip:port> <token> <file containing node token info>");
+ System.exit(1);
+ }
+
+ String ipPort = args[0];
+ String token = args[1];
+ String file = args[2];
+
+ String[] ipPortPair = ipPort.split(":");
+ EndPoint target = new EndPoint(ipPortPair[0], Integer.valueOf(ipPortPair[1]));
+ TokenInfoMessage tiMessage = new TokenInfoMessage( target, new BigInteger(token) );
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(bos);
+ TokenInfoMessage.serializer().serialize(tiMessage, dos);
+ /* Construct the token update message to be sent */
+ Message tokenUpdateMessage = new Message( new EndPoint(FBUtilities.getHostName(), port_), "", StorageService.tokenVerbHandler_, new Object[]{bos.toByteArray()} );
+
+ BufferedReader bufReader = new BufferedReader( new InputStreamReader( new FileInputStream(file) ) );
+ String line = null;
+
+ while ( ( line = bufReader.readLine() ) != null )
+ {
+ String[] nodeTokenPair = line.split(" ");
+ /* Add the node and the token pair into the header of this message. */
+ BigInteger nodeToken = new BigInteger(nodeTokenPair[1]);
+ tokenUpdateMessage.addHeader(nodeTokenPair[0], nodeToken.toByteArray());
+ }
+
+ System.out.println("Sending a token update message to " + target);
+ MessagingService.getMessagingInstance().sendOneWay(tokenUpdateMessage, target);
+ Thread.sleep(TokenUpdater.waitTime_);
+ System.out.println("Done sending the update message");
+ }
+
+ public static class TokenInfoMessage implements Serializable
+ {
+ private static ICompactSerializer<TokenInfoMessage> serializer_;
+ private static AtomicInteger idGen_ = new AtomicInteger(0);
+
+ static
+ {
+ serializer_ = new TokenInfoMessageSerializer();
+ }
+
+ static ICompactSerializer<TokenInfoMessage> serializer()
+ {
+ return serializer_;
+ }
+
+ private EndPoint target_;
+ private BigInteger token_;
+
+ TokenInfoMessage(EndPoint target, BigInteger token)
+ {
+ target_ = target;
+ token_ = token;
+ }
+
+ EndPoint getTarget()
+ {
+ return target_;
+ }
+
+ BigInteger getToken()
+ {
+ return token_;
+ }
+ }
+
+ public static class TokenInfoMessageSerializer implements ICompactSerializer<TokenInfoMessage>
+ {
+ public void serialize(TokenInfoMessage tiMessage, DataOutputStream dos) throws IOException
+ {
+ byte[] node = EndPoint.toBytes( tiMessage.getTarget() );
+ dos.writeInt(node.length);
+ dos.write(node);
+
+ byte[] token = tiMessage.getToken().toByteArray();
+ dos.writeInt( token.length );
+ dos.write(token);
+ }
+
+ public TokenInfoMessage deserialize(DataInputStream dis) throws IOException
+ {
+ byte[] target = new byte[dis.readInt()];
+ dis.readFully(target);
+
+ byte[] token = new byte[dis.readInt()];
+ dis.readFully(token);
+
+ return new TokenInfoMessage(EndPoint.fromBytes(target), new BigInteger(token));
+ }
+ }
+}