You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2014/02/04 20:34:09 UTC
git commit: ACCUMULO-2283 removing server-extras
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 55e864ce9 -> 4e331fabd
ACCUMULO-2283 removing server-extras
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4e331fab
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4e331fab
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4e331fab
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 4e331fabdb7a26bbfd6df7a2aa2bb62e09d52d82
Parents: 55e864c
Author: John Vines <vi...@apache.org>
Authored: Tue Feb 4 14:33:05 2014 -0500
Committer: John Vines <vi...@apache.org>
Committed: Tue Feb 4 14:34:00 2014 -0500
----------------------------------------------------------------------
assemble/pom.xml | 4 -
pom.xml | 13 --
server/extras/pom.xml | 106 ----------
.../apache/accumulo/utils/NamespaceRename.java | 211 -------------------
.../accumulo/utils/metanalysis/FilterMeta.java | 93 --------
.../accumulo/utils/metanalysis/FindTablet.java | 66 ------
.../accumulo/utils/metanalysis/IndexMeta.java | 183 ----------------
.../utils/metanalysis/LogFileInputFormat.java | 116 ----------
.../utils/metanalysis/LogFileOutputFormat.java | 66 ------
.../accumulo/utils/metanalysis/PrintEvents.java | 99 ---------
.../utils/metanalysis/package-info.java | 34 ---
11 files changed, 991 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/assemble/pom.xml
----------------------------------------------------------------------
diff --git a/assemble/pom.xml b/assemble/pom.xml
index b9d0b3e..9a45198 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -90,10 +90,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server-extras</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-start</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dd13d50..c5e8673 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,7 +90,6 @@
<module>server/native</module>
<module>server/tracer</module>
<module>server/tserver</module>
- <module>server/extras</module>
</modules>
<scm>
<connection>scm:git:git://git.apache.org/accumulo.git</connection>
@@ -348,18 +347,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server-extras</artifactId>
- <version>${project.version}</version>
- <classifier>el6</classifier>
- <type>rpm</type>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server-extras</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-start</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/pom.xml
----------------------------------------------------------------------
diff --git a/server/extras/pom.xml b/server/extras/pom.xml
deleted file mode 100644
index 8654c9f..0000000
--- a/server/extras/pom.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-project</artifactId>
- <version>1.6.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <artifactId>accumulo-server-extras</artifactId>
- <name>Extra Server Utilities</name>
- <description>A library of additional tools to work with Apache Accumulo.</description>
- <dependencies>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-fate</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server-base</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-tserver</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <profiles>
- <profile>
- <id>rpm</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>rpm-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>build-rpm</id>
- <goals>
- <goal>attached-rpm</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <requires>
- <require>jre >= 1.6.0</require>
- <require>accumulo-tserver = %{version}-%{release}</require>
- </requires>
- <mappings>
- <mapping>
- <directory>%{_javadir}/accumulo</directory>
- <artifact />
- </mapping>
- </mappings>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java b/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java
deleted file mode 100644
index bc0c834..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.utils;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.master.thrift.MasterGoalState;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.core.util.ColumnFQ;
-import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
-import org.apache.accumulo.server.Accumulo;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.fs.VolumeManagerImpl;
-import org.apache.accumulo.server.security.SystemCredentials;
-import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-
-public class NamespaceRename {
-
- static final Logger log = LoggerFactory.getLogger(NamespaceRename.class);
-
- static class Opts {
- @Parameter(names = {"--old", "-o"}, required = true)
- String oldName = null;
- @Parameter(names = {"--new", "-n"}, required = true)
- String newName = null;
- }
-
- /**
- * Utility to recovery from a name node restoration at a new location. For example, if you had been using "nn1" and the machine died but you were able to
- * restore the service on a different machine, "nn2" you could rewrite the metadata using
- * <pre>
- * accumulo org.apache.accumulo.server.util.NamespaceRename --old hdfs://nn1:9001 --new hdfs://nn2:9001
- * </pre>
- * @param args
- * @throws Exception
- */
- public static void main(String[] args) throws Exception {
- Opts opts = new Opts();
- JCommander cmdline = new JCommander(opts);
- cmdline.parse(args);
- log.info("Checking current configuration");
- AccumuloConfiguration configuration = ServerConfiguration.getSiteConfiguration();
- checkConfiguration(opts, configuration);
- Instance instance = HdfsZooInstance.getInstance();
- log.info("Waiting for HDFS and Zookeeper to be ready");
- VolumeManager fs = VolumeManagerImpl.get();
- Accumulo.waitForZookeeperAndHdfs(fs);
- log.info("Putting servers in SAFE_MODE");
- ZooReaderWriter zoo = ZooReaderWriter.getInstance();
- zoo.putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, MasterGoalState.SAFE_MODE.toString().getBytes(), NodeExistsPolicy.OVERWRITE);
- log.info("Updating root table write-ahead logs");
- updateZookeeper(opts, instance, zoo);
- log.info("Updating file references in the root table");
- updateMetaTable(opts, instance, RootTable.NAME);
- log.info("Updating file references in the metadata table");
- updateMetaTable(opts, instance, MetadataTable.NAME);
- zoo.putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(), NodeExistsPolicy.OVERWRITE);
- log.info("Namespace " + opts.oldName + " has been renamed " + opts.newName);
- }
-
- static final ColumnFQ DIRECTORY_COLUMN = MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
-
- private static void updateMetaTable(Opts opts, Instance instance, String tableName) throws Exception,
- MutationsRejectedException {
- log.info("Waiting for " + tableName + " to come online");
- Connector conn = getConnector(instance);
- Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
- scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LogColumnFamily.NAME);
- scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
- DIRECTORY_COLUMN.fetch(scanner);
- scanner.iterator().hasNext();
- log.info(tableName + " is online");
- BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
- for (Entry<Key,Value> entry : scanner) {
- Key key = entry.getKey();
- Mutation m = new Mutation(key.getRow());
- if (DIRECTORY_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) {
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp());
- String newName = rename(entry.getValue().toString(), opts);
- m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newName.getBytes()));
- bw.addMutation(m);
- } else if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp());
- String newName = rename(key.getColumnQualifier().toString(), opts);
- m.put(key.getColumnFamily(), new Text(newName), entry.getValue());
- bw.addMutation(m);
- } else if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LogColumnFamily.NAME)) {
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp());
- LogEntry update = convert(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()), opts);
- m.put(update.getColumnFamily(), update.getColumnQualifier(), update.getValue());
- bw.addMutation(m);
- }
- }
- bw.close();
- }
-
- static private LogEntry convert(LogEntry entry, Opts opts) {
- entry.filename = rename(entry.filename, opts);
- List<String> logSet = new ArrayList<String>();
- for (String log : entry.logSet) {
- logSet.add(rename(log, opts));
- }
- entry.logSet = logSet;
- return entry;
- }
-
- private static Connector getConnector(Instance instance) throws AccumuloException, AccumuloSecurityException {
- return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
- }
-
- private static void updateZookeeper(Opts opts, Instance instance, ZooReaderWriter zoo) throws KeeperException, InterruptedException,
- IOException {
- String root = ZooUtil.getRoot(instance);
- String rootTabletLocation = root + RootTable.ZROOT_TABLET_WALOGS;
- for (String walogName : zoo.getChildren(rootTabletLocation)) {
- LogEntry entry = new LogEntry();
- String logZPath = rootTabletLocation + "/" + walogName;
- byte[] data = zoo.getData(logZPath, null);
- entry.fromBytes(data);
- entry = convert(entry, opts);
- zoo.putPersistentData(logZPath, entry.toBytes(), NodeExistsPolicy.OVERWRITE);
- }
- String dirPath = root + RootTable.ZROOT_TABLET_PATH;
- byte[] dir = zoo.getData(dirPath, null);
- String newDir = rename(new String(dir), opts);
- zoo.putPersistentData(dirPath, newDir.getBytes(), NodeExistsPolicy.OVERWRITE);
- }
-
- private static String rename(String filename, Opts opts) {
- if (filename.startsWith(opts.oldName))
- return opts.newName + filename.substring(opts.oldName.length(), filename.length());
- return filename;
- }
-
- private static void checkConfiguration(Opts opts, AccumuloConfiguration configuration) throws IOException {
- if (opts.oldName.endsWith("/"))
- throw new RuntimeException(opts.oldName + " ends with a slash, do not include it");
- if (opts.newName.endsWith("/"))
- throw new RuntimeException(opts.newName + " ends with a slash, do not include it");
- String volumes = configuration.get(Property.INSTANCE_VOLUMES);
- if (volumes != null && !volumes.isEmpty()) {
- Set<String> volumeSet = new HashSet<String>(Arrays.asList(volumes.split(",")));
- if (volumeSet.contains(opts.oldName))
- throw new RuntimeException(Property.INSTANCE_VOLUMES.getKey() + " is set to " + volumes + " which still contains the old name " + opts.oldName);
- if (!volumeSet.contains(opts.newName))
- throw new RuntimeException(Property.INSTANCE_VOLUMES.getKey() + " is set to " + volumes + " which does not contain the new name " + opts.oldName);
- return;
- } else {
- String uri = configuration.get(Property.INSTANCE_DFS_URI);
- if (uri != null && !uri.isEmpty()) {
- if (!uri.startsWith(opts.newName))
- throw new RuntimeException(Property.INSTANCE_DFS_DIR.getKey() + " is set to " + uri + " which is not in " + opts.newName);
- return;
- }
- }
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- if (!fs.getUri().toString().equals(opts.newName))
- throw new RuntimeException("Default filesystem is " + fs.getUri() + " and the new name is " + opts.newName + ". Update your hadoop dfs configuration.");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java
deleted file mode 100644
index ac56c99..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.utils.metanalysis;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.tserver.logger.LogEvents;
-import org.apache.accumulo.tserver.logger.LogFileKey;
-import org.apache.accumulo.tserver.logger.LogFileValue;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * A map reduce job that takes a set of walogs and filters out all non metadata table events.
- */
-public class FilterMeta extends Configured implements Tool {
-
- public static class FilterMapper extends Mapper<LogFileKey,LogFileValue,LogFileKey,LogFileValue> {
- private Set<Integer> tabletIds;
-
- @Override
- protected void setup(Context context) throws java.io.IOException, java.lang.InterruptedException {
- tabletIds = new HashSet<Integer>();
- }
-
- @Override
- public void map(LogFileKey key, LogFileValue value, Context context) throws IOException, InterruptedException {
- if (key.event == LogEvents.OPEN) {
- context.write(key, value);
- } else if (key.event == LogEvents.DEFINE_TABLET && key.tablet.getTableId().toString().equals(MetadataTable.ID)) {
- tabletIds.add(key.tid);
- context.write(key, value);
- } else if ((key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) && tabletIds.contains(key.tid)) {
- context.write(key, value);
- }
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
-
- String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();
-
- @SuppressWarnings("deprecation")
- Job job = new Job(getConf(), jobName);
- job.setJarByClass(this.getClass());
-
- Path paths[] = new Path[args.length - 1];
- for (int i = 0; i < paths.length; i++) {
- paths[i] = new Path(args[i]);
- }
-
- job.setInputFormatClass(LogFileInputFormat.class);
- LogFileInputFormat.setInputPaths(job, paths);
-
- job.setOutputFormatClass(LogFileOutputFormat.class);
- LogFileOutputFormat.setOutputPath(job, new Path(args[args.length - 1]));
-
- job.setMapperClass(FilterMapper.class);
-
- job.setNumReduceTasks(0);
-
- job.waitForCompletion(true);
- return job.isSuccessful() ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(CachedConfiguration.getInstance(), new FilterMeta(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java
deleted file mode 100644
index 773c41a..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.utils.metanalysis;
-
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.TextUtil;
-import org.apache.accumulo.server.cli.ClientOpts;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Finds tablet creation events.
- */
-public class FindTablet {
-
- static public class Opts extends ClientOpts {
- @Parameter(names = {"-r", "--row"}, required = true, description = "find tablets that contain this row")
- String row = null;
-
- @Parameter(names = "--tableId", required = true, description = "table id")
- String tableId = null;
- }
-
- public static void main(String[] args) throws Exception {
- Opts opts = new Opts();
- opts.parseArgs(FindTablet.class.getName(), args);
-
- findContainingTablets(opts);
- }
-
- private static void findContainingTablets(Opts opts) throws Exception {
- Range range = new KeyExtent(new Text(opts.tableId), null, null).toMetadataRange();
-
- Scanner scanner = opts.getConnector().createScanner("createEvents", opts.auths);
- scanner.setRange(range);
-
- Text row = new Text(opts.row);
- for (Entry<Key,Value> entry : scanner) {
- KeyExtent ke = new KeyExtent(entry.getKey().getRow(), new Value(TextUtil.getBytes(entry.getKey().getColumnFamily())));
- if (ke.contains(row)) {
- System.out.println(entry.getKey().getColumnQualifier() + " " + ke + " " + entry.getValue());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java
deleted file mode 100644
index 448b871..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.utils.metanalysis;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.data.ColumnUpdate;
-import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.server.cli.ClientOpts;
-import org.apache.accumulo.tserver.logger.LogEvents;
-import org.apache.accumulo.tserver.logger.LogFileKey;
-import org.apache.accumulo.tserver.logger.LogFileValue;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Logger;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * A map reduce job that takes write ahead logs containing mutations for the metadata table and indexes them into Accumulo tables for analysis.
- *
- */
-
-public class IndexMeta extends Configured implements Tool {
-
- public static class IndexMapper extends Mapper<LogFileKey,LogFileValue,Text,Mutation> {
- private static final Text CREATE_EVENTS_TABLE = new Text("createEvents");
- private static final Text TABLET_EVENTS_TABLE = new Text("tabletEvents");
- private Map<Integer,KeyExtent> tabletIds = new HashMap<Integer,KeyExtent>();
- private String uuid = null;
-
- @Override
- protected void setup(Context context) throws java.io.IOException, java.lang.InterruptedException {
- tabletIds = new HashMap<Integer,KeyExtent>();
- uuid = null;
- }
-
- @Override
- public void map(LogFileKey key, LogFileValue value, Context context) throws IOException, InterruptedException {
- if (key.event == LogEvents.OPEN) {
- uuid = key.tserverSession;
- } else if (key.event == LogEvents.DEFINE_TABLET) {
- if (key.tablet.getTableId().toString().equals(MetadataTable.ID)) {
- tabletIds.put(key.tid, new KeyExtent(key.tablet));
- }
- } else if ((key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) && tabletIds.containsKey(key.tid)) {
- for (Mutation m : value.mutations) {
- index(context, m, uuid, tabletIds.get(key.tid));
- }
- }
- }
-
- void index(Context context, Mutation m, String logFile, KeyExtent metaTablet) throws IOException, InterruptedException {
- List<ColumnUpdate> columnsUpdates = m.getUpdates();
-
- Text prevRow = null;
- long timestamp = 0;
-
- if (m.getRow().length > 0 && m.getRow()[0] == '~') {
- return;
- }
-
- for (ColumnUpdate cu : columnsUpdates) {
- if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && !cu.isDeleted()) {
- prevRow = new Text(cu.getValue());
- }
-
- timestamp = cu.getTimestamp();
- }
-
- byte[] serMut = WritableUtils.toByteArray(m);
-
- if (prevRow != null) {
- Mutation createEvent = new Mutation(new Text(m.getRow()));
- createEvent.put(prevRow, new Text(String.format("%020d", timestamp)), new Value(metaTablet.toString().getBytes(Constants.UTF8)));
- context.write(CREATE_EVENTS_TABLE, createEvent);
- }
-
- Mutation tabletEvent = new Mutation(new Text(m.getRow()));
- tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("mut"), new Value(serMut));
- tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("mtab"), new Value(metaTablet.toString().getBytes(Constants.UTF8)));
- tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("log"), new Value(logFile.getBytes(Constants.UTF8)));
- context.write(TABLET_EVENTS_TABLE, tabletEvent);
- }
- }
-
- static class Opts extends ClientOpts {
- @Parameter(description = "<logfile> { <logfile> ...}")
- List<String> logFiles = new ArrayList<String>();
-
- public ClientConfiguration getConf() {
- return this.getClientConfiguration();
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Opts opts = new Opts();
- opts.parseArgs(IndexMeta.class.getName(), args);
-
- String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();
-
- @SuppressWarnings("deprecation")
- Job job = new Job(getConf(), jobName);
- job.setJarByClass(this.getClass());
-
- List<String> logFiles = Arrays.asList(args).subList(4, args.length);
- Path paths[] = new Path[logFiles.size()];
- int count = 0;
- for (String logFile : logFiles) {
- paths[count++] = new Path(logFile);
- }
-
- job.setInputFormatClass(LogFileInputFormat.class);
- LogFileInputFormat.setInputPaths(job, paths);
-
- job.setNumReduceTasks(0);
-
- job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setZooKeeperInstance(job, opts.getConf());
- AccumuloOutputFormat.setConnectorInfo(job, opts.principal, opts.getToken());
- AccumuloOutputFormat.setCreateTables(job, false);
-
- job.setMapperClass(IndexMapper.class);
-
- Connector conn = opts.getConnector();
-
- try {
- conn.tableOperations().create("createEvents");
- } catch (TableExistsException tee) {
- Logger.getLogger(IndexMeta.class).warn("Table createEvents exists");
- }
-
- try {
- conn.tableOperations().create("tabletEvents");
- } catch (TableExistsException tee) {
- Logger.getLogger(IndexMeta.class).warn("Table tabletEvents exists");
- }
-
- job.waitForCompletion(true);
- return job.isSuccessful() ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(CachedConfiguration.getInstance(), new IndexMeta(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java
deleted file mode 100644
index 603bf71..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.accumulo.utils.metanalysis;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import org.apache.accumulo.tserver.logger.LogFileKey;
-import org.apache.accumulo.tserver.logger.LogFileValue;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-
-/**
- * Input format for Accumulo write ahead logs
- */
-public class LogFileInputFormat extends FileInputFormat<LogFileKey,LogFileValue> {
-
- private static class LogFileRecordReader extends RecordReader<LogFileKey,LogFileValue> {
-
- private FSDataInputStream fsdis;
- private LogFileKey key;
- private LogFileValue value;
- private long length;
-
- @Override
- public void close() throws IOException {
- fsdis.close();
- }
-
- @Override
- public LogFileKey getCurrentKey() throws IOException, InterruptedException {
- return key;
- }
-
- @Override
- public LogFileValue getCurrentValue() throws IOException, InterruptedException {
- return value;
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- float progress = (length - fsdis.getPos()) / (float) length;
- if (progress < 0)
- return 0;
- return progress;
- }
-
- @Override
- public void initialize(InputSplit is, TaskAttemptContext context) throws IOException, InterruptedException {
- FileSplit fileSplit = (FileSplit) is;
-
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
-
- key = new LogFileKey();
- value = new LogFileValue();
-
- fsdis = fs.open(fileSplit.getPath());
- FileStatus status = fs.getFileStatus(fileSplit.getPath());
- length = status.getLen();
- }
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- if (key == null)
- return false;
-
- try {
- key.readFields(fsdis);
- value.readFields(fsdis);
- return true;
- } catch (EOFException ex) {
- key = null;
- value = null;
- return false;
- }
- }
-
- }
-
-
- @Override
- public RecordReader<LogFileKey,LogFileValue> createRecordReader(InputSplit arg0, TaskAttemptContext arg1) throws IOException, InterruptedException {
- return new LogFileRecordReader();
- }
-
- @Override
- protected boolean isSplitable(JobContext context, Path filename) {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java
deleted file mode 100644
index cccb89e..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.utils.metanalysis;
-
-import java.io.IOException;
-
-import org.apache.accumulo.tserver.logger.LogFileKey;
-import org.apache.accumulo.tserver.logger.LogFileValue;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-/**
- * Output format for Accumulo write ahead logs.
- */
-public class LogFileOutputFormat extends FileOutputFormat<LogFileKey,LogFileValue> {
-
- private static class LogFileRecordWriter extends RecordWriter<LogFileKey,LogFileValue> {
-
- private FSDataOutputStream out;
-
- public LogFileRecordWriter(Path outputPath) throws IOException {
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
-
- out = fs.create(outputPath);
- }
-
- @Override
- public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
- out.close();
- }
-
- @Override
- public void write(LogFileKey key, LogFileValue val) throws IOException, InterruptedException {
- key.write(out);
- val.write(out);
- }
-
- }
-
- @Override
- public RecordWriter<LogFileKey,LogFileValue> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
- Path outputPath = getDefaultWorkFile(context, "");
- return new LogFileRecordWriter(outputPath);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java
deleted file mode 100644
index fc6c18a..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.utils.metanalysis;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.ColumnUpdate;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.server.cli.ClientOpts;
-import org.apache.accumulo.tserver.logger.LogFileValue;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * Looks up and prints mutations indexed by IndexMeta
- */
-public class PrintEvents {
-
- static class Opts extends ClientOpts {
- @Parameter(names = {"-t", "--tableId"}, description = "table id", required = true)
- String tableId;
- @Parameter(names = {"-e", "--endRow"}, description = "end row")
- String endRow;
- @Parameter(names = {"-t", "--time"}, description = "time, in milliseconds", required = true)
- long time;
- }
-
- public static void main(String[] args) throws Exception {
- Opts opts = new Opts();
- opts.parseArgs(PrintEvents.class.getName(), args);
-
- Connector conn = opts.getConnector();
-
- printEvents(conn, opts.tableId, opts.endRow, opts.time);
- }
-
- private static void printEvents(Connector conn, String tableId, String endRow, Long time) throws Exception {
- Scanner scanner = conn.createScanner("tabletEvents", new Authorizations());
- String metaRow = tableId + (endRow == null ? "<" : ";" + endRow);
- scanner.setRange(new Range(new Key(metaRow, String.format("%020d", time)), true, new Key(metaRow).followingKey(PartialKey.ROW), false));
- int count = 0;
-
- String lastLog = null;
-
- loop1: for (Entry<Key,Value> entry : scanner) {
- if (entry.getKey().getColumnQualifier().toString().equals("log")) {
- if (lastLog == null || !lastLog.equals(entry.getValue().toString()))
- System.out.println("Log : " + entry.getValue());
- lastLog = entry.getValue().toString();
- } else if (entry.getKey().getColumnQualifier().toString().equals("mut")) {
- DataInputStream dis = new DataInputStream(new ByteArrayInputStream(entry.getValue().get()));
- Mutation m = new Mutation();
- m.readFields(dis);
-
- LogFileValue lfv = new LogFileValue();
- lfv.mutations = Collections.singletonList(m);
-
- System.out.println(LogFileValue.format(lfv, 1));
-
- List<ColumnUpdate> columnsUpdates = m.getUpdates();
- for (ColumnUpdate cu : columnsUpdates) {
- if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && count > 0) {
- System.out.println("Saw change to prevrow, stopping printing events.");
- break loop1;
- }
- }
- count++;
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java
----------------------------------------------------------------------
diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java
deleted file mode 100644
index 3c15884..0000000
--- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Provides programs to analyze metadata mutations written to write ahead logs.
- *
- * <p>
- * These programs can be used when write ahead logs are archived. The best way to find
- * which write ahead logs contain metadata mutations is to grep the tablet server logs.
- * Grep for events where walogs were added to metadata tablets, then take the unique set
- * of walogs.
- *
- * <p>
- * To use these programs, use IndexMeta to index the metadata mutations in walogs into
- * Accumulo tables. Then use FindTable and PrintEvents to analyze those indexes.
- * FilterMetaiallows filtering walogs down to just metadata events. This is useful for the
- * case where the walogs need to be exported from the cluster for analysis.
- *
- * @since 1.5
- */
-package org.apache.accumulo.utils.metanalysis;