You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2012/10/16 00:17:26 UTC
svn commit: r1398539 [1/4] - in /accumulo/branches/ACCUMULO-259: ./
assemble/ assemble/platform/ assemble/platform/debian/
assemble/platform/debian/init.d/ conf/examples/1GB/native-standalone/
conf/examples/1GB/standalone/ conf/examples/2GB/native-stan...
Author: vines
Date: Mon Oct 15 22:17:22 2012
New Revision: 1398539
URL: http://svn.apache.org/viewvc?rev=1398539&view=rev
Log:
Mergng ACCUMULO-259 up to date to resume work
Added:
accumulo/branches/ACCUMULO-259/assemble/platform/
- copied from r1398536, accumulo/trunk/assemble/platform/
accumulo/branches/ACCUMULO-259/assemble/platform/debian/
- copied from r1398536, accumulo/trunk/assemble/platform/debian/
accumulo/branches/ACCUMULO-259/assemble/platform/debian/gc-only-init.sh
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/gc-only-init.sh
accumulo/branches/ACCUMULO-259/assemble/platform/debian/init.d/
- copied from r1398536, accumulo/trunk/assemble/platform/debian/init.d/
accumulo/branches/ACCUMULO-259/assemble/platform/debian/init.d/accumulo-gc
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/init.d/accumulo-gc
accumulo/branches/ACCUMULO-259/assemble/platform/debian/init.d/accumulo-master
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/init.d/accumulo-master
accumulo/branches/ACCUMULO-259/assemble/platform/debian/init.d/accumulo-monitor
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/init.d/accumulo-monitor
accumulo/branches/ACCUMULO-259/assemble/platform/debian/init.d/accumulo-tracer
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/init.d/accumulo-tracer
accumulo/branches/ACCUMULO-259/assemble/platform/debian/init.d/accumulo-tserver
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/init.d/accumulo-tserver
accumulo/branches/ACCUMULO-259/assemble/platform/debian/master-only-init.sh
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/master-only-init.sh
accumulo/branches/ACCUMULO-259/assemble/platform/debian/monitor-only-init.sh
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/monitor-only-init.sh
accumulo/branches/ACCUMULO-259/assemble/platform/debian/stand-alone-init.sh
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/stand-alone-init.sh
accumulo/branches/ACCUMULO-259/assemble/platform/debian/tracer-only-init.sh
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/tracer-only-init.sh
accumulo/branches/ACCUMULO-259/assemble/platform/debian/tserver-only-init.sh
- copied unchanged from r1398536, accumulo/trunk/assemble/platform/debian/tserver-only-init.sh
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/data/OldMutation.java
- copied unchanged from r1398536, accumulo/trunk/core/src/test/java/org/apache/accumulo/core/data/OldMutation.java
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java
- copied unchanged from r1398536, accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java
accumulo/branches/ACCUMULO-259/docs/README_UBUNTU
- copied unchanged from r1398536, accumulo/trunk/docs/README_UBUNTU
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/data/
- copied from r1398536, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/data/
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/data/ServerColumnUpdate.java
- copied unchanged from r1398536, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/data/ServerColumnUpdate.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/data/ServerMutation.java
- copied unchanged from r1398536, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/data/ServerMutation.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletMutations.java
- copied unchanged from r1398536, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletMutations.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
- copied unchanged from r1398536, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
- copied unchanged from r1398536, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java
accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/data/
- copied from r1398536, accumulo/trunk/server/src/test/java/org/apache/accumulo/server/data/
accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/data/ServerMutationTest.java
- copied unchanged from r1398536, accumulo/trunk/server/src/test/java/org/apache/accumulo/server/data/ServerMutationTest.java
accumulo/branches/ACCUMULO-259/test/system/auto/simple/fateStartvation.py
- copied unchanged from r1398536, accumulo/trunk/test/system/auto/simple/fateStartvation.py
Removed:
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletMutations.java
Modified:
accumulo/branches/ACCUMULO-259/ (props changed)
accumulo/branches/ACCUMULO-259/assemble/ (props changed)
accumulo/branches/ACCUMULO-259/conf/examples/1GB/native-standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/1GB/standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/2GB/native-standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/2GB/standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/3GB/native-standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/3GB/standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/512MB/native-standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/conf/examples/512MB/standalone/accumulo-env.sh
accumulo/branches/ACCUMULO-259/core/ (props changed)
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/CreateTableCommand.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/EGrepCommand.java
accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GrepCommand.java
accumulo/branches/ACCUMULO-259/core/src/main/thrift/tabletserver.thrift
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/data/MutationTest.java
accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
accumulo/branches/ACCUMULO-259/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java (contents, props changed)
accumulo/branches/ACCUMULO-259/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java (contents, props changed)
accumulo/branches/ACCUMULO-259/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooUtil.java
accumulo/branches/ACCUMULO-259/pom.xml
accumulo/branches/ACCUMULO-259/server/ (props changed)
accumulo/branches/ACCUMULO-259/server/src/main/c++/Makefile
accumulo/branches/ACCUMULO-259/server/src/main/c++/mlock/Makefile
accumulo/branches/ACCUMULO-259/server/src/main/c++/nativeMap/Makefile
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/Accumulo.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/logger/LogFileValue.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/metanalysis/PrintEvents.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/MutationLog.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletTime.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousBatchWalker.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousMoru.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousScanner.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousWalk.java
accumulo/branches/ACCUMULO-259/server/src/main/java/org/apache/accumulo/server/test/scalability/Ingest.java
accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/logger/LogFileTest.java
accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancerTest.java
accumulo/branches/ACCUMULO-259/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
accumulo/branches/ACCUMULO-259/src/ (props changed)
accumulo/branches/ACCUMULO-259/test/system/continuous/continuous-env.sh.example
accumulo/branches/ACCUMULO-259/test/system/continuous/run-verify.sh
accumulo/branches/ACCUMULO-259/test/system/continuous/start-batchwalkers.sh
accumulo/branches/ACCUMULO-259/test/system/continuous/start-ingest.sh
accumulo/branches/ACCUMULO-259/test/system/continuous/start-scanners.sh
accumulo/branches/ACCUMULO-259/test/system/continuous/start-walkers.sh
accumulo/branches/ACCUMULO-259/test/system/randomwalk/ (props changed)
accumulo/branches/ACCUMULO-259/test/system/randomwalk/bin/start-local.sh
Propchange: accumulo/branches/ACCUMULO-259/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.4/src:r1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438
Merged /accumulo/trunk:r1391755-1398536
Merged /accumulo/branches/1.4:r1393868,1396065,1396572,1396616,1396758,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438
Propchange: accumulo/branches/ACCUMULO-259/assemble/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Mon Oct 15 22:17:22 2012
@@ -0,0 +1,16 @@
+/accumulo/branches/1.3/assemble:1309369,1328076,1330246,1330264,1330944,1349971,1354669
+/accumulo/branches/1.3/src/assemble:1309369,1328076,1330246,1349971,1354669
+/accumulo/branches/1.4/assemble:1305403-1382577,1382613,1388120,1388629,1393868,1396065,1396572,1396616,1396758,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438,1398514
+/accumulo/branches/1.4/src/assemble:1305403-1356900,1358206,1363430,1364778,1365213,1376679-1388120,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438,1398514
+/accumulo/branches/ACCUMULO-672/assemble:1357826,1357829,1357842,1357858,1358236,1359163
+/accumulo/trunk/assemble:1343822-1391624,1391755-1398536
+/accumulo/trunk/src/assemble:1329425,1332224,1332278,1332347,1333047,1333070,1341000,1342373,1350779,1351691,1356400,1359721
+/incubator/accumulo/branches/1.3/assemble:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632
+/incubator/accumulo/branches/1.3/src/assemble:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632
+/incubator/accumulo/branches/1.3.5rc/assemble:1209938
+/incubator/accumulo/branches/1.3.5rc/src/assemble:1209938
+/incubator/accumulo/branches/1.4/assemble:1201902-1305402
+/incubator/accumulo/branches/1.4/src/assemble:1201899-1305402
+/incubator/accumulo/branches/1.4.0rc/assemble:1304025,1305326
+/incubator/accumulo/branches/1.4.0rc/src/assemble:1304025,1305326
+/incubator/accumulo/trunk/src/assemble:1178656-1201898,1205476,1205570,1208726,1222413,1222719,1222725,1222733-1222734,1296160-1296495
Modified: accumulo/branches/ACCUMULO-259/conf/examples/1GB/native-standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/1GB/native-standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/1GB/native-standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/1GB/native-standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/1GB/standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/1GB/standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/1GB/standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/1GB/standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx384m -Xms384m -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx384m -Xms384m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/2GB/native-standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/2GB/native-standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/2GB/native-standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/2GB/native-standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx256m -Xms256m -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx256m -Xms256m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx256m -Xms256m"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx128m -Xms64m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx128m -Xms128m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/2GB/standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/2GB/standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/2GB/standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/2GB/standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx768m -Xms768m -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx768m -Xms768m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx256m -Xms256m"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx128m -Xms64m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx128m -Xms128m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/3GB/native-standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/3GB/native-standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/3GB/native-standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/3GB/native-standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx1g -Xms1g -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx1g -Xms1g"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx1g -Xms256m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx256m -Xms256m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/3GB/standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/3GB/standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/3GB/standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/3GB/standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx2g -Xms2g -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx1g -Xms1g"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx1g -Xms256m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx256m -Xms256m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/512MB/native-standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/512MB/native-standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/512MB/native-standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/512MB/native-standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx48m -Xms48m -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx48m -Xms48m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
Modified: accumulo/branches/ACCUMULO-259/conf/examples/512MB/standalone/accumulo-env.sh
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/conf/examples/512MB/standalone/accumulo-env.sh?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/conf/examples/512MB/standalone/accumulo-env.sh (original)
+++ accumulo/branches/ACCUMULO-259/conf/examples/512MB/standalone/accumulo-env.sh Mon Oct 15 22:17:22 2012
@@ -36,7 +36,7 @@ if [ -f ${ACCUMULO_HOME}/conf/accumulo.p
then
POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_HOME}/conf/accumulo.policy"
fi
-test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m -Xss160k"
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
Propchange: accumulo/branches/ACCUMULO-259/core/
------------------------------------------------------------------------------
Merged /accumulo/trunk/core:r1391755-1398536
Merged /accumulo/branches/1.4/core:r1393868,1396065,1396572,1396616,1396758,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438
Merged /accumulo/branches/1.4/src/core:r1396065,1396227,1396616,1396772,1397048,1397113,1397117,1397176,1397189,1397383,1397700,1397921,1398286,1398308,1398359,1398393,1398399,1398438
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java Mon Oct 15 22:17:22 2012
@@ -190,11 +190,7 @@ public class TableOperationsImpl extends
List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes()), ByteBuffer.wrap(timeType.name().getBytes()));
- Map<String,String> opts;
- if (limitVersion) {
- opts = IteratorUtil.generateInitialTableProperties();
- } else
- opts = Collections.emptyMap();
+ Map<String,String> opts = IteratorUtil.generateInitialTableProperties(limitVersion);
try {
doTableOperation(TableOperation.CREATE, args, opts);
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java Mon Oct 15 22:17:22 2012
@@ -137,6 +137,11 @@ public class ConnectorImpl extends Conne
}
@Override
+ public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config) {
+ return new MultiTableBatchWriterImpl(instance, credentials, config);
+ }
+
+ @Override
public Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException {
ArgumentChecker.notNull(tableName, authorizations);
return new ScannerImpl(instance, credentials, getTableId(tableName), authorizations);
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java Mon Oct 15 22:17:22 2012
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.A
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocations;
import org.apache.accumulo.core.client.impl.TabletLocatorImpl.TabletLocationObtainer;
import org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.ResultReceiver;
import org.apache.accumulo.core.data.Column;
@@ -44,6 +45,7 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
@@ -68,12 +70,12 @@ public class MetadataLocationObtainer im
}
@Override
- public List<TabletLocation> lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent) throws AccumuloSecurityException,
+ public TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent) throws AccumuloSecurityException,
AccumuloException {
-
- ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
-
+
try {
+ ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
+
OpTimer opTimer = null;
if (log.isTraceEnabled())
opTimer = new OpTimer(log, Level.TRACE).start("Looking up in " + src.tablet_extent.getTableId() + " row=" + TextUtil.truncate(row) + " extent="
@@ -98,12 +100,14 @@ public class MetadataLocationObtainer im
// System.out.println("results "+results.keySet());
- SortedMap<KeyExtent,Text> metadata = MetadataTable.getMetadataLocationEntries(results);
+ Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> metadata = MetadataTable.getMetadataLocationEntries(results);
- for (Entry<KeyExtent,Text> entry : metadata.entrySet()) {
+ for (Entry<KeyExtent,Text> entry : metadata.getFirst().entrySet()) {
list.add(new TabletLocation(entry.getKey(), entry.getValue().toString()));
}
+ return new TabletLocations(list, metadata.getSecond());
+
} catch (AccumuloServerException ase) {
if (log.isTraceEnabled())
log.trace(src.tablet_extent.getTableId() + " lookup failed, " + src.tablet_location + " server side exception");
@@ -118,7 +122,7 @@ public class MetadataLocationObtainer im
parent.invalidateCache(src.tablet_location);
}
- return list;
+ return null;
}
@Override
@@ -161,7 +165,7 @@ public class MetadataLocationObtainer im
throw e;
}
- SortedMap<KeyExtent,Text> metadata = MetadataTable.getMetadataLocationEntries(results);
+ SortedMap<KeyExtent,Text> metadata = MetadataTable.getMetadataLocationEntries(results).getFirst();
for (Entry<KeyExtent,Text> entry : metadata.entrySet()) {
list.add(new TabletLocation(entry.getKey(), entry.getValue().toString()));
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java Mon Oct 15 22:17:22 2012
@@ -126,6 +126,25 @@ public abstract class TabletLocator {
return tl;
}
+ public static class TabletLocations {
+
+ private final List<TabletLocation> locations;
+ private final List<KeyExtent> locationless;
+
+ public TabletLocations(List<TabletLocation> locations, List<KeyExtent> locationless) {
+ this.locations = locations;
+ this.locationless = locationless;
+ }
+
+ public List<TabletLocation> getLocations() {
+ return locations;
+ }
+
+ public List<KeyExtent> getLocationless() {
+ return locationless;
+ }
+ }
+
public static class TabletLocation implements Comparable<TabletLocation> {
private static WeakHashMap<String,WeakReference<String>> tabletLocs = new WeakHashMap<String,WeakReference<String>>();
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java Mon Oct 15 22:17:22 2012
@@ -91,7 +91,10 @@ public class TabletLocatorImpl extends T
private Lock wLock = rwLock.writeLock();
public static interface TabletLocationObtainer {
- List<TabletLocation> lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent) throws AccumuloSecurityException, AccumuloException;
+ /**
+ * @return null when unable to read information successfully
+ */
+ TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent) throws AccumuloSecurityException, AccumuloException;
List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>> map, TabletLocator parent) throws AccumuloSecurityException,
AccumuloException;
@@ -389,23 +392,31 @@ public class TabletLocatorImpl extends T
TabletLocation ptl = parent.locateTablet(metadataRow, false, retry);
if (ptl != null) {
- List<TabletLocation> locations = locationObtainer.lookupTablet(ptl, metadataRow, lastTabletRow, parent);
- if (locations.size() == 0 && !ptl.tablet_extent.isRootTablet()) {
- // try the next tablet
+ TabletLocations locations = locationObtainer.lookupTablet(ptl, metadataRow, lastTabletRow, parent);
+ while (locations != null && locations.getLocations().isEmpty() && locations.getLocationless().isEmpty()
+ && !ptl.tablet_extent.isRootTablet()) {
+ // try the next tablet, the current tablet does not have any tablets that overlap the row
Text er = ptl.tablet_extent.getEndRow();
if (er != null && er.compareTo(lastTabletRow) < 0) {
// System.out.println("er "+er+" ltr "+lastTabletRow);
ptl = parent.locateTablet(er, true, retry);
if (ptl != null)
locations = locationObtainer.lookupTablet(ptl, metadataRow, lastTabletRow, parent);
+ else
+ break;
+ } else {
+ break;
}
}
+ if (locations == null)
+ return;
+
// cannot assume the list contains contiguous key extents... so it is probably
// best to deal with each extent individually
Text lastEndRow = null;
- for (TabletLocation tabletLocation : locations) {
+ for (TabletLocation tabletLocation : locations.getLocations()) {
KeyExtent ke = tabletLocation.tablet_extent;
TabletLocation locToCache;
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java Mon Oct 15 22:17:22 2012
@@ -30,11 +30,17 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.fs.FileSystem;
public class MockAccumulo {
final Map<String,MockTable> tables = new HashMap<String,MockTable>();
final Map<String,String> systemProperties = new HashMap<String,String>();
Map<String,MockUser> users = new HashMap<String,MockUser>();
+ final FileSystem fs;
+
+ MockAccumulo(FileSystem fs) {
+ this.fs = fs;
+ }
{
MockUser root = new MockUser("root", new byte[] {}, Constants.NO_AUTHS);
@@ -43,6 +49,10 @@ public class MockAccumulo {
createTable("root", Constants.METADATA_TABLE_NAME, true, TimeType.LOGICAL);
}
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
void setProperty(String key, String value) {
systemProperties.put(key, value);
}
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java Mon Oct 15 22:17:22 2012
@@ -36,16 +36,18 @@ public class MockConnector extends Conne
String username;
private final MockAccumulo acu;
+ private final Instance instance;
- MockConnector(String username) {
- this(username, new MockAccumulo());
+ MockConnector(String username, Instance instance) {
+ this(username, new MockAccumulo(MockInstance.getDefaultFileSystem()), instance);
}
@SuppressWarnings("deprecation")
// Not really deprecated, just discouraging client use.
- MockConnector(String username, MockAccumulo acu) {
+ MockConnector(String username, MockAccumulo acu, Instance instance) {
this.username = username;
this.acu = acu;
+ this.instance = instance;
}
@Override
@@ -102,7 +104,7 @@ public class MockConnector extends Conne
@Override
public Instance getInstance() {
- return new MockInstance();
+ return instance;
}
@Override
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstance.java Mon Oct 15 22:17:22 2012
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.client.mock;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashMap;
@@ -30,7 +31,9 @@ import org.apache.accumulo.core.conf.Acc
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
public class MockInstance implements Instance {
@@ -41,16 +44,28 @@ public class MockInstance implements Ins
String instanceName;
public MockInstance() {
- acu = new MockAccumulo();
+ acu = new MockAccumulo(getDefaultFileSystem());
instanceName = "mock-instance";
}
+ static FileSystem getDefaultFileSystem() {
+ try {
+ return FileSystem.get(CachedConfiguration.getInstance());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
public MockInstance(String instanceName) {
+ this(instanceName, getDefaultFileSystem());
+ }
+
+ public MockInstance(String instanceName, FileSystem fs) {
synchronized (instances) {
if (instances.containsKey(instanceName))
acu = instances.get(instanceName);
else
- instances.put(instanceName, acu = new MockAccumulo());
+ instances.put(instanceName, acu = new MockAccumulo(fs));
}
this.instanceName = instanceName;
}
@@ -87,7 +102,7 @@ public class MockInstance implements Ins
@Override
public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
- Connector conn = new MockConnector(user, acu);
+ Connector conn = new MockConnector(user, acu, this);
conn.securityOperations().createUser(user, pass, new Authorizations());
return conn;
}
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java Mon Oct 15 22:17:22 2012
@@ -88,9 +88,9 @@ public class MockTable {
private TimeType timeType;
SortedSet<Text> splits = new TreeSet<Text>();
- MockTable(boolean useVersions, TimeType timeType) {
+ MockTable(boolean limitVersion, TimeType timeType) {
this.timeType = timeType;
- settings = IteratorUtil.generateInitialTableProperties();
+ settings = IteratorUtil.generateInitialTableProperties(limitVersion);
for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) {
String key = entry.getKey();
if (key.startsWith(Property.TABLE_PREFIX.getKey()))
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java Mon Oct 15 22:17:22 2012
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.client.mock;
+import java.io.DataInputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
@@ -36,9 +37,20 @@ import org.apache.accumulo.core.client.T
import org.apache.accumulo.core.client.admin.FindMax;
import org.apache.accumulo.core.client.admin.TableOperationsHelper;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
public class MockTableOperations extends TableOperationsHelper {
@@ -160,7 +172,92 @@ public class MockTableOperations extends
@Override
public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws IOException, AccumuloException,
AccumuloSecurityException, TableNotFoundException {
- throw new NotImplementedException();
+ long time = System.currentTimeMillis();
+ MockTable table = acu.tables.get(tableName);
+ if (table == null) {
+ throw new TableNotFoundException(null, tableName,
+ "The table was not found");
+ }
+ Path importPath = new Path(dir);
+ Path failurePath = new Path(failureDir);
+
+ FileSystem fs = acu.getFileSystem();
+ /*
+ * check preconditions
+ */
+ // directories are directories
+ if (fs.isFile(importPath)) {
+ throw new IOException("Import path must be a directory.");
+ }
+ if (fs.isFile(failurePath)) {
+ throw new IOException("Failure path must be a directory.");
+ }
+ // failures are writable
+ Path createPath = failurePath.suffix("/.createFile");
+ FSDataOutputStream createStream = null;
+ try {
+ createStream = fs.create(createPath);
+ } catch (IOException e) {
+ throw new IOException("Error path is not writable.");
+ } finally {
+ if (createStream != null) {
+ createStream.close();
+ }
+ }
+ fs.delete(createPath, false);
+ // failures are empty
+ FileStatus[] failureChildStats = fs.listStatus(failurePath);
+ if (failureChildStats.length > 0) {
+ throw new IOException("Error path must be empty.");
+ }
+ /*
+ * Begin the import - iterate the files in the path
+ */
+ for (FileStatus importStatus : fs.listStatus(importPath)) {
+ try {
+ FileSKVIterator importIterator = FileOperations.getInstance()
+ .openReader(importStatus.getPath().toString(), true, fs,
+ fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
+ while (importIterator.hasTop()) {
+ Key key = importIterator.getTopKey();
+ Value value = importIterator.getTopValue();
+ if (setTime) {
+ key.setTimestamp(time);
+ }
+ Mutation mutation = new Mutation(key.getRow());
+ if (!key.isDeleted()) {
+ mutation.put(key.getColumnFamily(), key.getColumnQualifier(),
+ new ColumnVisibility(key.getColumnVisibilityData().toArray()),
+ key.getTimestamp(), value);
+ } else {
+ mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(),
+ new ColumnVisibility(key.getColumnVisibilityData().toArray()),
+ key.getTimestamp());
+ }
+ table.addMutation(mutation);
+ importIterator.next();
+ }
+ } catch (Exception e) {
+ FSDataOutputStream failureWriter = null;
+ DataInputStream failureReader = null;
+ try {
+ failureWriter = fs.create(failurePath.suffix("/"
+ + importStatus.getPath().getName()));
+ failureReader = fs.open(importStatus.getPath());
+ int read = 0;
+ byte[] buffer = new byte[1024];
+ while (-1 != (read = failureReader.read(buffer))) {
+ failureWriter.write(buffer, 0, read);
+ }
+ } finally {
+ if (failureReader != null)
+ failureReader.close();
+ if (failureWriter != null)
+ failureWriter.close();
+ }
+ }
+ fs.delete(importStatus.getPath(), true);
+ }
}
@Override
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/ColumnUpdate.java Mon Oct 15 22:17:22 2012
@@ -31,11 +31,9 @@ public class ColumnUpdate {
private long timestamp;
private boolean hasTimestamp;
private byte[] val;
- private byte[] data;
- private int tsOffset;
private boolean deleted;
- public ColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val, byte[] data, int tsOffset) {
+ public ColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
this.columnFamily = cf;
this.columnQualifier = cq;
this.columnVisibility = cv;
@@ -43,25 +41,15 @@ public class ColumnUpdate {
this.timestamp = ts;
this.deleted = deleted;
this.val = val;
- this.data = data;
- this.tsOffset = tsOffset;
}
- public void setSystemTimestamp(long v) {
+ /**
+ * @deprecated use setTimestamp(long);
+ * @param timestamp
+ */
+ public void setSystemTimestamp(long timestamp) {
if (hasTimestamp)
throw new IllegalStateException("Cannot set system timestamp when user set a timestamp");
-
- int tso = this.tsOffset;
- data[tso++] = (byte) (v >>> 56);
- data[tso++] = (byte) (v >>> 48);
- data[tso++] = (byte) (v >>> 40);
- data[tso++] = (byte) (v >>> 32);
- data[tso++] = (byte) (v >>> 24);
- data[tso++] = (byte) (v >>> 16);
- data[tso++] = (byte) (v >>> 8);
- data[tso++] = (byte) (v >>> 0);
-
- this.timestamp = v;
}
public boolean hasTimestamp() {
@@ -100,4 +88,19 @@ public class ColumnUpdate {
return new String(Arrays.toString(columnFamily)) + ":" + new String(Arrays.toString(columnQualifier)) + " ["
+ new String(Arrays.toString(columnVisibility)) + "] " + (hasTimestamp ? timestamp : "NO_TIME_STAMP") + " " + Arrays.toString(val) + " " + deleted;
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ColumnUpdate))
+ return false;
+ ColumnUpdate upd = (ColumnUpdate)obj;
+ return Arrays.equals(getColumnFamily(), upd.getColumnFamily()) &&
+ Arrays.equals(getColumnQualifier(), upd.getColumnQualifier()) &&
+ Arrays.equals(getColumnVisibility(), upd.getColumnVisibility()) &&
+ isDeleted() == upd.isDeleted() &&
+ Arrays.equals(getValue(), upd.getValue()) &&
+ hasTimestamp() == upd.hasTimestamp() &&
+ getTimestamp() == upd.getTimestamp();
+ }
+
}
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/Mutation.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/Mutation.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/data/Mutation.java Mon Oct 15 22:17:22 2012
@@ -30,6 +30,7 @@ import org.apache.accumulo.core.util.Byt
import org.apache.accumulo.core.util.TextUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
/**
* <p>
@@ -55,11 +56,17 @@ public class Mutation implements Writabl
static final int VALUE_SIZE_COPY_CUTOFF = 1 << 15;
+ public static enum SERIALIZED_FORMAT {
+ VERSION1,
+ VERSION2
+ };
+
+ private boolean useOldDeserialize = false;
private byte[] row;
private byte[] data;
private int entries;
private List<byte[]> values;
-
+
// created this little class instead of using ByteArrayOutput stream and DataOutputStream
// because both are synchronized... lots of small syncs slow things down
private static class ByteBuffer {
@@ -100,32 +107,41 @@ public class Mutation implements Writabl
data[offset++] = 0;
}
- void add(long v) {
- reserve(8);
- data[offset++] = (byte) (v >>> 56);
- data[offset++] = (byte) (v >>> 48);
- data[offset++] = (byte) (v >>> 40);
- data[offset++] = (byte) (v >>> 32);
- data[offset++] = (byte) (v >>> 24);
- data[offset++] = (byte) (v >>> 16);
- data[offset++] = (byte) (v >>> 8);
- data[offset++] = (byte) (v >>> 0);
- }
-
- void add(int i) {
- reserve(4);
- data[offset++] = (byte) (i >>> 24);
- data[offset++] = (byte) (i >>> 16);
- data[offset++] = (byte) (i >>> 8);
- data[offset++] = (byte) (i >>> 0);
- }
-
public byte[] toArray() {
byte ret[] = new byte[offset];
System.arraycopy(data, 0, ret, 0, offset);
return ret;
}
+ public void writeVLong(long i) {
+ reserve(9);
+ if (i >= -112 && i <= 127) {
+ data[offset++] = (byte)i;
+ return;
+ }
+
+ int len = -112;
+ if (i < 0) {
+ i ^= -1L; // take one's complement'
+ len = -120;
+ }
+
+ long tmp = i;
+ while (tmp != 0) {
+ tmp = tmp >> 8;
+ len--;
+ }
+
+ data[offset++] = (byte)len;
+
+ len = (len < -120) ? -(len + 120) : -(len + 112);
+
+ for (int idx = len; idx != 0; idx--) {
+ int shiftbits = (idx - 1) * 8;
+ long mask = 0xFFL << shiftbits;
+ data[offset++] = (byte)((i & mask) >> shiftbits);
+ }
+ }
}
private static class SimpleReader {
@@ -135,10 +151,9 @@ public class Mutation implements Writabl
SimpleReader(byte b[]) {
this.data = b;
}
-
+
int readInt() {
return (data[offset++] << 24) + ((data[offset++] & 255) << 16) + ((data[offset++] & 255) << 8) + ((data[offset++] & 255) << 0);
-
}
long readLong() {
@@ -155,6 +170,20 @@ public class Mutation implements Writabl
return (data[offset++] == 1);
}
+ long readVLong() {
+ byte firstByte = data[offset++];
+ int len = WritableUtils.decodeVIntSize(firstByte);
+ if (len == 1) {
+ return firstByte;
+ }
+ long i = 0;
+ for (int idx = 0; idx < len-1; idx++) {
+ byte b = data[offset++];
+ i = i << 8;
+ i = i | (b & 0xFF);
+ }
+ return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
+ }
}
private ByteBuffer buffer;
@@ -202,12 +231,12 @@ public class Mutation implements Writabl
}
private void put(byte b[]) {
- buffer.add(b.length);
+ buffer.writeVLong(b.length);
buffer.add(b);
}
private void put(Text t) {
- buffer.add(t.getLength());
+ buffer.writeVLong(t.getLength());
buffer.add(t.getBytes(), 0, t.getLength());
}
@@ -216,11 +245,11 @@ public class Mutation implements Writabl
}
private void put(int i) {
- buffer.add(i);
+ buffer.writeVLong(i);
}
private void put(long l) {
- buffer.add(l);
+ buffer.writeVLong(l);
}
private void put(Text cf, Text cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
@@ -232,7 +261,8 @@ public class Mutation implements Writabl
put(cq);
put(cv);
put(hasts);
- put(ts);
+ if (hasts)
+ put(ts);
put(deleted);
if (val.length < VALUE_SIZE_COPY_CUTOFF) {
@@ -337,7 +367,7 @@ public class Mutation implements Writabl
put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, false, value);
}
- private byte[] readBytes(SimpleReader in) {
+ private byte[] oldReadBytes(SimpleReader in) {
int len = in.readInt();
if (len == 0)
return EMPTY_BYTES;
@@ -347,6 +377,16 @@ public class Mutation implements Writabl
return bytes;
}
+ private byte[] readBytes(SimpleReader in) {
+ int len = (int)in.readVLong();
+ if (len == 0)
+ return EMPTY_BYTES;
+
+ byte bytes[] = new byte[len];
+ in.readBytes(bytes);
+ return bytes;
+ }
+
public List<ColumnUpdate> getUpdates() {
serialize();
@@ -368,17 +408,51 @@ public class Mutation implements Writabl
return updates;
}
+ protected ColumnUpdate newColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
+ return new ColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
+ }
+
private ColumnUpdate deserializeColumnUpdate(SimpleReader in) {
+ if (useOldDeserialize)
+ return oldDeserializeColumnUpdate(in);
+ return newDeserializeColumnUpdate(in);
+ }
+
+ private ColumnUpdate oldDeserializeColumnUpdate(SimpleReader in) {
+ byte[] cf = oldReadBytes(in);
+ byte[] cq = oldReadBytes(in);
+ byte[] cv = oldReadBytes(in);
+ boolean hasts = in.readBoolean();
+ long ts = in.readLong();
+ boolean deleted = in.readBoolean();
+
+ byte[] val;
+ int valLen = in.readInt();
+
+ if (valLen < 0) {
+ val = values.get((-1 * valLen) - 1);
+ } else if (valLen == 0) {
+ val = EMPTY_BYTES;
+ } else {
+ val = new byte[valLen];
+ in.readBytes(val);
+ }
+
+ return newColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
+ }
+
+ private ColumnUpdate newDeserializeColumnUpdate(SimpleReader in) {
byte[] cf = readBytes(in);
byte[] cq = readBytes(in);
byte[] cv = readBytes(in);
boolean hasts = in.readBoolean();
- int tso = in.offset;
- long ts = in.readLong();
+ long ts = 0;
+ if (hasts)
+ ts = in.readVLong();
boolean deleted = in.readBoolean();
byte[] val;
- int valLen = in.readInt();
+ int valLen = (int)in.readVLong();
if (valLen < 0) {
val = values.get((-1 * valLen) - 1);
@@ -389,7 +463,7 @@ public class Mutation implements Writabl
in.readBytes(val);
}
- return new ColumnUpdate(cf, cq, cv, hasts, ts, deleted, val, data, tso);
+ return newColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
}
private int cachedValLens = -1;
@@ -416,7 +490,7 @@ public class Mutation implements Writabl
}
public long estimatedMemoryUsed() {
- return numBytes() + 230;
+ return numBytes() + 238;
}
/**
@@ -428,6 +502,12 @@ public class Mutation implements Writabl
@Override
public void readFields(DataInput in) throws IOException {
+ byte first = in.readByte();
+ if ((first & 0x80) != 0x80) {
+ oldReadFields(first, in);
+ return;
+ }
+
// Clear out cached column updates and value lengths so
// that we recalculate them based on the (potentially) new
// data we are about to read in.
@@ -435,7 +515,43 @@ public class Mutation implements Writabl
cachedValLens = -1;
buffer = null;
- int len = in.readInt();
+ int len = WritableUtils.readVInt(in);
+ row = new byte[len];
+ in.readFully(row);
+ len = WritableUtils.readVInt(in);
+ data = new byte[len];
+ in.readFully(data);
+ entries = WritableUtils.readVInt(in);
+
+ boolean valuesPresent = (first & 0x01) == 0x01;
+ if (!valuesPresent) {
+ values = null;
+ } else {
+ values = new ArrayList<byte[]>();
+ int numValues = WritableUtils.readVInt(in);
+ for (int i = 0; i < numValues; i++) {
+ len = WritableUtils.readVInt(in);
+ byte val[] = new byte[len];
+ in.readFully(val);
+ values.add(val);
+ }
+ }
+ }
+
+ public void oldReadFields(byte first, DataInput in) throws IOException {
+ // Clear out cached column updates and value lengths so
+ // that we recalculate them based on the (potentially) new
+ // data we are about to read in.
+ useOldDeserialize = true;
+ updates = null;
+ cachedValLens = -1;
+ buffer = null;
+ byte b = (byte)in.readByte();
+ byte c = (byte)in.readByte();
+ byte d = (byte)in.readByte();
+
+ int len = (((first & 0xff) << 24) | ((b & 0xff) << 16) |
+ ((c & 0xff) << 8) | (d & 0xff));
row = new byte[len];
in.readFully(row);
len = in.readInt();
@@ -458,27 +574,29 @@ public class Mutation implements Writabl
}
}
+
+
+
@Override
public void write(DataOutput out) throws IOException {
serialize();
- out.writeInt(row.length);
+ byte hasValues = (values == null) ? 0 : (byte)1;
+ out.write((byte)(0x80 | hasValues));
+
+ WritableUtils.writeVInt(out, row.length);
out.write(row);
- out.writeInt(data.length);
+ WritableUtils.writeVInt(out, data.length);
out.write(data);
- out.writeInt(entries);
+ WritableUtils.writeVInt(out, entries);
- if (values == null)
- out.writeBoolean(false);
- else {
- out.writeBoolean(true);
- out.writeInt(values.size());
+ if (hasValues > 0) {
+ WritableUtils.writeVInt(out, values.size());
for (int i = 0; i < values.size(); i++) {
byte val[] = values.get(i);
- out.writeInt(val.length);
+ WritableUtils.writeVInt(out, val.length);
out.write(val);
}
}
-
}
@Override
@@ -518,4 +636,8 @@ public class Mutation implements Writabl
return new TMutation(java.nio.ByteBuffer.wrap(row), java.nio.ByteBuffer.wrap(data), ByteBufferUtil.toByteBuffers(values), entries);
}
+ public SERIALIZED_FORMAT getSerializedFormat() {
+ return this.useOldDeserialize ? SERIALIZED_FORMAT.VERSION1 : SERIALIZED_FORMAT.VERSION2;
+ }
+
}
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java Mon Oct 15 22:17:22 2012
@@ -28,8 +28,10 @@ import org.apache.accumulo.core.file.blo
import org.apache.accumulo.core.file.rfile.RFile.Reader;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -47,7 +49,20 @@ public class PrintInfo {
Option histogramOption = new Option("h", "histogram", false, "print a histogram of the key-value sizes");
opts.addOption(histogramOption);
- CommandLine commandLine = new BasicParser().parse(opts, args);
+ CommandLine commandLine = null;
+ try {
+ commandLine = new BasicParser().parse(opts, args);
+ if (commandLine.getArgs().length == 0) {
+ throw new ParseException("No files were given");
+ }
+
+ } catch (ParseException e) {
+ System.err.println("Failed to parse command line : " + e.getMessage());
+ System.err.println();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("rfile-info <rfile> {<rfile>}", opts);
+ System.exit(-1);
+ }
boolean dump = commandLine.hasOption(dumpKeys.getOpt());
boolean doHistogram = commandLine.hasOption(histogramOption.getOpt());
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/FirstEntryInRowIterator.java Mon Oct 15 22:17:22 2012
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.Text;
public class FirstEntryInRowIterator extends SkippingIterator implements OptionDescriber {
// options
- private static final String NUM_SCANS_STRING_NAME = "scansBeforeSeek";
+ static final String NUM_SCANS_STRING_NAME = "scansBeforeSeek";
// iterator predecessor seek options to pass through
private Range latestRange;
@@ -75,7 +75,7 @@ public class FirstEntryInRowIterator ext
// this is only ever called immediately after getting "next" entry
@Override
protected void consume() throws IOException {
- if (lastRowFound == null)
+ if (finished == true || lastRowFound == null)
return;
int count = 0;
while (getSource().hasTop() && lastRowFound.equals(getSource().getTopKey().getRow())) {
@@ -92,11 +92,24 @@ public class FirstEntryInRowIterator ext
Key nextKey = getSource().getTopKey().followingKey(PartialKey.ROW);
if (!latestRange.afterEndKey(nextKey))
getSource().seek(new Range(nextKey, true, latestRange.getEndKey(), latestRange.isEndKeyInclusive()), latestColumnFamilies, latestInclusive);
+ else
+ {
+ finished = true;
+ break;
+ }
}
}
lastRowFound = getSource().hasTop() ? getSource().getTopKey().getRow(lastRowFound) : null;
}
+ private boolean finished = true;
+
+ @Override
+ public boolean hasTop()
+ {
+ return !finished && getSource().hasTop();
+ }
+
@Override
public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
// save parameters for future internal seeks
@@ -108,7 +121,8 @@ public class FirstEntryInRowIterator ext
Key startKey = range.getStartKey();
Range seekRange = new Range(startKey == null ? null : new Key(startKey.getRow()), true, range.getEndKey(), range.isEndKeyInclusive());
super.seek(seekRange, columnFamilies, inclusive);
-
+ finished = false;
+
if (getSource().hasTop()) {
lastRowFound = getSource().getTopKey().getRow();
if (range.beforeStartKey(getSource().getTopKey()))
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java Mon Oct 15 22:17:22 2012
@@ -63,13 +63,22 @@ public class IteratorUtil {
}
- public static Map<String,String> generateInitialTableProperties() {
+ /**
+ * Generate the initial (default) properties for a table
+ * @param limitVersion
+ * include a VersioningIterator at priority 20 that retains a single version of a given K/V pair.
+ * @return A map of Table properties
+ */
+ public static Map<String,String> generateInitialTableProperties(boolean limitVersion) {
TreeMap<String,String> props = new TreeMap<String,String>();
- for (IteratorScope iterScope : IteratorScope.values()) {
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers", "20," + VersioningIterator.class.getName());
- props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers.opt.maxVersions", "1");
+ if (limitVersion) {
+ for (IteratorScope iterScope : IteratorScope.values()) {
+ props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers", "20," + VersioningIterator.class.getName());
+ props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers.opt.maxVersions", "1");
+ }
}
+
return props;
}
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java?rev=1398539&r1=1398538&r2=1398539&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java Mon Oct 15 22:17:22 2012
@@ -46,12 +46,15 @@ public class RegExFilter extends Filter
return result;
}
+
+
public static final String ROW_REGEX = "rowRegex";
public static final String COLF_REGEX = "colfRegex";
public static final String COLQ_REGEX = "colqRegex";
public static final String VALUE_REGEX = "valueRegex";
public static final String OR_FIELDS = "orFields";
public static final String ENCODING = "encoding";
+ public static final String MATCH_SUBSTRING = "matchSubstring";
public static final String ENCODING_DEFAULT = "UTF-8";
@@ -60,6 +63,7 @@ public class RegExFilter extends Filter
private Matcher colqMatcher;
private Matcher valueMatcher;
private boolean orFields = false;
+ private boolean matchSubstring = false;
private String encoding = ENCODING_DEFAULT;
@@ -74,7 +78,7 @@ public class RegExFilter extends Filter
if (matcher != null) {
try {
matcher.reset(new String(bs.getBackingArray(), bs.offset(), bs.length(), encoding));
- return matcher.matches();
+ return matchSubstring ? matcher.find() : matcher.matches();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
@@ -86,7 +90,7 @@ public class RegExFilter extends Filter
if (matcher != null) {
try {
matcher.reset(new String(data, offset, len, encoding));
- return matcher.matches();
+ return matchSubstring ? matcher.find() : matcher.matches();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
@@ -97,10 +101,18 @@ public class RegExFilter extends Filter
@Override
public boolean accept(Key key, Value value) {
if (orFields)
- return matches(rowMatcher, key.getRowData()) || matches(colfMatcher, key.getColumnFamilyData()) || matches(colqMatcher, key.getColumnQualifierData())
- || matches(valueMatcher, value.get(), 0, value.get().length);
- return matches(rowMatcher, key.getRowData()) && matches(colfMatcher, key.getColumnFamilyData()) && matches(colqMatcher, key.getColumnQualifierData())
- && matches(valueMatcher, value.get(), 0, value.get().length);
+ return (
+ (matches(rowMatcher, rowMatcher == null ? null : key.getRowData())) ||
+ (matches(colfMatcher, colfMatcher == null ? null : key.getColumnFamilyData())) ||
+ (matches(colqMatcher, colqMatcher == null ? null : key.getColumnQualifierData())) ||
+ (matches(valueMatcher, value.get(), 0, value.get().length))
+ );
+ return (
+ (matches(rowMatcher, rowMatcher == null ? null : key.getRowData())) &&
+ (matches(colfMatcher, colfMatcher == null ? null : key.getColumnFamilyData())) &&
+ (matches(colqMatcher, colqMatcher == null ? null : key.getColumnQualifierData())) &&
+ (matches(valueMatcher, value.get(), 0, value.get().length))
+ );
}
@Override
@@ -136,6 +148,12 @@ public class RegExFilter extends Filter
orFields = false;
}
+ if (options.containsKey(MATCH_SUBSTRING)) {
+ matchSubstring = Boolean.parseBoolean(options.get(MATCH_SUBSTRING));
+ } else {
+ matchSubstring = false;
+ }
+
if (options.containsKey(ENCODING)) {
encoding = options.get(ENCODING);
}
@@ -151,6 +169,7 @@ public class RegExFilter extends Filter
io.addNamedOption(RegExFilter.COLQ_REGEX, "regular expression on column qualifier");
io.addNamedOption(RegExFilter.VALUE_REGEX, "regular expression on value");
io.addNamedOption(RegExFilter.OR_FIELDS, "use OR instread of AND when multiple regexes given");
+ io.addNamedOption(RegExFilter.MATCH_SUBSTRING, "match on substrings");
io.addNamedOption(RegExFilter.ENCODING, "character encoding of byte array value (default is " + ENCODING_DEFAULT + ")");
return io;
}
@@ -186,6 +205,8 @@ public class RegExFilter extends Filter
/**
* Encode the terms to match against in the iterator
+ * Same as calling setRegexs(IteratorSetting si, String rowTerm, String cfTerm, String cqTerm, String valueTerm, boolean orFields, boolean matchSubstring)
+ * with matchSubstring set to false
*
* @param si
* ScanIterator config to be updated
@@ -200,7 +221,28 @@ public class RegExFilter extends Filter
* @param orFields
* if true, any of the non-null terms can match to return the entry
*/
- public static void setRegexs(IteratorSetting si, String rowTerm, String cfTerm, String cqTerm, String valueTerm, boolean orFields) {
+ public static void setRegexs(IteratorSetting si, String rowTerm, String cfTerm, String cqTerm, String valueTerm, boolean orFields) {
+ setRegexs(si, rowTerm, cfTerm, cqTerm, valueTerm, orFields, false);
+ }
+
+ /**
+ * Encode the terms to match against in the iterator
+ *
+ * @param si
+ * ScanIterator config to be updated
+ * @param rowTerm
+ * the pattern to match against the Key's row. Not used if null.
+ * @param cfTerm
+ * the pattern to match against the Key's column family. Not used if null.
+ * @param cqTerm
+ * the pattern to match against the Key's column qualifier. Not used if null.
+ * @param valueTerm
+ * the pattern to match against the Key's value. Not used if null.
+ * @param matchSubstring
+ * if true then search expressions will match on partial strings
+ */
+ public static void setRegexs(IteratorSetting si, String rowTerm, String cfTerm, String cqTerm, String valueTerm, boolean orFields, boolean matchSubstring) {
+
if (rowTerm != null)
si.addOption(RegExFilter.ROW_REGEX, rowTerm);
if (cfTerm != null)
@@ -209,11 +251,14 @@ public class RegExFilter extends Filter
si.addOption(RegExFilter.COLQ_REGEX, cqTerm);
if (valueTerm != null)
si.addOption(RegExFilter.VALUE_REGEX, valueTerm);
- if (orFields) {
- si.addOption(RegExFilter.OR_FIELDS, "true");
- }
+ si.addOption(RegExFilter.OR_FIELDS, String.valueOf(orFields));
+ si.addOption(RegExFilter.MATCH_SUBSTRING, String.valueOf(matchSubstring));
+
}
+
+
+
/**
* Set the encoding string to use when interpreting characters
*