You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/03 20:55:50 UTC
[1/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Repository: accumulo-testing
Updated Branches:
refs/heads/master 89d6acbcb -> ac5b271ca
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/LongDirty.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/LongDirty.xml b/core/src/main/resources/randomwalk/modules/LongDirty.xml
new file mode 100644
index 0000000..480e57b
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/LongDirty.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="3600" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/LongEach.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/LongEach.xml b/core/src/main/resources/randomwalk/modules/LongEach.xml
new file mode 100644
index 0000000..5863341
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/LongEach.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="Image.xml"/>
+
+<node id="Image.xml" maxSec="3600" teardown="true">
+ <edge id="Sequential.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="3600" teardown="true">
+ <edge id="MultiTable.xml" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="3600" teardown="true">
+ <edge id="Shard.xml" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="3600" teardown="true">
+ <edge id="Concurrent.xml" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="3600" teardown="true">
+ <edge id="Security.xml" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="3600" teardown="true">
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="3600" teardown="true">
+ <edge id="Conditional.xml" weight="1"/>
+</node>
+
+<node id="Conditional.xml" maxSec="3600" teardown="true">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/MultiTable.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/MultiTable.xml b/core/src/main/resources/randomwalk/modules/MultiTable.xml
new file mode 100644
index 0000000..55f6590
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/MultiTable.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="mt" value="org.apache.accumulo.test.randomwalk.multitable"/>
+
+<fixture id="mt.MultiTableFixture"/>
+
+<init id="mt.CreateTable"/>
+
+<node id="mt.CreateTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="mt.CreateTable" weight="20"/>
+ <edge id="mt.Write" weight="100"/>
+ <edge id="mt.CopyTable" weight="5"/>
+ <edge id="mt.OfflineTable" weight="10"/>
+ <edge id="mt.DropTable" weight="3"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="mt.Write">
+ <edge id="mt.Write" weight="5000"/>
+ <edge id="mt.Commit" weight="1"/>
+</node>
+
+<node id="mt.Commit">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="mt.OfflineTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="mt.CopyTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="mt.DropTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Security.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Security.xml b/core/src/main/resources/randomwalk/modules/Security.xml
new file mode 100644
index 0000000..9e9ef9f
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Security.xml
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<module>
+
+ <package prefix="security"
+ value="org.apache.accumulo.test.randomwalk.security" />
+
+ <fixture id="security.SecurityFixture" />
+
+ <init id="dummy.NoUserNoTable" maxHops="1000000" />
+
+ <node id="security.CreateUser">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="CreateUserYesTable" src="security.CreateUser">
+ <edge id="dummy.YesUserYesTable" weight="1" />
+ </node>
+
+ <node id="CreateUserNoTable" src="security.CreateUser">
+ <edge id="dummy.YesUserNoTable" weight="1" />
+ </node>
+
+ <node id="security.DropUser">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="DropUserNoTable" src="security.DropUser">
+ <edge id="dummy.NoUserNoTable" weight="1" />
+ </node>
+
+ <node id="DropUserYesTable" src="security.DropUser">
+ <edge id="dummy.NoUserYesTable" weight="1" />
+ </node>
+
+ <node id="security.CreateTable">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="NoUserCreateTable" src="security.CreateTable">
+ <edge id="dummy.NoUserYesTable" weight="1" />
+ </node>
+
+ <node id="YesUserCreateTable" src="security.CreateTable">
+ <edge id="dummy.YesUserYesTable" weight="1" />
+ </node>
+
+ <node id="security.DropTable">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="NoUserDropTable" src="security.DropTable">
+ <edge id="dummy.NoUserNoTable" weight="1" />
+ </node>
+
+ <node id="YesUserDropTable" src="security.DropTable">
+ <edge id="dummy.YesUserNoTable" weight="1" />
+ </node>
+
+ <node id="security.AlterTable">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="RandomAuths" src="security.SetAuths">
+ <property key="auths" value="_random" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="SystemChangeSystemPass" src="security.ChangePass">
+ <property key="target" value="system" />
+ <property key="source" value="system" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="SystemChangeTablePass" src="security.ChangePass">
+ <property key="target" value="table" />
+ <property key="source" value="system" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="TableChangeTablePass" src="security.ChangePass">
+ <property key="target" value="table" />
+ <property key="source" value="table" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="AuthenticateSystemRight" src="security.Authenticate">
+ <property key="target" value="system" />
+ <property key="valid" value="true" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="AuthenticateSystemWrong" src="security.Authenticate">
+ <property key="target" value="system" />
+ <property key="valid" value="false" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="AuthenticateTableRight" src="security.Authenticate">
+ <property key="target" value="table" />
+ <property key="valid" value="true" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="AuthenticateTableWrong" src="security.Authenticate">
+ <property key="target" value="table" />
+ <property key="valid" value="false" />
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="security.Validate">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="security.AlterSystemPerm">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="security.AlterTablePerm">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="security.TableOp">
+ <edge id="alias.last" weight="1" />
+ </node>
+
+ <node id="dummy.NoUserNoTable">
+ <alias name="last" />
+ <property key="print" value="DEBUG" />
+ <edge id="CreateUserNoTable" weight="40" />
+ <edge id="security.DropUser" weight="10" />
+ <edge id="NoUserCreateTable" weight="40" />
+ <edge id="security.DropTable" weight="10" />
+ <edge id="security.AlterTable" weight="10" />
+ <edge id="SystemChangeSystemPass" weight="60" />
+ <edge id="SystemChangeTablePass" weight="10" />
+ <edge id="AuthenticateSystemRight" weight="50" />
+ <edge id="AuthenticateSystemWrong" weight="50" />
+ <edge id="AuthenticateTableRight" weight="10" />
+ <edge id="AuthenticateTableWrong" weight="10" />
+ <edge id="security.Validate" weight="20" />
+ <edge id="RandomAuths" weight="10" />
+ <edge id="security.AlterSystemPerm" weight="60" />
+ <edge id="security.AlterTablePerm" weight="10" />
+ <edge id="END" weight="1" />
+ </node>
+
+ <node id="dummy.YesUserNoTable">
+ <alias name="last" />
+ <property key="print" value="DEBUG" />
+ <edge id="security.CreateUser" weight="10" />
+ <edge id="DropUserNoTable" weight="30" />
+ <edge id="YesUserCreateTable" weight="40" />
+ <edge id="security.DropTable" weight="10" />
+ <edge id="security.AlterTable" weight="10" />
+ <edge id="SystemChangeSystemPass" weight="60" />
+ <edge id="SystemChangeTablePass" weight="60" />
+ <edge id="TableChangeTablePass" weight="60" />
+ <edge id="AuthenticateSystemRight" weight="50" />
+ <edge id="AuthenticateSystemWrong" weight="50" />
+ <edge id="AuthenticateTableRight" weight="50" />
+ <edge id="AuthenticateTableWrong" weight="50" />
+ <edge id="security.Validate" weight="20" />
+ <edge id="RandomAuths" weight="60" />
+ <edge id="security.AlterSystemPerm" weight="60" />
+ <edge id="security.AlterTablePerm" weight="10" />
+ <edge id="security.TableOp" weight="10" />
+ <edge id="END" weight="1" />
+ </node>
+
+ <node id="dummy.NoUserYesTable">
+ <alias name="last" />
+ <property key="print" value="DEBUG" />
+ <edge id="CreateUserYesTable" weight="40" />
+ <edge id="security.DropUser" weight="10" />
+ <edge id="security.CreateTable" weight="10" />
+ <edge id="NoUserDropTable" weight="30" />
+ <edge id="security.AlterTable" weight="60" />
+ <edge id="SystemChangeSystemPass" weight="60" />
+ <edge id="SystemChangeTablePass" weight="10" />
+ <edge id="AuthenticateSystemRight" weight="50" />
+ <edge id="AuthenticateSystemWrong" weight="50" />
+ <edge id="AuthenticateTableRight" weight="10" />
+ <edge id="AuthenticateTableWrong" weight="10" />
+ <edge id="security.Validate" weight="20" />
+ <edge id="RandomAuths" weight="10" />
+ <edge id="security.AlterSystemPerm" weight="60" />
+ <edge id="security.AlterTablePerm" weight="10" />
+ <edge id="END" weight="1" />
+ </node>
+
+ <node id="dummy.YesUserYesTable">
+ <alias name="last" />
+ <property key="print" value="DEBUG" />
+ <edge id="security.CreateUser" weight="10" />
+ <edge id="DropUserYesTable" weight="30" />
+ <edge id="security.CreateTable" weight="10" />
+ <edge id="YesUserDropTable" weight="30" />
+ <edge id="security.AlterTable" weight="60" />
+ <edge id="SystemChangeSystemPass" weight="60" />
+ <edge id="SystemChangeTablePass" weight="60" />
+ <edge id="TableChangeTablePass" weight="60" />
+ <edge id="AuthenticateSystemRight" weight="50" />
+ <edge id="AuthenticateSystemWrong" weight="50" />
+ <edge id="AuthenticateTableRight" weight="50" />
+ <edge id="AuthenticateTableWrong" weight="50" />
+ <edge id="security.Validate" weight="20" />
+ <edge id="RandomAuths" weight="60" />
+ <edge id="security.AlterSystemPerm" weight="60" />
+ <edge id="security.AlterTablePerm" weight="60" />
+ <edge id="security.TableOp" weight="120" />
+ <edge id="END" weight="1" />
+ </node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Sequential.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Sequential.xml b/core/src/main/resources/randomwalk/modules/Sequential.xml
new file mode 100644
index 0000000..454e75f
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Sequential.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="seq" value="org.apache.accumulo.test.randomwalk.sequential"/>
+
+<fixture id="seq.SequentialFixture"/>
+
+<init id="seq.Write"/>
+
+<node id="seq.Write">
+ <edge id="seq.Write" weight="10000"/>
+ <edge id="seq.Commit" weight="1"/>
+</node>
+
+<node id="seq.Commit">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="seq.Write" weight="50"/>
+ <edge id="seq.BatchVerify" weight="10"/>
+ <edge id="seq.MapRedVerify" weight="5"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="seq.BatchVerify">
+ <property key="maxVerify" value="10000"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="seq.MapRedVerify">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Shard.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Shard.xml b/core/src/main/resources/randomwalk/modules/Shard.xml
new file mode 100644
index 0000000..eb23f37
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Shard.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="shard" value="org.apache.accumulo.test.randomwalk.shard"/>
+
+<fixture id="shard.ShardFixture"/>
+
+<init id="shard.Insert"/>
+
+<node id="shard.Insert">
+ <edge id="shard.Insert" weight="10"/>
+ <edge id="shard.Commit" weight="1"/>
+</node>
+
+<node id="shard.Commit">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.Search">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.Reindex">
+ <edge id="Verify" weight="2"/>
+ <edge id="VerifyExit" weight="1"/>
+</node>
+
+<node id="shard.CloneIndex">
+ <edge id="Verify" weight="1"/>
+</node>
+
+<node id="shard.ExportIndex">
+ <edge id="Verify" weight="1"/>
+</node>
+
+<node id="Verify" src="shard.VerifyIndex">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="VerifyExit" src="shard.VerifyIndex">
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="shard.Delete">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.DeleteWord">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.DeleteSomeDocs">
+ <property key="pattern1" value=".0.*"/>
+ <property key="pattern2" value=".3.*"/>
+ <property key="pattern3" value=".7.*"/>
+ <property key="pattern4" value=".b.*"/>
+ <property key="pattern5" value="5.*"/>
+ <property key="pattern6" value="a.*"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.Flush">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.Grep">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.Split">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.Merge">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.BulkInsert">
+ <property key="minInsert" value="1000"/>
+ <property key="maxInsert" value="20000"/>
+ <property key="maxSplits" value="9"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="shard.CompactFilter">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+
+<node id="dummy.ToAll">
+ <edge id="shard.Reindex" weight="3"/>
+ <edge id="shard.Flush" weight="3"/>
+ <edge id="shard.CloneIndex" weight="3"/>
+ <edge id="shard.ExportIndex" weight="3"/>
+ <edge id="shard.Grep" weight="20"/>
+ <edge id="shard.Split" weight="40"/>
+ <edge id="shard.Merge" weight="20"/>
+ <edge id="shard.DeleteWord" weight="9"/>
+ <edge id="shard.CompactFilter" weight="9"/>
+ <edge id="shard.DeleteSomeDocs" weight="20"/>
+ <edge id="shard.BulkInsert" weight="3"/>
+ <edge id="shard.Delete" weight="486"/>
+ <edge id="shard.Insert" weight="4690"/>
+ <edge id="shard.Search" weight="4691"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/ShortClean.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/ShortClean.xml b/core/src/main/resources/randomwalk/modules/ShortClean.xml
new file mode 100644
index 0000000..19bb807
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/ShortClean.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="300" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/ShortDirty.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/ShortDirty.xml b/core/src/main/resources/randomwalk/modules/ShortDirty.xml
new file mode 100644
index 0000000..73fd988
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/ShortDirty.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="300" teardown="false">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/ShortEach.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/ShortEach.xml b/core/src/main/resources/randomwalk/modules/ShortEach.xml
new file mode 100644
index 0000000..fe7c857
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/ShortEach.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="Image.xml"/>
+
+<node id="Image.xml" maxSec="300" teardown="true">
+ <edge id="Sequential.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="300" teardown="true">
+ <edge id="MultiTable.xml" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="300" teardown="true">
+ <edge id="Shard.xml" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="300" teardown="true">
+ <edge id="Concurrent.xml" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="300" teardown="true">
+ <edge id="Security.xml" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="300" teardown="true">
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="300" teardown="true">
+ <edge id="Conditional.xml" weight="1"/>
+</node>
+
+<node id="Conditional.xml" maxSec="300" teardown="true">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/unit/Basic.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/unit/Basic.xml b/core/src/main/resources/randomwalk/modules/unit/Basic.xml
new file mode 100644
index 0000000..2dead02
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/unit/Basic.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+
+<init id="test.CreateTable"/>
+
+<node id="test.CreateTable">
+ <edge id="unit/Simple.xml" weight="1"/>
+</node>
+
+<node id="unit/Simple.xml">
+ <edge id="unit/Simple.xml" weight="3"/>
+ <edge id="test.DeleteTable" weight="1"/>
+</node>
+
+<node id="test.DeleteTable">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/unit/Simple.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/unit/Simple.xml b/core/src/main/resources/randomwalk/modules/unit/Simple.xml
new file mode 100644
index 0000000..cad940e
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/unit/Simple.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+
+<init id="dummy.all"/>
+
+<node id="dummy.all">
+ <edge id="test.Ingest" weight="1"/>
+ <edge id="test.Verify" weight="1"/>
+ <edge id="test.Scan" weight="1"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="test.Ingest">
+ <edge id="dummy.all" weight="1"/>
+</node>
+
+<node id="test.Verify">
+ <edge id="dummy.all" weight="1"/>
+</node>
+
+<node id="test.Scan">
+ <edge id="dummy.all" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/FrameworkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/FrameworkTest.java b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/FrameworkTest.java
new file mode 100644
index 0000000..b33385d
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/FrameworkTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+
+import org.apache.accumulo.testing.core.randomwalk.unit.CreateTable;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+public class FrameworkTest {
+
+ // Need to use fully qualified name here because of conflict with org.apache.accumulo.testing.core.randomwalk.Test
+ @org.junit.Test
+ public void testXML() throws SAXException, URISyntaxException, ParserConfigurationException, IOException {
+ SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ Schema moduleSchema = sf.newSchema(getFile("/randomwalk/module.xsd"));
+
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setSchema(moduleSchema);
+
+ DocumentBuilder docbuilder = dbf.newDocumentBuilder();
+ Document document = docbuilder.parse(getFile("/randomwalk/modules/unit/Basic.xml"));
+
+ assertNotEquals("Parsing randomwalk xml should result in nodes.", 0, document.getChildNodes().getLength());
+ }
+
+ private File getFile(String resource) throws URISyntaxException {
+ return new File(this.getClass().getResource(resource).toURI());
+ }
+
+ @org.junit.Test
+ public void testRWTest() {
+ Test t1 = new CreateTable();
+ assertEquals("org.apache.accumulo.testing.core.randomwalk.unit.CreateTable", t1.toString());
+
+ Test t2 = new CreateTable();
+ assertEquals("CreateTable test nodes were not equal.", t1, t2);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java
new file mode 100644
index 0000000..c288bd7
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/testing/core/randomwalk/ReplicationRandomWalkIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import static org.apache.accumulo.core.conf.Property.TSERV_ARCHIVE_WALOGS;
+import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.testing.core.randomwalk.concurrent.Replication;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+public class ReplicationRandomWalkIT extends ConfigurableMacBase {
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setProperty(TSERV_ARCHIVE_WALOGS, "false");
+ cfg.setProperty(TSERV_WALOG_MAX_SIZE, "1M");
+ cfg.setNumTservers(1);
+ }
+
+ @Test(timeout = 5 * 60 * 1000)
+ public void runReplicationRandomWalkStep() throws Exception {
+ Replication r = new Replication();
+
+ Environment env = new Environment(new Properties()) {
+ @Override
+ public String getUserName() {
+ return "root";
+ }
+
+ @Override
+ public String getPassword() {
+ return ROOT_PASSWORD;
+ }
+
+ @Override
+ public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
+ return ReplicationRandomWalkIT.this.getConnector();
+ }
+
+ };
+ r.visit(null, env, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
new file mode 100644
index 0000000..716f9db
--- /dev/null
+++ b/core/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=WARN, CA
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=[%t} %-5p %c %x - %m%n
+
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..a2efdf4
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,172 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>18</version>
+ </parent>
+
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-testing</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <name>Apache Accumulo Testing Parent</name>
+ <description>Testing suites for Apache Accumulo</description>
+
+ <modules>
+ <module>core</module>
+ </modules>
+
+ <properties>
+ <accumulo.version>1.8.0</accumulo.version>
+ <hadoop.version>2.6.4</hadoop.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
+ <slf4j.version>1.7.21</slf4j.version>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>14.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>1.48</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-fate</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-master</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-test</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <!-- Allows us to get the apache-ds bundle artifacts -->
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <version>3.0.1</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <optimize>true</optimize>
+ <encoding>UTF-8</encoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!-- Allows us to get the apache-ds bundle artifacts -->
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <inherited>true</inherited>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>run-integration-tests</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.5.0</version>
+ <configuration>
+ <cleanupDaemonThreads>false</cleanupDaemonThreads>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
[6/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Posted by mw...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Split.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Split.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Split.java
new file mode 100644
index 0000000..4ef212f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Split.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+public class Split extends SelectiveBulkTest {
+
+ @Override
+ protected void runLater(State state, Environment env) throws Exception {
+ SortedSet<Text> splits = new TreeSet<>();
+ Random rand = (Random) state.get("rand");
+ int count = rand.nextInt(20);
+ for (int i = 0; i < count; i++)
+ splits.add(new Text(String.format(BulkPlusOne.FMT, (rand.nextLong() & 0x7fffffffffffffffl) % BulkPlusOne.LOTS)));
+ log.info("splitting " + splits);
+ env.getConnector().tableOperations().addSplits(Setup.getTableName(), splits);
+ log.info("split for " + splits + " finished");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Verify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Verify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Verify.java
new file mode 100644
index 0000000..57aeff3
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Verify.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Verify extends Test {
+
+ static byte[] zero = new byte[] {'0'};
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ ThreadPoolExecutor threadPool = Setup.getThreadPool(state);
+ threadPool.shutdown();
+ int lastSize = 0;
+ while (!threadPool.isTerminated()) {
+ int size = threadPool.getQueue().size() + threadPool.getActiveCount();
+ log.info("Waiting for " + size + " nodes to complete");
+ if (size != lastSize)
+ makingProgress();
+ lastSize = size;
+ threadPool.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ if (!"true".equals(state.get("bulkImportSuccess"))) {
+ log.info("Not verifying bulk import test due to import failures");
+ return;
+ }
+
+ String user = env.getConnector().whoami();
+ Authorizations auths = env.getConnector().securityOperations().getUserAuthorizations(user);
+ Scanner scanner = env.getConnector().createScanner(Setup.getTableName(), auths);
+ scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
+ for (Entry<Key,Value> entry : scanner) {
+ byte[] value = entry.getValue().get();
+ if (!Arrays.equals(value, zero)) {
+ throw new Exception("Bad key at " + entry);
+ }
+ }
+
+ scanner.clearColumns();
+ scanner.fetchColumnFamily(BulkPlusOne.MARKER_CF);
+ RowIterator rowIter = new RowIterator(scanner);
+
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ long prev = 0;
+ Text rowText = null;
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+
+ if (rowText == null)
+ rowText = entry.getKey().getRow();
+
+ long curr = Long.parseLong(entry.getKey().getColumnQualifier().toString());
+
+ if (curr - 1 != prev)
+ throw new Exception("Bad marker count " + entry.getKey() + " " + entry.getValue() + " " + prev);
+
+ if (!entry.getValue().toString().equals("1"))
+ throw new Exception("Bad marker value " + entry.getKey() + " " + entry.getValue());
+
+ prev = curr;
+ }
+
+ if (BulkPlusOne.counter.get() != prev) {
+ throw new Exception("Row " + rowText + " does not have all markers " + BulkPlusOne.counter.get() + " " + prev);
+ }
+ }
+
+ log.info("Test successful on table " + Setup.getTableName());
+ env.getConnector().tableOperations().delete(Setup.getTableName());
+ }
+
+ public static void main(String args[]) throws Exception {
+ ClientOnRequiredTable opts = new ClientOnRequiredTable();
+ opts.parseArgs(Verify.class.getName(), args);
+ Scanner scanner = opts.getConnector().createScanner(opts.getTableName(), opts.auths);
+ scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
+ Text startBadRow = null;
+ Text lastBadRow = null;
+ Value currentBadValue = null;
+ for (Entry<Key,Value> entry : scanner) {
+ // System.out.println("Entry: " + entry);
+ byte[] value = entry.getValue().get();
+ if (!Arrays.equals(value, zero)) {
+ if (currentBadValue == null || entry.getValue().equals(currentBadValue)) {
+ // same value, keep skipping ahead
+ lastBadRow = new Text(entry.getKey().getRow());
+ if (startBadRow == null)
+ startBadRow = lastBadRow;
+ } else {
+ // new bad value, report
+ report(startBadRow, lastBadRow, currentBadValue);
+ startBadRow = lastBadRow = new Text(entry.getKey().getRow());
+ }
+ currentBadValue = new Value(entry.getValue());
+ } else {
+ // end of bad range, report
+ if (startBadRow != null) {
+ report(startBadRow, lastBadRow, currentBadValue);
+ }
+ startBadRow = lastBadRow = null;
+ currentBadValue = null;
+ }
+ }
+ if (startBadRow != null) {
+ report(startBadRow, lastBadRow, currentBadValue);
+ }
+ }
+
+ private static void report(Text startBadRow, Text lastBadRow, Value value) {
+ System.out.println("Bad value " + new String(value.get(), UTF_8));
+ System.out.println(" Range [" + startBadRow + " -> " + lastBadRow + "]");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/AddSplits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/AddSplits.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/AddSplits.java
new file mode 100644
index 0000000..18e0980
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/AddSplits.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class AddSplits extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+ tableNames = new ArrayList<>(tableNames);
+ tableNames.add(MetadataTable.NAME);
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ TreeSet<Text> splits = new TreeSet<>();
+
+ for (int i = 0; i < rand.nextInt(10) + 1; i++)
+ splits.add(new Text(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl)));
+
+ try {
+ conn.tableOperations().addSplits(tableName, splits);
+ log.debug("Added " + splits.size() + " splits " + tableName);
+ } catch (TableNotFoundException e) {
+ log.debug("AddSplits " + tableName + " failed, doesnt exist");
+ } catch (TableOfflineException e) {
+ log.debug("AddSplits " + tableName + " failed, offline");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
new file mode 100644
index 0000000..cebc146
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Apocalypse.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Apocalypse extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Process exec = Runtime.getRuntime().exec(new String[] {System.getenv("ACCUMULO_HOME") + "/test/system/randomwalk/bin/apocalypse.sh"});
+ if (exec.waitFor() != 0)
+ throw new RuntimeException("apocalypse.sh returned a non-zero response: " + exec.exitValue());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchScan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchScan.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchScan.java
new file mode 100644
index 0000000..970e4df
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchScan.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class BatchScan extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ BatchScanner bs = conn.createBatchScanner(tableName, Authorizations.EMPTY, 3);
+ List<Range> ranges = new ArrayList<>();
+ for (int i = 0; i < rand.nextInt(2000) + 1; i++)
+ ranges.add(new Range(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl)));
+
+ bs.setRanges(ranges);
+
+ try {
+ Iterator<Entry<Key,Value>> iter = bs.iterator();
+ while (iter.hasNext())
+ iter.next();
+ } finally {
+ bs.close();
+ }
+
+ log.debug("Wrote to " + tableName);
+ } catch (TableNotFoundException e) {
+ log.debug("BatchScan " + tableName + " failed, doesnt exist");
+ } catch (TableDeletedException tde) {
+ log.debug("BatchScan " + tableName + " failed, table deleted");
+ } catch (TableOfflineException e) {
+ log.debug("BatchScan " + tableName + " failed, offline");
+ } catch (RuntimeException e) {
+ if (e.getCause() instanceof AccumuloSecurityException) {
+ log.debug("BatchScan " + tableName + " failed, permission error");
+ } else {
+ throw e;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchWrite.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchWrite.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchWrite.java
new file mode 100644
index 0000000..39afec0
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BatchWrite.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class BatchWrite extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ try {
+ int numRows = rand.nextInt(100000);
+ for (int i = 0; i < numRows; i++) {
+ Mutation m = new Mutation(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl));
+ long val = rand.nextLong() & 0x7fffffffffffffffl;
+ for (int j = 0; j < 10; j++) {
+ m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(UTF_8)));
+ }
+
+ bw.addMutation(m);
+ }
+ } finally {
+ bw.close();
+ }
+
+ log.debug("Wrote to " + tableName);
+ } catch (TableNotFoundException e) {
+ log.debug("BatchWrite " + tableName + " failed, doesnt exist");
+ } catch (TableOfflineException e) {
+ log.debug("BatchWrite " + tableName + " failed, offline");
+ } catch (MutationsRejectedException mre) {
+ if (mre.getCause() instanceof TableDeletedException)
+ log.debug("BatchWrite " + tableName + " failed, table deleted");
+ else if (mre.getCause() instanceof TableOfflineException)
+ log.debug("BatchWrite " + tableName + " failed, offline");
+ else
+ throw mre;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
new file mode 100644
index 0000000..55fa8d6
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/BulkImport.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.file.streams.PositionedOutputs;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class BulkImport extends Test {
+
+ public static class RFileBatchWriter implements BatchWriter {
+
+ RFile.Writer writer;
+
+ public RFileBatchWriter(Configuration conf, FileSystem fs, String file) throws IOException {
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ CachableBlockFile.Writer cbw = new CachableBlockFile.Writer(PositionedOutputs.wrap(fs.create(new Path(file), false,
+ conf.getInt("io.file.buffer.size", 4096), (short) conf.getInt("dfs.replication", 3), conf.getLong("dfs.block.size", 1 << 26))), "gz", conf, aconf);
+ writer = new RFile.Writer(cbw, 100000);
+ writer.startDefaultLocalityGroup();
+ }
+
+ @Override
+ public void addMutation(Mutation m) throws MutationsRejectedException {
+ List<ColumnUpdate> updates = m.getUpdates();
+ for (ColumnUpdate cu : updates) {
+ Key key = new Key(m.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(), cu.getColumnVisibility(), 42, false, false);
+ Value val = new Value(cu.getValue(), false);
+
+ try {
+ writer.append(key, val);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
+ for (Mutation mutation : iterable)
+ addMutation(mutation);
+ }
+
+ @Override
+ public void flush() throws MutationsRejectedException {}
+
+ @Override
+ public void close() throws MutationsRejectedException {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs = FileSystem.get(conf);
+
+ String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl);
+
+ fs.mkdirs(new Path(bulkDir));
+ fs.mkdirs(new Path(bulkDir + "_f"));
+
+ try {
+ BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
+ try {
+ TreeSet<Long> rows = new TreeSet<>();
+ int numRows = rand.nextInt(100000);
+ for (int i = 0; i < numRows; i++) {
+ rows.add(rand.nextLong() & 0x7fffffffffffffffl);
+ }
+
+ for (Long row : rows) {
+ Mutation m = new Mutation(String.format("%016x", row));
+ long val = rand.nextLong() & 0x7fffffffffffffffl;
+ for (int j = 0; j < 10; j++) {
+ m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes(UTF_8)));
+ }
+
+ bw.addMutation(m);
+ }
+ } finally {
+ bw.close();
+ }
+
+ conn.tableOperations().importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean());
+
+ log.debug("BulkImported to " + tableName);
+ } catch (TableNotFoundException e) {
+ log.debug("BulkImport " + tableName + " failed, doesnt exist");
+ } catch (TableOfflineException toe) {
+ log.debug("BulkImport " + tableName + " failed, offline");
+ } finally {
+ fs.delete(new Path(bulkDir), true);
+ fs.delete(new Path(bulkDir + "_f"), true);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangeAuthorizations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangeAuthorizations.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangeAuthorizations.java
new file mode 100644
index 0000000..542c61e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangeAuthorizations.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class ChangeAuthorizations extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> userNames = (List<String>) state.get("users");
+
+ String userName = userNames.get(rand.nextInt(userNames.size()));
+ try {
+ List<byte[]> auths = new ArrayList<>(conn.securityOperations().getUserAuthorizations(userName).getAuthorizations());
+
+ if (rand.nextBoolean()) {
+ String authorization = String.format("a%d", rand.nextInt(5000));
+ log.debug("adding authorization " + authorization);
+ auths.add(authorization.getBytes(UTF_8));
+ } else {
+ if (auths.size() > 0) {
+ log.debug("removing authorization " + new String(auths.remove(0), UTF_8));
+ }
+ }
+ conn.securityOperations().changeUserAuthorizations(userName, new Authorizations(auths));
+ } catch (AccumuloSecurityException ex) {
+ log.debug("Unable to change user authorizations: " + ex.getCause());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangePermissions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangePermissions.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangePermissions.java
new file mode 100644
index 0000000..173af6e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ChangePermissions.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class ChangePermissions extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> userNames = (List<String>) state.get("users");
+ String userName = userNames.get(rand.nextInt(userNames.size()));
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ @SuppressWarnings("unchecked")
+ List<String> namespaces = (List<String>) state.get("namespaces");
+ String namespace = namespaces.get(rand.nextInt(namespaces.size()));
+
+ try {
+ int dice = rand.nextInt(3);
+ if (dice == 0)
+ changeSystemPermission(conn, rand, userName);
+ else if (dice == 1)
+ changeTablePermission(conn, rand, userName, tableName);
+ else if (dice == 2)
+ changeNamespacePermission(conn, rand, userName, namespace);
+ } catch (AccumuloSecurityException ex) {
+ log.debug("Unable to change user permissions: " + ex.getCause());
+ } catch (AccumuloException ex) {
+ Throwable cause = ex.getCause();
+ if (cause != null && cause instanceof ThriftTableOperationException) {
+ ThriftTableOperationException toe = (ThriftTableOperationException) cause.getCause();
+ if (toe.type == TableOperationExceptionType.NAMESPACE_NOTFOUND) {
+ log.debug("Unable to change user permissions: " + toe);
+ return;
+ }
+ }
+ }
+ }
+
+ private void changeTablePermission(Connector conn, Random rand, String userName, String tableName) throws AccumuloException, AccumuloSecurityException {
+
+ EnumSet<TablePermission> perms = EnumSet.noneOf(TablePermission.class);
+ for (TablePermission p : TablePermission.values()) {
+ if (conn.securityOperations().hasTablePermission(userName, tableName, p))
+ perms.add(p);
+ }
+
+ EnumSet<TablePermission> more = EnumSet.allOf(TablePermission.class);
+ more.removeAll(perms);
+
+ if (rand.nextBoolean() && more.size() > 0) {
+ List<TablePermission> moreList = new ArrayList<>(more);
+ TablePermission choice = moreList.get(rand.nextInt(moreList.size()));
+ log.debug("adding permission " + choice);
+ conn.securityOperations().grantTablePermission(userName, tableName, choice);
+ } else {
+ if (perms.size() > 0) {
+ List<TablePermission> permList = new ArrayList<>(perms);
+ TablePermission choice = permList.get(rand.nextInt(permList.size()));
+ log.debug("removing permission " + choice);
+ conn.securityOperations().revokeTablePermission(userName, tableName, choice);
+ }
+ }
+ }
+
+ private void changeSystemPermission(Connector conn, Random rand, String userName) throws AccumuloException, AccumuloSecurityException {
+ EnumSet<SystemPermission> perms = EnumSet.noneOf(SystemPermission.class);
+ for (SystemPermission p : SystemPermission.values()) {
+ if (conn.securityOperations().hasSystemPermission(userName, p))
+ perms.add(p);
+ }
+
+ EnumSet<SystemPermission> more = EnumSet.allOf(SystemPermission.class);
+ more.removeAll(perms);
+ more.remove(SystemPermission.GRANT);
+
+ if (rand.nextBoolean() && more.size() > 0) {
+ List<SystemPermission> moreList = new ArrayList<>(more);
+ SystemPermission choice = moreList.get(rand.nextInt(moreList.size()));
+ log.debug("adding permission " + choice);
+ conn.securityOperations().grantSystemPermission(userName, choice);
+ } else {
+ if (perms.size() > 0) {
+ List<SystemPermission> permList = new ArrayList<>(perms);
+ SystemPermission choice = permList.get(rand.nextInt(permList.size()));
+ log.debug("removing permission " + choice);
+ conn.securityOperations().revokeSystemPermission(userName, choice);
+ }
+ }
+ }
+
+ private void changeNamespacePermission(Connector conn, Random rand, String userName, String namespace) throws AccumuloException, AccumuloSecurityException {
+
+ EnumSet<NamespacePermission> perms = EnumSet.noneOf(NamespacePermission.class);
+ for (NamespacePermission p : NamespacePermission.values()) {
+ if (conn.securityOperations().hasNamespacePermission(userName, namespace, p))
+ perms.add(p);
+ }
+
+ EnumSet<NamespacePermission> more = EnumSet.allOf(NamespacePermission.class);
+ more.removeAll(perms);
+
+ if (rand.nextBoolean() && more.size() > 0) {
+ List<NamespacePermission> moreList = new ArrayList<>(more);
+ NamespacePermission choice = moreList.get(rand.nextInt(moreList.size()));
+ log.debug("adding permission " + choice);
+ conn.securityOperations().grantNamespacePermission(userName, namespace, choice);
+ } else {
+ if (perms.size() > 0) {
+ List<NamespacePermission> permList = new ArrayList<>(perms);
+ NamespacePermission choice = permList.get(rand.nextInt(permList.size()));
+ log.debug("removing permission " + choice);
+ conn.securityOperations().revokeNamespacePermission(userName, namespace, choice);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CheckPermission.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CheckPermission.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CheckPermission.java
new file mode 100644
index 0000000..4848423
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CheckPermission.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CheckPermission extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> userNames = (List<String>) state.get("users");
+ String userName = userNames.get(rand.nextInt(userNames.size()));
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ @SuppressWarnings("unchecked")
+ List<String> namespaces = (List<String>) state.get("namespaces");
+ String namespace = namespaces.get(rand.nextInt(namespaces.size()));
+
+ try {
+ int dice = rand.nextInt(2);
+ if (dice == 0) {
+ log.debug("Checking systerm permission " + userName);
+ conn.securityOperations().hasSystemPermission(userName, SystemPermission.values()[rand.nextInt(SystemPermission.values().length)]);
+ } else if (dice == 1) {
+ log.debug("Checking table permission " + userName + " " + tableName);
+ conn.securityOperations().hasTablePermission(userName, tableName, TablePermission.values()[rand.nextInt(TablePermission.values().length)]);
+ } else if (dice == 2) {
+ log.debug("Checking namespace permission " + userName + " " + namespace);
+ conn.securityOperations().hasNamespacePermission(userName, namespace, NamespacePermission.values()[rand.nextInt(NamespacePermission.values().length)]);
+ }
+
+ } catch (AccumuloSecurityException ex) {
+ log.debug("Unable to check permissions: " + ex.getCause());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CloneTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CloneTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CloneTable.java
new file mode 100644
index 0000000..9697aee
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CloneTable.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CloneTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String srcTableName = tableNames.get(rand.nextInt(tableNames.size()));
+ String newTableName = tableNames.get(rand.nextInt(tableNames.size()));
+ boolean flush = rand.nextBoolean();
+
+ try {
+ log.debug("Cloning table " + srcTableName + " " + newTableName + " " + flush);
+ conn.tableOperations().clone(srcTableName, newTableName, flush, new HashMap<String,String>(), new HashSet<String>());
+ } catch (TableExistsException e) {
+ log.debug("Clone " + srcTableName + " failed, " + newTableName + " exists");
+ } catch (TableNotFoundException e) {
+ log.debug("Clone " + srcTableName + " failed, doesnt exist");
+ } catch (IllegalArgumentException e) {
+ log.debug("Clone: " + e.toString());
+ } catch (AccumuloException e) {
+ Throwable cause = e.getCause();
+ if (cause != null && cause instanceof NamespaceNotFoundException)
+ log.debug("Clone: " + srcTableName + " to " + newTableName + " failed, namespace not found");
+ else
+ throw e;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Compact.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Compact.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Compact.java
new file mode 100644
index 0000000..cd2dc1a
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Compact.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Compact extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ List<Text> range = ConcurrentFixture.generateRange(rand);
+
+ try {
+ boolean wait = rand.nextBoolean();
+ conn.tableOperations().compact(tableName, range.get(0), range.get(1), false, wait);
+ log.debug((wait ? "compacted " : "initiated compaction ") + tableName + " from " + range.get(0) + " to " + range.get(1));
+ } catch (TableNotFoundException tne) {
+ log.debug("compact " + tableName + " from " + range.get(0) + " to " + range.get(1) + " failed, doesnt exist");
+ } catch (TableOfflineException toe) {
+ log.debug("compact " + tableName + " from " + range.get(0) + " to " + range.get(1) + " failed, offline");
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ConcurrentFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ConcurrentFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ConcurrentFixture.java
new file mode 100644
index 0000000..388e439
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ConcurrentFixture.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+/**
+ * When multiple instance of this test suite are run, all instances will operate on the same set of table names.
+ *
+ *
+ */
+
+public class ConcurrentFixture extends Fixture {
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {}
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {}
+
+ /**
+ *
+ * @param rand
+ * A Random to use
+ * @return A two element list with first being smaller than the second, but either value (or both) can be null
+ */
+ public static List<Text> generateRange(Random rand) {
+ ArrayList<Text> toRet = new ArrayList<>(2);
+
+ long firstLong = rand.nextLong();
+
+ long secondLong = rand.nextLong();
+ Text first = null, second = null;
+
+ // Having all negative values = null might be too frequent
+ if (firstLong >= 0)
+ first = new Text(String.format("%016x", firstLong & 0x7fffffffffffffffl));
+ if (secondLong >= 0)
+ second = new Text(String.format("%016x", secondLong & 0x7fffffffffffffffl));
+
+ if (first != null && second != null && first.compareTo(second) > 0) {
+ Text swap = first;
+ first = second;
+ second = swap;
+ }
+
+ toRet.add(first);
+ toRet.add(second);
+
+ return toRet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Config.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Config.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Config.java
new file mode 100644
index 0000000..a640def
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Config.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.Properties;
+import java.util.SortedSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.commons.math3.random.RandomDataGenerator;
+
+public class Config extends Test {
+
+ private static final String LAST_SETTING = "lastSetting";
+
+ private static final String LAST_TABLE_SETTING = "lastTableSetting";
+
+ private static final String LAST_NAMESPACE_SETTING = "lastNamespaceSetting";
+
+ static class Setting {
+ public Property property;
+ public long min;
+ public long max;
+
+ public Setting(Property property, long min, long max) {
+ this.property = property;
+ this.min = min;
+ this.max = max;
+ }
+ }
+
+ static Setting s(Property property, long min, long max) {
+ return new Setting(property, min, max);
+ }
+
+ // @formatter:off
+ Setting[] settings = {
+ s(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, 1, 10),
+ s(Property.TSERV_BULK_PROCESS_THREADS, 1, 10),
+ s(Property.TSERV_BULK_RETRY, 1, 10),
+ s(Property.TSERV_BULK_TIMEOUT, 10, 600),
+ s(Property.TSERV_BULK_ASSIGNMENT_THREADS, 1, 10),
+ s(Property.TSERV_DATACACHE_SIZE, 0, 1000000000L),
+ s(Property.TSERV_INDEXCACHE_SIZE, 0, 1000000000L),
+ s(Property.TSERV_CLIENT_TIMEOUT, 100, 10000),
+ s(Property.TSERV_MAJC_MAXCONCURRENT, 1, 10),
+ s(Property.TSERV_MAJC_DELAY, 100, 10000),
+ s(Property.TSERV_MAJC_THREAD_MAXOPEN, 3, 100),
+ s(Property.TSERV_MINC_MAXCONCURRENT, 1, 10),
+ s(Property.TSERV_DEFAULT_BLOCKSIZE, 100000, 10000000L),
+ s(Property.TSERV_MAX_IDLE, 10000, 500 * 1000),
+ s(Property.TSERV_MAXMEM, 1000000, 3 * 1024 * 1024 * 1024L),
+ s(Property.TSERV_READ_AHEAD_MAXCONCURRENT, 1, 25),
+ s(Property.TSERV_MIGRATE_MAXCONCURRENT, 1, 10),
+ s(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, 10000, 1024 * 1024),
+ s(Property.TSERV_RECOVERY_MAX_CONCURRENT, 1, 100),
+ s(Property.TSERV_SCAN_MAX_OPENFILES, 10, 1000),
+ s(Property.TSERV_THREADCHECK, 100, 10000),
+ s(Property.TSERV_MINTHREADS, 1, 100),
+ s(Property.TSERV_SESSION_MAXIDLE, 100, 5 * 60 * 1000),
+ s(Property.TSERV_SORT_BUFFER_SIZE, 1024 * 1024, 1024 * 1024 * 1024L),
+ s(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN, 5, 100),
+ s(Property.TSERV_WAL_BLOCKSIZE, 1024 * 1024, 1024 * 1024 * 1024 * 10L),
+ s(Property.TSERV_WORKQ_THREADS, 1, 10),
+ s(Property.MASTER_BULK_THREADPOOL_SIZE, 1, 10),
+ s(Property.MASTER_BULK_RETRIES, 1, 10),
+ s(Property.MASTER_BULK_TIMEOUT, 10, 600),
+ s(Property.MASTER_FATE_THREADPOOL_SIZE, 1, 100),
+ s(Property.MASTER_RECOVERY_DELAY, 0, 100),
+ s(Property.MASTER_LEASE_RECOVERY_WAITING_PERIOD, 0, 10),
+ s(Property.MASTER_RECOVERY_MAXTIME, 10, 1000),
+ s(Property.MASTER_THREADCHECK, 100, 10000),
+ s(Property.MASTER_MINTHREADS, 1, 200),};
+
+ Setting[] tableSettings = {
+ s(Property.TABLE_MAJC_RATIO, 1, 10),
+ s(Property.TABLE_MAJC_COMPACTALL_IDLETIME, 100, 10 * 60 * 60 * 1000L),
+ s(Property.TABLE_SPLIT_THRESHOLD, 10 * 1024, 10L * 1024 * 1024 * 1024),
+ s(Property.TABLE_MINC_COMPACT_IDLETIME, 100, 100 * 60 * 60 * 1000L),
+ s(Property.TABLE_SCAN_MAXMEM, 10 * 1024, 10 * 1024 * 1024),
+ s(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE, 10 * 1024, 10 * 1024 * 1024L),
+ s(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX, 10 * 1024, 10 * 1024 * 1024L),
+ s(Property.TABLE_FILE_REPLICATION, 0, 5),
+ s(Property.TABLE_FILE_MAX, 2, 50),};
+ // @formatter:on
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ // reset any previous setting
+ Object lastSetting = state.getOkIfAbsent(LAST_SETTING);
+ if (lastSetting != null) {
+ int choice = Integer.parseInt(lastSetting.toString());
+ Property property = settings[choice].property;
+ log.debug("Setting " + property.getKey() + " back to " + property.getDefaultValue());
+ env.getConnector().instanceOperations().setProperty(property.getKey(), property.getDefaultValue());
+ }
+ lastSetting = state.getOkIfAbsent(LAST_TABLE_SETTING);
+ if (lastSetting != null) {
+ String parts[] = lastSetting.toString().split(",");
+ String table = parts[0];
+ int choice = Integer.parseInt(parts[1]);
+ Property property = tableSettings[choice].property;
+ if (env.getConnector().tableOperations().exists(table)) {
+ log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue());
+ try {
+ env.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue());
+ } catch (AccumuloException ex) {
+ if (ex.getCause() instanceof ThriftTableOperationException) {
+ ThriftTableOperationException ttoe = (ThriftTableOperationException) ex.getCause();
+ if (ttoe.type == TableOperationExceptionType.NOTFOUND)
+ return;
+ }
+ throw ex;
+ }
+ }
+ }
+ lastSetting = state.getOkIfAbsent(LAST_NAMESPACE_SETTING);
+ if (lastSetting != null) {
+ String parts[] = lastSetting.toString().split(",");
+ String namespace = parts[0];
+ int choice = Integer.parseInt(parts[1]);
+ Property property = tableSettings[choice].property;
+ if (env.getConnector().namespaceOperations().exists(namespace)) {
+ log.debug("Setting " + property.getKey() + " on " + namespace + " back to " + property.getDefaultValue());
+ try {
+ env.getConnector().namespaceOperations().setProperty(namespace, property.getKey(), property.getDefaultValue());
+ } catch (AccumuloException ex) {
+ if (ex.getCause() instanceof ThriftTableOperationException) {
+ ThriftTableOperationException ttoe = (ThriftTableOperationException) ex.getCause();
+ if (ttoe.type == TableOperationExceptionType.NAMESPACE_NOTFOUND)
+ return;
+ }
+ throw ex;
+ }
+ }
+ }
+ state.remove(LAST_SETTING);
+ state.remove(LAST_TABLE_SETTING);
+ state.remove(LAST_NAMESPACE_SETTING);
+ RandomDataGenerator random = new RandomDataGenerator();
+ int dice = random.nextInt(0, 2);
+ if (dice == 0) {
+ changeTableSetting(random, state, env, props);
+ } else if (dice == 1) {
+ changeNamespaceSetting(random, state, env, props);
+ } else {
+ changeSetting(random, state, env, props);
+ }
+ }
+
+ private void changeTableSetting(RandomDataGenerator random, State state, Environment env, Properties props) throws Exception {
+ // pick a random property
+ int choice = random.nextInt(0, tableSettings.length - 1);
+ Setting setting = tableSettings[choice];
+
+ // pick a random table
+ SortedSet<String> tables = env.getConnector().tableOperations().list().tailSet("ctt").headSet("ctu");
+ if (tables.isEmpty())
+ return;
+ String table = random.nextSample(tables, 1)[0].toString();
+
+ // generate a random value
+ long newValue = random.nextLong(setting.min, setting.max);
+ state.set(LAST_TABLE_SETTING, table + "," + choice);
+ log.debug("Setting " + setting.property.getKey() + " on table " + table + " to " + newValue);
+ try {
+ env.getConnector().tableOperations().setProperty(table, setting.property.getKey(), "" + newValue);
+ } catch (AccumuloException ex) {
+ if (ex.getCause() instanceof ThriftTableOperationException) {
+ ThriftTableOperationException ttoe = (ThriftTableOperationException) ex.getCause();
+ if (ttoe.type == TableOperationExceptionType.NOTFOUND)
+ return;
+ }
+ throw ex;
+ }
+ }
+
+ private void changeNamespaceSetting(RandomDataGenerator random, State state, Environment env, Properties props) throws Exception {
+ // pick a random property
+ int choice = random.nextInt(0, tableSettings.length - 1);
+ Setting setting = tableSettings[choice];
+
+ // pick a random table
+ SortedSet<String> namespaces = env.getConnector().namespaceOperations().list().tailSet("nspc").headSet("nspd");
+ if (namespaces.isEmpty())
+ return;
+ String namespace = random.nextSample(namespaces, 1)[0].toString();
+
+ // generate a random value
+ long newValue = random.nextLong(setting.min, setting.max);
+ state.set(LAST_NAMESPACE_SETTING, namespace + "," + choice);
+ log.debug("Setting " + setting.property.getKey() + " on namespace " + namespace + " to " + newValue);
+ try {
+ env.getConnector().namespaceOperations().setProperty(namespace, setting.property.getKey(), "" + newValue);
+ } catch (AccumuloException ex) {
+ if (ex.getCause() instanceof ThriftTableOperationException) {
+ ThriftTableOperationException ttoe = (ThriftTableOperationException) ex.getCause();
+ if (ttoe.type == TableOperationExceptionType.NAMESPACE_NOTFOUND)
+ return;
+ }
+ throw ex;
+ }
+ }
+
+ private void changeSetting(RandomDataGenerator random, State state, Environment env, Properties props) throws Exception {
+ // pick a random property
+ int choice = random.nextInt(0, settings.length - 1);
+ Setting setting = settings[choice];
+ // generate a random value
+ long newValue = random.nextLong(setting.min, setting.max);
+ state.set(LAST_SETTING, "" + choice);
+ log.debug("Setting " + setting.property.getKey() + " to " + newValue);
+ env.getConnector().instanceOperations().setProperty(setting.property.getKey(), "" + newValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateNamespace.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateNamespace.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateNamespace.java
new file mode 100644
index 0000000..71250d8
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateNamespace.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.NamespaceExistsException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateNamespace extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> namespaces = (List<String>) state.get("namespaces");
+
+ String namespace = namespaces.get(rand.nextInt(namespaces.size()));
+
+ try {
+ conn.namespaceOperations().create(namespace);
+ log.debug("Created namespace " + namespace);
+ } catch (NamespaceExistsException e) {
+ log.debug("Create namespace " + namespace + " failed, it exists");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateTable.java
new file mode 100644
index 0000000..648732e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateTable.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ conn.tableOperations().create(tableName);
+ log.debug("Created table " + tableName);
+ } catch (TableExistsException e) {
+ log.debug("Create " + tableName + " failed, it exists");
+ } catch (AccumuloException e) {
+ if (e.getCause() != null && e.getCause() instanceof NamespaceNotFoundException)
+ log.debug("Create " + tableName + " failed, the namespace does not exist");
+ else
+ throw e;
+ } catch (IllegalArgumentException e) {
+ log.debug("Create: " + e.toString());
+ } catch (AccumuloSecurityException e) {
+ log.debug("Could not create table: " + e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateUser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateUser.java
new file mode 100644
index 0000000..708d48f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/CreateUser.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateUser extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> userNames = (List<String>) state.get("users");
+
+ String userName = userNames.get(rand.nextInt(userNames.size()));
+
+ try {
+ log.debug("Creating user " + userName);
+ conn.securityOperations().createLocalUser(userName, new PasswordToken(userName + "pass"));
+ } catch (AccumuloSecurityException ex) {
+ log.debug("Create user failed " + ex.getCause());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteNamespace.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteNamespace.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteNamespace.java
new file mode 100644
index 0000000..7f564ae
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteNamespace.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.NamespaceNotEmptyException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DeleteNamespace extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> namespaces = (List<String>) state.get("namespaces");
+
+ String namespace = namespaces.get(rand.nextInt(namespaces.size()));
+
+ try {
+ conn.namespaceOperations().delete(namespace);
+ log.debug("Deleted namespace " + namespace);
+ } catch (NamespaceNotFoundException e) {
+ log.debug("Delete namespace " + namespace + " failed, doesnt exist");
+ } catch (NamespaceNotEmptyException e) {
+ log.debug("Delete namespace " + namespace + " failed, not empty");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteRange.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteRange.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteRange.java
new file mode 100644
index 0000000..f8db0e7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteRange.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class DeleteRange extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ List<Text> range = new ArrayList<>();
+ do {
+ range.add(new Text(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl)));
+ range.add(new Text(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl)));
+ } while (range.get(0).equals(range.get(1)));
+ Collections.sort(range);
+ if (rand.nextInt(20) == 0)
+ range.set(0, null);
+ if (rand.nextInt(20) == 0)
+ range.set(1, null);
+
+ try {
+ conn.tableOperations().deleteRows(tableName, range.get(0), range.get(1));
+ log.debug("deleted rows (" + range.get(0) + " -> " + range.get(1) + "] in " + tableName);
+ } catch (TableNotFoundException tne) {
+ log.debug("deleted rows " + tableName + " failed, doesnt exist");
+ } catch (TableOfflineException toe) {
+ log.debug("deleted rows " + tableName + " failed, offline");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteTable.java
new file mode 100644
index 0000000..5937a29
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DeleteTable.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DeleteTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ conn.tableOperations().delete(tableName);
+ log.debug("Deleted table " + tableName);
+ } catch (TableNotFoundException e) {
+ log.debug("Delete " + tableName + " failed, doesnt exist");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DropUser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DropUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DropUser.java
new file mode 100644
index 0000000..2034d3d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/DropUser.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DropUser extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> userNames = (List<String>) state.get("users");
+
+ String userName = userNames.get(rand.nextInt(userNames.size()));
+
+ try {
+ log.debug("Dropping user " + userName);
+ conn.securityOperations().dropLocalUser(userName);
+ } catch (AccumuloSecurityException ex) {
+ log.debug("Unable to drop " + ex.getCause());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/IsolatedScan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/IsolatedScan.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/IsolatedScan.java
new file mode 100644
index 0000000..8a5483e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/IsolatedScan.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.RowIterator;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class IsolatedScan extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ RowIterator iter = new RowIterator(new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY)));
+
+ while (iter.hasNext()) {
+ PeekingIterator<Entry<Key,Value>> row = new PeekingIterator<>(iter.next());
+ Entry<Key,Value> kv = null;
+ if (row.hasNext())
+ kv = row.peek();
+ while (row.hasNext()) {
+ Entry<Key,Value> currentKV = row.next();
+ if (!kv.getValue().equals(currentKV.getValue()))
+ throw new Exception("values not equal " + kv + " " + currentKV);
+ }
+ }
+ log.debug("Isolated scan " + tableName);
+ } catch (TableDeletedException e) {
+ log.debug("Isolated scan " + tableName + " failed, table deleted");
+ } catch (TableNotFoundException e) {
+ log.debug("Isolated scan " + tableName + " failed, doesnt exist");
+ } catch (TableOfflineException e) {
+ log.debug("Isolated scan " + tableName + " failed, offline");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ListSplits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ListSplits.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ListSplits.java
new file mode 100644
index 0000000..a84c4fd
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ListSplits.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class ListSplits extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ Collection<Text> splits = conn.tableOperations().listSplits(tableName);
+ log.debug("Table " + tableName + " had " + splits.size() + " splits");
+ } catch (TableNotFoundException e) {
+ log.debug("listSplits " + tableName + " failed, doesnt exist");
+ } catch (AccumuloSecurityException ase) {
+ log.debug("listSplits " + tableName + " failed, " + ase.getMessage());
+ }
+ }
+}
[2/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Posted by mw...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
new file mode 100644
index 0000000..eacd36b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Test deleting documents by using a compaction filter iterator
+ */
+public class CompactFilter extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String docTableName = (String) state.get("docTableName");
+ Random rand = (Random) state.get("rand");
+
+ String deleteChar = Integer.toHexString(rand.nextInt(16)) + "";
+ String regex = "^[0-9a-f][" + deleteChar + "].*";
+
+ ArrayList<IteratorSetting> documentFilters = new ArrayList<>();
+
+ IteratorSetting is = new IteratorSetting(21, "ii", RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ RegExFilter.setNegate(is, true);
+ documentFilters.add(is);
+
+ long t1 = System.currentTimeMillis();
+ env.getConnector().tableOperations().compact(docTableName, null, null, documentFilters, true, true);
+ long t2 = System.currentTimeMillis();
+ long t3 = t2 - t1;
+
+ ArrayList<IteratorSetting> indexFilters = new ArrayList<>();
+
+ is = new IteratorSetting(21, RegExFilter.class);
+ RegExFilter.setRegexs(is, null, null, regex, null, false);
+ RegExFilter.setNegate(is, true);
+ indexFilters.add(is);
+
+ t1 = System.currentTimeMillis();
+ env.getConnector().tableOperations().compact(indexTableName, null, null, indexFilters, true, true);
+ t2 = System.currentTimeMillis();
+
+ log.debug("Filtered documents using compaction iterators " + regex + " " + (t3) + " " + (t2 - t1));
+
+ BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, new Authorizations(), 10);
+
+ List<Range> ranges = new ArrayList<>();
+ for (int i = 0; i < 16; i++) {
+ ranges.add(Range.prefix(new Text(Integer.toHexString(i) + "" + deleteChar)));
+ }
+
+ bscanner.setRanges(ranges);
+ Iterator<Entry<Key,Value>> iter = bscanner.iterator();
+
+ if (iter.hasNext()) {
+ throw new Exception("Saw unexpected document " + iter.next().getKey());
+ }
+
+ bscanner.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
new file mode 100644
index 0000000..e2c8bea
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Delete extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+
+ Entry<Key,Value> entry = Search.findRandomDocument(state, env, dataTableName, rand);
+ if (entry == null)
+ return;
+
+ String docID = entry.getKey().getRow().toString();
+ String doc = entry.getValue().toString();
+
+ Insert.unindexDocument(env.getMultiTableBatchWriter().getBatchWriter(indexTableName), doc, docID, numPartitions);
+
+ Mutation m = new Mutation(docID);
+ m.putDelete("doc", "");
+
+ env.getMultiTableBatchWriter().getBatchWriter(dataTableName).addMutation(m);
+
+ log.debug("Deleted document " + docID);
+
+ env.getMultiTableBatchWriter().flush();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
new file mode 100644
index 0000000..2b790bd
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+//a test created to test the batch deleter
+public class DeleteSomeDocs extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ // delete documents that where the document id matches a given pattern from doc and index table
+ // using the batch deleter
+
+ Random rand = (Random) state.get("rand");
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+
+ ArrayList<String> patterns = new ArrayList<>();
+
+ for (Object key : props.keySet())
+ if (key instanceof String && ((String) key).startsWith("pattern"))
+ patterns.add(props.getProperty((String) key));
+
+ String pattern = patterns.get(rand.nextInt(patterns.size()));
+ BatchWriterConfig bwc = new BatchWriterConfig();
+ BatchDeleter ibd = env.getConnector().createBatchDeleter(indexTableName, Authorizations.EMPTY, 8, bwc);
+ ibd.setRanges(Collections.singletonList(new Range()));
+
+ IteratorSetting iterSettings = new IteratorSetting(100, RegExFilter.class);
+ RegExFilter.setRegexs(iterSettings, null, null, pattern, null, false);
+
+ ibd.addScanIterator(iterSettings);
+
+ ibd.delete();
+
+ ibd.close();
+
+ BatchDeleter dbd = env.getConnector().createBatchDeleter(dataTableName, Authorizations.EMPTY, 8, bwc);
+ dbd.setRanges(Collections.singletonList(new Range()));
+
+ iterSettings = new IteratorSetting(100, RegExFilter.class);
+ RegExFilter.setRegexs(iterSettings, pattern, null, null, null, false);
+
+ dbd.addScanIterator(iterSettings);
+
+ dbd.delete();
+
+ dbd.close();
+
+ log.debug("Deleted documents w/ id matching '" + pattern + "'");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
new file mode 100644
index 0000000..544c35e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Delete all documents containing a particular word.
+ *
+ */
+
+public class DeleteWord extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String docTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+
+ String wordToDelete = Insert.generateRandomWord(rand);
+
+ // use index to find all documents containing word
+ Scanner scanner = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(new Text(wordToDelete));
+
+ ArrayList<Range> documentsToDelete = new ArrayList<>();
+
+ for (Entry<Key,Value> entry : scanner)
+ documentsToDelete.add(new Range(entry.getKey().getColumnQualifier()));
+
+ if (documentsToDelete.size() > 0) {
+ // use a batch scanner to fetch all documents
+ BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, Authorizations.EMPTY, 8);
+ bscanner.setRanges(documentsToDelete);
+
+ BatchWriter ibw = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
+ BatchWriter dbw = env.getMultiTableBatchWriter().getBatchWriter(docTableName);
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : bscanner) {
+ String docID = entry.getKey().getRow().toString();
+ String doc = entry.getValue().toString();
+
+ Insert.unindexDocument(ibw, doc, docID, numPartitions);
+
+ Mutation m = new Mutation(docID);
+ m.putDelete("doc", "");
+
+ dbw.addMutation(m);
+ count++;
+ }
+
+ bscanner.close();
+
+ env.getMultiTableBatchWriter().flush();
+
+ if (count != documentsToDelete.size()) {
+ throw new Exception("Batch scanner did not return expected number of docs " + count + " " + documentsToDelete.size());
+ }
+ }
+
+ log.debug("Deleted " + documentsToDelete.size() + " documents containing " + wordToDelete);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
new file mode 100644
index 0000000..d52198b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class ExportIndex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+
+ String exportDir = "/tmp/shard_export/" + indexTableName;
+ String copyDir = "/tmp/shard_export/" + tmpIndexTableName;
+
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+
+ fs.delete(new Path("/tmp/shard_export/" + indexTableName), true);
+ fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true);
+
+ // disable spits, so that splits can be compared later w/o worrying one table splitting and the other not
+ env.getConnector().tableOperations().setProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "20G");
+
+ long t1 = System.currentTimeMillis();
+
+ env.getConnector().tableOperations().flush(indexTableName, null, null, true);
+ env.getConnector().tableOperations().offline(indexTableName);
+
+ long t2 = System.currentTimeMillis();
+
+ env.getConnector().tableOperations().exportTable(indexTableName, exportDir);
+
+ long t3 = System.currentTimeMillis();
+
+ // copy files
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(new Path(exportDir, "distcp.txt")), UTF_8));
+ String file = null;
+ while ((file = reader.readLine()) != null) {
+ Path src = new Path(file);
+ Path dest = new Path(new Path(copyDir), src.getName());
+ FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance());
+ }
+
+ reader.close();
+
+ long t4 = System.currentTimeMillis();
+
+ env.getConnector().tableOperations().online(indexTableName);
+ env.getConnector().tableOperations().importTable(tmpIndexTableName, copyDir);
+
+ long t5 = System.currentTimeMillis();
+
+ fs.delete(new Path(exportDir), true);
+ fs.delete(new Path(copyDir), true);
+
+ HashSet<Text> splits1 = new HashSet<>(env.getConnector().tableOperations().listSplits(indexTableName));
+ HashSet<Text> splits2 = new HashSet<>(env.getConnector().tableOperations().listSplits(tmpIndexTableName));
+
+ if (!splits1.equals(splits2))
+ throw new Exception("Splits not equals " + indexTableName + " " + tmpIndexTableName);
+
+ HashMap<String,String> props1 = new HashMap<>();
+ for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(indexTableName))
+ props1.put(entry.getKey(), entry.getValue());
+
+ HashMap<String,String> props2 = new HashMap<>();
+ for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(tmpIndexTableName))
+ props2.put(entry.getKey(), entry.getValue());
+
+ if (!props1.equals(props2))
+ throw new Exception("Props not equals " + indexTableName + " " + tmpIndexTableName);
+
+ // unset the split threshold
+ env.getConnector().tableOperations().removeProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
+ env.getConnector().tableOperations().removeProperty(tmpIndexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
+
+ log.debug("Imported " + tmpIndexTableName + " from " + indexTableName + " flush: " + (t2 - t1) + "ms export: " + (t3 - t2) + "ms copy:" + (t4 - t3)
+ + "ms import:" + (t5 - t4) + "ms");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
new file mode 100644
index 0000000..e6ac574
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Flush extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ Random rand = (Random) state.get("rand");
+
+ String table;
+
+ if (rand.nextDouble() < .5)
+ table = indexTableName;
+ else
+ table = dataTableName;
+
+ env.getConnector().tableOperations().flush(table, null, null, true);
+ log.debug("Flushed " + table);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
new file mode 100644
index 0000000..5892626
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Grep extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ // pick a few randoms words... grep for those words and search the index
+ // ensure both return the same set of documents
+
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ Random rand = (Random) state.get("rand");
+
+ Text words[] = new Text[rand.nextInt(4) + 2];
+
+ for (int i = 0; i < words.length; i++) {
+ words[i] = new Text(Insert.generateRandomWord(rand));
+ }
+
+ BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 16);
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class.getName());
+ IntersectingIterator.setColumnFamilies(ii, words);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+
+ HashSet<Text> documentsFoundInIndex = new HashSet<>();
+
+ for (Entry<Key,Value> entry2 : bs) {
+ documentsFoundInIndex.add(entry2.getKey().getColumnQualifier());
+ }
+
+ bs.close();
+
+ bs = env.getConnector().createBatchScanner(dataTableName, Authorizations.EMPTY, 16);
+
+ for (int i = 0; i < words.length; i++) {
+ IteratorSetting more = new IteratorSetting(20 + i, "ii" + i, RegExFilter.class);
+ RegExFilter.setRegexs(more, null, null, null, "(^|(.*\\s))" + words[i] + "($|(\\s.*))", false);
+ bs.addScanIterator(more);
+ }
+
+ bs.setRanges(Collections.singleton(new Range()));
+
+ HashSet<Text> documentsFoundByGrep = new HashSet<>();
+
+ for (Entry<Key,Value> entry2 : bs) {
+ documentsFoundByGrep.add(entry2.getKey().getRow());
+ }
+
+ bs.close();
+
+ if (!documentsFoundInIndex.equals(documentsFoundByGrep)) {
+ throw new Exception("Set of documents found not equal for words " + Arrays.asList(words).toString() + " " + documentsFoundInIndex + " "
+ + documentsFoundByGrep);
+ }
+
+ log.debug("Grep and index agree " + Arrays.asList(words).toString() + " " + documentsFoundInIndex.size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
new file mode 100644
index 0000000..1b15323
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Insert extends Test {
+
+ static final int NUM_WORDS = 100000;
+ static final int MIN_WORDS_PER_DOC = 10;
+ static final int MAX_WORDS_PER_DOC = 3000;
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+ long nextDocID = (Long) state.get("nextDocID");
+
+ BatchWriter dataWriter = env.getMultiTableBatchWriter().getBatchWriter(dataTableName);
+ BatchWriter indexWriter = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
+
+ String docID = insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand);
+
+ log.debug("Inserted document " + docID);
+
+ state.set("nextDocID", Long.valueOf(nextDocID));
+ }
+
+ static String insertRandomDocument(long did, BatchWriter dataWriter, BatchWriter indexWriter, String indexTableName, String dataTableName, int numPartitions,
+ Random rand) throws TableNotFoundException, Exception, AccumuloException, AccumuloSecurityException {
+ String doc = createDocument(rand);
+
+ String docID = new StringBuilder(String.format("%016x", did)).reverse().toString();
+
+ saveDocument(dataWriter, docID, doc);
+ indexDocument(indexWriter, doc, docID, numPartitions);
+
+ return docID;
+ }
+
+ static void saveDocument(BatchWriter bw, String docID, String doc) throws Exception {
+
+ Mutation m = new Mutation(docID);
+ m.put("doc", "", doc);
+
+ bw.addMutation(m);
+ }
+
+ static String createDocument(Random rand) {
+ StringBuilder sb = new StringBuilder();
+
+ int numWords = rand.nextInt(MAX_WORDS_PER_DOC - MIN_WORDS_PER_DOC) + MIN_WORDS_PER_DOC;
+
+ for (int i = 0; i < numWords; i++) {
+ String word = generateRandomWord(rand);
+
+ if (i > 0)
+ sb.append(" ");
+
+ sb.append(word);
+ }
+
+ return sb.toString();
+ }
+
+ static String generateRandomWord(Random rand) {
+ return Integer.toString(rand.nextInt(NUM_WORDS), Character.MAX_RADIX);
+ }
+
+ static String genPartition(int partition) {
+ return String.format("%06x", Math.abs(partition));
+ }
+
+ static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
+ indexDocument(bw, doc, docId, numPartitions, false);
+ }
+
+ static void unindexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
+ indexDocument(bw, doc, docId, numPartitions, true);
+ }
+
+ static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions, boolean delete) throws Exception {
+
+ String[] tokens = doc.split("\\W+");
+
+ String partition = genPartition(doc.hashCode() % numPartitions);
+
+ Mutation m = new Mutation(partition);
+
+ HashSet<String> tokensSeen = new HashSet<>();
+
+ for (String token : tokens) {
+ token = token.toLowerCase();
+
+ if (!tokensSeen.contains(token)) {
+ tokensSeen.add(token);
+ if (delete)
+ m.putDelete(token, docId);
+ else
+ m.put(token, docId, new Value(new byte[0]));
+ }
+ }
+
+ if (m.size() > 0)
+ bw.addMutation(m);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
new file mode 100644
index 0000000..106ab3b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collection;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Merge extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+
+ Collection<Text> splits = env.getConnector().tableOperations().listSplits(indexTableName);
+ SortedSet<Text> splitSet = new TreeSet<>(splits);
+ log.debug("merging " + indexTableName);
+ env.getConnector().tableOperations().merge(indexTableName, null, null);
+ org.apache.accumulo.core.util.Merge merge = new org.apache.accumulo.core.util.Merge();
+ merge.mergomatic(env.getConnector(), indexTableName, null, null, 256 * 1024 * 1024, true);
+ splits = env.getConnector().tableOperations().listSplits(indexTableName);
+ if (splits.size() > splitSet.size()) {
+ // throw an excpetion so that test will die an no further changes to table will occur...
+ // this way table is left as is for debugging.
+ throw new Exception("There are more tablets after a merge: " + splits.size() + " was " + splitSet.size());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
new file mode 100644
index 0000000..ac0c872
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Reindex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+ String docTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+
+ Random rand = (Random) state.get("rand");
+
+ ShardFixture.createIndexTable(this.log, state, env, "_tmp", rand);
+
+ Scanner scanner = env.getConnector().createScanner(docTableName, Authorizations.EMPTY);
+ BatchWriter tbw = env.getConnector().createBatchWriter(tmpIndexTableName, new BatchWriterConfig());
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ String docID = entry.getKey().getRow().toString();
+ String doc = entry.getValue().toString();
+
+ Insert.indexDocument(tbw, doc, docID, numPartitions);
+
+ count++;
+ }
+
+ tbw.close();
+
+ log.debug("Reindexed " + count + " documents into " + tmpIndexTableName);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
new file mode 100644
index 0000000..c07397d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Search extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+
+ Random rand = (Random) state.get("rand");
+
+ Entry<Key,Value> entry = findRandomDocument(state, env, dataTableName, rand);
+ if (entry == null)
+ return;
+
+ Text docID = entry.getKey().getRow();
+ String doc = entry.getValue().toString();
+
+ String[] tokens = doc.split("\\W+");
+ int numSearchTerms = rand.nextInt(6);
+ if (numSearchTerms < 2)
+ numSearchTerms = 2;
+
+ HashSet<String> searchTerms = new HashSet<>();
+ while (searchTerms.size() < numSearchTerms)
+ searchTerms.add(tokens[rand.nextInt(tokens.length)]);
+
+ Text columns[] = new Text[searchTerms.size()];
+ int index = 0;
+ for (String term : searchTerms) {
+ columns[index++] = new Text(term);
+ }
+
+ log.debug("Looking up terms " + searchTerms + " expect to find " + docID);
+
+ BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 10);
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
+ IntersectingIterator.setColumnFamilies(ii, columns);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+
+ boolean sawDocID = false;
+
+ for (Entry<Key,Value> entry2 : bs) {
+ if (entry2.getKey().getColumnQualifier().equals(docID)) {
+ sawDocID = true;
+ break;
+ }
+ }
+
+ bs.close();
+
+ if (!sawDocID)
+ throw new Exception("Did not see doc " + docID + " in index. terms:" + searchTerms + " " + indexTableName + " " + dataTableName);
+ }
+
+ static Entry<Key,Value> findRandomDocument(State state, Environment env, String dataTableName, Random rand) throws Exception {
+ Scanner scanner = env.getConnector().createScanner(dataTableName, Authorizations.EMPTY);
+ scanner.setBatchSize(1);
+ scanner.setRange(new Range(Integer.toString(rand.nextInt(0xfffffff), 16), null));
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ if (!iter.hasNext())
+ return null;
+
+ return iter.next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
new file mode 100644
index 0000000..e20f64d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.net.InetAddress;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class ShardFixture extends Fixture {
+
+ static SortedSet<Text> genSplits(long max, int numTablets, String format) {
+
+ int numSplits = numTablets - 1;
+ long distance = max / numTablets;
+ long split = distance;
+
+ TreeSet<Text> splits = new TreeSet<>();
+
+ for (int i = 0; i < numSplits; i++) {
+ splits.add(new Text(String.format(format, split)));
+ split += distance;
+ }
+
+ return splits;
+ }
+
+ static void createIndexTable(Logger log, State state, Environment env, String suffix, Random rand) throws Exception {
+ Connector conn = env.getConnector();
+ String name = (String) state.get("indexTableName") + suffix;
+ int numPartitions = (Integer) state.get("numPartitions");
+ boolean enableCache = (Boolean) state.get("cacheIndex");
+ conn.tableOperations().create(name);
+
+ String tableId = conn.tableOperations().tableIdMap().get(name);
+ log.info("Created index table " + name + "(id:" + tableId + ")");
+
+ SortedSet<Text> splits = genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
+ conn.tableOperations().addSplits(name, splits);
+
+ log.info("Added " + splits.size() + " splits to " + name);
+
+ if (enableCache) {
+ conn.tableOperations().setProperty(name, Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
+ conn.tableOperations().setProperty(name, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
+
+ log.info("Enabled caching for table " + name);
+ }
+ }
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+ String pid = env.getPid();
+
+ Random rand = new Random();
+
+ int numPartitions = rand.nextInt(90) + 10;
+
+ state.set("indexTableName", String.format("ST_index_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
+ state.set("docTableName", String.format("ST_docs_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
+ state.set("numPartitions", Integer.valueOf(numPartitions));
+ state.set("cacheIndex", rand.nextDouble() < .5);
+ state.set("rand", rand);
+ state.set("nextDocID", Long.valueOf(0));
+
+ Connector conn = env.getConnector();
+
+ createIndexTable(this.log, state, env, "", rand);
+
+ String docTableName = (String) state.get("docTableName");
+ conn.tableOperations().create(docTableName);
+
+ String tableId = conn.tableOperations().tableIdMap().get(docTableName);
+ log.info("Created doc table " + docTableName + " (id:" + tableId + ")");
+
+ SortedSet<Text> splits = genSplits(0xff, rand.nextInt(32) + 1, "%02x");
+ conn.tableOperations().addSplits(docTableName, splits);
+
+ log.info("Added " + splits.size() + " splits to " + docTableName);
+
+ if (rand.nextDouble() < .5) {
+ conn.tableOperations().setProperty((String) state.get("docTableName"), Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+ log.info("Enabled bloom filters for table " + (String) state.get("docTableName"));
+ }
+ }
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {
+ // We have resources we need to clean up
+ if (env.isMultiTableBatchWriterInitialized()) {
+ MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Ignoring mutations that weren't flushed", e);
+ }
+
+ // Reset the MTBW on the state to null
+ env.resetMultiTableBatchWriter();
+ }
+
+ Connector conn = env.getConnector();
+
+ log.info("Deleting index and doc tables");
+
+ conn.tableOperations().delete((String) state.get("indexTableName"));
+ conn.tableOperations().delete((String) state.get("docTableName"));
+
+ log.debug("Exiting shard test");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
new file mode 100644
index 0000000..04c96f4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collection;
+
+import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
+import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SortTool extends Configured implements Tool {
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+ private String outputDir;
+ private String seqFile;
+ private String splitFile;
+ private Collection<Text> splits;
+
+ public SortTool(String seqFile, String outputDir, String splitFile, Collection<Text> splits) {
+ this.outputDir = outputDir;
+ this.seqFile = seqFile;
+ this.splitFile = splitFile;
+ this.splits = splits;
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
+ job.setJarByClass(this.getClass());
+
+ if (job.getJar() == null) {
+ log.error("M/R requires a jar file! Run mvn package.");
+ return 1;
+ }
+
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ SequenceFileInputFormat.setInputPaths(job, seqFile);
+
+ job.setPartitionerClass(KeyRangePartitioner.class);
+ KeyRangePartitioner.setSplitFile(job, splitFile);
+
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+
+ job.setNumReduceTasks(splits.size() + 1);
+
+ job.setOutputFormatClass(AccumuloFileOutputFormat.class);
+ AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
new file mode 100644
index 0000000..bef5104
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+import java.util.Random;
+import java.util.SortedSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Split extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+
+ SortedSet<Text> splitSet = ShardFixture.genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
+ log.debug("adding splits " + indexTableName);
+ env.getConnector().tableOperations().addSplits(indexTableName, splitSet);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
new file mode 100644
index 0000000..caba1d7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class VerifyIndex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+
+ // scan new and old index and verify identical
+ Scanner indexScanner1 = env.getConnector().createScanner(tmpIndexTableName, Authorizations.EMPTY);
+ Scanner indexScanner2 = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = indexScanner2.iterator();
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : indexScanner1) {
+ if (!iter.hasNext())
+ throw new Exception("index rebuild mismatch " + entry.getKey() + " " + indexTableName);
+
+ Key key1 = entry.getKey();
+ Key key2 = iter.next().getKey();
+
+ if (!key1.equals(key2, PartialKey.ROW_COLFAM_COLQUAL))
+ throw new Exception("index rebuild mismatch " + key1 + " " + key2 + " " + indexTableName + " " + tmpIndexTableName);
+ count++;
+ if (count % 1000 == 0)
+ makingProgress();
+ }
+
+ if (iter.hasNext())
+ throw new Exception("index rebuild mismatch " + iter.next().getKey() + " " + tmpIndexTableName);
+
+ log.debug("Verified " + count + " index entries ");
+
+ env.getConnector().tableOperations().delete(indexTableName);
+ env.getConnector().tableOperations().rename(tmpIndexTableName, indexTableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
new file mode 100644
index 0000000..df1df59
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
new file mode 100644
index 0000000..f7226cb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DeleteTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
new file mode 100644
index 0000000..5681402
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Ingest extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
new file mode 100644
index 0000000..c677cf9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Scan extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
new file mode 100644
index 0000000..95acf4f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Verify extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/module.xsd
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/module.xsd b/core/src/main/resources/randomwalk/module.xsd
new file mode 100644
index 0000000..bcdaaae
--- /dev/null
+++ b/core/src/main/resources/randomwalk/module.xsd
@@ -0,0 +1,69 @@
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+ <xsd:element name="module" type="ModuleType"/>
+
+ <xsd:complexType name="ModuleType">
+ <xsd:sequence>
+ <xsd:element name="package" type="PrefixType" minOccurs="0" maxOccurs="unbounded"/>
+ <xsd:element name="fixture" type="InitType" minOccurs="0" maxOccurs="1"/>
+ <xsd:element name="init" type="InitType"/>
+ <xsd:element name="node" type="NodeType" minOccurs="1" maxOccurs="unbounded"/>
+ </xsd:sequence>
+ </xsd:complexType>
+
+ <xsd:complexType name="PrefixType">
+ <xsd:attribute name="prefix" type="xsd:string"/>
+ <xsd:attribute name="value" type="xsd:string"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="InitType">
+ <xsd:attribute name="id" type="xsd:string"/>
+ <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="teardown" type="xsd:boolean"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="NodeType">
+ <xsd:sequence>
+ <xsd:element name="alias" type="AliasType" minOccurs="0" maxOccurs="unbounded"/>
+ <xsd:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
+ <xsd:element name="edge" type="EdgeType" minOccurs="1" maxOccurs="unbounded"/>
+ </xsd:sequence>
+ <xsd:attribute name="id" type="xsd:string"/>
+ <xsd:attribute name="src" type="xsd:string"/>
+ <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="teardown" type="xsd:boolean"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="EdgeType">
+ <xsd:attribute name="id" type="xsd:string"/>
+ <xsd:attribute name="weight" type="xsd:positiveInteger"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="AliasType">
+ <xsd:attribute name="name" type="xsd:string"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="PropertyType">
+ <xsd:attribute name="key" type="xsd:string"/>
+ <xsd:attribute name="value" type="xsd:string"/>
+ </xsd:complexType>
+
+</xsd:schema>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/All.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/All.xml b/core/src/main/resources/randomwalk/modules/All.xml
new file mode 100644
index 0000000..be5c815
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/All.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Conditional.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Conditional.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Bulk.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Bulk.xml b/core/src/main/resources/randomwalk/modules/Bulk.xml
new file mode 100644
index 0000000..35e2a67
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Bulk.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="bulk" value="org.apache.accumulo.test.randomwalk.bulk"/>
+
+<init id="bulk.Setup"/>
+
+<node id="bulk.Setup">
+ <edge id="bulk.BulkPlusOne" weight="1"/>
+</node>
+
+<node id="bulk.BulkPlusOne">
+ <edge id="bulk.Merge" weight="10"/>
+ <edge id="bulk.Split" weight="10"/>
+ <edge id="bulk.Compact" weight="10"/>
+ <edge id="bulk.ConsistencyCheck" weight="25"/>
+ <edge id="bulk.BulkMinusOne" weight="10"/>
+</node>
+
+<node id="bulk.BulkMinusOne">
+ <edge id="bulk.BulkPlusOne" weight="200"/>
+ <edge id="bulk.Verify" weight="1"/>
+</node>
+
+<node id="bulk.Merge">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Split">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Compact">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.ConsistencyCheck">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Verify">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
new file mode 100644
index 0000000..36ea53c
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="ct" value="org.apache.accumulo.test.randomwalk.concurrent"/>
+
+<fixture id="ct.ConcurrentFixture"/>
+
+<init id="ct.Setup"/>
+
+<node id="dummy.ToAll">
+ <edge id="ct.CreateTable" weight="1000"/>
+ <edge id="ct.BatchWrite" weight="1000"/>
+ <edge id="ct.BatchScan" weight="1000"/>
+ <edge id="ct.CloneTable" weight="1000"/>
+ <edge id="ct.DeleteTable" weight="1000"/>
+ <edge id="ct.RenameTable" weight="1000"/>
+ <edge id="ct.ScanTable" weight="500"/>
+ <edge id="ct.Replication" weight="500"/>
+ <edge id="ct.IsolatedScan" weight="500"/>
+ <edge id="ct.AddSplits" weight="1000"/>
+ <edge id="ct.ListSplits" weight="1000"/>
+ <edge id="ct.OfflineTable" weight="1000"/>
+ <edge id="ct.Merge" weight="1000"/>
+ <edge id="ct.Compact" weight="1000"/>
+ <edge id="ct.BulkImport" weight="1000"/>
+ <edge id="ct.DeleteRange" weight="1000"/>
+ <edge id="ct.CreateUser" weight="1000"/>
+ <edge id="ct.DropUser" weight="1000"/>
+ <edge id="ct.ChangeAuthorizations" weight="1000"/>
+ <edge id="ct.ChangePermissions" weight="1000"/>
+ <edge id="ct.CheckPermission" weight="1000"/>
+ <edge id="ct.StopTabletServer" weight="100"/>
+ <edge id="ct.StartAll" weight="1000"/>
+ <edge id="ct.Shutdown" weight="10"/>
+ <edge id="ct.Config" weight="1000"/>
+ <edge id="ct.CreateNamespace" weight="1000"/>
+ <edge id="ct.DeleteNamespace" weight="100"/>
+ <edge id="ct.RenameNamespace" weight="100"/>
+ <edge id="ct.Apocalypse" weight="10"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="ct.Setup">
+ <property key="numTables" value="9"/>
+ <property key="numNamespaces" value="2"/>
+ <edge id="ct.CreateTable" weight="1"/>
+</node>
+
+<node id="ct.BatchScan">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.BatchWrite">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CloneTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CreateTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.RenameTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ScanTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Replication">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.IsolatedScan">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.AddSplits">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ListSplits">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.OfflineTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Merge">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Compact">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.BulkImport">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteRange">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CreateUser">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DropUser">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ChangeAuthorizations">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ChangePermissions">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CheckPermission">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.StopTabletServer">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.StartAll">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Config">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Shutdown">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.Apocalypse">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.CreateNamespace">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteNamespace">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.RenameNamespace">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Conditional.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Conditional.xml b/core/src/main/resources/randomwalk/modules/Conditional.xml
new file mode 100644
index 0000000..54ff7ab
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Conditional.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="ct" value="org.apache.accumulo.test.randomwalk.conditional"/>
+
+
+<init id="ct.Setup"/>
+
+<node id="dummy.ToAll">
+ <edge id="ct.Compact" weight="1"/>
+ <edge id="ct.Flush" weight="1"/>
+ <edge id="ct.Merge" weight="1"/>
+ <edge id="ct.Split" weight="1"/>
+ <edge id="ct.Transfer" weight="100000"/>
+ <edge id="ct.Verify" weight="2"/>
+</node>
+
+<node id="ct.Setup">
+ <property key="numAccts" value="10000"/>
+ <property key="numBanks" value="1000"/>
+ <edge id="ct.Init" weight="1"/>
+</node>
+
+<node id="ct.Init">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Compact">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Flush">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Merge">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Split">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Transfer">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Verify">
+ <edge id="dummy.ToAll" weight="1000"/>
+ <edge id="ct.TearDown" weight="1"/>
+</node>
+
+
+<node id="ct.TearDown">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Image.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Image.xml b/core/src/main/resources/randomwalk/modules/Image.xml
new file mode 100644
index 0000000..7561895
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Image.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="image" value="org.apache.accumulo.test.randomwalk.image"/>
+
+<fixture id="image.ImageFixture"/>
+
+<init id="BigWrite"/>
+
+<node id="BigWrite" src="image.Write">
+ <property key="minSize" value="400000"/>
+ <property key="maxSize" value="1000000"/>
+ <edge id="BigWrite" weight="167"/>
+ <edge id="SmallWrite" weight="833"/>
+ <edge id="image.Commit" weight="1"/>
+</node>
+
+<node id="SmallWrite" src="image.Write">
+ <property key="minSize" value="4000"/>
+ <property key="maxSize" value="10000"/>
+ <edge id="BigWrite" weight="167"/>
+ <edge id="SmallWrite" weight="833"/>
+ <edge id="image.Commit" weight="1"/>
+</node>
+
+<node id="image.Commit">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="SmallWrite" weight="300"/>
+ <edge id="BigWrite" weight="300"/>
+ <edge id="image.Verify" weight="150"/>
+ <edge id="image.ScanMeta" weight="150"/>
+ <edge id="image.TableOp" weight="30"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="image.Verify">
+ <property key="maxVerify" value="500"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="image.ScanMeta">
+ <property key="minScan" value="100"/>
+ <property key="maxScan" value="500"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="image.TableOp">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/LongClean.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/LongClean.xml b/core/src/main/resources/randomwalk/modules/LongClean.xml
new file mode 100644
index 0000000..217afb0
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/LongClean.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
[5/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Posted by mw...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Merge.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Merge.java
new file mode 100644
index 0000000..87a48f9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Merge.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Merge extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+ tableNames = new ArrayList<>(tableNames);
+ tableNames.add(MetadataTable.NAME);
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ List<Text> range = ConcurrentFixture.generateRange(rand);
+
+ try {
+ conn.tableOperations().merge(tableName, range.get(0), range.get(1));
+ log.debug("merged " + tableName + " from " + range.get(0) + " to " + range.get(1));
+ } catch (TableOfflineException toe) {
+ log.debug("merge " + tableName + " from " + range.get(0) + " to " + range.get(1) + " failed, table is not online");
+ } catch (TableNotFoundException tne) {
+ log.debug("merge " + tableName + " from " + range.get(0) + " to " + range.get(1) + " failed, doesnt exist");
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
new file mode 100644
index 0000000..fd01d98
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/OfflineTable.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class OfflineTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ conn.tableOperations().offline(tableName, rand.nextBoolean());
+ log.debug("Offlined " + tableName);
+ sleepUninterruptibly(rand.nextInt(200), TimeUnit.MILLISECONDS);
+ conn.tableOperations().online(tableName, rand.nextBoolean());
+ log.debug("Onlined " + tableName);
+ } catch (TableNotFoundException tne) {
+ log.debug("offline or online failed " + tableName + ", doesnt exist");
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameNamespace.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameNamespace.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameNamespace.java
new file mode 100644
index 0000000..dab41bf
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameNamespace.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.NamespaceExistsException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class RenameNamespace extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> namespaces = (List<String>) state.get("namespaces");
+
+ String srcName = namespaces.get(rand.nextInt(namespaces.size()));
+ String newName = namespaces.get(rand.nextInt(namespaces.size()));
+
+ try {
+ conn.namespaceOperations().rename(srcName, newName);
+ log.debug("Renamed namespace " + srcName + " " + newName);
+ } catch (NamespaceExistsException e) {
+ log.debug("Rename namespace " + srcName + " failed, " + newName + " exists");
+ } catch (NamespaceNotFoundException e) {
+ log.debug("Rename namespace " + srcName + " failed, doesn't exist");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameTable.java
new file mode 100644
index 0000000..4c5a52f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/RenameTable.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class RenameTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String srcTableName = tableNames.get(rand.nextInt(tableNames.size()));
+ String newTableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ String srcNamespace = "", newNamespace = "";
+
+ int index = srcTableName.indexOf('.');
+ if (-1 != index) {
+ srcNamespace = srcTableName.substring(0, index);
+ }
+
+ index = newTableName.indexOf('.');
+ if (-1 != index) {
+ newNamespace = newTableName.substring(0, index);
+ }
+
+ try {
+ conn.tableOperations().rename(srcTableName, newTableName);
+ log.debug("Renamed table " + srcTableName + " " + newTableName);
+ } catch (TableExistsException e) {
+ log.debug("Rename " + srcTableName + " failed, " + newTableName + " exists");
+ } catch (TableNotFoundException e) {
+ Throwable cause = e.getCause();
+ if (null != cause) {
+ // Rename has to have failed on the destination namespace, because the source namespace
+ // couldn't be deleted with our table in it
+ if (cause.getClass().isAssignableFrom(NamespaceNotFoundException.class)) {
+ log.debug("Rename failed because new namespace doesn't exist: " + newNamespace, cause);
+ // Avoid the final src/dest namespace check
+ return;
+ }
+ }
+
+ log.debug("Rename " + srcTableName + " failed, doesnt exist");
+ } catch (IllegalArgumentException e) {
+ log.debug("Rename: " + e.toString());
+ } catch (AccumuloException e) {
+ // Catch the expected failure when we try to rename a table into a new namespace
+ if (!srcNamespace.equals(newNamespace)) {
+ return;
+ }
+ log.debug("Rename " + srcTableName + " failed.", e);
+ }
+
+ if (!srcNamespace.equals(newNamespace)) {
+ log.error("RenameTable operation should have failed when renaming across namespaces.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
new file mode 100644
index 0000000..189d743
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Replication.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import static org.apache.accumulo.core.conf.Property.MASTER_REPLICATION_SCAN_INTERVAL;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_NAME;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_PEERS;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_PEER_PASSWORD;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_PEER_USER;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_WORK_ASSIGNMENT_SLEEP;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_WORK_PROCESSOR_DELAY;
+import static org.apache.accumulo.core.conf.Property.REPLICATION_WORK_PROCESSOR_PERIOD;
+import static org.apache.accumulo.core.conf.Property.TABLE_REPLICATION;
+import static org.apache.accumulo.core.conf.Property.TABLE_REPLICATION_TARGET;
+import static org.apache.accumulo.server.replication.ReplicaSystemFactory.getPeerConfigurationValue;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.io.Text;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class Replication extends Test {
+
+ final int ROWS = 1000;
+ final int COLS = 50;
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ final Connector c = env.getConnector();
+ final Instance inst = c.getInstance();
+ final String instName = inst.getInstanceName();
+ final InstanceOperations iOps = c.instanceOperations();
+ final TableOperations tOps = c.tableOperations();
+
+ // Replicate to ourselves
+ iOps.setProperty(REPLICATION_NAME.getKey(), instName);
+ iOps.setProperty(REPLICATION_PEERS.getKey() + instName, getPeerConfigurationValue(AccumuloReplicaSystem.class, instName + "," + inst.getZooKeepers()));
+ iOps.setProperty(REPLICATION_PEER_USER.getKey() + instName, env.getUserName());
+ iOps.setProperty(REPLICATION_PEER_PASSWORD.getKey() + instName, env.getPassword());
+ // Tweak some replication parameters to make the replication go faster
+ iOps.setProperty(MASTER_REPLICATION_SCAN_INTERVAL.getKey(), "1s");
+ iOps.setProperty(REPLICATION_WORK_ASSIGNMENT_SLEEP.getKey(), "1s");
+ iOps.setProperty(REPLICATION_WORK_PROCESSOR_DELAY.getKey(), "1s");
+ iOps.setProperty(REPLICATION_WORK_PROCESSOR_PERIOD.getKey(), "1s");
+
+ // Ensure the replication table is online
+ ReplicationTable.setOnline(c);
+ boolean online = ReplicationTable.isOnline(c);
+ for (int i = 0; i < 10; i++) {
+ if (online)
+ break;
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ online = ReplicationTable.isOnline(c);
+ }
+ assertTrue("Replication table was not online", online);
+
+ // Make a source and destination table
+ final String sourceTable = ("repl-source-" + UUID.randomUUID()).replace('-', '_');
+ final String destTable = ("repl-dest-" + UUID.randomUUID()).replace('-', '_');
+ final String tables[] = new String[] {sourceTable, destTable};
+
+ for (String tableName : tables) {
+ log.debug("creating " + tableName);
+ tOps.create(tableName);
+ }
+
+ // Point the source to the destination
+ final String destID = tOps.tableIdMap().get(destTable);
+ tOps.setProperty(sourceTable, TABLE_REPLICATION.getKey(), "true");
+ tOps.setProperty(sourceTable, TABLE_REPLICATION_TARGET.getKey() + instName, destID);
+
+ // zookeeper propagation wait
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+
+ // Maybe split the tables
+ Random rand = new Random(System.currentTimeMillis());
+ for (String tableName : tables) {
+ if (rand.nextBoolean()) {
+ splitTable(tOps, tableName);
+ }
+ }
+
+ // write some checkable data
+ BatchWriter bw = c.createBatchWriter(sourceTable, null);
+ for (int row = 0; row < ROWS; row++) {
+ Mutation m = new Mutation(itos(row));
+ for (int col = 0; col < COLS; col++) {
+ m.put("", itos(col), "");
+ }
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ // attempt to force the WAL to roll so replication begins
+ final Set<String> origRefs = c.replicationOperations().referencedFiles(sourceTable);
+ // write some data we will ignore
+ while (true) {
+ final Set<String> updatedFileRefs = c.replicationOperations().referencedFiles(sourceTable);
+ updatedFileRefs.retainAll(origRefs);
+ log.debug("updateFileRefs size " + updatedFileRefs.size());
+ if (updatedFileRefs.isEmpty()) {
+ break;
+ }
+ bw = c.createBatchWriter(sourceTable, null);
+ for (int row = 0; row < ROWS; row++) {
+ Mutation m = new Mutation(itos(row));
+ for (int col = 0; col < COLS; col++) {
+ m.put("ignored", itos(col), "");
+ }
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+
+ // wait a little while for replication to take place
+ sleepUninterruptibly(30, TimeUnit.SECONDS);
+
+ // check the data
+ Scanner scanner = c.createScanner(destTable, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(new Text(""));
+ int row = 0;
+ int col = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ assertEquals(row, Integer.parseInt(entry.getKey().getRow().toString()));
+ assertEquals(col, Integer.parseInt(entry.getKey().getColumnQualifier().toString()));
+ col++;
+ if (col == COLS) {
+ row++;
+ col = 0;
+ }
+ }
+ assertEquals(ROWS, row);
+ assertEquals(0, col);
+
+ // cleanup
+ for (String tableName : tables) {
+ log.debug("Deleting " + tableName);
+ tOps.delete(tableName);
+ }
+ }
+
+ // junit isn't a dependency
+ private void assertEquals(int expected, int actual) {
+ if (expected != actual)
+ throw new RuntimeException(String.format("%d fails to match expected value %d", actual, expected));
+ }
+
+ // junit isn't a dependency
+ private void assertTrue(String string, boolean test) {
+ if (!test)
+ throw new RuntimeException(string);
+ }
+
+ private static String itos(int i) {
+ return String.format("%08d", i);
+ }
+
+ private void splitTable(TableOperations tOps, String tableName) throws Exception {
+ SortedSet<Text> splits = new TreeSet<>();
+ for (int i = 1; i <= 9; i++) {
+ splits.add(new Text(itos(i * (ROWS / 10))));
+ }
+ log.debug("Adding splits to " + tableName);
+ tOps.addSplits(tableName, splits);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ScanTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ScanTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ScanTable.java
new file mode 100644
index 0000000..ab89bea
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/ScanTable.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableDeletedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class ScanTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ Random rand = (Random) state.get("rand");
+
+ @SuppressWarnings("unchecked")
+ List<String> tableNames = (List<String>) state.get("tables");
+
+ String tableName = tableNames.get(rand.nextInt(tableNames.size()));
+
+ try {
+ Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ while (iter.hasNext()) {
+ iter.next();
+ }
+ log.debug("Scanned " + tableName);
+ } catch (TableDeletedException e) {
+ log.debug("Scan " + tableName + " failed, table deleted");
+ } catch (TableNotFoundException e) {
+ log.debug("Scan " + tableName + " failed, doesnt exist");
+ } catch (TableOfflineException e) {
+ log.debug("Scan " + tableName + " failed, offline");
+ } catch (RuntimeException e) {
+ if (e.getCause() instanceof AccumuloSecurityException) {
+ log.debug("BatchScan " + tableName + " failed, permission error");
+ } else {
+ throw e;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Setup.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Setup.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Setup.java
new file mode 100644
index 0000000..164fd4f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Setup.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Setup extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Random rand = new Random();
+ state.set("rand", rand);
+
+ int numTables = Integer.parseInt(props.getProperty("numTables", "9"));
+ int numNamespaces = Integer.parseInt(props.getProperty("numNamespaces", "2"));
+ log.debug("numTables = " + numTables);
+ log.debug("numNamespaces = " + numNamespaces);
+ List<String> tables = new ArrayList<>();
+ List<String> namespaces = new ArrayList<>();
+
+ for (int i = 0; i < numNamespaces; i++) {
+ namespaces.add(String.format("nspc_%03d", i));
+ }
+
+ // Make tables in the default namespace
+ double tableCeil = Math.ceil((double) numTables / (numNamespaces + 1));
+ for (int i = 0; i < tableCeil; i++) {
+ tables.add(String.format("ctt_%03d", i));
+ }
+
+ // Make tables in each namespace
+ double tableFloor = Math.floor(numTables / (numNamespaces + 1));
+ for (String n : namespaces) {
+ for (int i = 0; i < tableFloor; i++) {
+ tables.add(String.format(n + ".ctt_%03d", i));
+ }
+ }
+
+ state.set("tables", tables);
+ state.set("namespaces", namespaces);
+
+ int numUsers = Integer.parseInt(props.getProperty("numUsers", "5"));
+ log.debug("numUsers = " + numUsers);
+ List<String> users = new ArrayList<>();
+ for (int i = 0; i < numUsers; i++)
+ users.add(String.format("user%03d", i));
+ state.set("users", users);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
new file mode 100644
index 0000000..dc2e670
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/Shutdown.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.trace.Tracer;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.server.AccumuloServerContext;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class Shutdown extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ log.info("shutting down");
+ SetGoalState.main(new String[] {MasterGoalState.CLEAN_STOP.name()});
+
+ while (!env.getConnector().instanceOperations().getTabletServers().isEmpty()) {
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+
+ while (true) {
+ try {
+ AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()));
+ Client client = MasterClient.getConnection(context);
+ client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
+ } catch (Exception e) {
+ // assume this is due to server shutdown
+ break;
+ }
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+
+ log.info("servers stopped");
+ sleepUninterruptibly(10, TimeUnit.SECONDS);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
new file mode 100644
index 0000000..df30487
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StartAll.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
+import org.apache.accumulo.core.master.thrift.MasterGoalState;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.trace.Tracer;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.server.AccumuloServerContext;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class StartAll extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ log.info("Starting all servers");
+ SetGoalState.main(new String[] {MasterGoalState.NORMAL.name()});
+ Process exec = Runtime.getRuntime().exec(new String[] {System.getenv().get("ACCUMULO_HOME") + "/bin/start-all.sh"});
+ exec.waitFor();
+ while (true) {
+ try {
+ AccumuloServerContext context = new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance()));
+ Client client = MasterClient.getConnection(context);
+ MasterMonitorInfo masterStats = client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
+ if (!masterStats.tServerInfo.isEmpty())
+ break;
+ } catch (Exception ex) {
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
new file mode 100644
index 0000000..8210dc4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/concurrent/StopTabletServer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.concurrent;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+public class StopTabletServer extends Test {
+
+ Set<TServerInstance> getTServers(Instance instance) throws KeeperException, InterruptedException {
+ Set<TServerInstance> result = new HashSet<>();
+ ZooReader rdr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ String base = ZooUtil.getRoot(instance) + Constants.ZTSERVERS;
+ for (String child : rdr.getChildren(base)) {
+ try {
+ List<String> children = rdr.getChildren(base + "/" + child);
+ if (children.size() > 0) {
+ Collections.sort(children);
+ Stat stat = new Stat();
+ byte[] data = rdr.getData(base + "/" + child + "/" + children.get(0), stat);
+ if (!"master".equals(new String(data, UTF_8))) {
+ result.add(new TServerInstance(AddressUtil.parseAddress(child, false), stat.getEphemeralOwner()));
+ }
+ }
+ } catch (KeeperException.NoNodeException ex) {
+ // someone beat us too it
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ Instance instance = env.getInstance();
+
+ List<TServerInstance> currentServers = new ArrayList<>(getTServers(instance));
+ Collections.shuffle(currentServers);
+ Runtime runtime = Runtime.getRuntime();
+ if (currentServers.size() > 1) {
+ TServerInstance victim = currentServers.get(0);
+ log.info("Stopping " + victim.hostPort());
+ Process exec = runtime.exec(new String[] {System.getenv("ACCUMULO_HOME") + "/bin/accumulo", "admin", "stop", victim.hostPort()});
+ if (exec.waitFor() != 0)
+ throw new RuntimeException("admin stop returned a non-zero response: " + exec.exitValue());
+ Set<TServerInstance> set = getTServers(instance);
+ if (set.contains(victim))
+ throw new RuntimeException("Failed to stop " + victim);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Compact.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Compact.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Compact.java
new file mode 100644
index 0000000..b0aa7e1
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Compact.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class Compact extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String table = state.getString("tableName");
+ Random rand = (Random) state.get("rand");
+ Connector conn = env.getConnector();
+ Text row1 = new Text(Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))));
+ Text row2 = new Text(Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))));
+
+ if (row1.compareTo(row2) >= 0) {
+ row1 = null;
+ row2 = null;
+ }
+
+ log.debug("compacting " + row1 + " " + row2);
+ conn.tableOperations().compact(table, row1, row2, rand.nextBoolean(), rand.nextBoolean());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Flush.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Flush.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Flush.java
new file mode 100644
index 0000000..2c5448d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Flush.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class Flush extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String table = state.getString("tableName");
+ Random rand = (Random) state.get("rand");
+ Connector conn = env.getConnector();
+ Text row1 = new Text(Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))));
+ Text row2 = new Text(Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))));
+
+ if (row1.compareTo(row2) >= 0) {
+ row1 = null;
+ row2 = null;
+ }
+
+ log.debug("flushing " + row1 + " " + row2);
+ conn.tableOperations().flush(table, row1, row2, rand.nextBoolean());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
new file mode 100644
index 0000000..50a1e52
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Init.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriter.Status;
+import org.apache.accumulo.core.data.Condition;
+import org.apache.accumulo.core.data.ConditionalMutation;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class Init extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ int numBanks = (Integer) state.get("numBanks");
+ int numAccts = (Integer) state.get("numAccts");
+
+ // add some splits to spread ingest out a little
+ TreeSet<Text> splits = new TreeSet<>();
+ for (int i = 1; i < 10; i++)
+ splits.add(new Text(Utils.getBank((int) (numBanks * .1 * i))));
+ env.getConnector().tableOperations().addSplits((String) state.get("tableName"), splits);
+ log.debug("Added splits " + splits);
+
+ ArrayList<Integer> banks = new ArrayList<>();
+ for (int i = 0; i < numBanks; i++)
+ banks.add(i);
+ // shuffle for case when multiple threads are adding banks
+ Collections.shuffle(banks, (Random) state.get("rand"));
+
+ ConditionalWriter cw = (ConditionalWriter) state.get("cw");
+
+ for (int i : banks) {
+ ConditionalMutation m = new ConditionalMutation(Utils.getBank(i));
+ int acceptedCount = 0;
+ for (int j = 0; j < numAccts; j++) {
+ String cf = Utils.getAccount(j);
+ m.addCondition(new Condition(cf, "seq"));
+ m.put(cf, "bal", "100");
+ m.put(cf, "seq", Utils.getSeq(0));
+
+ if (j % 1000 == 0 && j > 0) {
+ Status status = cw.write(m).getStatus();
+
+ while (status == Status.UNKNOWN)
+ status = cw.write(m).getStatus();
+
+ if (status == Status.ACCEPTED)
+ acceptedCount++;
+ m = new ConditionalMutation(Utils.getBank(i));
+ }
+
+ }
+ if (m.getConditions().size() > 0) {
+ Status status = cw.write(m).getStatus();
+ while (status == Status.UNKNOWN)
+ status = cw.write(m).getStatus();
+
+ if (status == Status.ACCEPTED)
+ acceptedCount++;
+ }
+
+ log.debug("Added bank " + Utils.getBank(i) + " " + acceptedCount);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Merge.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Merge.java
new file mode 100644
index 0000000..2f5d52b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Merge.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class Merge extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String table = state.getString("tableName");
+ Random rand = (Random) state.get("rand");
+ Connector conn = env.getConnector();
+ Text row1 = new Text(Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))));
+ Text row2 = new Text(Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))));
+
+ if (row1.compareTo(row2) >= 0) {
+ row1 = null;
+ row2 = null;
+ }
+
+ log.debug("merging " + row1 + " " + row2);
+ conn.tableOperations().merge(table, row1, row2);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Setup.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Setup.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Setup.java
new file mode 100644
index 0000000..1e4ad01
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Setup.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriterConfig;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Setup extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Random rand = new Random();
+ state.set("rand", rand);
+
+ int numBanks = Integer.parseInt(props.getProperty("numBanks", "1000"));
+ log.debug("numBanks = " + numBanks);
+ state.set("numBanks", numBanks);
+
+ int numAccts = Integer.parseInt(props.getProperty("numAccts", "10000"));
+ log.debug("numAccts = " + numAccts);
+ state.set("numAccts", numAccts);
+
+ String tableName = "banks";
+ state.set("tableName", tableName);
+
+ try {
+ env.getConnector().tableOperations().create(tableName);
+ log.debug("created table " + tableName);
+ boolean blockCache = rand.nextBoolean();
+ env.getConnector().tableOperations().setProperty(tableName, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), blockCache + "");
+ log.debug("set " + Property.TABLE_BLOCKCACHE_ENABLED.getKey() + " " + blockCache);
+ } catch (TableExistsException tee) {}
+
+ ConditionalWriter cw = env.getConnector().createConditionalWriter(tableName, new ConditionalWriterConfig().setMaxWriteThreads(1));
+ state.set("cw", cw);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Split.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Split.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Split.java
new file mode 100644
index 0000000..8ea9aab
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Split.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class Split extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String table = state.getString("tableName");
+ Random rand = (Random) state.get("rand");
+ Connector conn = env.getConnector();
+ String row = Utils.getBank(rand.nextInt((Integer) state.get("numBanks")));
+
+ log.debug("adding split " + row);
+ conn.tableOperations().addSplits(table, new TreeSet<>(Arrays.asList(new Text(row))));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/TearDown.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/TearDown.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/TearDown.java
new file mode 100644
index 0000000..cf72607
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/TearDown.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+/**
+ *
+ */
+public class TearDown extends Test {
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ ConditionalWriter cw = (ConditionalWriter) state.get("cw");
+ cw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Transfer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Transfer.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Transfer.java
new file mode 100644
index 0000000..73a7d91
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Transfer.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriter.Status;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Condition;
+import org.apache.accumulo.core.data.ConditionalMutation;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.commons.math3.distribution.ZipfDistribution;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class Transfer extends Test {
+
+ private static class Account {
+ int seq;
+ int bal;
+
+ void setBal(String s) {
+ bal = Integer.parseInt(s);
+ }
+
+ void setSeq(String s) {
+ seq = Integer.parseInt(s);
+ }
+
+ @Override
+ public String toString() {
+ return seq + " " + bal;
+ }
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String table = state.getString("tableName");
+ Random rand = (Random) state.get("rand");
+ Connector conn = env.getConnector();
+
+ int numAccts = (Integer) state.get("numAccts");
+ // note: non integer exponents are slow
+
+ ZipfDistribution zdiBanks = new ZipfDistribution((Integer) state.get("numBanks"), 1);
+ String bank = Utils.getBank(zdiBanks.inverseCumulativeProbability(rand.nextDouble()));
+ ZipfDistribution zdiAccts = new ZipfDistribution(numAccts, 1);
+ String acct1 = Utils.getAccount(zdiAccts.inverseCumulativeProbability(rand.nextDouble()));
+ String acct2 = Utils.getAccount(zdiAccts.inverseCumulativeProbability(rand.nextDouble()));
+ while (acct2.equals(acct1)) {
+ // intentionally not using zipf distribution to pick on retry
+ acct2 = Utils.getAccount(rand.nextInt(numAccts));
+ }
+
+ // TODO document how data should be read when using ConditionalWriter
+ try (Scanner scanner = new IsolatedScanner(conn.createScanner(table, Authorizations.EMPTY))) {
+
+ scanner.setRange(new Range(bank));
+ scanner.fetchColumnFamily(new Text(acct1));
+ scanner.fetchColumnFamily(new Text(acct2));
+
+ Account a1 = new Account();
+ Account a2 = new Account();
+ Account a;
+
+ for (Entry<Key,Value> entry : scanner) {
+ String cf = entry.getKey().getColumnFamilyData().toString();
+ String cq = entry.getKey().getColumnQualifierData().toString();
+
+ if (cf.equals(acct1))
+ a = a1;
+ else if (cf.equals(acct2))
+ a = a2;
+ else
+ throw new Exception("Unexpected column fam: " + cf);
+
+ if (cq.equals("bal"))
+ a.setBal(entry.getValue().toString());
+ else if (cq.equals("seq"))
+ a.setSeq(entry.getValue().toString());
+ else
+ throw new Exception("Unexpected column qual: " + cq);
+ }
+
+ int amt = rand.nextInt(50);
+
+ log.debug("transfer req " + bank + " " + amt + " " + acct1 + " " + a1 + " " + acct2 + " " + a2);
+
+ if (a1.bal >= amt) {
+ ConditionalMutation cm = new ConditionalMutation(bank, new Condition(acct1, "seq").setValue(Utils.getSeq(a1.seq)),
+ new Condition(acct2, "seq").setValue(Utils.getSeq(a2.seq)));
+ cm.put(acct1, "bal", (a1.bal - amt) + "");
+ cm.put(acct2, "bal", (a2.bal + amt) + "");
+ cm.put(acct1, "seq", Utils.getSeq(a1.seq + 1));
+ cm.put(acct2, "seq", Utils.getSeq(a2.seq + 1));
+
+ ConditionalWriter cw = (ConditionalWriter) state.get("cw");
+ Status status = cw.write(cm).getStatus();
+ while (status == Status.UNKNOWN) {
+ log.debug("retrying transfer " + status);
+ status = cw.write(cm).getStatus();
+ }
+ log.debug("transfer result " + bank + " " + status + " " + a1 + " " + a2);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Utils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Utils.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Utils.java
new file mode 100644
index 0000000..5436c22
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Utils.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+/**
+ *
+ */
+public class Utils {
+
+ static String getBank(int b) {
+ return String.format("b%03d", b);
+ }
+
+ static String getAccount(int a) {
+ return "acct" + String.format("%06d", a);
+ }
+
+ static String getSeq(int s) {
+ return String.format("%06d", s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Verify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Verify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Verify.java
new file mode 100644
index 0000000..fa516f1
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/conditional/Verify.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.conditional;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.ColumnSliceFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+/**
+ *
+ */
+public class Verify extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String table = state.getString("tableName");
+ Connector conn = env.getConnector();
+
+ int numAccts = (Integer) state.get("numAccts");
+
+ for (int i = 0; i < (Integer) state.get("numBanks"); i++)
+ verifyBank(table, conn, Utils.getBank(i), numAccts);
+
+ }
+
+ private void verifyBank(String table, Connector conn, String row, int numAccts) throws TableNotFoundException, Exception {
+ log.debug("Verifying bank " + row);
+
+ int count = 0;
+ int sum = 0;
+ int min = Integer.MAX_VALUE;
+ int max = Integer.MIN_VALUE;
+
+ // TODO do not use IsolatedScanner, just enable isolation on scanner
+ try (Scanner scanner = new IsolatedScanner(conn.createScanner(table, Authorizations.EMPTY))) {
+
+ scanner.setRange(new Range(row));
+ IteratorSetting iterConf = new IteratorSetting(100, "cqsl", ColumnSliceFilter.class);
+ ColumnSliceFilter.setSlice(iterConf, "bal", true, "bal", true);
+ scanner.clearScanIterators();
+ scanner.addScanIterator(iterConf);
+
+ for (Entry<Key,Value> entry : scanner) {
+ int bal = Integer.parseInt(entry.getValue().toString());
+ sum += bal;
+ if (bal > max)
+ max = bal;
+ if (bal < min)
+ min = bal;
+ count++;
+ }
+
+ }
+
+ if (count > 0 && sum != numAccts * 100) {
+ throw new Exception("Sum is off " + sum);
+ }
+
+ log.debug("Verified " + row + " count = " + count + " sum = " + sum + " min = " + min + " max = " + max);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Commit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Commit.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Commit.java
new file mode 100644
index 0000000..09774ff
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Commit.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.image;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Commit extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ env.getMultiTableBatchWriter().flush();
+
+ log.debug("Committed " + state.getLong("numWrites") + " writes. Total writes: " + state.getLong("totalWrites"));
+ state.set("numWrites", Long.valueOf(0));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java
new file mode 100644
index 0000000..687b2d1
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ImageFixture.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.image;
+
+import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+public class ImageFixture extends Fixture {
+
+ String imageTableName;
+ String indexTableName;
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {
+
+ Connector conn = env.getConnector();
+ Instance instance = env.getInstance();
+
+ SortedSet<Text> splits = new TreeSet<>();
+ for (int i = 1; i < 256; i++) {
+ splits.add(new Text(String.format("%04x", i << 8)));
+ }
+
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+ String pid = env.getPid();
+
+ imageTableName = String.format("img_%s_%s_%d", hostname, pid, System.currentTimeMillis());
+ state.set("imageTableName", imageTableName);
+
+ indexTableName = String.format("img_ndx_%s_%s_%d", hostname, pid, System.currentTimeMillis());
+ state.set("indexTableName", indexTableName);
+
+ try {
+ conn.tableOperations().create(imageTableName);
+ conn.tableOperations().addSplits(imageTableName, splits);
+ log.debug("Created table " + imageTableName + " (id:" + Tables.getNameToIdMap(instance).get(imageTableName) + ")");
+ } catch (TableExistsException e) {
+ log.error("Table " + imageTableName + " already exists.");
+ throw e;
+ }
+
+ try {
+ conn.tableOperations().create(indexTableName);
+ log.debug("Created table " + indexTableName + " (id:" + Tables.getNameToIdMap(instance).get(indexTableName) + ")");
+ } catch (TableExistsException e) {
+ log.error("Table " + imageTableName + " already exists.");
+ throw e;
+ }
+
+ Random rand = new Random();
+ if (rand.nextInt(10) < 5) {
+ // setup locality groups
+ Map<String,Set<Text>> groups = getLocalityGroups();
+
+ conn.tableOperations().setLocalityGroups(imageTableName, groups);
+ log.debug("Configured locality groups for " + imageTableName + " groups = " + groups);
+ }
+
+ state.set("numWrites", Long.valueOf(0));
+ state.set("totalWrites", Long.valueOf(0));
+ state.set("verified", Integer.valueOf(0));
+ state.set("lastIndexRow", new Text(""));
+ }
+
+ static Map<String,Set<Text>> getLocalityGroups() {
+ Map<String,Set<Text>> groups = new HashMap<>();
+
+ HashSet<Text> lg1 = new HashSet<>();
+ lg1.add(Write.CONTENT_COLUMN_FAMILY);
+ groups.put("lg1", lg1);
+
+ HashSet<Text> lg2 = new HashSet<>();
+ lg2.add(Write.META_COLUMN_FAMILY);
+ groups.put("lg2", lg2);
+ return groups;
+ }
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {
+ // We have resources we need to clean up
+ if (env.isMultiTableBatchWriterInitialized()) {
+ MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Ignoring mutations that weren't flushed", e);
+ }
+
+ // Reset the MTBW on the state to null
+ env.resetMultiTableBatchWriter();
+ }
+
+ // Now we can safely delete the tables
+ log.debug("Dropping tables: " + imageTableName + " " + indexTableName);
+
+ Connector conn = env.getConnector();
+
+ conn.tableOperations().delete(imageTableName);
+ conn.tableOperations().delete(indexTableName);
+
+ log.debug("Final total of writes: " + state.getLong("totalWrites"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ScanMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ScanMeta.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ScanMeta.java
new file mode 100644
index 0000000..dbd89e8
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/ScanMeta.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.image;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class ScanMeta extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ // scan just the metadata of the images table to find N hashes... use the batch scanner to lookup those N hashes in the index table
+ // this scan will test locality groups....
+
+ String indexTableName = state.getString("indexTableName");
+ String imageTableName = state.getString("imageTableName");
+
+ String uuid = UUID.randomUUID().toString();
+
+ Connector conn = env.getConnector();
+
+ Scanner imageScanner = conn.createScanner(imageTableName, new Authorizations());
+
+ imageScanner.setRange(new Range(new Text(uuid), null));
+ imageScanner.fetchColumn(Write.META_COLUMN_FAMILY, Write.SHA1_COLUMN_QUALIFIER);
+
+ int minScan = Integer.parseInt(props.getProperty("minScan"));
+ int maxScan = Integer.parseInt(props.getProperty("maxScan"));
+
+ Random rand = new Random();
+ int numToScan = rand.nextInt(maxScan - minScan) + minScan;
+
+ Map<Text,Text> hashes = new HashMap<>();
+
+ Iterator<Entry<Key,Value>> iter = imageScanner.iterator();
+
+ while (iter.hasNext() && numToScan > 0) {
+
+ Entry<Key,Value> entry = iter.next();
+
+ hashes.put(new Text(entry.getValue().get()), entry.getKey().getRow());
+
+ numToScan--;
+ }
+
+ log.debug("Found " + hashes.size() + " hashes starting at " + uuid);
+
+ if (hashes.isEmpty()) {
+ return;
+ }
+
+ // use batch scanner to verify all of these exist in index
+ BatchScanner indexScanner = conn.createBatchScanner(indexTableName, Authorizations.EMPTY, 3);
+ ArrayList<Range> ranges = new ArrayList<>();
+ for (Text row : hashes.keySet()) {
+ ranges.add(new Range(row));
+ }
+
+ indexScanner.setRanges(ranges);
+
+ Map<Text,Text> hashes2 = new HashMap<>();
+
+ for (Entry<Key,Value> entry : indexScanner)
+ hashes2.put(entry.getKey().getRow(), new Text(entry.getValue().get()));
+
+ log.debug("Looked up " + ranges.size() + " ranges, found " + hashes2.size());
+
+ if (!hashes.equals(hashes2)) {
+ log.error("uuids from doc table : " + hashes.values());
+ log.error("uuids from index : " + hashes2.values());
+ throw new Exception("Mismatch between document table and index " + indexTableName + " " + imageTableName);
+ }
+
+ indexScanner.close();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/TableOp.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/TableOp.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/TableOp.java
new file mode 100644
index 0000000..1d14a90
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/TableOp.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.image;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class TableOp extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ // choose a table
+ Random rand = new Random();
+ String tableName;
+ if (rand.nextInt(10) < 8) {
+ tableName = state.getString("imageTableName");
+ } else {
+ tableName = state.getString("indexTableName");
+ }
+
+ // check if chosen table exists
+ Connector conn = env.getConnector();
+ TableOperations tableOps = conn.tableOperations();
+ if (tableOps.exists(tableName) == false) {
+ log.error("Table " + tableName + " does not exist!");
+ return;
+ }
+
+ // choose a random action
+ int num = rand.nextInt(10);
+ if (num > 6) {
+ log.debug("Retrieving info for " + tableName);
+ tableOps.getLocalityGroups(tableName);
+ tableOps.getProperties(tableName);
+ tableOps.listSplits(tableName);
+ tableOps.list();
+ } else {
+ log.debug("Clearing locator cache for " + tableName);
+ tableOps.clearLocatorCache(tableName);
+ }
+
+ if (rand.nextInt(10) < 3) {
+ Map<String,Set<Text>> groups = tableOps.getLocalityGroups(state.getString("imageTableName"));
+
+ if (groups.size() == 0) {
+ log.debug("Adding locality groups to " + state.getString("imageTableName"));
+ groups = ImageFixture.getLocalityGroups();
+ } else {
+ log.debug("Removing locality groups from " + state.getString("imageTableName"));
+ groups = new HashMap<>();
+ }
+
+ tableOps.setLocalityGroups(state.getString("imageTableName"), groups);
+ }
+ }
+}
[4/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Posted by mw...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Verify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Verify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Verify.java
new file mode 100644
index 0000000..f3caf15
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Verify.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.image;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.security.MessageDigest;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Verify extends Test {
+
+ String indexTableName;
+ String imageTableName;
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ Random rand = new Random();
+
+ int maxVerify = Integer.parseInt(props.getProperty("maxVerify"));
+ int numVerifications = rand.nextInt(maxVerify - 1) + 1;
+
+ indexTableName = state.getString("indexTableName");
+ imageTableName = state.getString("imageTableName");
+
+ Connector conn = env.getConnector();
+
+ Scanner indexScanner = conn.createScanner(indexTableName, new Authorizations());
+ Scanner imageScanner = conn.createScanner(imageTableName, new Authorizations());
+
+ String uuid = UUID.randomUUID().toString();
+
+ MessageDigest alg = MessageDigest.getInstance("SHA-1");
+ alg.update(uuid.getBytes(UTF_8));
+
+ indexScanner.setRange(new Range(new Text(alg.digest()), null));
+ indexScanner.setBatchSize(numVerifications);
+
+ Text curRow = null;
+ int count = 0;
+ for (Entry<Key,Value> entry : indexScanner) {
+
+ curRow = entry.getKey().getRow();
+ String rowToVerify = entry.getValue().toString();
+
+ verifyRow(imageScanner, rowToVerify);
+
+ count++;
+ if (count == numVerifications) {
+ break;
+ }
+ }
+
+ if (count != numVerifications && curRow != null) {
+ Text lastRow = (Text) state.get("lastIndexRow");
+ if (lastRow.compareTo(curRow) != 0) {
+ log.error("Verified only " + count + " of " + numVerifications + " - curRow " + curRow + " lastKey " + lastRow);
+ }
+ }
+
+ int verified = ((Integer) state.get("verified")).intValue() + numVerifications;
+ log.debug("Verified " + numVerifications + " - Total " + verified);
+ state.set("verified", Integer.valueOf(verified));
+ }
+
+ public void verifyRow(Scanner scanner, String row) throws Exception {
+
+ scanner.setRange(new Range(new Text(row)));
+ scanner.clearColumns();
+ scanner.fetchColumnFamily(Write.CONTENT_COLUMN_FAMILY);
+ scanner.fetchColumn(Write.META_COLUMN_FAMILY, Write.SHA1_COLUMN_QUALIFIER);
+
+ Iterator<Entry<Key,Value>> scanIter = scanner.iterator();
+
+ if (scanIter.hasNext() == false) {
+ log.error("Found row(" + row + ") in " + indexTableName + " but not " + imageTableName);
+ return;
+ }
+
+ // get image
+ Entry<Key,Value> entry = scanIter.next();
+ byte[] imageBytes = entry.getValue().get();
+
+ MessageDigest alg = MessageDigest.getInstance("SHA-1");
+ alg.update(imageBytes);
+ byte[] localHash = alg.digest();
+
+ // get stored hash
+ entry = scanIter.next();
+ byte[] storedHash = entry.getValue().get();
+
+ if (localHash.length != storedHash.length) {
+ throw new Exception("Hash lens do not match for " + row);
+ }
+
+ for (int i = 0; i < localHash.length; i++) {
+ if (localHash[i] != storedHash[i]) {
+ throw new Exception("Hashes do not match for " + row);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Write.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Write.java
new file mode 100644
index 0000000..f7a2781
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/image/Write.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.image;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.security.MessageDigest;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Write extends Test {
+
+ static final Text UUID_COLUMN_QUALIFIER = new Text("uuid");
+ static final Text COUNT_COLUMN_QUALIFIER = new Text("count");
+ static final Text SHA1_COLUMN_QUALIFIER = new Text("sha1");
+ static final Text IMAGE_COLUMN_QUALIFIER = new Text("image");
+ static final Text META_COLUMN_FAMILY = new Text("meta");
+ static final Text CONTENT_COLUMN_FAMILY = new Text("content");
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+
+ BatchWriter imagesBW = mtbw.getBatchWriter(state.getString("imageTableName"));
+ BatchWriter indexBW = mtbw.getBatchWriter(state.getString("indexTableName"));
+
+ String uuid = UUID.randomUUID().toString();
+ Mutation m = new Mutation(new Text(uuid));
+
+ // create a fake image between 4KB and 1MB
+ int maxSize = Integer.parseInt(props.getProperty("maxSize"));
+ int minSize = Integer.parseInt(props.getProperty("minSize"));
+
+ Random rand = new Random();
+ int numBytes = rand.nextInt(maxSize - minSize) + minSize;
+ byte[] imageBytes = new byte[numBytes];
+ rand.nextBytes(imageBytes);
+ m.put(CONTENT_COLUMN_FAMILY, IMAGE_COLUMN_QUALIFIER, new Value(imageBytes));
+
+ // store size
+ m.put(META_COLUMN_FAMILY, new Text("size"), new Value(String.format("%d", numBytes).getBytes(UTF_8)));
+
+ // store hash
+ MessageDigest alg = MessageDigest.getInstance("SHA-1");
+ alg.update(imageBytes);
+ byte[] hash = alg.digest();
+ m.put(META_COLUMN_FAMILY, SHA1_COLUMN_QUALIFIER, new Value(hash));
+
+ // update write counts
+ state.set("numWrites", state.getLong("numWrites") + 1);
+ Long totalWrites = state.getLong("totalWrites") + 1;
+ state.set("totalWrites", totalWrites);
+
+ // set count
+ m.put(META_COLUMN_FAMILY, COUNT_COLUMN_QUALIFIER, new Value(String.format("%d", totalWrites).getBytes(UTF_8)));
+
+ // add mutation
+ imagesBW.addMutation(m);
+
+ // now add mutation to index
+ Text row = new Text(hash);
+ m = new Mutation(row);
+ m.put(META_COLUMN_FAMILY, UUID_COLUMN_QUALIFIER, new Value(uuid.getBytes(UTF_8)));
+
+ indexBW.addMutation(m);
+
+ Text lastRow = (Text) state.get("lastIndexRow");
+ if (lastRow.compareTo(row) < 0) {
+ state.set("lastIndexRow", new Text(row));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Commit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Commit.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Commit.java
new file mode 100644
index 0000000..4ac6b47
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Commit.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Commit extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ env.getMultiTableBatchWriter().flush();
+
+ Long numWrites = state.getLong("numWrites");
+ Long totalWrites = state.getLong("totalWrites") + numWrites;
+
+ log.debug("Committed " + numWrites + " writes. Total writes: " + totalWrites);
+
+ state.set("totalWrites", totalWrites);
+ state.set("numWrites", Long.valueOf(0));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
new file mode 100644
index 0000000..6552161
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ToolRunner;
+
+public class CopyTable extends Test {
+
+ private final TreeSet<Text> splits;
+
+ public CopyTable() {
+ splits = new TreeSet<>();
+ for (int i = 1; i < 10; i++) {
+ splits.add(new Text(Integer.toString(i)));
+ }
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ @SuppressWarnings("unchecked")
+ List<String> tables = (List<String>) state.get("tableList");
+ if (tables.isEmpty())
+ return;
+
+ Random rand = new Random();
+ String srcTableName = tables.remove(rand.nextInt(tables.size()));
+
+ int nextId = ((Integer) state.get("nextId")).intValue();
+ String dstTableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId);
+
+ String[] args = new String[8];
+ args[0] = "-libjars";
+ args[1] = getMapReduceJars();
+ args[2] = env.getUserName();
+ args[3] = env.getPassword();
+ if (null == args[3]) {
+ args[3] = env.getKeytab();
+ }
+ args[4] = srcTableName;
+ args[5] = env.getInstance().getInstanceName();
+ args[6] = env.getConfigProperty("ZOOKEEPERS");
+ args[7] = dstTableName;
+
+ log.debug("copying " + srcTableName + " to " + dstTableName);
+
+ env.getConnector().tableOperations().create(dstTableName);
+
+ env.getConnector().tableOperations().addSplits(dstTableName, splits);
+
+ if (ToolRunner.run(CachedConfiguration.getInstance(), new CopyTool(), args) != 0) {
+ log.error("Failed to run map/red verify");
+ return;
+ }
+
+ String tableId = Tables.getNameToIdMap(env.getInstance()).get(dstTableName);
+ log.debug("copied " + srcTableName + " to " + dstTableName + " (id - " + tableId + " )");
+
+ tables.add(dstTableName);
+
+ env.getConnector().tableOperations().delete(srcTableName);
+ log.debug("dropped " + srcTableName);
+
+ nextId++;
+ state.set("nextId", Integer.valueOf(nextId));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java
new file mode 100644
index 0000000..02da2e0
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTool.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Tool;
+import org.apache.log4j.Logger;
+
+public class CopyTool extends Configured implements Tool {
+ protected final Logger log = Logger.getLogger(this.getClass());
+
+ public static class SeqMapClass extends Mapper<Key,Value,Text,Mutation> {
+ @Override
+ public void map(Key key, Value val, Context output) throws IOException, InterruptedException {
+ Mutation m = new Mutation(key.getRow());
+ m.put(key.getColumnFamily(), key.getColumnQualifier(), val);
+ output.write(null, m);
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
+ job.setJarByClass(this.getClass());
+
+ if (job.getJar() == null) {
+ log.error("M/R requires a jar file! Run mvn package.");
+ return 1;
+ }
+
+ ClientConfiguration clientConf = new ClientConfiguration().withInstance(args[3]).withZkHosts(args[4]);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ AccumuloInputFormat.setInputTableName(job, args[2]);
+ AccumuloInputFormat.setScanAuthorizations(job, Authorizations.EMPTY);
+ AccumuloInputFormat.setZooKeeperInstance(job, clientConf);
+
+ final String principal;
+ final AuthenticationToken token;
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ // Use the Kerberos creds to request a DelegationToken for MapReduce to use
+ // We could use the specified keytab (args[1]), but we're already logged in and don't need to, so we can just use the current user
+ KerberosToken kt = new KerberosToken();
+ try {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ if (!user.hasKerberosCredentials()) {
+ throw new IllegalStateException("Expected current user to have Kerberos credentials");
+ }
+
+ // Get the principal via UGI
+ principal = user.getUserName();
+
+ // Connector w/ the Kerberos creds
+ ZooKeeperInstance inst = new ZooKeeperInstance(clientConf);
+ Connector conn = inst.getConnector(principal, kt);
+
+ // Do the explicit check to see if the user has the permission to get a delegation token
+ if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
+ log.error(principal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name()
+ + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's"
+ + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials.");
+ throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token");
+ }
+
+ // Fetch a delegation token from Accumulo
+ token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+
+ } catch (Exception e) {
+ final String msg = "Failed to acquire DelegationToken for use with MapReduce";
+ log.error(msg, e);
+ throw new RuntimeException(msg, e);
+ }
+ } else {
+ // Simple principal + password
+ principal = args[0];
+ token = new PasswordToken(args[1]);
+ }
+
+ AccumuloInputFormat.setConnectorInfo(job, principal, token);
+ AccumuloOutputFormat.setConnectorInfo(job, principal, token);
+
+ job.setMapperClass(SeqMapClass.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(Mutation.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.setCreateTables(job, true);
+ AccumuloOutputFormat.setDefaultTableName(job, args[5]);
+ AccumuloOutputFormat.setZooKeeperInstance(job, clientConf);
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CreateTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CreateTable.java
new file mode 100644
index 0000000..4250003
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CreateTable.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class CreateTable extends Test {
+
+ private final TreeSet<Text> splits;
+
+ public CreateTable() {
+ splits = new TreeSet<>();
+ for (int i = 1; i < 10; i++) {
+ splits.add(new Text(Integer.toString(i)));
+ }
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+
+ int nextId = ((Integer) state.get("nextId")).intValue();
+ String tableName = String.format("%s_%d", state.getString("tableNamePrefix"), nextId);
+ try {
+ conn.tableOperations().create(tableName);
+ // Add some splits to make the server's life easier
+ conn.tableOperations().addSplits(tableName, splits);
+ String tableId = Tables.getNameToIdMap(env.getInstance()).get(tableName);
+ log.debug("created " + tableName + " (id:" + tableId + ")");
+ // Add some splits to make the server's life easier
+ conn.tableOperations().addSplits(tableName, splits);
+ log.debug("created " + splits.size() + " splits on " + tableName);
+ @SuppressWarnings("unchecked")
+ List<String> tables = (List<String>) state.get("tableList");
+ tables.add(tableName);
+ } catch (TableExistsException e) {
+ log.warn("Failed to create " + tableName + " as it already exists");
+ }
+
+ nextId++;
+ state.set("nextId", Integer.valueOf(nextId));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/DropTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/DropTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/DropTable.java
new file mode 100644
index 0000000..61904ca
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/DropTable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DropTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ @SuppressWarnings("unchecked")
+ List<String> tables = (List<String>) state.get("tableList");
+
+ // don't drop a table if we only have one table or less
+ if (tables.size() <= 1) {
+ return;
+ }
+
+ Random rand = new Random();
+ String tableName = tables.remove(rand.nextInt(tables.size()));
+
+ try {
+ env.getConnector().tableOperations().delete(tableName);
+ log.debug("Dropped " + tableName);
+ } catch (TableNotFoundException e) {
+ log.error("Tried to drop table " + tableName + " but could not be found!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/MultiTableFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/MultiTableFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/MultiTableFixture.java
new file mode 100644
index 0000000..5a7a415
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/MultiTableFixture.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.net.InetAddress;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+
+public class MultiTableFixture extends Fixture {
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {
+
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+
+ state.set("tableNamePrefix", String.format("multi_%s_%s_%d", hostname, env.getPid(), System.currentTimeMillis()));
+ state.set("nextId", Integer.valueOf(0));
+ state.set("numWrites", Long.valueOf(0));
+ state.set("totalWrites", Long.valueOf(0));
+ state.set("tableList", new CopyOnWriteArrayList<String>());
+ }
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {
+ // We have resources we need to clean up
+ if (env.isMultiTableBatchWriterInitialized()) {
+ MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Ignoring mutations that weren't flushed", e);
+ }
+
+ // Reset the MTBW on the state to null
+ env.resetMultiTableBatchWriter();
+ }
+
+ Connector conn = env.getConnector();
+
+ @SuppressWarnings("unchecked")
+ List<String> tables = (List<String>) state.get("tableList");
+
+ for (String tableName : tables) {
+ try {
+ conn.tableOperations().delete(tableName);
+ log.debug("Dropping table " + tableName);
+ } catch (TableNotFoundException e) {
+ log.warn("Tried to drop table " + tableName + " but could not be found!");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/OfflineTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/OfflineTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/OfflineTable.java
new file mode 100644
index 0000000..70a6b21
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/OfflineTable.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class OfflineTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ @SuppressWarnings("unchecked")
+ List<String> tables = (List<String>) state.get("tableList");
+
+ if (tables.size() <= 0) {
+ return;
+ }
+
+ Random rand = new Random();
+ String tableName = tables.get(rand.nextInt(tables.size()));
+
+ env.getConnector().tableOperations().offline(tableName, rand.nextBoolean());
+ log.debug("Table " + tableName + " offline ");
+ env.getConnector().tableOperations().online(tableName, rand.nextBoolean());
+ log.debug("Table " + tableName + " online ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Write.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Write.java
new file mode 100644
index 0000000..3c0c792
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/Write.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.multitable;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.security.MessageDigest;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Write extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ @SuppressWarnings("unchecked")
+ List<String> tables = (List<String>) state.get("tableList");
+
+ if (tables.isEmpty()) {
+ log.debug("No tables to ingest into");
+ return;
+ }
+
+ Random rand = new Random();
+ String tableName = tables.get(rand.nextInt(tables.size()));
+
+ BatchWriter bw = null;
+ try {
+ bw = env.getMultiTableBatchWriter().getBatchWriter(tableName);
+ } catch (TableOfflineException e) {
+ log.error("Table " + tableName + " is offline!");
+ return;
+ } catch (TableNotFoundException e) {
+ log.error("Table " + tableName + " not found!");
+ return;
+ }
+
+ Text meta = new Text("meta");
+ String uuid = UUID.randomUUID().toString();
+
+ Mutation m = new Mutation(new Text(uuid));
+
+ // create a fake payload between 4KB and 16KB
+ int numBytes = rand.nextInt(12000) + 4000;
+ byte[] payloadBytes = new byte[numBytes];
+ rand.nextBytes(payloadBytes);
+ m.put(meta, new Text("payload"), new Value(payloadBytes));
+
+ // store size
+ m.put(meta, new Text("size"), new Value(String.format("%d", numBytes).getBytes(UTF_8)));
+
+ // store hash
+ MessageDigest alg = MessageDigest.getInstance("SHA-1");
+ alg.update(payloadBytes);
+ m.put(meta, new Text("sha1"), new Value(alg.digest()));
+
+ // add mutation
+ bw.addMutation(m);
+
+ state.set("numWrites", state.getLong("numWrites") + 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterSystemPerm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterSystemPerm.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterSystemPerm.java
new file mode 100644
index 0000000..c075541
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterSystemPerm.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class AlterSystemPerm extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getConnector();
+ WalkingSecurity ws = new WalkingSecurity(state, env);
+
+ String action = props.getProperty("task", "toggle");
+ String perm = props.getProperty("perm", "random");
+
+ String targetUser = WalkingSecurity.get(state, env).getSysUserName();
+
+ SystemPermission sysPerm;
+ if (perm.equals("random")) {
+ Random r = new Random();
+ int i = r.nextInt(SystemPermission.values().length);
+ sysPerm = SystemPermission.values()[i];
+ } else
+ sysPerm = SystemPermission.valueOf(perm);
+
+ boolean hasPerm = ws.hasSystemPermission(targetUser, sysPerm);
+
+ // toggle
+ if (!"take".equals(action) && !"give".equals(action)) {
+ if (hasPerm != conn.securityOperations().hasSystemPermission(targetUser, sysPerm))
+ throw new AccumuloException("Test framework and accumulo are out of sync!");
+ if (hasPerm)
+ action = "take";
+ else
+ action = "give";
+ }
+
+ if ("take".equals(action)) {
+ try {
+ conn.securityOperations().revokeSystemPermission(targetUser, sysPerm);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case GRANT_INVALID:
+ if (sysPerm.equals(SystemPermission.GRANT))
+ return;
+ throw new AccumuloException("Got GRANT_INVALID when not dealing with GRANT", ae);
+ case PERMISSION_DENIED:
+ throw new AccumuloException("Test user doesn't have root", ae);
+ case USER_DOESNT_EXIST:
+ throw new AccumuloException("System user doesn't exist and they SHOULD.", ae);
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ ws.revokeSystemPermission(targetUser, sysPerm);
+ } else if ("give".equals(action)) {
+ try {
+ conn.securityOperations().grantSystemPermission(targetUser, sysPerm);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case GRANT_INVALID:
+ if (sysPerm.equals(SystemPermission.GRANT))
+ return;
+ throw new AccumuloException("Got GRANT_INVALID when not dealing with GRANT", ae);
+ case PERMISSION_DENIED:
+ throw new AccumuloException("Test user doesn't have root", ae);
+ case USER_DOESNT_EXIST:
+ throw new AccumuloException("System user doesn't exist and they SHOULD.", ae);
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ ws.grantSystemPermission(targetUser, sysPerm);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java
new file mode 100644
index 0000000..0e613d8
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTable.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.net.InetAddress;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class AlterTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getInstance().getConnector(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken());
+
+ String tableName = WalkingSecurity.get(state, env).getTableName();
+ String namespaceName = WalkingSecurity.get(state, env).getNamespaceName();
+
+ boolean exists = WalkingSecurity.get(state, env).getTableExists();
+ boolean hasPermission = WalkingSecurity.get(state, env).canAlterTable(WalkingSecurity.get(state, env).getSysCredentials(), tableName, namespaceName);
+ String newTableName = String.format("security_%s_%s_%d", InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_"), env.getPid(),
+ System.currentTimeMillis());
+
+ renameTable(conn, state, env, tableName, newTableName, hasPermission, exists);
+ }
+
+ public static void renameTable(Connector conn, State state, Environment env, String oldName, String newName, boolean hasPermission, boolean tableExists)
+ throws AccumuloSecurityException, AccumuloException, TableExistsException {
+ try {
+ conn.tableOperations().rename(oldName, newName);
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
+ if (hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else
+ return;
+ } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
+ if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ return;
+ }
+ throw new AccumuloException("Got unexpected ae error code", ae);
+ } catch (TableNotFoundException tnfe) {
+ if (tableExists)
+ throw new TableExistsException(null, oldName, "Got a TableNotFoundException but it should exist", tnfe);
+ else
+ return;
+ }
+ WalkingSecurity.get(state, env).setTableName(newName);
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java
new file mode 100644
index 0000000..381a801
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/AlterTablePerm.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class AlterTablePerm extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ alter(state, env, props);
+ }
+
+ public static void alter(State state, Environment env, Properties props) throws Exception {
+ String action = props.getProperty("task", "toggle");
+ String perm = props.getProperty("perm", "random");
+ String sourceUserProp = props.getProperty("source", "system");
+ String targetUser = props.getProperty("target", "table");
+ boolean tabExists = WalkingSecurity.get(state, env).getTableExists();
+
+ String target;
+ if ("table".equals(targetUser))
+ target = WalkingSecurity.get(state, env).getTabUserName();
+ else
+ target = WalkingSecurity.get(state, env).getSysUserName();
+
+ boolean exists = WalkingSecurity.get(state, env).userExists(target);
+ boolean tableExists = WalkingSecurity.get(state, env).getTableExists();
+
+ TablePermission tabPerm;
+ if (perm.equals("random")) {
+ Random r = new Random();
+ int i = r.nextInt(TablePermission.values().length);
+ tabPerm = TablePermission.values()[i];
+ } else
+ tabPerm = TablePermission.valueOf(perm);
+ String tableName = WalkingSecurity.get(state, env).getTableName();
+ boolean hasPerm = WalkingSecurity.get(state, env).hasTablePermission(target, tableName, tabPerm);
+ boolean canGive;
+ String sourceUser;
+ AuthenticationToken sourceToken;
+ if ("system".equals(sourceUserProp)) {
+ sourceUser = WalkingSecurity.get(state, env).getSysUserName();
+ sourceToken = WalkingSecurity.get(state, env).getSysToken();
+ } else if ("table".equals(sourceUserProp)) {
+ sourceUser = WalkingSecurity.get(state, env).getTabUserName();
+ sourceToken = WalkingSecurity.get(state, env).getTabToken();
+ } else {
+ sourceUser = env.getUserName();
+ sourceToken = env.getToken();
+ }
+ Connector conn = env.getInstance().getConnector(sourceUser, sourceToken);
+
+ canGive = WalkingSecurity.get(state, env).canGrantTable(new Credentials(sourceUser, sourceToken).toThrift(env.getInstance()), target,
+ WalkingSecurity.get(state, env).getTableName(), WalkingSecurity.get(state, env).getNamespaceName());
+
+ // toggle
+ if (!"take".equals(action) && !"give".equals(action)) {
+ try {
+ boolean res;
+ if (hasPerm != (res = env.getConnector().securityOperations().hasTablePermission(target, tableName, tabPerm)))
+ throw new AccumuloException("Test framework and accumulo are out of sync for user " + conn.whoami() + " for perm " + tabPerm.name()
+ + " with local vs. accumulo being " + hasPerm + " " + res);
+
+ if (hasPerm)
+ action = "take";
+ else
+ action = "give";
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case USER_DOESNT_EXIST:
+ if (exists)
+ throw new AccumuloException("Framework and Accumulo are out of sync, we think user exists", ae);
+ else
+ return;
+ case TABLE_DOESNT_EXIST:
+ if (tabExists)
+ throw new AccumuloException(conn.whoami(), ae);
+ else
+ return;
+ default:
+ throw ae;
+ }
+ }
+ }
+
+ boolean trans = WalkingSecurity.get(state, env).userPassTransient(conn.whoami());
+ if ("take".equals(action)) {
+ try {
+ conn.securityOperations().revokeTablePermission(target, tableName, tabPerm);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case GRANT_INVALID:
+ throw new AccumuloException("Got a grant invalid on non-System.GRANT option", ae);
+ case PERMISSION_DENIED:
+ if (canGive)
+ throw new AccumuloException(conn.whoami() + " failed to revoke permission to " + target + " when it should have worked", ae);
+ return;
+ case USER_DOESNT_EXIST:
+ if (exists)
+ throw new AccumuloException("Table user doesn't exist and they SHOULD.", ae);
+ return;
+ case TABLE_DOESNT_EXIST:
+ if (tableExists)
+ throw new AccumuloException("Table doesn't exist but it should", ae);
+ return;
+ case BAD_CREDENTIALS:
+ if (!trans)
+ throw new AccumuloException("Bad credentials for user " + conn.whoami());
+ return;
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ WalkingSecurity.get(state, env).revokeTablePermission(target, tableName, tabPerm);
+ } else if ("give".equals(action)) {
+ try {
+ conn.securityOperations().grantTablePermission(target, tableName, tabPerm);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case GRANT_INVALID:
+ throw new AccumuloException("Got a grant invalid on non-System.GRANT option", ae);
+ case PERMISSION_DENIED:
+ if (canGive)
+ throw new AccumuloException(conn.whoami() + " failed to give permission to " + target + " when it should have worked", ae);
+ return;
+ case USER_DOESNT_EXIST:
+ if (exists)
+ throw new AccumuloException("Table user doesn't exist and they SHOULD.", ae);
+ return;
+ case TABLE_DOESNT_EXIST:
+ if (tableExists)
+ throw new AccumuloException("Table doesn't exist but it should", ae);
+ return;
+ case BAD_CREDENTIALS:
+ if (!trans)
+ throw new AccumuloException("Bad credentials for user " + conn.whoami());
+ return;
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ WalkingSecurity.get(state, env).grantTablePermission(target, tableName, tabPerm);
+ }
+
+ if (!exists)
+ throw new AccumuloException("User shouldn't have existed, but apparantly does");
+ if (!tableExists)
+ throw new AccumuloException("Table shouldn't have existed, but apparantly does");
+ if (!canGive)
+ throw new AccumuloException(conn.whoami() + " shouldn't have been able to grant privilege");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java
new file mode 100644
index 0000000..a9548c4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Authenticate.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Arrays;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Authenticate extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ authenticate(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken(), state, env, props);
+ }
+
+ public static void authenticate(String principal, AuthenticationToken token, State state, Environment env, Properties props) throws Exception {
+ String targetProp = props.getProperty("target");
+ boolean success = Boolean.parseBoolean(props.getProperty("valid"));
+
+ Connector conn = env.getInstance().getConnector(principal, token);
+
+ String target;
+
+ if (targetProp.equals("table")) {
+ target = WalkingSecurity.get(state, env).getTabUserName();
+ } else {
+ target = WalkingSecurity.get(state, env).getSysUserName();
+ }
+ boolean exists = WalkingSecurity.get(state, env).userExists(target);
+ // Copy so if failed it doesn't mess with the password stored in state
+ byte[] password = Arrays.copyOf(WalkingSecurity.get(state, env).getUserPassword(target), WalkingSecurity.get(state, env).getUserPassword(target).length);
+ boolean hasPermission = WalkingSecurity.get(state, env).canAskAboutUser(new Credentials(principal, token).toThrift(env.getInstance()), target);
+
+ if (!success)
+ for (int i = 0; i < password.length; i++)
+ password[i]++;
+
+ boolean result;
+
+ try {
+ result = conn.securityOperations().authenticateUser(target, new PasswordToken(password));
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ if (exists && hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else
+ return;
+ default:
+ throw new AccumuloException("Unexpected exception!", ae);
+ }
+ }
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ if (result != (success && exists))
+ throw new AccumuloException("Authentication " + (result ? "succeeded" : "failed") + " when it should have "
+ + ((success && exists) ? "succeeded" : "failed") + " while the user exists? " + exists);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java
new file mode 100644
index 0000000..e58db32
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/ChangePass.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class ChangePass extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String target = props.getProperty("target");
+ String source = props.getProperty("source");
+
+ String principal;
+ AuthenticationToken token;
+ if (source.equals("system")) {
+ principal = WalkingSecurity.get(state, env).getSysUserName();
+ token = WalkingSecurity.get(state, env).getSysToken();
+ } else {
+ principal = WalkingSecurity.get(state, env).getTabUserName();
+ token = WalkingSecurity.get(state, env).getTabToken();
+ }
+ Connector conn = env.getInstance().getConnector(principal, token);
+
+ boolean hasPerm;
+ boolean targetExists;
+ if (target.equals("table")) {
+ target = WalkingSecurity.get(state, env).getTabUserName();
+ } else
+ target = WalkingSecurity.get(state, env).getSysUserName();
+
+ targetExists = WalkingSecurity.get(state, env).userExists(target);
+
+ hasPerm = WalkingSecurity.get(state, env).canChangePassword(new Credentials(principal, token).toThrift(env.getInstance()), target);
+
+ Random r = new Random();
+
+ byte[] newPassw = new byte[r.nextInt(50) + 1];
+ for (int i = 0; i < newPassw.length; i++)
+ newPassw[i] = (byte) ((r.nextInt(26) + 65) & 0xFF);
+
+ PasswordToken newPass = new PasswordToken(newPassw);
+ try {
+ conn.securityOperations().changeLocalUserPassword(target, newPass);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ if (hasPerm)
+ throw new AccumuloException("Change failed when it should have succeeded to change " + target + "'s password", ae);
+ return;
+ case USER_DOESNT_EXIST:
+ if (targetExists)
+ throw new AccumuloException("User " + target + " doesn't exist and they SHOULD.", ae);
+ return;
+ case BAD_CREDENTIALS:
+ if (!WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ throw new AccumuloException("Bad credentials for user " + conn.whoami());
+ return;
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ WalkingSecurity.get(state, env).changePassword(target, newPass);
+ // Waiting 1 second for password to propogate through Zk
+ Thread.sleep(1000);
+ if (!hasPerm)
+ throw new AccumuloException("Password change succeeded when it should have failed for " + source + " changing the password for " + target + ".");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java
new file mode 100644
index 0000000..c0b1cd9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateTable.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getInstance().getConnector(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken());
+
+ String tableName = WalkingSecurity.get(state, env).getTableName();
+
+ boolean exists = WalkingSecurity.get(state, env).getTableExists();
+ boolean hasPermission = WalkingSecurity.get(state, env).canCreateTable(WalkingSecurity.get(state, env).getSysCredentials(), null, null);
+
+ try {
+ conn.tableOperations().create(tableName);
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
+ if (hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else {
+ // create table anyway for sake of state
+ try {
+ env.getConnector().tableOperations().create(tableName);
+ WalkingSecurity.get(state, env).initTable(tableName);
+ } catch (TableExistsException tee) {
+ if (exists)
+ return;
+ else
+ throw new AccumuloException("Test and Accumulo are out of sync");
+ }
+ return;
+ }
+ } else
+ throw new AccumuloException("Got unexpected error", ae);
+ } catch (TableExistsException tee) {
+ if (!exists)
+ throw new TableExistsException(null, tableName, "Got a TableExistsException but it shouldn't have existed", tee);
+ else
+ return;
+ }
+ WalkingSecurity.get(state, env).initTable(tableName);
+ for (TablePermission tp : TablePermission.values())
+ WalkingSecurity.get(state, env).grantTablePermission(conn.whoami(), tableName, tp);
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java
new file mode 100644
index 0000000..f604928
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/CreateUser.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateUser extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getInstance().getConnector(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken());
+
+ String tableUserName = WalkingSecurity.get(state, env).getTabUserName();
+
+ boolean exists = WalkingSecurity.get(state, env).userExists(tableUserName);
+ boolean hasPermission = WalkingSecurity.get(state, env).canCreateUser(WalkingSecurity.get(state, env).getSysCredentials(), tableUserName);
+ PasswordToken tabUserPass = new PasswordToken("Super Sekret Table User Password");
+ try {
+ conn.securityOperations().createLocalUser(tableUserName, tabUserPass);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ if (hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else {
+ // create user anyway for sake of state
+ if (!exists) {
+ env.getConnector().securityOperations().createLocalUser(tableUserName, tabUserPass);
+ WalkingSecurity.get(state, env).createUser(tableUserName, tabUserPass);
+ Thread.sleep(1000);
+ }
+ return;
+ }
+ case USER_EXISTS:
+ if (!exists)
+ throw new AccumuloException("Got security exception when the user shouldn't have existed", ae);
+ else
+ return;
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ WalkingSecurity.get(state, env).createUser(tableUserName, tabUserPass);
+ Thread.sleep(1000);
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java
new file mode 100644
index 0000000..235c9ba
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropTable.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DropTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ dropTable(state, env, props);
+ }
+
+ public static void dropTable(State state, Environment env, Properties props) throws Exception {
+ String sourceUser = props.getProperty("source", "system");
+ String principal;
+ AuthenticationToken token;
+ if (sourceUser.equals("table")) {
+ principal = WalkingSecurity.get(state, env).getTabUserName();
+ token = WalkingSecurity.get(state, env).getTabToken();
+ } else {
+ principal = WalkingSecurity.get(state, env).getSysUserName();
+ token = WalkingSecurity.get(state, env).getSysToken();
+ }
+ Connector conn = env.getInstance().getConnector(principal, token);
+
+ String tableName = WalkingSecurity.get(state, env).getTableName();
+ String namespaceName = WalkingSecurity.get(state, env).getNamespaceName();
+
+ boolean exists = WalkingSecurity.get(state, env).getTableExists();
+ boolean hasPermission = WalkingSecurity.get(state, env).canDeleteTable(new Credentials(principal, token).toThrift(env.getInstance()), tableName,
+ namespaceName);
+
+ try {
+ conn.tableOperations().delete(tableName);
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
+ if (hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else {
+ // Drop anyway for sake of state
+ env.getConnector().tableOperations().delete(tableName);
+ WalkingSecurity.get(state, env).cleanTablePermissions(tableName);
+ return;
+ }
+ } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
+ if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ return;
+ }
+ throw new AccumuloException("Got unexpected ae error code", ae);
+ } catch (TableNotFoundException tnfe) {
+ if (exists)
+ throw new TableExistsException(null, tableName, "Got a TableNotFOundException but it should have existed", tnfe);
+ else
+ return;
+ }
+ WalkingSecurity.get(state, env).cleanTablePermissions(tableName);
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java
new file mode 100644
index 0000000..8d6080b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/DropUser.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DropUser extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getInstance().getConnector(WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getSysToken());
+
+ String tableUserName = WalkingSecurity.get(state, env).getTabUserName();
+
+ boolean exists = WalkingSecurity.get(state, env).userExists(tableUserName);
+ boolean hasPermission = WalkingSecurity.get(state, env).canDropUser(WalkingSecurity.get(state, env).getSysCredentials(), tableUserName);
+
+ try {
+ conn.securityOperations().dropLocalUser(tableUserName);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ if (hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else {
+ if (exists) {
+ env.getConnector().securityOperations().dropLocalUser(tableUserName);
+ WalkingSecurity.get(state, env).dropUser(tableUserName);
+ }
+ return;
+ }
+
+ case USER_DOESNT_EXIST:
+ if (exists)
+ throw new AccumuloException("Got user DNE exception when user should exists.", ae);
+ else
+ return;
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ WalkingSecurity.get(state, env).dropUser(tableUserName);
+ Thread.sleep(1000);
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java
new file mode 100644
index 0000000..edfd15f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityFixture.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.net.InetAddress;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+
+public class SecurityFixture extends Fixture {
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {
+ String secTableName, systemUserName, tableUserName, secNamespaceName;
+ // A best-effort sanity check to guard against not password-based auth
+ ClientConfiguration clientConf = ClientConfiguration.loadDefault();
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ throw new IllegalStateException("Security module currently cannot support Kerberos/SASL instances");
+ }
+
+ Connector conn = env.getConnector();
+
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+
+ systemUserName = String.format("system_%s", hostname);
+ tableUserName = String.format("table_%s", hostname);
+ secTableName = String.format("security_%s", hostname);
+ secNamespaceName = String.format("securityNs_%s", hostname);
+
+ if (conn.tableOperations().exists(secTableName))
+ conn.tableOperations().delete(secTableName);
+ Set<String> users = conn.securityOperations().listLocalUsers();
+ if (users.contains(tableUserName))
+ conn.securityOperations().dropLocalUser(tableUserName);
+ if (users.contains(systemUserName))
+ conn.securityOperations().dropLocalUser(systemUserName);
+
+ PasswordToken sysUserPass = new PasswordToken("sysUser");
+ conn.securityOperations().createLocalUser(systemUserName, sysUserPass);
+
+ WalkingSecurity.get(state, env).setTableName(secTableName);
+ WalkingSecurity.get(state, env).setNamespaceName(secNamespaceName);
+ state.set("rootUserPass", env.getToken());
+
+ WalkingSecurity.get(state, env).setSysUserName(systemUserName);
+ WalkingSecurity.get(state, env).createUser(systemUserName, sysUserPass);
+
+ WalkingSecurity.get(state, env).changePassword(tableUserName, new PasswordToken(new byte[0]));
+
+ WalkingSecurity.get(state, env).setTabUserName(tableUserName);
+
+ for (TablePermission tp : TablePermission.values()) {
+ WalkingSecurity.get(state, env).revokeTablePermission(systemUserName, secTableName, tp);
+ WalkingSecurity.get(state, env).revokeTablePermission(tableUserName, secTableName, tp);
+ }
+ for (SystemPermission sp : SystemPermission.values()) {
+ WalkingSecurity.get(state, env).revokeSystemPermission(systemUserName, sp);
+ WalkingSecurity.get(state, env).revokeSystemPermission(tableUserName, sp);
+ }
+ WalkingSecurity.get(state, env).changeAuthorizations(tableUserName, new Authorizations());
+ }
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {
+ log.debug("One last validate");
+ Validate.validate(state, env, log);
+ Connector conn = env.getConnector();
+
+ if (WalkingSecurity.get(state, env).getTableExists()) {
+ String secTableName = WalkingSecurity.get(state, env).getTableName();
+ log.debug("Dropping tables: " + secTableName);
+
+ conn.tableOperations().delete(secTableName);
+ }
+
+ if (WalkingSecurity.get(state, env).getNamespaceExists()) {
+ String secNamespaceName = WalkingSecurity.get(state, env).getNamespaceName();
+ log.debug("Dropping namespace: " + secNamespaceName);
+
+ conn.namespaceOperations().delete(secNamespaceName);
+ }
+
+ if (WalkingSecurity.get(state, env).userExists(WalkingSecurity.get(state, env).getTabUserName())) {
+ String tableUserName = WalkingSecurity.get(state, env).getTabUserName();
+ log.debug("Dropping user: " + tableUserName);
+
+ conn.securityOperations().dropLocalUser(tableUserName);
+ }
+ String systemUserName = WalkingSecurity.get(state, env).getSysUserName();
+ log.debug("Dropping user: " + systemUserName);
+ conn.securityOperations().dropLocalUser(systemUserName);
+ WalkingSecurity.clearInstance();
+
+ // Allow user drops to propagate, in case a new security test starts
+ Thread.sleep(2000);
+ }
+}
[3/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Posted by mw...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityHelper.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityHelper.java
new file mode 100644
index 0000000..93c7f02
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SecurityHelper.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.fs.FileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SecurityHelper {
+ private static final Logger log = LoggerFactory.getLogger(SecurityHelper.class);
+
+ private static final String tableName = "secTableName";
+ private static final String masterName = "sysUserName";
+ private static final String tUserName = "tabUserName";
+
+ private static final String masterPass = "sysUserPass";
+ private static final String tUserPass = "tabUserPass";
+
+ private static final String tUserExists = "tabUserExists";
+ private static final String tableExists = "secTableExists";
+
+ private static final String masterConn = "sysUserConn";
+
+ private static final String authsMap = "authorizationsCountMap";
+ private static final String lastKey = "lastMutationKey";
+ private static final String filesystem = "securityFileSystem";
+
+ public static String getTableName(State state) {
+ return state.getString(tableName);
+ }
+
+ public static void setTableName(State state, String tName) {
+ state.set(tableName, tName);
+ }
+
+ public static String getSysUserName(State state) {
+ return state.getString(masterName);
+ }
+
+ public static void setSysUserName(State state, String sysUserName) {
+ state.set(masterName, sysUserName);
+ }
+
+ public static String getTabUserName(State state) {
+ return state.getString(tUserName);
+ }
+
+ public static void setTabUserName(State state, String tabUserName) {
+ state.set(tUserName, tabUserName);
+ }
+
+ public static byte[] getSysUserPass(State state) {
+ return (byte[]) state.get(masterPass);
+ }
+
+ public static void setSysUserPass(State state, byte[] sysUserPass) {
+ log.debug("Setting system user pass to " + new String(sysUserPass, UTF_8));
+ state.set(masterPass, sysUserPass);
+ state.set(masterPass + "time", System.currentTimeMillis());
+
+ }
+
+ public static boolean sysUserPassTransient(State state) {
+ return System.currentTimeMillis() - state.getLong(masterPass + "time") < 1000;
+ }
+
+ public static byte[] getTabUserPass(State state) {
+ return (byte[]) state.get(tUserPass);
+ }
+
+ public static void setTabUserPass(State state, byte[] tabUserPass) {
+ log.debug("Setting table user pass to " + new String(tabUserPass, UTF_8));
+ state.set(tUserPass, tabUserPass);
+ state.set(tUserPass + "time", System.currentTimeMillis());
+ }
+
+ public static boolean tabUserPassTransient(State state) {
+ return System.currentTimeMillis() - state.getLong(tUserPass + "time") < 1000;
+ }
+
+ public static boolean getTabUserExists(State state) {
+ return Boolean.parseBoolean(state.getString(tUserExists));
+ }
+
+ public static void setTabUserExists(State state, boolean exists) {
+ state.set(tUserExists, Boolean.toString(exists));
+ }
+
+ public static boolean getTableExists(State state) {
+ return Boolean.parseBoolean(state.getString(tableExists));
+ }
+
+ public static void setTableExists(State state, boolean exists) {
+ state.set(tableExists, Boolean.toString(exists));
+ }
+
+ public static Connector getSystemConnector(State state) {
+ return (Connector) state.get(masterConn);
+ }
+
+ public static void setSystemConnector(State state, Connector conn) {
+ state.set(masterConn, conn);
+ }
+
+ public static boolean getTabPerm(State state, String userName, TablePermission tp) {
+ return Boolean.parseBoolean(state.getString("Tab" + userName + tp.name()));
+ }
+
+ public static void setTabPerm(State state, String userName, TablePermission tp, boolean value) {
+ log.debug((value ? "Gave" : "Took") + " the table permission " + tp.name() + (value ? " to" : " from") + " user " + userName);
+ state.set("Tab" + userName + tp.name(), Boolean.toString(value));
+ if (tp.equals(TablePermission.READ) || tp.equals(TablePermission.WRITE))
+ state.set("Tab" + userName + tp.name() + "time", System.currentTimeMillis());
+
+ }
+
+ public static boolean getSysPerm(State state, String userName, SystemPermission tp) {
+ return Boolean.parseBoolean(state.getString("Sys" + userName + tp.name()));
+ }
+
+ public static void setSysPerm(State state, String userName, SystemPermission tp, boolean value) {
+ log.debug((value ? "Gave" : "Took") + " the system permission " + tp.name() + (value ? " to" : " from") + " user " + userName);
+ state.set("Sys" + userName + tp.name(), Boolean.toString(value));
+ }
+
+ public static Authorizations getUserAuths(State state, String target) {
+ return (Authorizations) state.get(target + "_auths");
+ }
+
+ public static void setUserAuths(State state, String target, Authorizations auths) {
+ state.set(target + "_auths", auths);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Map<String,Integer> getAuthsMap(State state) {
+ return (Map<String,Integer>) state.get(authsMap);
+ }
+
+ public static void setAuthsMap(State state, Map<String,Integer> map) {
+ state.set(authsMap, map);
+ }
+
+ public static String[] getAuthsArray() {
+ return new String[] {"Fishsticks", "PotatoSkins", "Ribs", "Asparagus", "Paper", "Towels", "Lint", "Brush", "Celery"};
+ }
+
+ public static String getLastKey(State state) {
+ return state.getString(lastKey);
+ }
+
+ public static void setLastKey(State state, String key) {
+ state.set(lastKey, key);
+ }
+
+ public static void increaseAuthMap(State state, String s, int increment) {
+ Integer curVal = getAuthsMap(state).get(s);
+ if (curVal == null) {
+ curVal = Integer.valueOf(0);
+ getAuthsMap(state).put(s, curVal);
+ }
+ curVal += increment;
+ }
+
+ public static FileSystem getFs(State state) {
+ FileSystem fs = null;
+ try {
+ fs = (FileSystem) state.get(filesystem);
+ } catch (RuntimeException re) {}
+
+ if (fs == null) {
+ try {
+ fs = FileSystem.get(CachedConfiguration.getInstance());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ state.set(filesystem, fs);
+ }
+ return fs;
+ }
+
+ public static boolean inAmbiguousZone(State state, String userName, TablePermission tp) {
+ if (tp.equals(TablePermission.READ) || tp.equals(TablePermission.WRITE)) {
+ Long setTime = (Long) state.get("Tab" + userName + tp.name() + "time");
+ if (System.currentTimeMillis() < (setTime + 1000))
+ return true;
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java
new file mode 100644
index 0000000..54ab69f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/SetAuths.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class SetAuths extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String authsString = props.getProperty("auths", "_random");
+
+ String targetUser = props.getProperty("system");
+ String target;
+ String authPrincipal;
+ AuthenticationToken authToken;
+ if ("table".equals(targetUser)) {
+ target = WalkingSecurity.get(state, env).getTabUserName();
+ authPrincipal = WalkingSecurity.get(state, env).getSysUserName();
+ authToken = WalkingSecurity.get(state, env).getSysToken();
+ } else {
+ target = WalkingSecurity.get(state, env).getSysUserName();
+ authPrincipal = env.getUserName();
+ authToken = env.getToken();
+ }
+ Connector conn = env.getInstance().getConnector(authPrincipal, authToken);
+
+ boolean exists = WalkingSecurity.get(state, env).userExists(target);
+ boolean hasPermission = WalkingSecurity.get(state, env).canChangeAuthorizations(new Credentials(authPrincipal, authToken).toThrift(env.getInstance()),
+ target);
+
+ Authorizations auths;
+ if (authsString.equals("_random")) {
+ String[] possibleAuths = WalkingSecurity.get(state, env).getAuthsArray();
+
+ Random r = new Random();
+ int i = r.nextInt(possibleAuths.length);
+ String[] authSet = new String[i];
+ int length = possibleAuths.length;
+ for (int j = 0; j < i; j++) {
+ int nextRand = r.nextInt(length);
+ authSet[j] = possibleAuths[nextRand];
+ length--;
+ possibleAuths[nextRand] = possibleAuths[length];
+ possibleAuths[length] = authSet[j];
+ }
+ auths = new Authorizations(authSet);
+ } else {
+ auths = new Authorizations(authsString.split(","));
+ }
+
+ try {
+ conn.securityOperations().changeUserAuthorizations(target, auths);
+ } catch (AccumuloSecurityException ae) {
+ switch (ae.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ if (hasPermission)
+ throw new AccumuloException("Got a security exception when I should have had permission.", ae);
+ else
+ return;
+ case USER_DOESNT_EXIST:
+ if (exists)
+ throw new AccumuloException("Got security exception when the user should have existed", ae);
+ else
+ return;
+ default:
+ throw new AccumuloException("Got unexpected exception", ae);
+ }
+ }
+ WalkingSecurity.get(state, env).changeAuthorizations(target, auths);
+ if (!hasPermission)
+ throw new AccumuloException("Didn't get Security Exception when we should have");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java
new file mode 100644
index 0000000..d3335c4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/TableOp.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+public class TableOp extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Connector conn = env.getInstance().getConnector(WalkingSecurity.get(state, env).getTabUserName(), WalkingSecurity.get(state, env).getTabToken());
+
+ String action = props.getProperty("action", "_random");
+ TablePermission tp;
+ if ("_random".equalsIgnoreCase(action)) {
+ Random r = new Random();
+ tp = TablePermission.values()[r.nextInt(TablePermission.values().length)];
+ } else {
+ tp = TablePermission.valueOf(action);
+ }
+
+ boolean tableExists = WalkingSecurity.get(state, env).getTableExists();
+ String tableName = WalkingSecurity.get(state, env).getTableName();
+ String namespaceName = WalkingSecurity.get(state, env).getNamespaceName();
+
+ switch (tp) {
+ case READ: {
+ boolean canRead = WalkingSecurity.get(state, env).canScan(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName);
+ Authorizations auths = WalkingSecurity.get(state, env).getUserAuthorizations(WalkingSecurity.get(state, env).getTabCredentials());
+ boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp);
+ boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(conn.whoami());
+
+ Scanner scan = null;
+ try {
+ scan = conn.createScanner(tableName, conn.securityOperations().getUserAuthorizations(conn.whoami()));
+ int seen = 0;
+ Iterator<Entry<Key,Value>> iter = scan.iterator();
+ while (iter.hasNext()) {
+ Entry<Key,Value> entry = iter.next();
+ Key k = entry.getKey();
+ seen++;
+ if (!auths.contains(k.getColumnVisibilityData()) && !ambiguousAuths)
+ throw new AccumuloException("Got data I should not be capable of seeing: " + k + " table " + tableName);
+ }
+ if (!canRead && !ambiguousZone)
+ throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName);
+ for (Entry<String,Integer> entry : WalkingSecurity.get(state, env).getAuthsMap().entrySet()) {
+ if (auths.contains(entry.getKey().getBytes(UTF_8)))
+ seen = seen - entry.getValue();
+ }
+ if (seen != 0 && !ambiguousAuths)
+ throw new AccumuloException("Got mismatched amounts of data");
+ } catch (TableNotFoundException tnfe) {
+ if (tableExists)
+ throw new AccumuloException("Accumulo and test suite out of sync: table " + tableName, tnfe);
+ return;
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
+ if (canRead && !ambiguousZone)
+ throw new AccumuloException("Table read permission out of sync with Accumulo: table " + tableName, ae);
+ else
+ return;
+ }
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_AUTHORIZATIONS)) {
+ if (ambiguousAuths)
+ return;
+ else
+ throw new AccumuloException("Mismatched authorizations! ", ae);
+ }
+ throw new AccumuloException("Unexpected exception!", ae);
+ } catch (RuntimeException re) {
+ if (re.getCause() instanceof AccumuloSecurityException
+ && ((AccumuloSecurityException) re.getCause()).getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
+ if (canRead && !ambiguousZone)
+ throw new AccumuloException("Table read permission out of sync with Accumulo: table " + tableName, re.getCause());
+ else
+ return;
+ }
+ if (re.getCause() instanceof AccumuloSecurityException
+ && ((AccumuloSecurityException) re.getCause()).getSecurityErrorCode().equals(SecurityErrorCode.BAD_AUTHORIZATIONS)) {
+ if (ambiguousAuths)
+ return;
+ else
+ throw new AccumuloException("Mismatched authorizations! ", re.getCause());
+ }
+
+ throw new AccumuloException("Unexpected exception!", re);
+ } finally {
+ if (scan != null) {
+ scan.close();
+ scan = null;
+ }
+
+ }
+
+ break;
+ }
+ case WRITE:
+ boolean canWrite = WalkingSecurity.get(state, env).canWrite(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName);
+ boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp);
+
+ String key = WalkingSecurity.get(state, env).getLastKey() + "1";
+ Mutation m = new Mutation(new Text(key));
+ for (String s : WalkingSecurity.get(state, env).getAuthsArray()) {
+ m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(UTF_8)));
+ }
+ BatchWriter writer = null;
+ try {
+ try {
+ writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1));
+ } catch (TableNotFoundException tnfe) {
+ if (tableExists)
+ throw new AccumuloException("Table didn't exist when it should have: " + tableName);
+ return;
+ }
+ boolean works = true;
+ try {
+ writer.addMutation(m);
+ writer.close();
+ } catch (MutationsRejectedException mre) {
+ // Currently no method for detecting reason for mre. Waiting on ACCUMULO-670
+ // For now, just wait a second and go again if they can write!
+ if (!canWrite)
+ return;
+
+ if (ambiguousZone) {
+ Thread.sleep(1000);
+ try {
+ writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxWriteThreads(1));
+ writer.addMutation(m);
+ writer.close();
+ writer = null;
+ } catch (MutationsRejectedException mre2) {
+ throw new AccumuloException("Mutation exception!", mre2);
+ }
+ }
+ }
+ if (works)
+ for (String s : WalkingSecurity.get(state, env).getAuthsArray())
+ WalkingSecurity.get(state, env).increaseAuthMap(s, 1);
+ } finally {
+ if (writer != null) {
+ writer.close();
+ writer = null;
+ }
+ }
+ break;
+ case BULK_IMPORT:
+ key = WalkingSecurity.get(state, env).getLastKey() + "1";
+ SortedSet<Key> keys = new TreeSet<>();
+ for (String s : WalkingSecurity.get(state, env).getAuthsArray()) {
+ Key k = new Key(key, "", "", s);
+ keys.add(k);
+ }
+ Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
+ Path fail = new Path(dir.toString() + "_fail");
+ FileSystem fs = WalkingSecurity.get(state, env).getFs();
+ FileSKVWriter f = FileOperations.getInstance().newWriterBuilder().forFile(dir + "/securityBulk." + RFile.EXTENSION, fs, fs.getConf())
+ .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build();
+ f.startDefaultLocalityGroup();
+ fs.mkdirs(fail);
+ for (Key k : keys)
+ f.append(k, new Value("Value".getBytes(UTF_8)));
+ f.close();
+ try {
+ conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true);
+ } catch (TableNotFoundException tnfe) {
+ if (tableExists)
+ throw new AccumuloException("Table didn't exist when it should have: " + tableName);
+ return;
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
+ if (WalkingSecurity.get(state, env).canBulkImport(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName))
+ throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName);
+ return;
+ } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
+ if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami()))
+ return;
+ }
+ throw new AccumuloException("Unexpected exception!", ae);
+ }
+ for (String s : WalkingSecurity.get(state, env).getAuthsArray())
+ WalkingSecurity.get(state, env).increaseAuthMap(s, 1);
+ fs.delete(dir, true);
+ fs.delete(fail, true);
+
+ if (!WalkingSecurity.get(state, env).canBulkImport(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName))
+ throw new AccumuloException("Bulk Import succeeded when it should have failed: " + dir + " table " + tableName);
+ break;
+ case ALTER_TABLE:
+ AlterTable.renameTable(conn, state, env, tableName, tableName + "plus",
+ WalkingSecurity.get(state, env).canAlterTable(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName), tableExists);
+ break;
+
+ case GRANT:
+ props.setProperty("task", "grant");
+ props.setProperty("perm", "random");
+ props.setProperty("source", "table");
+ props.setProperty("target", "system");
+ AlterTablePerm.alter(state, env, props);
+ break;
+
+ case DROP_TABLE:
+ props.setProperty("source", "table");
+ DropTable.dropTable(state, env, props);
+ break;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java
new file mode 100644
index 0000000..c28f28d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/Validate.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.security.SecurityErrorCode;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.log4j.Logger;
+
+public class Validate extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ validate(state, env, log);
+ }
+
+ public static void validate(State state, Environment env, Logger log) throws Exception {
+ Connector conn = env.getConnector();
+
+ boolean tableExists = WalkingSecurity.get(state, env).getTableExists();
+ boolean cloudTableExists = conn.tableOperations().list().contains(WalkingSecurity.get(state, env).getTableName());
+ if (tableExists != cloudTableExists)
+ throw new AccumuloException("Table existance out of sync");
+
+ boolean tableUserExists = WalkingSecurity.get(state, env).userExists(WalkingSecurity.get(state, env).getTabUserName());
+ boolean cloudTableUserExists = conn.securityOperations().listLocalUsers().contains(WalkingSecurity.get(state, env).getTabUserName());
+ if (tableUserExists != cloudTableUserExists)
+ throw new AccumuloException("Table User existance out of sync");
+
+ Properties props = new Properties();
+ props.setProperty("target", "system");
+ Authenticate.authenticate(env.getUserName(), env.getToken(), state, env, props);
+ props.setProperty("target", "table");
+ Authenticate.authenticate(env.getUserName(), env.getToken(), state, env, props);
+
+ for (String user : new String[] {WalkingSecurity.get(state, env).getSysUserName(), WalkingSecurity.get(state, env).getTabUserName()}) {
+ for (SystemPermission sp : SystemPermission.values()) {
+ boolean hasSp = WalkingSecurity.get(state, env).hasSystemPermission(user, sp);
+ boolean accuHasSp;
+ try {
+ accuHasSp = conn.securityOperations().hasSystemPermission(user, sp);
+ log.debug("Just checked to see if user " + user + " has system perm " + sp.name() + " with answer " + accuHasSp);
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.USER_DOESNT_EXIST)) {
+ if (tableUserExists)
+ throw new AccumuloException("Got user DNE error when they should", ae);
+ else
+ continue;
+ } else
+ throw new AccumuloException("Unexpected exception!", ae);
+ }
+ if (hasSp != accuHasSp)
+ throw new AccumuloException(user + " existance out of sync for system perm " + sp + " hasSp/CloudhasSP " + hasSp + " " + accuHasSp);
+ }
+
+ for (TablePermission tp : TablePermission.values()) {
+ boolean hasTp = WalkingSecurity.get(state, env).hasTablePermission(user, WalkingSecurity.get(state, env).getTableName(), tp);
+ boolean accuHasTp;
+ try {
+ accuHasTp = conn.securityOperations().hasTablePermission(user, WalkingSecurity.get(state, env).getTableName(), tp);
+ log.debug("Just checked to see if user " + user + " has table perm " + tp.name() + " with answer " + accuHasTp);
+ } catch (AccumuloSecurityException ae) {
+ if (ae.getSecurityErrorCode().equals(SecurityErrorCode.USER_DOESNT_EXIST)) {
+ if (tableUserExists)
+ throw new AccumuloException("Got user DNE error when they should", ae);
+ else
+ continue;
+ } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.TABLE_DOESNT_EXIST)) {
+ if (tableExists)
+ throw new AccumuloException("Got table DNE when it should", ae);
+ else
+ continue;
+ } else
+ throw new AccumuloException("Unexpected exception!", ae);
+ }
+ if (hasTp != accuHasTp)
+ throw new AccumuloException(user + " existance out of sync for table perm " + tp + " hasTp/CloudhasTP " + hasTp + " " + accuHasTp);
+ }
+
+ }
+
+ Authorizations accuAuths;
+ Authorizations auths;
+ try {
+ auths = WalkingSecurity.get(state, env).getUserAuthorizations(WalkingSecurity.get(state, env).getTabCredentials());
+ accuAuths = conn.securityOperations().getUserAuthorizations(WalkingSecurity.get(state, env).getTabUserName());
+ } catch (ThriftSecurityException ae) {
+ if (ae.getCode() == org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.USER_DOESNT_EXIST) {
+ if (tableUserExists)
+ throw new AccumuloException("Table user didn't exist when they should.", ae);
+ else
+ return;
+ }
+ throw new AccumuloException("Unexpected exception!", ae);
+ }
+ if (!auths.equals(accuAuths))
+ throw new AccumuloException("Table User authorizations out of sync");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/WalkingSecurity.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/WalkingSecurity.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/WalkingSecurity.java
new file mode 100644
index 0000000..302d6ec
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/security/WalkingSecurity.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.security;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.NamespacePermission;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.server.AccumuloServerContext;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.server.security.handler.Authorizor;
+import org.apache.accumulo.server.security.handler.PermissionHandler;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.fs.FileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class WalkingSecurity extends SecurityOperation implements Authorizor, Authenticator, PermissionHandler {
+ State state = null;
+ Environment env = null;
+ private static final Logger log = LoggerFactory.getLogger(WalkingSecurity.class);
+
+ private static final String tableName = "SecurityTableName";
+ private static final String namespaceName = "SecurityNamespaceName";
+ private static final String userName = "UserName";
+
+ private static final String userPass = "UserPass";
+ private static final String userExists = "UserExists";
+ private static final String tableExists = "TableExists";
+ private static final String namespaceExists = "NamespaceExists";
+
+ private static final String connector = "UserConnection";
+
+ private static final String authsMap = "authorizationsCountMap";
+ private static final String lastKey = "lastMutationKey";
+ private static final String filesystem = "securityFileSystem";
+
+ private static WalkingSecurity instance = null;
+
+ public WalkingSecurity(AccumuloServerContext context, Authorizor author, Authenticator authent, PermissionHandler pm) {
+ super(context, author, authent, pm);
+ }
+
+ public WalkingSecurity(State state2, Environment env2) {
+ super(new AccumuloServerContext(new ServerConfigurationFactory(HdfsZooInstance.getInstance())));
+ this.state = state2;
+ this.env = env2;
+ authorizor = this;
+ authenticator = this;
+ permHandle = this;
+ }
+
+ public static WalkingSecurity get(State state, Environment env) {
+ if (instance == null || instance.state != state) {
+ instance = new WalkingSecurity(state, env);
+ state.set(tableExists, Boolean.toString(false));
+ state.set(namespaceExists, Boolean.toString(false));
+ state.set(authsMap, new HashMap<String,Integer>());
+ }
+
+ return instance;
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ throw new UnsupportedOperationException("nope");
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator one, PermissionHandler two) {
+ return this.getClass().equals(one.getClass()) && this.getClass().equals(two.getClass());
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authenticator one, Authorizor two) {
+ return this.getClass().equals(one.getClass()) && this.getClass().equals(two.getClass());
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authorizor one, PermissionHandler two) {
+ return this.getClass().equals(one.getClass()) && this.getClass().equals(two.getClass());
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials rootuser, String token) throws ThriftSecurityException {
+ throw new UnsupportedOperationException("nope");
+ }
+
+ @Override
+ public void changeAuthorizations(String user, Authorizations authorizations) throws AccumuloSecurityException {
+ state.set(user + "_auths", authorizations);
+ state.set("Auths-" + user + '-' + "time", System.currentTimeMillis());
+ }
+
+ @Override
+ public Authorizations getCachedUserAuthorizations(String user) throws AccumuloSecurityException {
+ return (Authorizations) state.get(user + "_auths");
+ }
+
+ public boolean ambiguousAuthorizations(String userName) {
+ Long setTime = state.getLong("Auths-" + userName + '-' + "time");
+ if (setTime == null)
+ throw new RuntimeException("WTF? Auths-" + userName + '-' + "time is null");
+ if (System.currentTimeMillis() < (setTime + 1000))
+ return true;
+ return false;
+ }
+
+ @Override
+ public void initUser(String user) throws AccumuloSecurityException {
+ changeAuthorizations(user, new Authorizations());
+ }
+
+ @Override
+ public Set<String> listUsers() throws AccumuloSecurityException {
+ Set<String> userList = new TreeSet<>();
+ for (String user : new String[] {getSysUserName(), getTabUserName()}) {
+ if (userExists(user))
+ userList.add(user);
+ }
+ return userList;
+ }
+
+ @Override
+ public boolean authenticateUser(String principal, AuthenticationToken token) {
+ PasswordToken pass = (PasswordToken) state.get(principal + userPass);
+ boolean ret = pass.equals(token);
+ return ret;
+ }
+
+ @Override
+ public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ state.set(principal + userExists, Boolean.toString(true));
+ changePassword(principal, token);
+ cleanUser(principal);
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ state.set(user + userExists, Boolean.toString(false));
+ cleanUser(user);
+ if (user.equals(getTabUserName()))
+ state.set("table" + connector, null);
+ }
+
+ @Override
+ public void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ state.set(principal + userPass, token);
+ state.set(principal + userPass + "time", System.currentTimeMillis());
+ }
+
+ @Override
+ public boolean userExists(String user) {
+ return Boolean.parseBoolean(state.getString(user + userExists));
+ }
+
+ @Override
+ public boolean hasSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ boolean res = Boolean.parseBoolean(state.getString("Sys-" + user + '-' + permission.name()));
+ return res;
+ }
+
+ @Override
+ public boolean hasCachedSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ return hasSystemPermission(user, permission);
+ }
+
+ @Override
+ public boolean hasTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return Boolean.parseBoolean(state.getString("Tab-" + user + '-' + permission.name()));
+ }
+
+ @Override
+ public boolean hasCachedTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ return hasTablePermission(user, table, permission);
+ }
+
+ @Override
+ public boolean hasNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ return Boolean.parseBoolean(state.getString("Nsp-" + user + '-' + permission.name()));
+ }
+
+ @Override
+ public boolean hasCachedNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ return hasNamespacePermission(user, namespace, permission);
+ }
+
+ @Override
+ public void grantSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ setSysPerm(state, user, permission, true);
+ }
+
+ @Override
+ public void revokeSystemPermission(String user, SystemPermission permission) throws AccumuloSecurityException {
+ setSysPerm(state, user, permission, false);
+ }
+
+ @Override
+ public void grantTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ setTabPerm(state, user, permission, table, true);
+ }
+
+ private static void setSysPerm(State state, String userName, SystemPermission tp, boolean value) {
+ log.debug((value ? "Gave" : "Took") + " the system permission " + tp.name() + (value ? " to" : " from") + " user " + userName);
+ state.set("Sys-" + userName + '-' + tp.name(), Boolean.toString(value));
+ }
+
+ private void setTabPerm(State state, String userName, TablePermission tp, String table, boolean value) {
+ if (table.equals(userName))
+ throw new RuntimeException("This is also fucked up");
+ log.debug((value ? "Gave" : "Took") + " the table permission " + tp.name() + (value ? " to" : " from") + " user " + userName);
+ state.set("Tab-" + userName + '-' + tp.name(), Boolean.toString(value));
+ if (tp.equals(TablePermission.READ) || tp.equals(TablePermission.WRITE))
+ state.set("Tab-" + userName + '-' + tp.name() + '-' + "time", System.currentTimeMillis());
+ }
+
+ @Override
+ public void revokeTablePermission(String user, String table, TablePermission permission) throws AccumuloSecurityException, TableNotFoundException {
+ setTabPerm(state, user, permission, table, false);
+ }
+
+ @Override
+ public void grantNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ setNspPerm(state, user, permission, namespace, true);
+ }
+
+ private void setNspPerm(State state, String userName, NamespacePermission tnp, String namespace, boolean value) {
+ if (namespace.equals(userName))
+ throw new RuntimeException("I don't even know");
+ log.debug((value ? "Gave" : "Took") + " the table permission " + tnp.name() + (value ? " to" : " from") + " user " + userName);
+ state.set("Nsp-" + userName + '-' + tnp.name(), Boolean.toString(value));
+ if (tnp.equals(NamespacePermission.READ) || tnp.equals(NamespacePermission.WRITE))
+ state.set("Nsp-" + userName + '-' + tnp.name() + '-' + "time", System.currentTimeMillis());
+ }
+
+ @Override
+ public void revokeNamespacePermission(String user, String namespace, NamespacePermission permission) throws AccumuloSecurityException,
+ NamespaceNotFoundException {
+ setNspPerm(state, user, permission, namespace, false);
+ }
+
+ @Override
+ public void cleanTablePermissions(String table) throws AccumuloSecurityException, TableNotFoundException {
+ for (String user : new String[] {getSysUserName(), getTabUserName()}) {
+ for (TablePermission tp : TablePermission.values()) {
+ revokeTablePermission(user, table, tp);
+ }
+ }
+ state.set(tableExists, Boolean.toString(false));
+ }
+
+ @Override
+ public void cleanNamespacePermissions(String namespace) throws AccumuloSecurityException, NamespaceNotFoundException {
+ for (String user : new String[] {getSysUserName(), getNspUserName()}) {
+ for (NamespacePermission tnp : NamespacePermission.values()) {
+ revokeNamespacePermission(user, namespace, tnp);
+ }
+ }
+ state.set(namespaceExists, Boolean.toString(false));
+ }
+
+ @Override
+ public void cleanUser(String user) throws AccumuloSecurityException {
+ if (getTableExists())
+ for (TablePermission tp : TablePermission.values())
+ try {
+ revokeTablePermission(user, getTableName(), tp);
+ } catch (TableNotFoundException e) {
+ // ignore
+ }
+ for (SystemPermission sp : SystemPermission.values())
+ revokeSystemPermission(user, sp);
+ }
+
+ public String getTabUserName() {
+ return state.getString("table" + userName);
+ }
+
+ public String getSysUserName() {
+ return state.getString("system" + userName);
+ }
+
+ public String getNspUserName() {
+ return state.getString("namespace" + userName);
+ }
+
+ public void setTabUserName(String name) {
+ state.set("table" + userName, name);
+ state.set(name + userExists, Boolean.toString(false));
+ }
+
+ public void setNspUserName(String name) {
+ state.set("namespace" + userName, name);
+ state.set(name + userExists, Boolean.toString(false));
+ }
+
+ public void setSysUserName(String name) {
+ state.set("system" + userName, name);
+ }
+
+ public String getTableName() {
+ return state.getString(tableName);
+ }
+
+ public String getNamespaceName() {
+ return state.getString(namespaceName);
+ }
+
+ public boolean getTableExists() {
+ return Boolean.parseBoolean(state.getString(tableExists));
+ }
+
+ public boolean getNamespaceExists() {
+ return Boolean.parseBoolean(state.getString(namespaceExists));
+ }
+
+ public TCredentials getSysCredentials() {
+ return new Credentials(getSysUserName(), getSysToken()).toThrift(this.env.getInstance());
+ }
+
+ public TCredentials getTabCredentials() {
+ return new Credentials(getTabUserName(), getTabToken()).toThrift(this.env.getInstance());
+ }
+
+ public AuthenticationToken getSysToken() {
+ return new PasswordToken(getSysPassword());
+ }
+
+ public AuthenticationToken getTabToken() {
+ return new PasswordToken(getTabPassword());
+ }
+
+ public byte[] getUserPassword(String user) {
+ Object obj = state.get(user + userPass);
+ if (obj instanceof PasswordToken) {
+ return ((PasswordToken) obj).getPassword();
+ }
+ return null;
+ }
+
+ public byte[] getSysPassword() {
+ Object obj = state.get(getSysUserName() + userPass);
+ if (obj instanceof PasswordToken) {
+ return ((PasswordToken) obj).getPassword();
+ }
+ return null;
+ }
+
+ public byte[] getTabPassword() {
+ Object obj = state.get(getTabUserName() + userPass);
+ if (obj instanceof PasswordToken) {
+ return ((PasswordToken) obj).getPassword();
+ }
+ return null;
+ }
+
+ public boolean userPassTransient(String user) {
+ return System.currentTimeMillis() - state.getLong(user + userPass + "time") < 1000;
+ }
+
+ public void setTableName(String tName) {
+ state.set(tableName, tName);
+ }
+
+ public void setNamespaceName(String nsName) {
+ state.set(namespaceName, nsName);
+ }
+
+ @Override
+ public void initTable(String table) throws AccumuloSecurityException {
+ state.set(tableExists, Boolean.toString(true));
+ state.set(tableName, table);
+ }
+
+ public String[] getAuthsArray() {
+ return new String[] {"Fishsticks", "PotatoSkins", "Ribs", "Asparagus", "Paper", "Towels", "Lint", "Brush", "Celery"};
+ }
+
+ public boolean inAmbiguousZone(String userName, TablePermission tp) {
+ if (tp.equals(TablePermission.READ) || tp.equals(TablePermission.WRITE)) {
+ Long setTime = state.getLong("Tab-" + userName + '-' + tp.name() + '-' + "time");
+ if (setTime == null)
+ throw new RuntimeException("WTF? Tab-" + userName + '-' + tp.name() + '-' + "time is null");
+ if (System.currentTimeMillis() < (setTime + 1000))
+ return true;
+ }
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String,Integer> getAuthsMap() {
+ return (Map<String,Integer>) state.get(authsMap);
+ }
+
+ public String getLastKey() {
+ return state.getString(lastKey);
+ }
+
+ public void increaseAuthMap(String s, int increment) {
+ Integer curVal = getAuthsMap().get(s);
+ if (curVal == null) {
+ curVal = Integer.valueOf(0);
+ getAuthsMap().put(s, curVal);
+ }
+ curVal += increment;
+ }
+
+ public FileSystem getFs() {
+ FileSystem fs = null;
+ try {
+ fs = (FileSystem) state.get(filesystem);
+ } catch (RuntimeException re) {}
+
+ if (fs == null) {
+ try {
+ fs = FileSystem.get(CachedConfiguration.getInstance());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ state.set(filesystem, fs);
+ }
+ return fs;
+ }
+
+ @Override
+ public boolean canAskAboutUser(TCredentials credentials, String user) throws ThriftSecurityException {
+ try {
+ return super.canAskAboutUser(credentials, user);
+ } catch (ThriftSecurityException tse) {
+ if (tse.getCode().equals(SecurityErrorCode.PERMISSION_DENIED))
+ return false;
+ throw tse;
+ }
+ }
+
+ @Override
+ public boolean validTokenClass(String tokenClass) {
+ return tokenClass.equals(PasswordToken.class.getName());
+ }
+
+ public static void clearInstance() {
+ instance = null;
+ }
+
+ @Override
+ public Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes() {
+ Set<Class<? extends AuthenticationToken>> cs = new HashSet<>();
+ cs.add(PasswordToken.class);
+ return cs;
+ }
+
+ @Override
+ public boolean isValidAuthorizations(String user, List<ByteBuffer> auths) throws AccumuloSecurityException {
+ Collection<ByteBuffer> userauths = getCachedUserAuthorizations(user).getAuthorizationsBB();
+ for (ByteBuffer auth : auths)
+ if (!userauths.contains(auth))
+ return false;
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/BatchVerify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/BatchVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/BatchVerify.java
new file mode 100644
index 0000000..ec50285
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/BatchVerify.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.sequential;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class BatchVerify extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ Random rand = new Random();
+
+ long numWrites = state.getLong("numWrites");
+ int maxVerify = Integer.parseInt(props.getProperty("maxVerify", "2000"));
+ long numVerify = rand.nextInt(maxVerify - 1) + 1;
+
+ if (numVerify > (numWrites / 4)) {
+ numVerify = numWrites / 4;
+ }
+
+ Connector conn = env.getConnector();
+ BatchScanner scanner = conn.createBatchScanner(state.getString("seqTableName"), new Authorizations(), 2);
+
+ try {
+ int count = 0;
+ List<Range> ranges = new ArrayList<>();
+ while (count < numVerify) {
+ long rangeStart = rand.nextInt((int) numWrites);
+ long rangeEnd = rangeStart + 99;
+ if (rangeEnd > (numWrites - 1)) {
+ rangeEnd = numWrites - 1;
+ }
+ count += rangeEnd - rangeStart + 1;
+ ranges.add(new Range(new Text(String.format("%010d", rangeStart)), new Text(String.format("%010d", rangeEnd))));
+ }
+
+ ranges = Range.mergeOverlapping(ranges);
+ if (ranges.size() > 1) {
+ Collections.sort(ranges);
+ }
+
+ if (count == 0 || ranges.size() == 0)
+ return;
+
+ log.debug(String.format("scanning %d rows in the following %d ranges:", count, ranges.size()));
+ for (Range r : ranges) {
+ log.debug(r);
+ }
+
+ scanner.setRanges(ranges);
+
+ List<Key> keys = new ArrayList<>();
+ for (Entry<Key,Value> entry : scanner) {
+ keys.add(entry.getKey());
+ }
+
+ log.debug("scan returned " + keys.size() + " rows. now verifying...");
+
+ Collections.sort(keys);
+
+ Iterator<Key> iterator = keys.iterator();
+ int curKey = Integer.parseInt(iterator.next().getRow().toString());
+ boolean done = false;
+ for (Range r : ranges) {
+ int start = Integer.parseInt(r.getStartKey().getRow().toString());
+ int end = Integer.parseInt(String.copyValueOf(r.getEndKey().getRow().toString().toCharArray(), 0, 10));
+ for (int i = start; i <= end; i++) {
+
+ if (done) {
+ log.error("missing key " + i);
+ break;
+ }
+
+ while (curKey < i) {
+ log.error("extra key " + curKey);
+ if (iterator.hasNext() == false) {
+ done = true;
+ break;
+ }
+ curKey = Integer.parseInt(iterator.next().getRow().toString());
+ }
+
+ if (curKey > i) {
+ log.error("missing key " + i);
+ }
+
+ if (iterator.hasNext()) {
+ curKey = Integer.parseInt(iterator.next().getRow().toString());
+ } else {
+ done = true;
+ }
+ }
+ }
+
+ log.debug("verify is now complete");
+ } finally {
+ scanner.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Commit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Commit.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Commit.java
new file mode 100644
index 0000000..6865557
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Commit.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.sequential;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Commit extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ env.getMultiTableBatchWriter().flush();
+
+ log.debug("Committed " + state.getLong("numWrites") + " writes. Total writes: " + state.getLong("totalWrites"));
+ state.set("numWrites", Long.valueOf(0));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
new file mode 100644
index 0000000..58d44d4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.sequential;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.util.ToolRunner;
+
+public class MapRedVerify extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String[] args = new String[8];
+ args[0] = "-libjars";
+ args[1] = getMapReduceJars();
+ args[2] = env.getUserName();
+ args[3] = env.getPassword();
+ if (null == args[3]) {
+ args[3] = env.getKeytab();
+ }
+ args[4] = state.getString("seqTableName");
+ args[5] = env.getInstance().getInstanceName();
+ args[6] = env.getConfigProperty("ZOOKEEPERS");
+ args[7] = args[4] + "_MR";
+
+ if (ToolRunner.run(CachedConfiguration.getInstance(), new MapRedVerifyTool(), args) != 0) {
+ log.error("Failed to run map/red verify");
+ return;
+ }
+
+ Scanner outputScanner = env.getConnector().createScanner(args[7], Authorizations.EMPTY);
+ outputScanner.setRange(new Range());
+
+ int count = 0;
+ Key lastKey = null;
+ for (Entry<Key,Value> entry : outputScanner) {
+ Key current = entry.getKey();
+ if (lastKey != null && lastKey.getColumnFamily().equals(current.getRow())) {
+ log.info(entry.getKey());
+ count++;
+ }
+ lastKey = current;
+ }
+
+ if (count > 1) {
+ log.error("Gaps in output");
+ }
+
+ log.debug("Dropping table: " + args[7]);
+ Connector conn = env.getConnector();
+ conn.tableOperations().delete(args[7]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java
new file mode 100644
index 0000000..6f785ce
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerifyTool.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.sequential;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Tool;
+import org.apache.log4j.Logger;
+
+public class MapRedVerifyTool extends Configured implements Tool {
+ protected final Logger log = Logger.getLogger(this.getClass());
+
+ public static class SeqMapClass extends Mapper<Key,Value,NullWritable,IntWritable> {
+ @Override
+ public void map(Key row, Value data, Context output) throws IOException, InterruptedException {
+ Integer num = Integer.valueOf(row.getRow().toString());
+ output.write(NullWritable.get(), new IntWritable(num.intValue()));
+ }
+ }
+
+ public static class SeqReduceClass extends Reducer<NullWritable,IntWritable,Text,Mutation> {
+ @Override
+ public void reduce(NullWritable ignore, Iterable<IntWritable> values, Context output) throws IOException, InterruptedException {
+ Iterator<IntWritable> iterator = values.iterator();
+
+ if (iterator.hasNext() == false) {
+ return;
+ }
+
+ int start = iterator.next().get();
+ int index = start;
+ while (iterator.hasNext()) {
+ int next = iterator.next().get();
+ if (next != index + 1) {
+ writeMutation(output, start, index);
+ start = next;
+ }
+ index = next;
+ }
+ writeMutation(output, start, index);
+ }
+
+ public void writeMutation(Context output, int start, int end) throws IOException, InterruptedException {
+ Mutation m = new Mutation(new Text(String.format("%010d", start)));
+ m.put(new Text(String.format("%010d", end)), new Text(""), new Value(new byte[0]));
+ output.write(null, m);
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
+ job.setJarByClass(this.getClass());
+
+ if (job.getJar() == null) {
+ log.error("M/R requires a jar file! Run mvn package.");
+ return 1;
+ }
+
+ ClientConfiguration clientConf = ClientConfiguration.loadDefault().withInstance(args[3]).withZkHosts(args[4]);
+
+ AccumuloInputFormat.setInputTableName(job, args[2]);
+ AccumuloInputFormat.setZooKeeperInstance(job, clientConf);
+ AccumuloOutputFormat.setDefaultTableName(job, args[5]);
+ AccumuloOutputFormat.setZooKeeperInstance(job, clientConf);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ // Better be logged in
+ KerberosToken token = new KerberosToken();
+ try {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ if (!user.hasKerberosCredentials()) {
+ throw new IllegalStateException("Expected current user to have Kerberos credentials");
+ }
+
+ String newPrincipal = user.getUserName();
+
+ ZooKeeperInstance inst = new ZooKeeperInstance(clientConf);
+ Connector conn = inst.getConnector(newPrincipal, token);
+
+ // Do the explicit check to see if the user has the permission to get a delegation token
+ if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
+ log.error(newPrincipal + " doesn't have the " + SystemPermission.OBTAIN_DELEGATION_TOKEN.name()
+ + " SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's"
+ + " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials.");
+ throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token");
+ }
+
+ // Fetch a delegation token from Accumulo
+ AuthenticationToken dt = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+
+ // Set the delegation token instead of the kerberos token
+ AccumuloInputFormat.setConnectorInfo(job, newPrincipal, dt);
+ AccumuloOutputFormat.setConnectorInfo(job, newPrincipal, dt);
+ } catch (Exception e) {
+ final String msg = "Failed to acquire DelegationToken for use with MapReduce";
+ log.error(msg, e);
+ throw new RuntimeException(msg, e);
+ }
+ } else {
+ AccumuloInputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1]));
+ AccumuloOutputFormat.setConnectorInfo(job, args[0], new PasswordToken(args[1]));
+ }
+
+ job.setMapperClass(SeqMapClass.class);
+ job.setMapOutputKeyClass(NullWritable.class);
+ job.setMapOutputValueClass(IntWritable.class);
+
+ job.setReducerClass(SeqReduceClass.class);
+ job.setNumReduceTasks(1);
+
+ job.setOutputFormatClass(AccumuloOutputFormat.class);
+ AccumuloOutputFormat.setCreateTables(job, true);
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java
new file mode 100644
index 0000000..de8af18
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/SequentialFixture.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.sequential;
+
+import java.net.InetAddress;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+
+public class SequentialFixture extends Fixture {
+
+ String seqTableName;
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {
+
+ Connector conn = env.getConnector();
+ Instance instance = env.getInstance();
+
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+
+ seqTableName = String.format("sequential_%s_%s_%d", hostname, env.getPid(), System.currentTimeMillis());
+ state.set("seqTableName", seqTableName);
+
+ try {
+ conn.tableOperations().create(seqTableName);
+ log.debug("Created table " + seqTableName + " (id:" + Tables.getNameToIdMap(instance).get(seqTableName) + ")");
+ } catch (TableExistsException e) {
+ log.warn("Table " + seqTableName + " already exists!");
+ throw e;
+ }
+ conn.tableOperations().setProperty(seqTableName, "table.scan.max.memory", "1K");
+
+ state.set("numWrites", Long.valueOf(0));
+ state.set("totalWrites", Long.valueOf(0));
+ }
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {
+ // We have resources we need to clean up
+ if (env.isMultiTableBatchWriterInitialized()) {
+ MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Ignoring mutations that weren't flushed", e);
+ }
+
+ // Reset the MTBW on the state to null
+ env.resetMultiTableBatchWriter();
+ }
+
+ log.debug("Dropping tables: " + seqTableName);
+
+ Connector conn = env.getConnector();
+
+ conn.tableOperations().delete(seqTableName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Write.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Write.java
new file mode 100644
index 0000000..80e0e38
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/Write.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.sequential;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Write extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ BatchWriter bw = env.getMultiTableBatchWriter().getBatchWriter(state.getString("seqTableName"));
+
+ state.set("numWrites", state.getLong("numWrites") + 1);
+
+ Long totalWrites = state.getLong("totalWrites") + 1;
+ if (totalWrites % 10000 == 0) {
+ log.debug("Total writes: " + totalWrites);
+ }
+ state.set("totalWrites", totalWrites);
+
+ Mutation m = new Mutation(new Text(String.format("%010d", totalWrites)));
+ m.put(new Text("cf"), new Text("cq"), new Value("val".getBytes(UTF_8)));
+ bw.addMutation(m);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
new file mode 100644
index 0000000..86afd8f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/BulkInsert.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ToolRunner;
+
+public class BulkInsert extends Test {
+
+ class SeqfileBatchWriter implements BatchWriter {
+
+ SequenceFile.Writer writer;
+
+ SeqfileBatchWriter(Configuration conf, FileSystem fs, String file) throws IOException {
+ writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(fs.makeQualified(new Path(file))), SequenceFile.Writer.keyClass(Key.class),
+ SequenceFile.Writer.valueClass(Value.class));
+ }
+
+ @Override
+ public void addMutation(Mutation m) throws MutationsRejectedException {
+ List<ColumnUpdate> updates = m.getUpdates();
+ for (ColumnUpdate cu : updates) {
+ Key key = new Key(m.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(), cu.getColumnVisibility(), Long.MAX_VALUE, false, false);
+ Value val = new Value(cu.getValue(), false);
+
+ try {
+ writer.append(key, val);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void addMutations(Iterable<Mutation> iterable) throws MutationsRejectedException {
+ for (Mutation mutation : iterable)
+ addMutation(mutation);
+ }
+
+ @Override
+ public void flush() throws MutationsRejectedException {}
+
+ @Override
+ public void close() throws MutationsRejectedException {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+ long nextDocID = (Long) state.get("nextDocID");
+
+ int minInsert = Integer.parseInt(props.getProperty("minInsert"));
+ int maxInsert = Integer.parseInt(props.getProperty("maxInsert"));
+ int numToInsert = rand.nextInt(maxInsert - minInsert) + minInsert;
+
+ int maxSplits = Integer.parseInt(props.getProperty("maxSplits"));
+
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs = FileSystem.get(conf);
+
+ String rootDir = "/tmp/shard_bulk/" + dataTableName;
+
+ fs.mkdirs(new Path(rootDir));
+
+ BatchWriter dataWriter = new SeqfileBatchWriter(conf, fs, rootDir + "/data.seq");
+ BatchWriter indexWriter = new SeqfileBatchWriter(conf, fs, rootDir + "/index.seq");
+
+ for (int i = 0; i < numToInsert; i++) {
+ String docID = Insert.insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand);
+ log.debug("Bulk inserting document " + docID);
+ }
+
+ state.set("nextDocID", Long.valueOf(nextDocID));
+
+ dataWriter.close();
+ indexWriter.close();
+
+ sort(state, env, fs, dataTableName, rootDir + "/data.seq", rootDir + "/data_bulk", rootDir + "/data_work", maxSplits);
+ sort(state, env, fs, indexTableName, rootDir + "/index.seq", rootDir + "/index_bulk", rootDir + "/index_work", maxSplits);
+
+ bulkImport(fs, state, env, dataTableName, rootDir, "data");
+ bulkImport(fs, state, env, indexTableName, rootDir, "index");
+
+ fs.delete(new Path(rootDir), true);
+ }
+
+ private void bulkImport(FileSystem fs, State state, Environment env, String tableName, String rootDir, String prefix) throws Exception {
+ while (true) {
+ String bulkDir = rootDir + "/" + prefix + "_bulk";
+ String failDir = rootDir + "/" + prefix + "_failure";
+ Path failPath = new Path(failDir);
+ fs.delete(failPath, true);
+ fs.mkdirs(failPath);
+ env.getConnector().tableOperations().importDirectory(tableName, bulkDir, failDir, true);
+
+ FileStatus[] failures = fs.listStatus(failPath);
+ if (failures != null && failures.length > 0) {
+ log.warn("Failed to bulk import some files, retrying ");
+
+ for (FileStatus failure : failures) {
+ if (!failure.getPath().getName().endsWith(".seq"))
+ fs.rename(failure.getPath(), new Path(new Path(bulkDir), failure.getPath().getName()));
+ else
+ log.debug("Ignoring " + failure.getPath());
+ }
+ sleepUninterruptibly(3, TimeUnit.SECONDS);
+ } else
+ break;
+ }
+ }
+
+ private void sort(State state, Environment env, FileSystem fs, String tableName, String seqFile, String outputDir, String workDir, int maxSplits)
+ throws Exception {
+
+ PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDir + "/splits.txt"))), false, UTF_8.name());
+
+ Connector conn = env.getConnector();
+
+ Collection<Text> splits = conn.tableOperations().listSplits(tableName, maxSplits);
+ for (Text split : splits)
+ out.println(Base64.getEncoder().encodeToString(TextUtil.getBytes(split)));
+
+ out.close();
+
+ SortTool sortTool = new SortTool(seqFile, outputDir, workDir + "/splits.txt", splits);
+
+ String[] args = new String[2];
+ args[0] = "-libjars";
+ args[1] = getMapReduceJars();
+
+ if (ToolRunner.run(CachedConfiguration.getInstance(), sortTool, args) != 0) {
+ throw new Exception("Failed to run map/red verify");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CloneIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CloneIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CloneIndex.java
new file mode 100644
index 0000000..c47d2a8
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CloneIndex.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CloneIndex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+
+ long t1 = System.currentTimeMillis();
+ env.getConnector().tableOperations().flush(indexTableName, null, null, true);
+ long t2 = System.currentTimeMillis();
+ env.getConnector().tableOperations().clone(indexTableName, tmpIndexTableName, false, new HashMap<String,String>(), new HashSet<String>());
+ long t3 = System.currentTimeMillis();
+
+ log.debug("Cloned " + tmpIndexTableName + " from " + indexTableName + " flush: " + (t2 - t1) + "ms clone: " + (t3 - t2) + "ms");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Commit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Commit.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Commit.java
new file mode 100644
index 0000000..06e8b44
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Commit.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Commit extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ env.getMultiTableBatchWriter().flush();
+ log.debug("Committed inserts ");
+ }
+
+}
[7/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
Posted by mw...@apache.org.
ACCUMULO-4510 Adding Randomwalk code from Accumulo
* All tests are in core module which creates shaded jar
* Shaded jar can be used on its own or submitted to YARN via Twill
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/ac5b271c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/ac5b271c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/ac5b271c
Branch: refs/heads/master
Commit: ac5b271caec45083751efad7298852969bec387e
Parents: 89d6acb
Author: Mike Walch <mw...@apache.org>
Authored: Thu Dec 22 10:39:30 2016 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Tue Jan 3 15:50:50 2017 -0500
----------------------------------------------------------------------
.gitignore | 6 +
core/.gitignore | 2 +
core/pom.xml | 129 ++++
.../testing/core/randomwalk/Environment.java | 248 ++++++++
.../testing/core/randomwalk/Fixture.java | 28 +
.../testing/core/randomwalk/Framework.java | 109 ++++
.../testing/core/randomwalk/Module.java | 624 +++++++++++++++++++
.../accumulo/testing/core/randomwalk/Node.java | 100 +++
.../accumulo/testing/core/randomwalk/State.java | 129 ++++
.../accumulo/testing/core/randomwalk/Test.java | 28 +
.../core/randomwalk/bulk/BulkImportTest.java | 85 +++
.../core/randomwalk/bulk/BulkMinusOne.java | 35 ++
.../core/randomwalk/bulk/BulkPlusOne.java | 117 ++++
.../testing/core/randomwalk/bulk/BulkTest.java | 44 ++
.../testing/core/randomwalk/bulk/Compact.java | 34 +
.../core/randomwalk/bulk/ConsistencyCheck.java | 57 ++
.../testing/core/randomwalk/bulk/Merge.java | 61 ++
.../core/randomwalk/bulk/SelectiveBulkTest.java | 42 ++
.../core/randomwalk/bulk/SelectiveQueueing.java | 50 ++
.../testing/core/randomwalk/bulk/Setup.java | 82 +++
.../testing/core/randomwalk/bulk/Split.java | 41 ++
.../testing/core/randomwalk/bulk/Verify.java | 148 +++++
.../core/randomwalk/concurrent/AddSplits.java | 62 ++
.../core/randomwalk/concurrent/Apocalypse.java | 34 +
.../core/randomwalk/concurrent/BatchScan.java | 84 +++
.../core/randomwalk/concurrent/BatchWrite.java | 82 +++
.../core/randomwalk/concurrent/BulkImport.java | 151 +++++
.../concurrent/ChangeAuthorizations.java | 63 ++
.../concurrent/ChangePermissions.java | 156 +++++
.../randomwalk/concurrent/CheckPermission.java | 70 +++
.../core/randomwalk/concurrent/CloneTable.java | 66 ++
.../core/randomwalk/concurrent/Compact.java | 57 ++
.../concurrent/ConcurrentFixture.java | 73 +++
.../core/randomwalk/concurrent/Config.java | 235 +++++++
.../randomwalk/concurrent/CreateNamespace.java | 49 ++
.../core/randomwalk/concurrent/CreateTable.java | 61 ++
.../core/randomwalk/concurrent/CreateUser.java | 49 ++
.../randomwalk/concurrent/DeleteNamespace.java | 52 ++
.../core/randomwalk/concurrent/DeleteRange.java | 66 ++
.../core/randomwalk/concurrent/DeleteTable.java | 49 ++
.../core/randomwalk/concurrent/DropUser.java | 48 ++
.../randomwalk/concurrent/IsolatedScan.java | 74 +++
.../core/randomwalk/concurrent/ListSplits.java | 54 ++
.../core/randomwalk/concurrent/Merge.java | 59 ++
.../randomwalk/concurrent/OfflineTable.java | 56 ++
.../randomwalk/concurrent/RenameNamespace.java | 53 ++
.../core/randomwalk/concurrent/RenameTable.java | 90 +++
.../core/randomwalk/concurrent/Replication.java | 203 ++++++
.../core/randomwalk/concurrent/ScanTable.java | 72 +++
.../core/randomwalk/concurrent/Setup.java | 71 +++
.../core/randomwalk/concurrent/Shutdown.java | 63 ++
.../core/randomwalk/concurrent/StartAll.java | 58 ++
.../randomwalk/concurrent/StopTabletServer.java | 84 +++
.../core/randomwalk/conditional/Compact.java | 48 ++
.../core/randomwalk/conditional/Flush.java | 48 ++
.../core/randomwalk/conditional/Init.java | 94 +++
.../core/randomwalk/conditional/Merge.java | 49 ++
.../core/randomwalk/conditional/Setup.java | 60 ++
.../core/randomwalk/conditional/Split.java | 45 ++
.../core/randomwalk/conditional/TearDown.java | 35 ++
.../core/randomwalk/conditional/Transfer.java | 135 ++++
.../core/randomwalk/conditional/Utils.java | 35 ++
.../core/randomwalk/conditional/Verify.java | 89 +++
.../testing/core/randomwalk/image/Commit.java | 35 ++
.../core/randomwalk/image/ImageFixture.java | 134 ++++
.../testing/core/randomwalk/image/ScanMeta.java | 111 ++++
.../testing/core/randomwalk/image/TableOp.java | 81 +++
.../testing/core/randomwalk/image/Verify.java | 131 ++++
.../testing/core/randomwalk/image/Write.java | 97 +++
.../core/randomwalk/multitable/Commit.java | 40 ++
.../core/randomwalk/multitable/CopyTable.java | 92 +++
.../core/randomwalk/multitable/CopyTool.java | 131 ++++
.../core/randomwalk/multitable/CreateTable.java | 67 ++
.../core/randomwalk/multitable/DropTable.java | 51 ++
.../multitable/MultiTableFixture.java | 74 +++
.../randomwalk/multitable/OfflineTable.java | 47 ++
.../core/randomwalk/multitable/Write.java | 89 +++
.../randomwalk/security/AlterSystemPerm.java | 101 +++
.../core/randomwalk/security/AlterTable.java | 74 +++
.../randomwalk/security/AlterTablePerm.java | 180 ++++++
.../core/randomwalk/security/Authenticate.java | 82 +++
.../core/randomwalk/security/ChangePass.java | 94 +++
.../core/randomwalk/security/CreateTable.java | 75 +++
.../core/randomwalk/security/CreateUser.java | 70 +++
.../core/randomwalk/security/DropTable.java | 87 +++
.../core/randomwalk/security/DropUser.java | 68 ++
.../randomwalk/security/SecurityFixture.java | 120 ++++
.../randomwalk/security/SecurityHelper.java | 215 +++++++
.../core/randomwalk/security/SetAuths.java | 100 +++
.../core/randomwalk/security/TableOp.java | 257 ++++++++
.../core/randomwalk/security/Validate.java | 124 ++++
.../randomwalk/security/WalkingSecurity.java | 505 +++++++++++++++
.../core/randomwalk/sequential/BatchVerify.java | 132 ++++
.../core/randomwalk/sequential/Commit.java | 36 ++
.../randomwalk/sequential/MapRedVerify.java | 79 +++
.../randomwalk/sequential/MapRedVerifyTool.java | 156 +++++
.../sequential/SequentialFixture.java | 80 +++
.../core/randomwalk/sequential/Write.java | 50 ++
.../core/randomwalk/shard/BulkInsert.java | 191 ++++++
.../core/randomwalk/shard/CloneIndex.java | 45 ++
.../testing/core/randomwalk/shard/Commit.java | 33 +
.../core/randomwalk/shard/CompactFilter.java | 94 +++
.../testing/core/randomwalk/shard/Delete.java | 58 ++
.../core/randomwalk/shard/DeleteSomeDocs.java | 80 +++
.../core/randomwalk/shard/DeleteWord.java | 97 +++
.../core/randomwalk/shard/ExportIndex.java | 118 ++++
.../testing/core/randomwalk/shard/Flush.java | 45 ++
.../testing/core/randomwalk/shard/Grep.java | 97 +++
.../testing/core/randomwalk/shard/Insert.java | 136 ++++
.../testing/core/randomwalk/shard/Merge.java | 49 ++
.../testing/core/randomwalk/shard/Reindex.java | 66 ++
.../testing/core/randomwalk/shard/Search.java | 105 ++++
.../core/randomwalk/shard/ShardFixture.java | 137 ++++
.../testing/core/randomwalk/shard/SortTool.java | 75 +++
.../testing/core/randomwalk/shard/Split.java | 41 ++
.../core/randomwalk/shard/VerifyIndex.java | 71 +++
.../core/randomwalk/unit/CreateTable.java | 30 +
.../core/randomwalk/unit/DeleteTable.java | 29 +
.../testing/core/randomwalk/unit/Ingest.java | 29 +
.../testing/core/randomwalk/unit/Scan.java | 29 +
.../testing/core/randomwalk/unit/Verify.java | 29 +
core/src/main/resources/randomwalk/module.xsd | 69 ++
.../main/resources/randomwalk/modules/All.xml | 65 ++
.../main/resources/randomwalk/modules/Bulk.xml | 61 ++
.../resources/randomwalk/modules/Concurrent.xml | 181 ++++++
.../randomwalk/modules/Conditional.xml | 74 +++
.../main/resources/randomwalk/modules/Image.xml | 70 +++
.../resources/randomwalk/modules/LongClean.xml | 60 ++
.../resources/randomwalk/modules/LongDirty.xml | 60 ++
.../resources/randomwalk/modules/LongEach.xml | 54 ++
.../resources/randomwalk/modules/MultiTable.xml | 60 ++
.../resources/randomwalk/modules/Security.xml | 224 +++++++
.../resources/randomwalk/modules/Sequential.xml | 51 ++
.../main/resources/randomwalk/modules/Shard.xml | 123 ++++
.../resources/randomwalk/modules/ShortClean.xml | 60 ++
.../resources/randomwalk/modules/ShortDirty.xml | 60 ++
.../resources/randomwalk/modules/ShortEach.xml | 54 ++
.../resources/randomwalk/modules/unit/Basic.xml | 37 ++
.../randomwalk/modules/unit/Simple.xml | 43 ++
.../testing/core/randomwalk/FrameworkTest.java | 67 ++
.../randomwalk/ReplicationRandomWalkIT.java | 66 ++
core/src/test/resources/log4j.properties | 21 +
pom.xml | 172 +++++
143 files changed, 12562 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f534230
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+/.classpath
+/.project
+/.settings/
+/target/
+/*.iml
+/.idea
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/.gitignore
----------------------------------------------------------------------
diff --git a/core/.gitignore b/core/.gitignore
new file mode 100644
index 0000000..17bb010
--- /dev/null
+++ b/core/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/*.iml
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
new file mode 100644
index 0000000..83998d5
--- /dev/null
+++ b/core/pom.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-testing</artifactId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>accumulo-testing-core</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Accumulo Testing Core</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-fate</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-master</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-test</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>create-shade-jar</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>testing-shade-jar</id>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>shaded</shadedClassifierName>
+ <artifactSet>
+ <excludes>
+ <exclude>org.apache.accumulo:accumulo-native</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
new file mode 100644
index 0000000..5684353
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The test environment that is available for randomwalk tests. This includes configuration properties that are available to any randomwalk test and facilities
+ * for creating client-side objects. This class is not thread-safe.
+ */
+public class Environment {
+ /**
+ * The configuration property key for a username.
+ */
+ public static final String KEY_USERNAME = "USERNAME";
+ /**
+ * The configuration property key for a password.
+ */
+ public static final String KEY_PASSWORD = "PASSWORD";
+ /**
+ * The configuration property key for a keytab
+ */
+ public static final String KEY_KEYTAB = "KEYTAB";
+ /**
+ * The configuration property key for the instance name.
+ */
+ public static final String KEY_INSTANCE = "INSTANCE";
+ /**
+ * The configuration property key for the comma-separated list of ZooKeepers.
+ */
+ public static final String KEY_ZOOKEEPERS = "ZOOKEEPERS";
+ /**
+ * The configuration property key for the maximum memory for the multi-table batch writer.
+ */
+ public static final String KEY_MAX_MEM = "MAX_MEM";
+ /**
+ * The configuration property key for the maximum latency, in milliseconds, for the multi-table batch writer.
+ */
+ public static final String KEY_MAX_LATENCY = "MAX_LATENCY";
+ /**
+ * The configuration property key for the number of write threads for the multi-table batch writer.
+ */
+ public static final String KEY_NUM_THREADS = "NUM_THREADS";
+
+ private static final Logger log = LoggerFactory.getLogger(Environment.class);
+
+ private final Properties p;
+ private Instance instance = null;
+ private Connector connector = null;
+ private MultiTableBatchWriter mtbw = null;
+
+ /**
+ * Creates a new test environment.
+ *
+ * @param p
+ * configuration properties
+ * @throws NullPointerException
+ * if p is null
+ */
+ public Environment(Properties p) {
+ requireNonNull(p);
+ this.p = p;
+ }
+
+ /**
+ * Gets a copy of the configuration properties.
+ *
+ * @return a copy of the configuration properties
+ */
+ Properties copyConfigProperties() {
+ return new Properties(p);
+ }
+
+ /**
+ * Gets a configuration property.
+ *
+ * @param key
+ * key
+ * @return property value
+ */
+ public String getConfigProperty(String key) {
+ return p.getProperty(key);
+ }
+
+ /**
+ * Gets the configured username.
+ *
+ * @return username
+ */
+ public String getUserName() {
+ return p.getProperty(KEY_USERNAME);
+ }
+
+ /**
+ * Gets the configured password.
+ *
+ * @return password
+ */
+ public String getPassword() {
+ return p.getProperty(KEY_PASSWORD);
+ }
+
+ /**
+ * Gets the configured keytab.
+ *
+ * @return path to keytab
+ */
+ public String getKeytab() {
+ return p.getProperty(KEY_KEYTAB);
+ }
+
+ /**
+ * Gets this process's ID.
+ *
+ * @return pid
+ */
+ public String getPid() {
+ return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
+ }
+
+ /**
+ * Gets an authentication token based on the configured password.
+ *
+ * @return authentication token
+ */
+ public AuthenticationToken getToken() {
+ String password = getPassword();
+ if (null != password) {
+ return new PasswordToken(getPassword());
+ }
+ String keytab = getKeytab();
+ if (null != keytab) {
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists() || !keytabFile.isFile()) {
+ throw new IllegalArgumentException("Provided keytab is not a normal file: " + keytab);
+ }
+ try {
+ UserGroupInformation.loginUserFromKeytab(getUserName(), keytabFile.getAbsolutePath());
+ return new KerberosToken();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to login", e);
+ }
+ }
+ throw new IllegalArgumentException("Must provide password or keytab in configuration");
+ }
+
+ /**
+ * Gets an Accumulo instance object. The same instance is reused after the first call.
+ *
+ * @return instance
+ */
+ public Instance getInstance() {
+ if (instance == null) {
+ String instance = p.getProperty(KEY_INSTANCE);
+ String zookeepers = p.getProperty(KEY_ZOOKEEPERS);
+ this.instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers));
+ }
+ return instance;
+ }
+
+ /**
+ * Gets an Accumulo connector. The same connector is reused after the first call.
+ *
+ * @return connector
+ */
+ public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
+ if (connector == null) {
+ connector = getInstance().getConnector(getUserName(), getToken());
+ }
+ return connector;
+ }
+
+ /**
+ * Gets a multitable batch writer. The same object is reused after the first call unless it is reset.
+ *
+ * @return multitable batch writer
+ * @throws NumberFormatException
+ * if any of the numeric batch writer configuration properties cannot be parsed
+ * @throws NumberFormatException
+ * if any configuration property cannot be parsed
+ */
+ public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException {
+ if (mtbw == null) {
+ long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM));
+ long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY));
+ int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS));
+ mtbw = getConnector().createMultiTableBatchWriter(
+ new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads));
+ }
+ return mtbw;
+ }
+
+ /**
+ * Checks if a multitable batch writer has been created by this wrapper.
+ *
+ * @return true if multitable batch writer is already created
+ */
+ public boolean isMultiTableBatchWriterInitialized() {
+ return mtbw != null;
+ }
+
+ /**
+ * Clears the multitable batch writer previously created and remembered by this wrapper.
+ */
+ public void resetMultiTableBatchWriter() {
+ if (mtbw == null)
+ return;
+ if (!mtbw.isClosed()) {
+ log.warn("Setting non-closed MultiTableBatchWriter to null (leaking resources)");
+ }
+ mtbw = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Fixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Fixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Fixture.java
new file mode 100644
index 0000000..5ac280e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Fixture.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import org.apache.log4j.Logger;
+
+public abstract class Fixture {
+
+ protected final Logger log = Logger.getLogger(this.getClass());
+
+ public abstract void setUp(State state, Environment env) throws Exception;
+
+ public abstract void tearDown(State state, Environment env) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
new file mode 100644
index 0000000..1a5700e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.HashMap;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+public class Framework {
+
+ private static final Logger log = Logger.getLogger(Framework.class);
+ private HashMap<String,Node> nodes = new HashMap<>();
+ private static final Framework INSTANCE = new Framework();
+
+ /**
+ * @return Singleton instance of Framework
+ */
+ public static Framework getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Run random walk framework
+ *
+ * @param startName
+ * Full name of starting graph or test
+ */
+ public int run(String startName, State state, Environment env) {
+
+ try {
+ Node node = getNode(startName);
+ node.visit(state, env, new Properties());
+ } catch (Exception e) {
+ log.error("Error during random walk", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ /**
+ * Creates node (if it does not already exist) and inserts into map
+ *
+ * @param id
+ * Name of node
+ * @return Node specified by id
+ */
+ public Node getNode(String id) throws Exception {
+
+ // check for node in nodes
+ if (nodes.containsKey(id)) {
+ return nodes.get(id);
+ }
+
+ // otherwise create and put in nodes
+ Node node = null;
+ if (id.endsWith(".xml")) {
+ node = new Module(new File("/randomwalk/modules/" + id));
+ } else {
+ node = (Test) Class.forName(id).newInstance();
+ }
+ nodes.put(id, node);
+ return node;
+ }
+
+ static class Opts extends org.apache.accumulo.core.cli.Help {
+ @Parameter(names = "--configDir", required = true, description = "directory containing the test configuration")
+ String configDir;
+ @Parameter(names = "--module", required = true, description = "the name of the module to run")
+ String module;
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ if (args.length != 2) {
+ System.out.println("Usage: Framework <propsPath> <module>");
+ System.exit(-1);
+ }
+
+ Properties props = new Properties();
+ FileInputStream fis = new FileInputStream(args[0]);
+ props.load(fis);
+ fis.close();
+
+ State state = new State();
+ Environment env = new Environment(props);
+ int retval = getInstance().run(args[1], state, env);
+
+ System.exit(retval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
new file mode 100644
index 0000000..1a3d059
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
@@ -0,0 +1,624 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+/**
+ * A module is directed graph of tests
+ */
+public class Module extends Node {
+
+ private static final Logger log = Logger.getLogger(Module.class);
+
+ private class Dummy extends Node {
+
+ String name;
+
+ Dummy(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) {
+ String print;
+ if ((print = props.getProperty("print")) != null) {
+ Level level = Level.toLevel(print);
+ log.log(level, name);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ private class Alias extends Node {
+
+ Node target;
+ String targetId;
+ String id;
+
+ Alias(String id) {
+ target = null;
+ this.id = id;
+ }
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ throw new Exception("You don't visit aliases!");
+ }
+
+ @Override
+ public String toString() {
+ return id;
+ }
+
+ public void update(String node) throws Exception {
+ targetId = node;
+ target = getNode(node);
+ }
+
+ public Node get() {
+ return target;
+ }
+
+ public String getTargetId() {
+ return targetId;
+ }
+ }
+
+ private HashMap<String,Node> nodes = new HashMap<>();
+ private HashMap<String,Properties> localProps = new HashMap<>();
+
+ private class Edge {
+ String nodeId;
+ int weight;
+ }
+
+ private class AdjList {
+
+ private List<Edge> edges = new ArrayList<>();
+ private int totalWeight = 0;
+ private Random rand = new Random();
+
+ /**
+ * Adds a neighbor node and weight of edge
+ */
+ private void addEdge(String nodeId, int weight) {
+
+ totalWeight += weight;
+
+ Edge e = new Edge();
+ e.nodeId = nodeId;
+ e.weight = weight;
+ edges.add(e);
+ }
+
+ /**
+ * Chooses a random neighbor node
+ *
+ * @return Node or null if no edges
+ */
+ private String randomNeighbor() throws Exception {
+
+ String nodeId = null;
+ rand = new Random();
+
+ int randNum = rand.nextInt(totalWeight) + 1;
+ int sum = 0;
+
+ for (Edge e : edges) {
+ nodeId = e.nodeId;
+ sum += e.weight;
+ if (randNum <= sum) {
+ break;
+ }
+ }
+ return nodeId;
+ }
+ }
+
+ private HashMap<String,String> prefixes = new HashMap<>();
+ private HashMap<String,AdjList> adjMap = new HashMap<>();
+ private HashMap<String,Set<String>> aliasMap = new HashMap<>();
+ private final File xmlFile;
+ private String initNodeId;
+ private Fixture fixture = null;
+
+ public Module(File xmlFile) throws Exception {
+ this.xmlFile = xmlFile;
+ loadFromXml();
+ }
+
+ @Override
+ public void visit(final State state, final Environment env, Properties props) throws Exception {
+ int maxHops, maxSec;
+ boolean teardown;
+
+ Properties initProps = getProps("_init");
+ initProps.putAll(props);
+ String prop;
+ if ((prop = initProps.getProperty("maxHops")) == null || prop.equals("0") || prop.equals(""))
+ maxHops = Integer.MAX_VALUE;
+ else
+ maxHops = Integer.parseInt(initProps.getProperty("maxHops", "0"));
+
+ if ((prop = initProps.getProperty("maxSec")) == null || prop.equals("0") || prop.equals(""))
+ maxSec = Integer.MAX_VALUE;
+ else
+ maxSec = Integer.parseInt(initProps.getProperty("maxSec", "0"));
+
+ if ((prop = initProps.getProperty("teardown")) == null || prop.equals("true") || prop.equals(""))
+ teardown = true;
+ else
+ teardown = false;
+
+ if (fixture != null) {
+ fixture.setUp(state, env);
+ }
+
+ ExecutorService service = new SimpleThreadPool(1, "RandomWalk Runner");
+
+ try {
+ Node initNode = getNode(initNodeId);
+
+ boolean test = false;
+ if (initNode instanceof Test) {
+ startTimer(initNode);
+ test = true;
+ }
+ initNode.visit(state, env, getProps(initNodeId));
+ if (test)
+ stopTimer(initNode);
+
+ // update aliases
+ Set<String> aliases;
+ if ((aliases = aliasMap.get(initNodeId)) != null)
+ for (String alias : aliases) {
+ ((Alias) nodes.get(alias)).update(initNodeId);
+ }
+
+ String curNodeId = initNodeId;
+ int numHops = 0;
+ long startTime = System.currentTimeMillis() / 1000;
+ while (true) {
+ // check if END state was reached
+ if (curNodeId.equalsIgnoreCase("END")) {
+ log.debug("reached END state");
+ break;
+ }
+ // check if maxSec was reached
+ long curTime = System.currentTimeMillis() / 1000;
+ if ((curTime - startTime) > maxSec) {
+ log.debug("reached maxSec(" + maxSec + ")");
+ break;
+ }
+
+ // The number of seconds before the test should exit
+ long secondsRemaining = maxSec - (curTime - startTime);
+
+ // check if maxHops was reached
+ if (numHops > maxHops) {
+ log.debug("reached maxHops(" + maxHops + ")");
+ break;
+ }
+ numHops++;
+
+ if (!adjMap.containsKey(curNodeId) && !curNodeId.startsWith("alias.")) {
+ throw new Exception("Reached node(" + curNodeId + ") without outgoing edges in module(" + this + ")");
+ }
+ AdjList adj = adjMap.get(curNodeId);
+ String nextNodeId = adj.randomNeighbor();
+ final Node nextNode;
+ Node nextNodeOrAlias = getNode(nextNodeId);
+ if (nextNodeOrAlias instanceof Alias) {
+ nextNodeId = ((Alias) nextNodeOrAlias).getTargetId();
+ nextNode = ((Alias) nextNodeOrAlias).get();
+ } else {
+ nextNode = nextNodeOrAlias;
+ }
+ final Properties nodeProps = getProps(nextNodeId);
+ try {
+ test = false;
+ if (nextNode instanceof Test) {
+ startTimer(nextNode);
+ test = true;
+ }
+
+ // Wrap the visit of the next node in the module in a callable that returns a thrown exception
+ FutureTask<Exception> task = new FutureTask<>(new Callable<Exception>() {
+
+ @Override
+ public Exception call() throws Exception {
+ try {
+ nextNode.visit(state, env, nodeProps);
+ return null;
+ } catch (Exception e) {
+ return e;
+ }
+ }
+
+ });
+
+ // Run the task (should execute immediately)
+ service.submit(task);
+
+ Exception nodeException;
+ try {
+ // Bound the time we'll wait for the node to complete
+ nodeException = task.get(secondsRemaining, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ log.warn("Interrupted waiting for " + nextNode.getClass().getSimpleName() + " to complete. Exiting.", e);
+ break;
+ } catch (ExecutionException e) {
+ log.error("Caught error executing " + nextNode.getClass().getSimpleName(), e);
+ throw e;
+ } catch (TimeoutException e) {
+ log.info("Timed out waiting for " + nextNode.getClass().getSimpleName() + " to complete (waited " + secondsRemaining + " seconds). Exiting.", e);
+ break;
+ }
+
+ // The RandomWalk node throw an Exception that that Callable handed back
+ // Throw it and let the Module perform cleanup
+ if (null != nodeException) {
+ throw nodeException;
+ }
+
+ if (test)
+ stopTimer(nextNode);
+ } catch (Exception e) {
+ log.debug("Connector belongs to user: " + env.getConnector().whoami());
+ log.debug("Exception occured at: " + System.currentTimeMillis());
+ log.debug("Properties for node: " + nextNodeId);
+ for (Entry<Object,Object> entry : nodeProps.entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
+ log.debug("Overall Configuration Properties");
+ for (Entry<Object,Object> entry : env.copyConfigProperties().entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
+ log.debug("State information");
+ for (String key : new TreeSet<>(state.getMap().keySet())) {
+ Object value = state.getMap().get(key);
+ String logMsg = " " + key + ": ";
+ if (value == null)
+ logMsg += "null";
+ else if (value instanceof String || value instanceof Map || value instanceof Collection || value instanceof Number)
+ logMsg += value;
+ else if (value instanceof byte[])
+ logMsg += new String((byte[]) value, UTF_8);
+ else if (value instanceof PasswordToken)
+ logMsg += new String(((PasswordToken) value).getPassword(), UTF_8);
+ else
+ logMsg += value.getClass() + " - " + value;
+
+ log.debug(logMsg);
+ }
+ throw new Exception("Error running node " + nextNodeId, e);
+ }
+
+ // update aliases
+ if ((aliases = aliasMap.get(curNodeId)) != null)
+ for (String alias : aliases) {
+ ((Alias) nodes.get(alias)).update(curNodeId);
+ }
+
+ curNodeId = nextNodeId;
+ }
+ } finally {
+ if (null != service) {
+ service.shutdownNow();
+ }
+ }
+
+ if (teardown && (fixture != null)) {
+ log.debug("tearing down module");
+ fixture.tearDown(state, env);
+ }
+ }
+
+ Thread timer = null;
+ final int time = 5 * 1000 * 60;
+ AtomicBoolean runningLong = new AtomicBoolean(false);
+ long systemTime;
+
+ /**
+ *
+ */
+ private void startTimer(final Node initNode) {
+ runningLong.set(false);
+ timer = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ systemTime = System.currentTimeMillis();
+ Thread.sleep(time);
+ } catch (InterruptedException ie) {
+ return;
+ }
+ long timeSinceLastProgress = System.currentTimeMillis() - initNode.lastProgress();
+ if (timeSinceLastProgress > time) {
+ log.warn("Node " + initNode + " has been running for " + timeSinceLastProgress / 1000.0 + " seconds. You may want to look into it.");
+ runningLong.set(true);
+ }
+ }
+ });
+ initNode.makingProgress();
+ timer.start();
+ }
+
+ /**
+ *
+ */
+ private void stopTimer(Node nextNode) {
+ synchronized (timer) {
+ timer.interrupt();
+ try {
+ timer.join();
+ } catch (InterruptedException e) {
+ log.error("Failed to join timer '" + timer.getName() + "'.", e);
+ }
+ }
+ if (runningLong.get())
+ log.warn("Node " + nextNode + ", which was running long, has now completed after " + (System.currentTimeMillis() - systemTime) / 1000.0 + " seconds");
+ }
+
+ @Override
+ public String toString() {
+ return xmlFile.toString();
+ }
+
+ private String getFullName(String name) {
+
+ int index = name.indexOf(".");
+ if (index == -1 || name.endsWith(".xml")) {
+ return name;
+ }
+
+ String id = name.substring(0, index);
+
+ if (!prefixes.containsKey(id)) {
+ log.warn("Id (" + id + ") was not found in prefixes");
+ return name;
+ }
+
+ return prefixes.get(id).concat(name.substring(index + 1));
+ }
+
+ private Node createNode(String id, String src) throws Exception {
+
+ // check if id indicates dummy node
+ if (id.equalsIgnoreCase("END") || id.startsWith("dummy")) {
+ if (nodes.containsKey(id) == false) {
+ nodes.put(id, new Dummy(id));
+ }
+ return nodes.get(id);
+ }
+
+ if (id.startsWith("alias")) {
+ if (nodes.containsKey(id) == false) {
+ nodes.put(id, new Alias(id));
+ }
+ return nodes.get(id);
+ }
+
+ // grab node from framework based on its id or src
+ Node node;
+ if (src == null || src.isEmpty()) {
+ node = Framework.getInstance().getNode(getFullName(id));
+ } else {
+ node = Framework.getInstance().getNode(getFullName(src));
+ }
+
+ // add to node to this module's map
+ nodes.put(id, node);
+
+ return node;
+ }
+
+ private Node getNode(String id) throws Exception {
+
+ if (nodes.containsKey(id)) {
+ return nodes.get(id);
+ }
+
+ if (id.equalsIgnoreCase("END")) {
+ nodes.put(id, new Dummy(id));
+ return nodes.get(id);
+ }
+
+ return Framework.getInstance().getNode(getFullName(id));
+ }
+
+ private Properties getProps(String nodeId) {
+ if (localProps.containsKey(nodeId)) {
+ return localProps.get(nodeId);
+ }
+ return new Properties();
+ }
+
+ private void loadFromXml() throws Exception {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder docbuilder;
+ Document d = null;
+
+ // set the schema
+ SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ Schema moduleSchema = sf.newSchema(this.getClass().getClassLoader().getResource("randomwalk/module.xsd"));
+ dbf.setSchema(moduleSchema);
+
+ // parse the document
+ try {
+ docbuilder = dbf.newDocumentBuilder();
+ d = docbuilder.parse(xmlFile);
+ } catch (Exception e) {
+ log.error("Failed to parse: " + xmlFile, e);
+ throw new Exception("Failed to parse: " + xmlFile);
+ }
+
+ // parse packages
+ NodeList nodelist = d.getDocumentElement().getElementsByTagName("package");
+ for (int i = 0; i < nodelist.getLength(); i++) {
+ Element el = (Element) nodelist.item(i);
+ String value = el.getAttribute("value");
+ if (!value.endsWith(".")) {
+ value = value.concat(".");
+ }
+ prefixes.put(el.getAttribute("prefix"), value);
+ }
+
+ // parse fixture node
+ nodelist = d.getDocumentElement().getElementsByTagName("fixture");
+ if (nodelist.getLength() > 0) {
+ Element fixtureEl = (Element) nodelist.item(0);
+ fixture = (Fixture) Class.forName(getFullName(fixtureEl.getAttribute("id"))).newInstance();
+ }
+
+ // parse initial node
+ Element initEl = (Element) d.getDocumentElement().getElementsByTagName("init").item(0);
+ initNodeId = initEl.getAttribute("id");
+ Properties initProps = new Properties();
+ String attr = initEl.getAttribute("maxHops");
+
+ if (attr != null)
+ initProps.setProperty("maxHops", attr);
+ attr = initEl.getAttribute("maxSec");
+
+ if (attr != null)
+ initProps.setProperty("maxSec", attr);
+ attr = initEl.getAttribute("teardown");
+
+ if (attr != null)
+ initProps.setProperty("teardown", attr);
+ localProps.put("_init", initProps);
+
+ // parse all nodes
+ nodelist = d.getDocumentElement().getElementsByTagName("node");
+ for (int i = 0; i < nodelist.getLength(); i++) {
+
+ Element nodeEl = (Element) nodelist.item(i);
+
+ // get attributes
+ String id = nodeEl.getAttribute("id");
+ if (adjMap.containsKey(id)) {
+ throw new Exception("Module already contains: " + id);
+ }
+ String src = nodeEl.getAttribute("src");
+
+ // create node
+ createNode(id, src);
+
+ // set some attributes in properties for later use
+ Properties props = new Properties();
+ props.setProperty("maxHops", nodeEl.getAttribute("maxHops"));
+ props.setProperty("maxSec", nodeEl.getAttribute("maxSec"));
+ props.setProperty("teardown", nodeEl.getAttribute("teardown"));
+
+ // parse aliases
+ NodeList aliaslist = nodeEl.getElementsByTagName("alias");
+ Set<String> aliases = new TreeSet<>();
+ for (int j = 0; j < aliaslist.getLength(); j++) {
+ Element propEl = (Element) aliaslist.item(j);
+
+ if (!propEl.hasAttribute("name")) {
+ throw new Exception("Node " + id + " has alias with no identifying name");
+ }
+
+ String key = "alias." + propEl.getAttribute("name");
+
+ aliases.add(key);
+ createNode(key, null);
+ }
+ if (aliases.size() > 0)
+ aliasMap.put(id, aliases);
+
+ // parse properties of nodes
+ NodeList proplist = nodeEl.getElementsByTagName("property");
+ for (int j = 0; j < proplist.getLength(); j++) {
+ Element propEl = (Element) proplist.item(j);
+
+ if (!propEl.hasAttribute("key") || !propEl.hasAttribute("value")) {
+ throw new Exception("Node " + id + " has property with no key or value");
+ }
+
+ String key = propEl.getAttribute("key");
+
+ if (key.equals("maxHops") || key.equals("maxSec") || key.equals("teardown")) {
+ throw new Exception("The following property can only be set in attributes: " + key);
+ }
+
+ props.setProperty(key, propEl.getAttribute("value"));
+ }
+ localProps.put(id, props);
+
+ // parse edges of nodes
+ AdjList edges = new AdjList();
+ adjMap.put(id, edges);
+ NodeList edgelist = nodeEl.getElementsByTagName("edge");
+ if (edgelist.getLength() == 0) {
+ throw new Exception("Node " + id + " has no edges!");
+ }
+ for (int j = 0; j < edgelist.getLength(); j++) {
+ Element edgeEl = (Element) edgelist.item(j);
+
+ String edgeID = edgeEl.getAttribute("id");
+
+ if (!edgeEl.hasAttribute("weight")) {
+ throw new Exception("Edge with id=" + edgeID + " is missing weight");
+ }
+
+ int weight = Integer.parseInt(edgeEl.getAttribute("weight"));
+ edges.addEdge(edgeID, weight);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
new file mode 100644
index 0000000..b2c2f97
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Node.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Represents a point in graph of RandomFramework
+ */
+public abstract class Node {
+
+ protected final Logger log = Logger.getLogger(this.getClass());
+ long progress = System.currentTimeMillis();
+
+ /**
+ * Visits node
+ *
+ * @param state
+ * Random walk state passed between nodes
+ * @param env
+ * test environment
+ */
+ public abstract void visit(State state, Environment env, Properties props) throws Exception;
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null)
+ return false;
+ return toString().equals(o.toString());
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ synchronized public void makingProgress() {
+ progress = System.currentTimeMillis();
+ }
+
+ synchronized public long lastProgress() {
+ return progress;
+ }
+
+ protected String getMapReduceJars() {
+
+ String acuHome = System.getenv("ACCUMULO_HOME");
+ String zkHome = System.getenv("ZOOKEEPER_HOME");
+
+ if (acuHome == null || zkHome == null) {
+ throw new RuntimeException("ACCUMULO or ZOOKEEPER home not set!");
+ }
+
+ String retval = null;
+
+ File zkLib = new File(zkHome);
+ String[] files = zkLib.list();
+ if (files != null) {
+ for (int i = 0; i < files.length; i++) {
+ String f = files[i];
+ if (f.matches("^zookeeper-.+jar$")) {
+ if (retval == null) {
+ retval = String.format("%s/%s", zkLib.getAbsolutePath(), f);
+ } else {
+ retval += String.format(",%s/%s", zkLib.getAbsolutePath(), f);
+ }
+ }
+ }
+ }
+
+ File libdir = new File(acuHome + "/lib");
+ for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math3 libthrift htrace-core".split(" ")) {
+ retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
+ }
+
+ return retval;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/State.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/State.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/State.java
new file mode 100644
index 0000000..b619674
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/State.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+import java.util.HashMap;
+
+/**
+ * A structure for storing state kept during a test. This class is not thread-safe.
+ */
+public class State {
+
+ private HashMap<String,Object> stateMap = new HashMap<>();
+
+ /**
+ * Creates new empty state.
+ */
+ public State() {}
+
+ /**
+ * Sets a state object.
+ *
+ * @param key
+ * key for object
+ * @param value
+ * object
+ */
+ public void set(String key, Object value) {
+ stateMap.put(key, value);
+ }
+
+ /**
+ * Removes a state object.
+ *
+ * @param key
+ * key for object
+ */
+ public void remove(String key) {
+ stateMap.remove(key);
+ }
+
+ /**
+ * Gets a state object.
+ *
+ * @param key
+ * key for object
+ * @return value object
+ * @throws RuntimeException
+ * if state object is not present
+ */
+ public Object get(String key) {
+ if (stateMap.containsKey(key) == false) {
+ throw new RuntimeException("State does not contain " + key);
+ }
+ return stateMap.get(key);
+ }
+
+ /**
+ * Gets a state object, returning null if it is absent.
+ *
+ * @param key
+ * key for object
+ * @return value object, or null if not present
+ */
+ public Object getOkIfAbsent(String key) {
+ return stateMap.get(key);
+ }
+
+ /**
+ * Gets the map of state objects. The backing map for state is returned, so changes to it affect the state.
+ *
+ * @return state map
+ */
+ HashMap<String,Object> getMap() {
+ return stateMap;
+ }
+
+ /**
+ * Gets a state object as a string.
+ *
+ * @param key
+ * key for object
+ * @return value as string
+ * @throws ClassCastException
+ * if the value object is not a string
+ */
+ public String getString(String key) {
+ return (String) stateMap.get(key);
+ }
+
+ /**
+ * Gets a state object as an integer.
+ *
+ * @param key
+ * key for object
+ * @return value as integer
+ * @throws ClassCastException
+ * if the value object is not an integer
+ */
+ public Integer getInteger(String key) {
+ return (Integer) stateMap.get(key);
+ }
+
+ /**
+ * Gets a state object as a long.
+ *
+ * @param key
+ * key for object
+ * @return value as long
+ * @throws ClassCastException
+ * if the value object is not a long
+ */
+ public Long getLong(String key) {
+ return (Long) stateMap.get(key);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Test.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Test.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Test.java
new file mode 100644
index 0000000..a8e117a
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Test.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk;
+
+/**
+ * Tests are extended by users to perform actions on accumulo and are a node of the graph
+ */
+public abstract class Test extends Node {
+
+ @Override
+ public String toString() {
+ return getClass().getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkImportTest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkImportTest.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkImportTest.java
new file mode 100644
index 0000000..317a294
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkImportTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+
+/**
+ * If we have a sufficient back-up of imports, let them work off before adding even more bulk-imports. Imports of PlusOne must always be balanced with imports
+ * of MinusOne.
+ */
+public abstract class BulkImportTest extends BulkTest {
+
+ public static final String SKIPPED_IMPORT = "skipped.import", TRUE = Boolean.TRUE.toString(), FALSE = Boolean.FALSE.toString();
+
+ @Override
+ public void visit(final State state, Environment env, Properties props) throws Exception {
+ /**
+ * Each visit() is performed sequentially and then submitted to the threadpool which will have async execution. As long as we're checking the state and
+ * making decisions about what to do before we submit something to the thread pool, we're fine.
+ */
+
+ String lastImportSkipped = state.getString(SKIPPED_IMPORT);
+ // We have a marker in the state for the previous insert, we have to balance skipping BulkPlusOne
+ // with skipping the new BulkMinusOne to make sure that we maintain consistency
+ if (null != lastImportSkipped) {
+ if (!getClass().equals(BulkMinusOne.class)) {
+ throw new IllegalStateException("Should not have a skipped import marker for a class other than " + BulkMinusOne.class.getName() + " but was "
+ + getClass().getName());
+ }
+
+ if (TRUE.equals(lastImportSkipped)) {
+ log.debug("Last import was skipped, skipping this import to ensure consistency");
+ state.remove(SKIPPED_IMPORT);
+
+ // Wait 30s to balance the skip of a BulkPlusOne/BulkMinusOne pair
+ log.debug("Waiting 30s before continuing");
+ try {
+ Thread.sleep(30 * 1000);
+ } catch (InterruptedException e) {}
+
+ return;
+ } else {
+ // last import was not skipped, remove the marker
+ state.remove(SKIPPED_IMPORT);
+ }
+ }
+
+ if (shouldQueueMoreImports(state, env)) {
+ super.visit(state, env, props);
+ } else {
+ log.debug("Not queuing more imports this round because too many are already queued");
+ state.set(SKIPPED_IMPORT, TRUE);
+ // Don't sleep here, let the sleep happen when we skip the next BulkMinusOne
+ }
+ }
+
+ private boolean shouldQueueMoreImports(State state, Environment env) throws Exception {
+ // Only selectively import when it's BulkPlusOne. If we did a BulkPlusOne,
+ // we must also do a BulkMinusOne to keep the table consistent
+ if (getClass().equals(BulkPlusOne.class)) {
+ // Only queue up more imports if the number of queued tasks already
+ // exceeds the number of tservers by 50x
+ return SelectiveQueueing.shouldQueueOperation(state, env);
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkMinusOne.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkMinusOne.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkMinusOne.java
new file mode 100644
index 0000000..a9bb8f9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkMinusOne.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+
+public class BulkMinusOne extends BulkImportTest {
+
+ private static final Value negOne = new Value("-1".getBytes(UTF_8));
+
+ @Override
+ protected void runLater(State state, Environment env) throws Exception {
+ log.info("Decrementing");
+ BulkPlusOne.bulkLoadLots(log, state, env, negOne);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkPlusOne.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkPlusOne.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkPlusOne.java
new file mode 100644
index 0000000..239e93e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkPlusOne.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.client.IteratorSetting.Column;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class BulkPlusOne extends BulkImportTest {
+
+ public static final int LOTS = 100000;
+ public static final int COLS = 10;
+ public static final int HEX_SIZE = (int) Math.ceil(Math.log(LOTS) / Math.log(16));
+ public static final String FMT = "r%0" + HEX_SIZE + "x";
+ public static final List<Column> COLNAMES = new ArrayList<>();
+ public static final Text CHECK_COLUMN_FAMILY = new Text("cf");
+ static {
+ for (int i = 0; i < COLS; i++) {
+ COLNAMES.add(new Column(CHECK_COLUMN_FAMILY, new Text(String.format("%03d", i))));
+ }
+ }
+ public static final Text MARKER_CF = new Text("marker");
+ static final AtomicLong counter = new AtomicLong();
+
+ private static final Value ONE = new Value("1".getBytes());
+
+ static void bulkLoadLots(Logger log, State state, Environment env, Value value) throws Exception {
+ final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
+ final Path fail = new Path(dir.toString() + "_fail");
+ final DefaultConfiguration defaultConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ final Random rand = (Random) state.get("rand");
+ final FileSystem fs = (FileSystem) state.get("fs");
+ fs.mkdirs(fail);
+ final int parts = rand.nextInt(10) + 1;
+
+ TreeSet<Integer> startRows = new TreeSet<>();
+ startRows.add(0);
+ while (startRows.size() < parts)
+ startRows.add(rand.nextInt(LOTS));
+
+ List<String> printRows = new ArrayList<>(startRows.size());
+ for (Integer row : startRows)
+ printRows.add(String.format(FMT, row));
+
+ String markerColumnQualifier = String.format("%07d", counter.incrementAndGet());
+ log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier);
+
+ List<Integer> rows = new ArrayList<>(startRows);
+ rows.add(LOTS);
+
+ for (int i = 0; i < parts; i++) {
+ String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION;
+ FileSKVWriter f = FileOperations.getInstance().newWriterBuilder().forFile(fileName, fs, fs.getConf()).withTableConfiguration(defaultConfiguration)
+ .build();
+ f.startDefaultLocalityGroup();
+ int start = rows.get(i);
+ int end = rows.get(i + 1);
+ for (int j = start; j < end; j++) {
+ Text row = new Text(String.format(FMT, j));
+ for (Column col : COLNAMES) {
+ f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value);
+ }
+ f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE);
+ }
+ f.close();
+ }
+ env.getConnector().tableOperations().importDirectory(Setup.getTableName(), dir.toString(), fail.toString(), true);
+ fs.delete(dir, true);
+ FileStatus[] failures = fs.listStatus(fail);
+ if (failures != null && failures.length > 0) {
+ state.set("bulkImportSuccess", "false");
+ throw new Exception(failures.length + " failure files found importing files from " + dir);
+ }
+ fs.delete(fail, true);
+ log.debug("Finished bulk import, start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier);
+ }
+
+ @Override
+ protected void runLater(State state, Environment env) throws Exception {
+ log.info("Incrementing");
+ bulkLoadLots(log, state, env, ONE);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkTest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkTest.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkTest.java
new file mode 100644
index 0000000..dc11501
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/BulkTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public abstract class BulkTest extends Test {
+
+ @Override
+ public void visit(final State state, final Environment env, Properties props) throws Exception {
+ Setup.run(state, new Runnable() {
+ @Override
+ public void run() {
+ try {
+ runLater(state, env);
+ } catch (Throwable ex) {
+ log.error(ex, ex);
+ }
+ }
+
+ });
+ }
+
+ abstract protected void runLater(State state, Environment env) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Compact.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Compact.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Compact.java
new file mode 100644
index 0000000..356a7c9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Compact.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+public class Compact extends SelectiveBulkTest {
+
+ @Override
+ protected void runLater(State state, Environment env) throws Exception {
+ final Text[] points = Merge.getRandomTabletRange(state);
+ final String rangeString = Merge.rangeToString(points);
+ log.info("Compacting " + rangeString);
+ env.getConnector().tableOperations().compact(Setup.getTableName(), points[0], points[1], false, true);
+ log.info("Compaction " + rangeString + " finished");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/ConsistencyCheck.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/ConsistencyCheck.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/ConsistencyCheck.java
new file mode 100644
index 0000000..eb21f30
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/ConsistencyCheck.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+public class ConsistencyCheck extends SelectiveBulkTest {
+
+ @Override
+ protected void runLater(State state, Environment env) throws Exception {
+ Random rand = (Random) state.get("rand");
+ Text row = Merge.getRandomRow(rand);
+ log.info("Checking " + row);
+ String user = env.getConnector().whoami();
+ Authorizations auths = env.getConnector().securityOperations().getUserAuthorizations(user);
+ try (Scanner scanner = new IsolatedScanner(env.getConnector().createScanner(Setup.getTableName(), auths))) {
+ scanner.setRange(new Range(row));
+ scanner.fetchColumnFamily(BulkPlusOne.CHECK_COLUMN_FAMILY);
+ Value v = null;
+ Key first = null;
+ for (Entry<Key,Value> entry : scanner) {
+ if (v == null) {
+ v = entry.getValue();
+ first = entry.getKey();
+ }
+ if (!v.equals(entry.getValue()))
+ throw new RuntimeException("Inconsistent value at " + entry.getKey() + " was " + entry.getValue() + " should be " + v + " first read at " + first);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Merge.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Merge.java
new file mode 100644
index 0000000..ebce171
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Merge.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+
+public class Merge extends SelectiveBulkTest {
+
+ @Override
+ protected void runLater(State state, Environment env) throws Exception {
+ Text[] points = getRandomTabletRange(state);
+ log.info("merging " + rangeToString(points));
+ env.getConnector().tableOperations().merge(Setup.getTableName(), points[0], points[1]);
+ log.info("merging " + rangeToString(points) + " complete");
+ }
+
+ public static String rangeToString(Text[] points) {
+ return "(" + (points[0] == null ? "-inf" : points[0]) + " -> " + (points[1] == null ? "+inf" : points[1]) + "]";
+ }
+
+ public static Text getRandomRow(Random rand) {
+ return new Text(String.format(BulkPlusOne.FMT, (rand.nextLong() & 0x7fffffffffffffffl) % BulkPlusOne.LOTS));
+ }
+
+ public static Text[] getRandomTabletRange(State state) {
+ Random rand = (Random) state.get("rand");
+ Text points[] = {getRandomRow(rand), getRandomRow(rand),};
+ Arrays.sort(points);
+ if (rand.nextInt(10) == 0) {
+ points[0] = null;
+ }
+ if (rand.nextInt(10) == 0) {
+ points[1] = null;
+ }
+ if (rand.nextInt(20) == 0) {
+ points[0] = null;
+ points[1] = null;
+ }
+ return points;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveBulkTest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveBulkTest.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveBulkTest.java
new file mode 100644
index 0000000..a708942
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveBulkTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+
+/**
+ * Selectively runs the actual {@link BulkTest} based on the number of active TServers and the number of queued operations.
+ */
+public abstract class SelectiveBulkTest extends BulkTest {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ if (SelectiveQueueing.shouldQueueOperation(state, env)) {
+ super.visit(state, env, props);
+ } else {
+ log.debug("Skipping queueing of " + getClass().getSimpleName() + " because of excessive queued tasks already");
+ log.debug("Waiting 30 seconds before continuing");
+ try {
+ Thread.sleep(30 * 1000);
+ } catch (InterruptedException e) {}
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveQueueing.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveQueueing.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveQueueing.java
new file mode 100644
index 0000000..59cf8aa
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/SelectiveQueueing.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Chooses whether or not an operation should be queued based on the current thread pool queue length and the number of available TServers.
+ */
+public class SelectiveQueueing {
+ private static final Logger log = LoggerFactory.getLogger(SelectiveQueueing.class);
+
+ public static boolean shouldQueueOperation(State state, Environment env) throws Exception {
+ final ThreadPoolExecutor pool = (ThreadPoolExecutor) state.get("pool");
+ long queuedThreads = pool.getTaskCount() - pool.getActiveCount() - pool.getCompletedTaskCount();
+ final Connector conn = env.getConnector();
+ int numTservers = conn.instanceOperations().getTabletServers().size();
+
+ if (!shouldQueue(queuedThreads, numTservers)) {
+ log.info("Not queueing because of " + queuedThreads + " outstanding tasks");
+ return false;
+ }
+
+ return true;
+ }
+
+ private static boolean shouldQueue(long queuedThreads, int numTservers) {
+ return queuedThreads < numTservers * 50;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
new file mode 100644
index 0000000..f3c3fdf
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/bulk/Setup.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.bulk;
+
+import java.net.InetAddress;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.iterators.LongCombiner;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.SimpleThreadPool;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.fs.FileSystem;
+
+public class Setup extends Test {
+
+ private static final int MAX_POOL_SIZE = 8;
+ static String tableName = null;
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ Random rand = new Random();
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+ String pid = env.getPid();
+ tableName = String.format("bulk_%s_%s_%d", hostname, pid, System.currentTimeMillis());
+ log.info("Starting bulk test on " + tableName);
+
+ TableOperations tableOps = env.getConnector().tableOperations();
+ try {
+ if (!tableOps.exists(getTableName())) {
+ tableOps.create(getTableName());
+ IteratorSetting is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setEncodingType(is, LongCombiner.Type.STRING);
+ SummingCombiner.setCombineAllColumns(is, true);
+ tableOps.attachIterator(getTableName(), is);
+ }
+ } catch (TableExistsException ex) {
+ // expected if there are multiple walkers
+ }
+ state.set("rand", rand);
+ state.set("fs", FileSystem.get(CachedConfiguration.getInstance()));
+ state.set("bulkImportSuccess", "true");
+ BulkPlusOne.counter.set(0l);
+
+ ThreadPoolExecutor e = new SimpleThreadPool(MAX_POOL_SIZE, "bulkImportPool");
+ state.set("pool", e);
+ }
+
+ public static String getTableName() {
+ return tableName;
+ }
+
+ public static ThreadPoolExecutor getThreadPool(State state) {
+ return (ThreadPoolExecutor) state.get("pool");
+ }
+
+ public static void run(State state, Runnable r) {
+ getThreadPool(state).submit(r);
+ }
+
+}