You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by le...@apache.org on 2016/01/05 13:17:07 UTC
[1/2] gora git commit: GORA-453 Implement bootstraping of GoraCI
nodes with Chef and the JClouds ComputeService this closes #47
Repository: gora
Updated Branches:
refs/heads/master e56eebb9d -> 0e6a30958
GORA-453 Implement bootstraping of GoraCI nodes with Chef and the JClouds ComputeService this closes #47
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/0501812a
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/0501812a
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/0501812a
Branch: refs/heads/master
Commit: 0501812a6d21dcfbfa48d65a1e4d9c787b5f3aee
Parents: a11bf62
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Tue Dec 1 18:07:05 2015 -0800
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Tue Jan 5 04:18:23 2016 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +
bin/gora | 9 +-
.../gora/compiler/utils/LicenseHeaders.java | 300 +++++++++----------
gora-goraci/README | 132 ++++----
gora-goraci/src/main/avro/cinode.json | 6 +-
gora-goraci/src/main/avro/flushed.json | 2 +-
.../goraci/chef/ChefSoftwareProvisioning.java | 177 +++++++++++
.../apache/gora/goraci/chef/package-info.java | 21 ++
.../apache/gora/goraci/generated/CINode.java | 17 +-
.../apache/gora/goraci/generated/Flushed.java | 17 +-
.../rackspace/RackspaceOrchestration.java | 85 ++++--
gora-goraci/src/main/resources/gora.properties | 16 +-
12 files changed, 497 insertions(+), 287 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dbe4415..33f1fb3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@
Current Development
+* GORA-452 Upgrade goraci prior to Chef and the ComputeService implementations
+
* GORA-439 Remove Unused Method Parameters (Furkan KAMACI)
* GORA-240 Tests for MemStore (lewismc)
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/bin/gora
----------------------------------------------------------------------
diff --git a/bin/gora b/bin/gora
index 8c37272..8231159 100755
--- a/bin/gora
+++ b/bin/gora
@@ -46,7 +46,8 @@ if [ $# = 0 ]; then
echo " goracompiler Run Compiler"
echo " specificcompiler Run Avro Specific Compiler"
echo " dynamocompiler Run Gora DynamoDB Compiler"
- echo " goracisetup Run the GoraCI Rackspace orchestration setup"
+ echo " goracirackspace Run the GoraCI Rackspace orchestration setup"
+ echo " goracichef Run the GoraCI Chef software provisioning setup"
echo " logmanager Run the tutorial log manager"
echo " loganalytics Run the tutorial log analytics"
echo " loganalyticsspark Run the tutorial log analytics spark"
@@ -123,10 +124,14 @@ elif [ "$COMMAND" = "dynamocompiler" ] ; then
MODULE=gora-dynamodb
CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/
CLASS=org.apache.gora.dynamodb.compiler.GoraDynamoDBCompiler
-elif [ "$COMMAND" = "goracisetup" ] ; then
+elif [ "$COMMAND" = "goracirackspace" ] ; then
MODULE=gora-goraci
CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/
CLASS=org.apache.gora.goraci.rackspace.RackspaceOrchestration
+elif [ "$COMMAND" = "goracichef" ] ; then
+ MODULE=gora-goraci
+ CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/
+ CLASS=org.apache.gora.goraci.chef.ChefSoftwareProvisioning
elif [ "$COMMAND" = "logmanager" ] ; then
MODULE=gora-tutorial
CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-compiler/src/main/java/org/apache/gora/compiler/utils/LicenseHeaders.java
----------------------------------------------------------------------
diff --git a/gora-compiler/src/main/java/org/apache/gora/compiler/utils/LicenseHeaders.java b/gora-compiler/src/main/java/org/apache/gora/compiler/utils/LicenseHeaders.java
index 1bba8a0..155e2d2 100644
--- a/gora-compiler/src/main/java/org/apache/gora/compiler/utils/LicenseHeaders.java
+++ b/gora-compiler/src/main/java/org/apache/gora/compiler/utils/LicenseHeaders.java
@@ -35,12 +35,12 @@ public class LicenseHeaders {
* Chosen license to be included within the generated classes
*/
private String licenseName;
-
+
/**
* Licenses supported by Gora Compilers
*/
private static final String supportedLicenses[] = {"ASLv2", "AGPLv3", "CDDLv1", "FDLv13", "GPLv1", "GPLv2", "GPLv3", "LGPLv21", "LGPLv3"};
-
+
/**
* HashMap containing supported licenses' names and their corresponding text.
*/
@@ -49,175 +49,175 @@ public class LicenseHeaders {
// ASLv2 license header
@SuppressWarnings("unused")
private static final String ASLv2 =
- "/**\n" +
- " *Licensed to the Apache Software Foundation (ASF) under one\n" +
- " *or more contributor license agreements. See the NOTICE file\n" +
- " *distributed with this work for additional information\n" +
- " *regarding copyright ownership. The ASF licenses this file\n" +
- " *to you under the Apache License, Version 2.0 (the\"\n" +
- " *License\"); you may not use this file except in compliance\n" +
- " *with the License. You may obtain a copy of the License at\n" +
- " *\n " +
- " * http://www.apache.org/licenses/LICENSE-2.0\n" +
- " * \n" +
- " *Unless required by applicable law or agreed to in writing, software\n" +
- " *distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
- " *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
- " *See the License for the specific language governing permissions and\n" +
- " *limitations under the License.\n" +
- " */\n";
-
+ "/**\n" +
+ " *Licensed to the Apache Software Foundation (ASF) under one\n" +
+ " *or more contributor license agreements. See the NOTICE file\n" +
+ " *distributed with this work for additional information\n" +
+ " *regarding copyright ownership. The ASF licenses this file\n" +
+ " *to you under the Apache License, Version 2.0 (the\"\n" +
+ " *License\"); you may not use this file except in compliance\n" +
+ " *with the License. You may obtain a copy of the License at\n" +
+ " *\n " +
+ " * http://www.apache.org/licenses/LICENSE-2.0\n" +
+ " * \n" +
+ " *Unless required by applicable law or agreed to in writing, software\n" +
+ " *distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
+ " *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
+ " *See the License for the specific language governing permissions and\n" +
+ " *limitations under the License.\n" +
+ " */\n";
+
// AGPLv3 license header
@SuppressWarnings("unused")
private static final String AGPLv3 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU Affero General Public License as published by\n" +
- " * the Free Software Foundation, either version 3 of the License, or\n" +
- " * (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " */\n";
-
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU Affero General Public License as published by\n" +
+ " * the Free Software Foundation, either version 3 of the License, or\n" +
+ " * (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " */\n";
+
// CDDLv1 license header
@SuppressWarnings("unused")
private static final String CDDLv1 =
- "/**\n" +
- " * COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0\n" +
- " *\n " +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the Common Development and Distrubtion License as\n" +
- " * published by the Sun Microsystems, either version 1.0 of the\n" +
- " * License, or (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Lesser Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the Common Development and Distrubtion\n" +
- " * License along with this program. If not, see\n" +
- " * <http://www.gnu.org/licenses/gpl-1.0.html>.\n" +
- " */\n";
-
+ "/**\n" +
+ " * COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0\n" +
+ " *\n " +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the Common Development and Distrubtion License as\n" +
+ " * published by the Sun Microsystems, either version 1.0 of the\n" +
+ " * License, or (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Lesser Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the Common Development and Distrubtion\n" +
+ " * License along with this program. If not, see\n" +
+ " * <http://www.gnu.org/licenses/gpl-1.0.html>.\n" +
+ " */\n";
+
// FDLv1.3 license header
@SuppressWarnings("unused")
private static final String FDLv13 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU Free Documentation License as published by\n" +
- " * the Free Software Foundation, either version 1.3 of the License, or\n" +
- " * (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the GNU Free Documentation License\n" +
- " * along with this program. If not, see <http://www.gnu.org/licenses/>.\n" +
- " */\n";
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU Free Documentation License as published by\n" +
+ " * the Free Software Foundation, either version 1.3 of the License, or\n" +
+ " * (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the GNU Free Documentation License\n" +
+ " * along with this program. If not, see <http://www.gnu.org/licenses/>.\n" +
+ " */\n";
// GPLv1 license header
@SuppressWarnings("unused")
private static final String GPLv1 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU General Public License as\n" +
- " * published by the Free Software Foundation, either version 1 of the\n" +
- " * License, or (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the GNU General Public\n" +
- " * License along with this program. If not, see\n" +
- " * <http://www.gnu.org/licenses/gpl-1.0.html>.\n" +
- " */\n";
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU General Public License as\n" +
+ " * published by the Free Software Foundation, either version 1 of the\n" +
+ " * License, or (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the GNU General Public\n" +
+ " * License along with this program. If not, see\n" +
+ " * <http://www.gnu.org/licenses/gpl-1.0.html>.\n" +
+ " */\n";
// GPLv2 license header
@SuppressWarnings("unused")
private static final String GPLv2 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU General Public License as\n" +
- " * published by the Free Software Foundation, either version 2 of the\n" +
- " * License, or (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the GNU General Public\n" +
- " * License along with this program. If not, see\n" +
- " * <http://www.gnu.org/licenses/gpl-2.0.html>.\n" +
- " */\n";
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU General Public License as\n" +
+ " * published by the Free Software Foundation, either version 2 of the\n" +
+ " * License, or (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the GNU General Public\n" +
+ " * License along with this program. If not, see\n" +
+ " * <http://www.gnu.org/licenses/gpl-2.0.html>.\n" +
+ " */\n";
// GPLv3 license header
@SuppressWarnings("unused")
private static final String GPLv3 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU General Public License as\n" +
- " * published by the Free Software Foundation, either version 3 of the\n" +
- " * License, or (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the GNU General Public\n" +
- " * License along with this program. If not, see\n" +
- " * <http://www.gnu.org/licenses/gpl-3.0.html>.\n" +
- " */\n";
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU General Public License as\n" +
+ " * published by the Free Software Foundation, either version 3 of the\n" +
+ " * License, or (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the GNU General Public\n" +
+ " * License along with this program. If not, see\n" +
+ " * <http://www.gnu.org/licenses/gpl-3.0.html>.\n" +
+ " */\n";
// LGPLv21 license header
@SuppressWarnings("unused")
private static final String LGPLv21 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU Lesser General Public License as\n" +
- " * published by the Free Software Foundation, either version 2.1 of the\n" +
- " * License, or (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the GNU Lesser General Public\n" +
- " * License along with this program. If not, see\n" +
- " * <http://www.gnu.org/licenses/lgpl-2.1.html>.\n" +
- " */\n";
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU Lesser General Public License as\n" +
+ " * published by the Free Software Foundation, either version 2.1 of the\n" +
+ " * License, or (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the GNU Lesser General Public\n" +
+ " * License along with this program. If not, see\n" +
+ " * <http://www.gnu.org/licenses/lgpl-2.1.html>.\n" +
+ " */\n";
// LGPLv3 license header
@SuppressWarnings("unused")
private static final String LGPLv3 =
- "/**\n" +
- " * This program is free software: you can redistribute it and/or modify\n" +
- " * it under the terms of the GNU Lesser General Public License as\n" +
- " * published by the Free Software Foundation, either version 3 of the\n" +
- " * License, or (at your option) any later version.\n" +
- " *\n " +
- " * This program is distributed in the hope that it will be useful,\n" +
- " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
- " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
- " * GNU General Public License for more details.\n" +
- " *\n " +
- " * You should have received a copy of the GNU Lesser General Public\n" +
- " * License along with this program. If not, see\n" +
- " * <http://www.gnu.org/licenses/lgpl-3.0.html>.\n" +
- " */\n";
-
+ "/**\n" +
+ " * This program is free software: you can redistribute it and/or modify\n" +
+ " * it under the terms of the GNU Lesser General Public License as\n" +
+ " * published by the Free Software Foundation, either version 3 of the\n" +
+ " * License, or (at your option) any later version.\n" +
+ " *\n " +
+ " * This program is distributed in the hope that it will be useful,\n" +
+ " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n" +
+ " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" +
+ " * GNU General Public License for more details.\n" +
+ " *\n " +
+ " * You should have received a copy of the GNU Lesser General Public\n" +
+ " * License along with this program. If not, see\n" +
+ " * <http://www.gnu.org/licenses/lgpl-3.0.html>.\n" +
+ " */\n";
+
/**
- * @param license
+ * @param pLicenseName the license header you wish to create with this object
*/
public LicenseHeaders(String pLicenseName) {
this.initializeRelations();
@@ -230,10 +230,10 @@ public class LicenseHeaders {
public void initializeRelations(){
relatedLicenses = new HashMap<>();
try {
- for (String licenseValue : supportedLicenses) {
- String var = (String) this.getClass().getDeclaredField(licenseValue).get(licenseValue);
- relatedLicenses.put(licenseValue,var);
- }
+ for (String licenseValue : supportedLicenses) {
+ String var = (String) this.getClass().getDeclaredField(licenseValue).get(licenseValue);
+ relatedLicenses.put(licenseValue,var);
+ }
} catch (SecurityException | NoSuchFieldException | IllegalArgumentException | IllegalAccessException e) {
LOG.error(e.getMessage());
throw new RuntimeException(e);
@@ -242,23 +242,23 @@ public class LicenseHeaders {
/**
*Set the license header for the LicenseHeader object.
- *
+ *@param pLicenseName the license name to set the header to
*/
public void setLicenseName(String pLicenseName) {
this.licenseName = pLicenseName;
}
-
+
/**
* Get the license header for the LicenseHeader object.
- * @return
+ * @return get the license name (performing some checking to prevent NPE, etc.) you are advised to use this option
*/
public String getLicense() {
return relatedLicenses.get(licenseName)!=null?relatedLicenses.get(licenseName):"";
}
-
+
/**
* Get the license name for the LicenseHeader object.
- * @return
+ * @return get the license name
*/
public String getLicenseName(){
return licenseName;
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/README
----------------------------------------------------------------------
diff --git a/gora-goraci/README b/gora-goraci/README
index 54c60ed..b9f0c1d 100644
--- a/gora-goraci/README
+++ b/gora-goraci/README
@@ -1,15 +1,12 @@
-=================================
-= =
-= GORACI README =
-= @author Keith Turner =
-=================================
+#GORACI README
+@author Keith Turner
-BACKGROUND
-------------
+## BACKGROUND
-Apache Accumulo [0] has a simple test suite that verifies that data is not lost
-at scale. This test suite is called continuous ingest [5]. This test runs
-many ingest clients that continually create linked lists containing 25 million
+[Apache Accumulo](http://accumulo.apache.org) has a simple test suite that verifies that data is not lost
+at scale.
+This test suite is called [continuous ingest](http://svn.apache.org/viewvc/accumulo/tags/1.4.0/test/system/continuous/ScaleTest.odp?view=co).
+This test runs many ingest clients that continually create linked lists containing 25 million
nodes. At some point the clients are stopped and a map reduce job is run to
ensure no linked list has a hole. A hole indicates data was lost.
@@ -17,11 +14,12 @@ The nodes in the linked list are random. This causes each linked list to
spread across the table. Therefore if one part of a table loses data, then it
will be detected by references in another part of the table.
-This project is a version of the test suite written using Apache Gora [1].
+This project is a version of the test suite written using [Apache Gora](http://gora.apache.org).
Goraci has been tested against Accumulo and HBase.
-THE ANATOMY OF GORACI TESTS
-----------------------------
+More documentation on GoraCI can be found at the [goraci-integration-testsing-suite](http://gora.apache.org/current/index.html#goraci-integration-testsing-suite)
+
+## THE ANATOMY OF GORACI TESTS
Below is rough sketch of how data is written. For specific details look at the
Generator code (src/main/java/org.apache.gora.goraci/Generator.java)
@@ -54,46 +52,36 @@ million. The reason for this is that circular linked list are generated every
list not having references. The loss of an unreferenced node can not be
detected.
-BUILDING GORACI
----------------
-
-This code currently depends on an unreleased version of Gora. To build Gora
-0.2 run the following commands.
-
- svn export http://svn.apache.org/repos/asf/gora/trunk gora
- cd gora
- mvn install -DskipTests
-
-After this you can build org.apache.gora.goraci.
+## BUILDING GORACI
- git clone git://github.com/keith-turner/org.apache.gora.goraci.git
- cd org.apache.gora.goraci
- mvn compile
-
-The maven pom file has some profiles that attempt to make it easier to run
-org.apache.gora.goraci against different gora backends by copying the jars you need into lib.
+GoraCI can be built as part of the main source via
+```
+mvn install -DskipTests
+```
+Maven artifacts also exist... see [Maven Central](http://search.maven.org/#search|ga|1|a%3A%22gora-goraci%22)
+The Maven pom file has some profiles that attempt to make it easier to run
+GoraCI against different gora backends by copying the jars you need into lib.
Before packaging its important to edit gora.properties and set it correctly
for your datastore. To run against accumulo do the following.
-
+```
vim src/main/resources/gora.properties (set Accumulo properties)
- mvn package -Paccumulo-1.4
-
+ mvn package -Paccumulo-1.6.4
+```
To run against hbase, do the following.
-
+```
vim src/main/resources/gora.properties (set HBase properties)
- mvn package -Phbase-0.92
-
+ mvn package -Phbase-1.1.2
+```
To run against cassandra, do the following.
-
+```
vim src/main/resources/gora.properties (set Cassandra properties)
- mvn package -Pcassandra-1.1.2
-
+ mvn package -Pcassandra-2.0.2
+```
For other datastores mentioned in gora.properties, you will need to copy the
appropriate deps into lib. Feel free to update the pom with other profiles and
-send me pull request.
+send a pull request.
-JAVA CLASS DESCRIPTION
------------------
+## JAVA CLASS DESCRIPTION
Below is a description of the Java programs
@@ -112,39 +100,37 @@ Below is a description of the Java programs
org.apache.gora.goraci.sh is a helper script that you can use to run the above programs. It
assumes all needed jars are in the lib dir. It does not need the package name.
You can just run "./org.apache.gora.goraci.sh Generator", below is an example.
-
+```
$ ./org.apache.gora.goraci.sh Generator
Usage : Generator <num mappers> <num nodes>
-
+```
For Gora to work, it needs a gora.properties file on the classpath and a
mapping file on the classpath, the contents of both are datastore specific,
-more details can be found here [2]. You can edit the ones in src/main/resources
+more details can be found at [gora-conf](http://gora.apache.org/current/gora-conf.html). You can edit the ones in src/main/resources
and build the org.apache.gora.goraci-${version}-SNAPSHOT.jar with those. Alternatively remove
those and put them on the classpath through some other means.
-GORA AND HADOOP
------------------
+## GORA AND HADOOP
Gora uses Avro which uses a Json library that Hadoop has an old version of.
The two libraries jackson-core and jackson-mapper need to be updated in
<HADOOP_HOME>/lib and <HADOOP_HOME>/share/hadoop/lib/. I updated these to
jackson-core-asl-1.4.2.jar and jackson-mapper-asl-1.4.2.jar. For details see
-HADOOP-6945 [3].
+[HADOOP-6945](https://issues.apache.org/jira/browse/HADOOP-6945).
-GORACI AND HBASE
------------------
+## GORACI AND HBASE
To improve performance running read jobs such as the Verify step, enable
scanner caching on the command line. For example:
-
+```
$ ./gorachi.sh Verify -Dhbase.client.scanner.caching=1000 \
-Dmapred.map.tasks.speculative.execution=false verify_dir 1000
-
+```
Dependent on how you have your hadoop and hbase deployed, you may need to
change the gorachi.sh script around some. Here is one suggestion that may help
in the case where your hadoop and hbase configuration are other than under the
hadoop and hbase home directories.
-
+```
diff --git a/org.apache.gora.goraci.sh b/org.apache.gora.goraci.sh
index db1562a..31c3c94 100755
--- a/org.apache.gora.goraci.sh
@@ -157,16 +143,15 @@ hadoop and hbase home directories.
-
-
+CLASSPATH="${HBASE_CONF_DIR}" hadoop --config "${HADOOP_CONF_DIR} jar "$GORACI_HOME/lib/org.apache.gora.goraci-0.0.1-SNAPSHOT.jar" $CLASS -files "${HBASE_CONF_DIR}/hbase-site.xml" -libjars "$LIBJARS" "$@"
-
+```
You will need to define HBASE_CONF_DIR and HADOOP_CONF_DIR before you run your
org.apache.gora.goraci jobs. For example:
-
+```
$ export HADOOP_CONF_DIR=/home/you/hadoop-conf
$ export HBASE_CONF_DIR=/home/you/hbase-conf
$ PATH=/home/you/hadoop-1.0.2/bin:$PATH ./org.apache.gora.goraci.sh Generator 1000 1000000
-
-CONCURRENCY
-------------
+```
+## CONCURRENCY
Its possible to run verification at the same time as generation. To do this
supply the -c option to Generator and Verify. This will cause Genertor to
@@ -184,8 +169,7 @@ written, when the node is later referenced it will appear to be missing. The
-c option basically filters out newer information using data written to the
secondary table.
-CONCLUSIONS
-------------
+## CONCLUSIONS
This test suite does not do everything that the Accumulo test suite does,
mainly it does not collect statistics and generate reports. The reports
@@ -194,7 +178,7 @@ are useful for assesing performance.
Below shows running a test of the test. Ingest one linked list, deleted a node
in it, ensure the verifaction map reduce job notices that the node is missing.
Not all output is shown, just the important parts.
-
+```
$ ./org.apache.gora.goraci.sh Generator 1 25000000
$ ./org.apache.gora.goraci.sh Print -s 2000000000000000 -l 1
2000001f65dbd238:30350f9ae6f6e8f7:000004265852:ef09f9dd-75b1-4c16-9f14-0fa84f3029b6
@@ -209,48 +193,42 @@ Not all output is shown, just the important parts.
11/12/20 17:12:31 INFO mapred.JobClient: UNREFERENCED=1
$ hadoop fs -cat gci_verify_1/part\*
30350f9ae6f6e8f7 2000001f65dbd238
-
+```
The map reduce job found the one undefined node and gave the node that
referenced it.
Below are some timing statistics for running org.apache.gora.goraci on a 10 node cluster.
-
+```
Store | Task | Time | Undef | Unref | Ref
----------------+------------------------+---------+--------+-------+------------
accumulo-1.4.0 | Generator 10 100000000 | 40m 16s | N/A | N/A | N/A
accumulo-1.4.0 | Verify /tmp/goraci1 40 | 6m 7s | 0 | 0 | 1000000000
hbase-0.92.1 | Generator 10 100000000 | 2h 44m | N/A | N/A | N/A
hbase-0.92.1 | Verify /tmp/goraci2 40 | 6m 34s | 0 | 0 | 1000000000
-
+```
Hbase and Accumulo are configured differently out-of-the-box. We used the Accumulo
3G, native configuration examples in the conf/examples directory.
To provide a comparable memory footprint, we increased the HBase jvm to "-Xmx4000m",
and turned on compression for the ci table:
-
+```
create 'ci', {NAME=>'meta', COMPRESSION=>'GZ'}
-
+```
We also turned down the replication of write-ahead logs to be comparable to Accumulo:
-
+```
<property>
<name>hbase.regionserver.hlog.replication</name>
<value>2</value>
</property>
-
+```
For the accumulo run, we set the split threshold to 512M:
-
+```
shell> config -t ci -s table.split.threshold=512M
-
+```
This was done so that Accumulo would end up with 64 tablets, which is the
number of regions hbase had. The number of tablets/regions determines how
much parallelism there is in the map phase of the verify step.
Sometimes when this test suite is run against HBase data is lost. This issue
-is being tracked under HBASE-5754 [4].
+is being tracked under [HBASE-5754](https://issues.apache.org/jira/browse/HBASE-5754).
-[0] http://accumulo.apache.org
-[1] http://gora.apache.org
-[2] http://gora.apache.org/docs/current/gora-conf.html
-[3] https://issues.apache.org/jira/browse/HADOOP-6945
-[4] https://issues.apache.org/jira/browse/HBASE-5754
-[5] http://svn.apache.org/viewvc/accumulo/tags/1.4.0/test/system/continuous/ScaleTest.odp?view=co
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/avro/cinode.json
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/avro/cinode.json b/gora-goraci/src/main/avro/cinode.json
index 207bb18..ae48e17 100644
--- a/gora-goraci/src/main/avro/cinode.json
+++ b/gora-goraci/src/main/avro/cinode.json
@@ -3,8 +3,8 @@
"name": "CINode",
"namespace": "org.apache.gora.goraci.generated",
"fields" : [
- {"name": "prev", "type": "long"},
- {"name": "client", "type": "string"},
- {"name": "count", "type": "long"}
+ {"name": "prev", "type": "long","default":0},
+ {"name": "client", "type": ["null","string"],"default":null},
+ {"name": "count", "type": "long","default":0}
]
}
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/avro/flushed.json
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/avro/flushed.json b/gora-goraci/src/main/avro/flushed.json
index 74872e6..e1b7559 100644
--- a/gora-goraci/src/main/avro/flushed.json
+++ b/gora-goraci/src/main/avro/flushed.json
@@ -3,6 +3,6 @@
"name": "Flushed",
"namespace": "org.apache.gora.goraci.generated",
"fields" : [
- {"name": "count", "type": "long"}
+ {"name": "count", "type": "long","default":0}
]
}
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/java/org/apache/gora/goraci/chef/ChefSoftwareProvisioning.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/chef/ChefSoftwareProvisioning.java b/gora-goraci/src/main/java/org/apache/gora/goraci/chef/ChefSoftwareProvisioning.java
new file mode 100644
index 0000000..a27ef0e
--- /dev/null
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/chef/ChefSoftwareProvisioning.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.goraci.chef;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.concat;
+import static org.jclouds.compute.options.TemplateOptions.Builder.runScript;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.gora.goraci.rackspace.RackspaceOrchestration;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.jclouds.ContextBuilder;
+import org.jclouds.chef.ChefContext;
+import org.jclouds.chef.config.ChefProperties;
+import org.jclouds.chef.domain.BootstrapConfig;
+import org.jclouds.chef.domain.CookbookVersion;
+import org.jclouds.chef.predicates.CookbookVersionPredicates;
+import org.jclouds.chef.util.RunListBuilder;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.RunNodesException;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.domain.JsonBall;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.sshj.config.SshjSshClientModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import com.google.inject.Module;
+
+/**
+ *This class contains all of the Chef software provisioning code required to
+ *provision nodes for use within GoraCI.
+ *For configuration options required you should see the chef properties within
+ *<code>gora.properties</code>.
+ */
+public class ChefSoftwareProvisioning {
+
+ private static String CHEF_CLIENT = "chef.client";
+
+ private static String CHEF_ORGANIZATION = "chef.organization";
+
+ private static String RS_CONTINENT = "rackspace.continent";
+
+ private static String RS_USERNAME = "rackspace.username";
+
+ private static String RS_APIKEY = "rackspace.apikey";
+
+ private static String RS_REGION = "rackspace.region";
+
+ private static final Logger LOG = LoggerFactory.getLogger(RackspaceOrchestration.class);
+
+ /**
+ * Default constructor
+ */
+ public ChefSoftwareProvisioning() {
+ }
+
+ private static void performChefComputeServiceBootstrapping(Properties properties) throws IOException, InstantiationException, IllegalAccessException {
+ // Get the credentials that will be used to authenticate to the Chef server
+ String rsContinent = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(),
+ RS_CONTINENT, "rackspace-cloudservers-us");
+ String rsUser = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_USERNAME, "asf-gora");
+ String rsApiKey = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_APIKEY, null);
+ String rsRegion = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_REGION, "DFW");
+ String client = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), CHEF_CLIENT, System.getProperty("user.name"));
+ String organization = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), CHEF_ORGANIZATION, null);
+ String pemFile = System.getProperty("user.home") + "/.chef/" + client + ".pem";
+ String credential = Files.toString(new File(pemFile), Charsets.UTF_8);
+
+ // Provide the validator information to let the nodes to auto-register themselves
+ // in the Chef server during bootstrap
+ String validator = organization + "-validator";
+ String validatorPemFile = System.getProperty("user.home") + "/.chef/" + validator + ".pem";
+ String validatorCredential = Files.toString(new File(validatorPemFile), Charsets.UTF_8);
+
+ Properties chefConfig = new Properties();
+ chefConfig.put(ChefProperties.CHEF_VALIDATOR_NAME, validator);
+ chefConfig.put(ChefProperties.CHEF_VALIDATOR_CREDENTIAL, validatorCredential);
+
+ // Create the connection to the Chef server
+ ChefContext chefContext = ContextBuilder.newBuilder("chef")
+ .endpoint("https://api.opscode.com/organizations/" + organization)
+ .credentials(client, credential)
+ .overrides(chefConfig)
+ .buildView(ChefContext.class);
+
+ // Create the connection to the compute provider. Note that ssh will be used to bootstrap chef
+ ComputeServiceContext computeContext = ContextBuilder.newBuilder(rsContinent)
+ .endpoint(rsRegion)
+ .credentials(rsUser, rsApiKey)
+ .modules(ImmutableSet.<Module> of(new SshjSshClientModule()))
+ .buildView(ComputeServiceContext.class);
+
+ // Group all nodes in both Chef and the compute provider by this group
+ String group = "jclouds-chef-goraci";
+
+ // Set the recipe to install and the configuration values to override
+ String recipe = "apache2";
+ JsonBall attributes = new JsonBall("{\"apache\": {\"listen_ports\": \"8080\"}}");
+
+ // Check to see if the recipe you want exists
+ List<String> runlist = null;
+ Iterable< ? extends CookbookVersion> cookbookVersions =
+ chefContext.getChefService().listCookbookVersions();
+ if (any(cookbookVersions, CookbookVersionPredicates.containsRecipe(recipe))) {
+ runlist = new RunListBuilder().addRecipe(recipe).build();
+ }
+ for (Iterator<String> iterator = runlist.iterator(); iterator.hasNext();) {
+ String string = (String) iterator.next();
+ LOG.info(string);
+ }
+
+ // Update the chef service with the run list you wish to apply to all nodes in the group
+ // and also provide the json configuration used to customize the desired values
+ BootstrapConfig config = BootstrapConfig.builder().runList(runlist).attributes(attributes).build();
+ chefContext.getChefService().updateBootstrapConfigForGroup(group, config);
+
+ // Build the script that will bootstrap the node
+ Statement bootstrap = chefContext.getChefService().createBootstrapScriptForGroup(group);
+
+ TemplateBuilder templateBuilder = computeContext.getComputeService().templateBuilder();
+ templateBuilder.options(runScript(bootstrap));
+ // Run a node on the compute provider that bootstraps chef
+ try {
+ Set< ? extends NodeMetadata> nodes =
+ computeContext.getComputeService().createNodesInGroup(group, 1, templateBuilder.build());
+ for (NodeMetadata nodeMetadata : nodes) {
+ LOG.info("<< node %s: %s%n", nodeMetadata.getId(),
+ concat(nodeMetadata.getPrivateAddresses(), nodeMetadata.getPublicAddresses()));
+ }
+ } catch (RunNodesException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+
+ // Release resources
+ chefContext.close();
+ computeContext.close();
+
+ }
+
+ /**
+ * @param args
+ * @throws IOException
+ * @throws IllegalAccessException
+ * @throws InstantiationException
+ */
+ public static void main(String[] args) throws InstantiationException, IllegalAccessException, IOException {
+ Properties props = DataStoreFactory.createProps();
+ ChefSoftwareProvisioning.performChefComputeServiceBootstrapping(props);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/java/org/apache/gora/goraci/chef/package-info.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/chef/package-info.java b/gora-goraci/src/main/java/org/apache/gora/goraci/chef/package-info.java
new file mode 100644
index 0000000..946e0e1
--- /dev/null
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/chef/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *This package contains all of the Chef software provisioning code required to
+ *provision nodes for use within GoraCI.
+ */
+package org.apache.gora.goraci.chef;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/java/org/apache/gora/goraci/generated/CINode.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/generated/CINode.java b/gora-goraci/src/main/java/org/apache/gora/goraci/generated/CINode.java
index d8396f9..3773929 100644
--- a/gora-goraci/src/main/java/org/apache/gora/goraci/generated/CINode.java
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/generated/CINode.java
@@ -1,23 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Autogenerated by Avro
*
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * DO NOT EDIT DIRECTLY
*/
package org.apache.gora.goraci.generated;
@SuppressWarnings("all")
public class CINode extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"CINode\",\"namespace\":\"org.apache.gora.goraci.generated\",\"fields\":[{\"name\":\"prev\",\"type\":\"long\"},{\"name\":\"client\",\"type\":\"string\"},{\"name\":\"count\",\"type\":\"long\"}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"CINode\",\"namespace\":\"org.apache.gora.goraci.generated\",\"fields\":[{\"name\":\"prev\",\"type\":\"long\",\"default\":0},{\"name\":\"client\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"count\",\"type\":\"long\",\"default\":0}]}");
/** Enum containing all data bean's fields. */
public static enum Field {
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/java/org/apache/gora/goraci/generated/Flushed.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/generated/Flushed.java b/gora-goraci/src/main/java/org/apache/gora/goraci/generated/Flushed.java
index b5222cf..7c3a85d 100644
--- a/gora-goraci/src/main/java/org/apache/gora/goraci/generated/Flushed.java
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/generated/Flushed.java
@@ -1,23 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Autogenerated by Avro
*
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * DO NOT EDIT DIRECTLY
*/
package org.apache.gora.goraci.generated;
@SuppressWarnings("all")
public class Flushed extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Flushed\",\"namespace\":\"org.apache.gora.goraci.generated\",\"fields\":[{\"name\":\"count\",\"type\":\"long\"}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Flushed\",\"namespace\":\"org.apache.gora.goraci.generated\",\"fields\":[{\"name\":\"count\",\"type\":\"long\",\"default\":0}]}");
/** Enum containing all data bean's fields. */
public static enum Field {
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
index aea05ea..29b8e8b 100644
--- a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
+++ b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java
@@ -18,12 +18,25 @@ package org.apache.gora.goraci.rackspace;
import java.io.File;
import java.io.IOException;
+import java.util.List;
import java.util.NoSuchElementException;
import java.util.Properties;
+import java.util.Set;
import org.apache.gora.memory.store.MemStore;
import org.apache.gora.store.DataStoreFactory;
import org.jclouds.ContextBuilder;
+import org.jclouds.chef.ChefContext;
+import org.jclouds.chef.config.ChefProperties;
+import org.jclouds.chef.domain.BootstrapConfig;
+import org.jclouds.chef.domain.CookbookVersion;
+import org.jclouds.chef.predicates.CookbookVersionPredicates;
+import org.jclouds.chef.util.RunListBuilder;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.RunNodesException;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.domain.JsonBall;
import org.jclouds.openstack.nova.v2_0.NovaApi;
import org.jclouds.openstack.nova.v2_0.domain.Flavor;
import org.jclouds.openstack.nova.v2_0.domain.Image;
@@ -35,15 +48,24 @@ import org.jclouds.openstack.nova.v2_0.features.ImageApi;
import org.jclouds.openstack.nova.v2_0.features.ServerApi;
import org.jclouds.openstack.nova.v2_0.options.CreateServerOptions;
import org.jclouds.openstack.nova.v2_0.predicates.ServerPredicates;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.sshj.config.SshjSshClientModule;
+import static org.jclouds.compute.options.TemplateOptions.Builder.runScript;
import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableSet;
import com.google.common.io.Files;
+import com.google.inject.Module;
+
+import static com.google.common.collect.Iterables.any;
+import static com.google.common.collect.Iterables.concat;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* <p>This is the main class for initiating Rackspace cloud
- * topography for use within the GoraCI job. A wealth of settings
+ * topography for use within GoraCI jobs. A wealth of settings
* are configurable from within <code>gora.properties</code>.</p>
* <p>For
* further documentation on the Rackspace Orchestration please see the
@@ -75,8 +97,21 @@ public class RackspaceOrchestration<K> {
/* we can optionally upload our own keys for security, otherwise
* we can have Rackspace autocreate one for us*/
private static String RS_PUBKEY = "rackspace.uploadpubkey";
+
+ private static NovaApi novaApi = null;
+
+ private static String rsContinent = null;
+
+ private static String rsUser = null;
+
+ private static String rsApiKey = null;
+
+ private static String rsRegion = null;
/**
+ * Right now the Rackspace Orchestration and Services Provisioning
+ * requires no arguments. It can be invoked from
+ * <code>$GORA_HOME/bin/gora goracisetup</code>
* @param args
* @throws IllegalAccessException
* @throws InstantiationException
@@ -84,29 +119,33 @@ public class RackspaceOrchestration<K> {
*/
public static void main(String[] args) throws NoSuchElementException, InstantiationException, IllegalAccessException, IOException {
Properties properties = DataStoreFactory.createProps();
- String rsContinent = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(),
+ performRackspaceOrchestration(properties);
+ }
+
+ private static void performRackspaceOrchestration(Properties properties) throws InstantiationException, IllegalAccessException, IOException {
+ rsContinent = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(),
RS_CONTINENT, "rackspace-cloudservers-us");
- String rsUser = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_USERNAME, "asf-gora");
- String rsApiKey = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_APIKEY, null);
- String rs_region = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_REGION, "DFW");
- String rs_flavourId = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_FLAVORID, null);
- String rs_imageId = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_IMAGEID, null);
- int num_servers = Integer.parseInt(DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_NUM_SERVERS, "10"));
+ rsUser = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_USERNAME, "asf-gora");
+ rsApiKey = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_APIKEY, null);
+ rsRegion = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_REGION, "DFW");
+ String rsFlavourId = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_FLAVORID, null);
+ String rsImageId = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_IMAGEID, null);
+ int numServers = Integer.parseInt(DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_NUM_SERVERS, "10"));
String serverName = DataStoreFactory.findProperty(properties, MemStore.class.newInstance(), RS_SERVERNAME, "goraci_test_server");
- NovaApi novaApi = ContextBuilder.newBuilder(rsContinent).credentials(rsUser, rsApiKey).buildApi(NovaApi.class);
- LOG.info("Defining Rackspace cloudserver continent as: {}, and region: {}.", rsContinent, rs_region);
+ novaApi = ContextBuilder.newBuilder(rsContinent).credentials(rsUser, rsApiKey).buildApi(NovaApi.class);
+ LOG.info("Defining Rackspace cloudserver continent as: {}, and region: {}.", rsContinent, rsRegion);
//Choose operating system
- ImageApi imageApi = novaApi.getImageApiForZone(rs_region);
- Image image = imageApi.get(rs_imageId);
+ ImageApi imageApi = novaApi.getImageApi(rsRegion);
+ Image image = imageApi.get(rsImageId);
//Choose hardware configuration
- FlavorApi flavorApi = novaApi.getFlavorApiForZone(rs_region);
- Flavor flavor = flavorApi.get(rs_flavourId);
- LOG.info("Defining Rackspace cloudserver flavors as: {}, with image id's {}", rs_flavourId, rs_imageId);
+ FlavorApi flavorApi = novaApi.getFlavorApi(rsRegion);
+ Flavor flavor = flavorApi.get(rsFlavourId);
+ LOG.info("Defining Rackspace cloudserver flavors as: {}, with image id's {}", rsFlavourId, rsImageId);
//Manage keypairs
- KeyPairApi keyPairApi = novaApi.getKeyPairExtensionForZone(rs_region).get();
+ KeyPairApi keyPairApi = novaApi.getKeyPairApi(rsRegion).get();
File keyPairFile = null;
String publicKey = null;
@@ -137,20 +176,20 @@ public class RackspaceOrchestration<K> {
keyPairApi.createWithPublicKey("goraci-keypair", publicKey);
}
}
-
- ServerApi serverApi = novaApi.getServerApiForZone(rs_region);
+
+ ServerApi serverApi = novaApi.getServerApi(rsRegion);
CreateServerOptions options = CreateServerOptions.Builder.keyPairName("goraci-keypair");
ServerCreated serverCreated = null;
- for (int i = 0; i < num_servers; i++) {
+ for (int i = 0; i < numServers; i++) {
if (serverName != null) {
- serverCreated = serverApi.create(serverName + num_servers, rs_imageId, rs_flavourId, options);
+ serverCreated = serverApi.create(serverName + i, rsImageId, rsFlavourId, options);
} else {
- serverCreated = serverApi.create("goraci_test_server" + num_servers, image.getId(), flavor.getId(), options);
+ serverCreated = serverApi.create("goraci_test_server" + i, image.getId(), flavor.getId(), options);
}
ServerPredicates.awaitActive(serverApi).apply(serverCreated.getId());
- LOG.info("Provisioned node " + i + 1 + " of " + num_servers + " with name: " + serverName + i);
+ LOG.info("Provisioned node {} of {} with name {}", new Object[]{i + 1, numServers, serverName + i});
}
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/gora/blob/0501812a/gora-goraci/src/main/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-goraci/src/main/resources/gora.properties b/gora-goraci/src/main/resources/gora.properties
index a6ba5be..735345d 100644
--- a/gora-goraci/src/main/resources/gora.properties
+++ b/gora-goraci/src/main/resources/gora.properties
@@ -13,10 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+###########################
+# Rackspace Configuration #
+###########################
# Available continents are cloudservers-us and cloudservers-uk
gora.datastore.rackspace.continent=rackspace-cloudservers-us
-gora.datastore.rackspace.username=asf-gora
+gora.datastore.rackspace.username=
gora.datastore.rackspace.apikey=
#Currently you can pick from the following regions
# * US Northern Virginia - IAD=Suppliers.ofInstance(https://iad.servers.api.rackspacecloud.com/v2/910883)
@@ -54,10 +57,17 @@ gora.datastore.rackspace.imageid=6110edfe-8589-4bb1-aa27-385f12242627
# * performance2-120 - 120 GB Performance
gora.datastore.rackspace.flavorid=6
gora.datastore.rackspace.servername=goraci_test_server
-gora.datastore.rackspace.num.servers=20
+gora.datastore.rackspace.num.servers=5
# we can optionally upload our own keys for security, otherwise we can have Rackspace
# create one for us
-gora.datastore.rackspace.uploadpubkey=false
+gora.datastore.rackspace.uploadpubkey=true
+
+######################
+# Chef Configuration #
+######################
+
+gora.datastore.chef.client=
+gora.datastore.chef.organization=
###############################
# Default Accumulo properties #
[2/2] gora git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/gora
Posted by le...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/gora
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/0e6a3095
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/0e6a3095
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/0e6a3095
Branch: refs/heads/master
Commit: 0e6a30958bd523976e184206e120091f900a3d78
Parents: 0501812 e56eebb
Author: Lewis John McGibbney <le...@jpl.nasa.gov>
Authored: Tue Jan 5 04:19:38 2016 -0800
Committer: Lewis John McGibbney <le...@jpl.nasa.gov>
Committed: Tue Jan 5 04:19:38 2016 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------