You are viewing a plain text version of this content. The canonical link for it is here.
Posted to olio-commits@incubator.apache.org by sh...@apache.org on 2009/02/17 17:43:26 UTC
svn commit: r745169 [2/3] - in /incubator/olio/workload/php/trunk/src: com/
org/ org/apache/ org/apache/olio/ org/apache/olio/workload/
org/apache/olio/workload/driver/ org/apache/olio/workload/fsloader/
org/apache/olio/workload/harness/ org/apache/oli...
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: Person.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import com.sun.faban.driver.util.Random;
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.util.RandomUtil;
+import org.apache.olio.workload.util.ScaleFactors;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+import org.apache.olio.workload.loader.framework.ThreadResource;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+
+/**
+ * Person loader
+ */
+public class Person extends Loadable {
+
+ private static final String STATEMENT = "insert into PERSON (username, " +
+ "password, firstname, lastname, email, telephone, imageurl, " +
+ "imagethumburl, summary, timezone, ADDRESS_addressid)" +
+ "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ static Logger logger = Logger.getLogger(Person.class.getName());
+
+ String[] fields = new String[10];
+ int addressId;
+
+ public String getClearStatement() {
+ return "truncate table PERSON";
+ }
+
+ public void prepare() {
+ int id = getSequence();
+ ++id;
+ ThreadResource tr = ThreadResource.getInstance();
+ Random r = tr.getRandom();
+ StringBuilder b = tr.getBuffer();
+ fields[0] = UserName.getUserName(id);
+ fields[1] = String.valueOf(id);
+ fields[2] = RandomUtil.randomName(r, b, 2, 12).toString();
+ b.setLength(0);
+ fields[3] = RandomUtil.randomName(r, b, 5, 15).toString();
+ fields[4] = r.makeCString(3, 10);
+ fields[4] = fields[2] + '_' + fields[3] + '@' + fields[4] + ".com";
+ b.setLength(0);
+ fields[5] = RandomUtil.randomPhone(r, b);
+ fields[6] = "p" + id + ".jpg";
+ fields[7] = "p" + id + "t.jpg";
+ fields[8] = RandomUtil.randomText(r, 250, 2500);
+ fields[9] = "PST";
+ addressId = r.random(1, ScaleFactors.users);
+ }
+
+ public void load() {
+ ThreadConnection c = ThreadConnection.getInstance();
+ try {
+ PreparedStatement s = c.prepareStatement(STATEMENT);
+ for (int i = 0; i < fields.length; i++)
+ s.setString(i + 1, fields[i]);
+ s.setInt(11, addressId);
+ c.addBatch();
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, e.getMessage(), e);
+ }
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: SocialEvent.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import com.sun.faban.driver.util.Random;
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.util.RandomUtil;
+import org.apache.olio.workload.util.ScaleFactors;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+import org.apache.olio.workload.loader.framework.ThreadResource;
+
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.DateFormat;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * SocialEvent loader.
+ */
+public class SocialEvent extends Loadable {
+
+ public static final Date BASE_DATE = new Date(System.currentTimeMillis());
+
+ private static final String STATEMENT = "insert into SOCIALEVENT " +
+ "(title, description, submitterUserName, imageurl, " +
+ "imagethumburl, literatureurl, telephone, timezone, " +
+ "eventtimestamp, eventdate,summary, createdtimestamp, ADDRESS_addressid, " +
+ "totalscore, numberofvotes, disabled) " +
+ "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+ private static final String[] EVT_MINUTES = { "00", "15", "30", "45" };
+
+ static Logger logger = Logger.getLogger(SocialEvent.class.getName());
+
+ String[] fields = new String[11];
+ Date createdTimestamp;
+ int[] ifields = new int[4];
+
+ public String getClearStatement() {
+ return "truncate table SOCIALEVENT";
+ }
+
+ public void prepare() {
+ int id = getSequence();
+ ++id;
+ ThreadResource tr = ThreadResource.getInstance();
+ Random r = tr.getRandom();
+ StringBuilder buffer = tr.getBuffer();
+ fields[0] = RandomUtil.randomText(r, 15, 20); //title
+ fields[1] = RandomUtil.randomText(r, 50, 495); // description
+ fields[2] = UserName.getUserName(r.random(1, ScaleFactors.users));
+ fields[3] = "e" + id + ".jpg"; // imageurl
+ fields[4] = "e" + id + "t.jpg"; // imagethumburl
+ fields[5] = "e" + id + "l.pdf"; //literatureurl
+ fields[6] = RandomUtil.randomPhone(r, buffer); //phone
+ fields[7] = RandomUtil.randomTimeZone(r); // timezone
+ DateFormat dateFormat = tr.getDateFormat(); // eventtimestamp
+ String eventDate = dateFormat.format(
+ r.makeDateInInterval(BASE_DATE, 0, 540));
+ int eventHr = r.random(7, 21);
+ String eventMin = EVT_MINUTES[r.random(0, 3)]; // eventtimestamp
+ fields[8] = String.format("%s %02d:%s:00",
+ eventDate, eventHr, eventMin);
+ fields[9] = eventDate; // eventdate
+ fields[10] = RandomUtil.randomText(r, 50, 90); //summary
+ createdTimestamp = r.makeDateInInterval( //createdtimestamp
+ BASE_DATE, -540, 0);
+ ifields[0] = r.random(1, ScaleFactors.users); // addressId
+ // The rest is initialized to 0 anyway, leave it that way.
+ }
+
+ public void load() {
+ ThreadConnection c = ThreadConnection.getInstance();
+ try {
+ PreparedStatement s = c.prepareStatement(STATEMENT);
+ for (int i = 0; i < fields.length; i++)
+ if (fields[i] != null)
+ s.setString(i + 1, fields[i]);
+ else
+ s.setNull(i + 1, Types.VARCHAR);
+ s.setDate(12, createdTimestamp);
+ for (int i = 0; i < ifields.length; i++)
+ s.setInt(13 + i, ifields[i]);
+ c.addBatch();
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, e.getMessage(), e);
+ }
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: Tag.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.Loader;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * The tag loader.
+ */
+public class Tag extends Loadable {
+
+ // Note that the tag id in the database is autoincrement and may
+ // not coincide with this tag id/name when using multi-thread loading.
+ private static final String STATEMENT = "insert into SOCIALEVENTTAG " +
+ "(tag, refcount) values (?, ?)";
+
+ static Logger logger = Logger.getLogger(Tag.class.getName());
+
+ String tag;
+
+ public String getClearStatement() {
+ return "truncate table SOCIALEVENTTAG";
+ }
+
+ public void prepare() {
+ int id = getSequence();
+ ++id;
+ tag = UserName.getUserName(id);
+ }
+
+
+ public void load() {
+ ThreadConnection c = ThreadConnection.getInstance();
+ try {
+ PreparedStatement s = c.prepareStatement(STATEMENT);
+ s.setString(1, tag);
+ s.setInt(2, 0); // Initialize it to 0 first, count and add later.
+ c.addBatch();
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, e.getMessage(), e);
+ Loader.increaseErrorCount();
+ }
+ }
+
+ /**
+ * For tags, we won't know the refcount till all the data is loaded.
+ * So we update the table at postload.
+ */
+ public void postLoad() {
+ ThreadConnection c = ThreadConnection.getInstance();
+ try {
+ c.prepareStatement("update SOCIALEVENTTAG set refcount = " +
+ "(select count(*) from SOCIALEVENTTAG_SOCIALEVENT " +
+ "where socialeventtagid = " +
+ "SOCIALEVENTTAG.socialeventtagid)");
+ c.executeUpdate();
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, e.getMessage(), e);
+ }
+
+
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: Loadable.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+public abstract class Loadable {
+
+ // Sequence is set by the pool.
+ int sequence;
+
+ protected Loader loader = Loader.getInstance(getClass());
+ LoadablePool<? extends Loadable> pool;
+
+ /**
+ * Obtains the sequence, starting from 0, of this loader.
+ *
+ * @return The sequence of this loadable.
+ */
+ protected int getSequence() {
+ return sequence;
+ }
+
+ public abstract String getClearStatement();
+
+ public abstract void prepare();
+
+ public abstract void load();
+
+ public void postLoad() {
+ // Empty. We do not make it abstract.
+ // A majority of LoadObjects do not need this.
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+package org.apache.olio.workload.loader.framework;
+
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author akara
+ */
+public class LoadablePool<T extends Loadable> {
+
+ private static Logger logger = Logger.getLogger(LoadablePool.class.getName());
+ LinkedBlockingDeque<T> pool = new LinkedBlockingDeque<T>();
+ int sequence;
+ int count = 0;
+ int size;
+ Class<T> clazz;
+
+ public LoadablePool(int size, Class<T> clazz) {
+ this.size = size;
+ this.clazz = clazz;
+ }
+
+ public T getLoadable() throws Exception {
+ T loadable = pool.poll();
+ if (loadable == null) {
+ if (count < size) {
+ loadable = clazz.newInstance();
+ loadable.pool = this;
+ ++count;
+ } else {
+ for (;;) {
+ try {
+ loadable = pool.take();
+ break;
+ } catch (InterruptedException ex) {
+ logger.log(Level.WARNING, "getLoader interrupted", ex);
+ }
+ }
+ }
+ }
+ loadable.sequence = sequence++;
+ return loadable;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void putLoader(Loadable loadable) {
+ for (;;) {
+ try {
+ // User a LIFO model to keep the hot objects in cache.
+ pool.putFirst((T) loadable);
+ break;
+ } catch (InterruptedException ex) {
+ logger.log(Level.WARNING, "putLoader interrupted!", ex);
+ }
+ }
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: Loader.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+import java.sql.SQLException;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.ArrayList;
+
+/**
+ * The loader, one instance per Loadable type loaded, is responsible
+ * for loading the data into the database in the most efficient manner.
+ * We use queues and thread pools to ensure multiple threads are loading
+ * concurrently in large batches.
+ *
+ * @author Akara Sucharitakul
+ */
+public class Loader {
+
+ /** The batch size of a single batch. */
+ public static final int BATCHSIZE = 1000;
+
+ /** The recycling pool size is 3 times the size of the batch. */
+ public static final int POOLSIZE = 3 * BATCHSIZE;
+
+ /** The number of errors before exiting. */
+ public static final int ERROR_THRESHOLD = 50;
+
+ public static final int LOAD_THREADS = 5;
+
+ private static Logger logger =
+ Logger.getLogger(Loader.class.getName());
+
+ private static AtomicInteger errorCount = new AtomicInteger();
+
+ private static ConcurrentHashMap<String, Loader> typeMap =
+ new ConcurrentHashMap<String, Loader>();
+
+ private static ConcurrentHashMap<Class, LoadablePool> poolMap =
+ new ConcurrentHashMap<Class, LoadablePool>();
+
+ private static ArrayList<Thread> mainLoaders = new ArrayList<Thread>();
+
+ // This is a single processing pool for processing data preps.
+ private static ExecutorService processor =
+ Executors.newCachedThreadPool();
+
+ private String name;
+ AtomicInteger loadCount;
+
+ LoadablePool<? extends Loadable> loadablePool;
+
+ // A Loadable type database loading pool.
+ ExecutorService pool;
+ ConcurrentLinkedQueue<Loadable> queue;
+
+ /**
+ * Obtains the instance of the loader for a given loadable type.
+ * @param clazz The loadable type
+ * @return The loader for this type name, or a new loader if none exists
+ */
+ static Loader getInstance(Class<? extends Loadable> clazz) {
+ // We may need to change this to a configurable thread pool size
+ // on a per-type basis. This is the only place to change.
+
+ String name = clazz.getName();
+ Loader loader = new Loader();
+ Loader oldEntry = typeMap.putIfAbsent(name, loader);
+
+ if (oldEntry != null)
+ loader = oldEntry;
+
+ loader.validate(name);
+ return loader;
+ }
+
+ private synchronized void validate(String name) {
+ if (this.name == null)
+ this.name = name;
+ if (loadCount == null)
+ loadCount = new AtomicInteger(0);
+ if (queue == null)
+ queue = new ConcurrentLinkedQueue<Loadable>();
+
+ // We may need to change this to a configurable thread pool size
+ // on a per-type basis. This is the only place to change.
+ if (pool == null)
+ pool = Executors.newFixedThreadPool(LOAD_THREADS);
+ // pool = Executors.newCachedThreadPool();
+ }
+
+ private static <T extends Loadable> LoadablePool<T>
+ getLoadablePool(Class<T> clazz) {
+ LoadablePool<T> pool = new LoadablePool<T>(3 * BATCHSIZE, clazz);
+ @SuppressWarnings("unchecked")
+ LoadablePool<T> oldEntry = poolMap.putIfAbsent(clazz, pool);
+
+ if (oldEntry != null) {
+ pool = oldEntry;
+ }
+
+ return pool;
+ }
+
+ /**
+ * Sets the URL for the connection to the database.
+ * @param url The connection URL
+ */
+ public static void setConnectionURL(String url) {
+ ThreadConnection.connectionURL = url;
+ }
+
+ public static void setJDBCDriverClassName(String driver)
+ throws ClassNotFoundException, IllegalAccessException,
+ InstantiationException {
+ // Just load the DB driver class.
+ Class.forName(driver).newInstance();
+ }
+
+ /**
+ * Uses the loadable to clear the database through the loadable's
+ * clear statement.
+ * @param clazz The loadable class to use
+ */
+ public static void clear(Class<? extends Loadable> clazz) {
+ Loadable loadable = null;
+ try {
+ loadable = clazz.newInstance();
+ } catch (Exception ex) {
+ logger.log(Level.SEVERE, "Error instantiating loader class.", ex);
+ increaseErrorCount();
+ }
+
+ if (loadable != null) {
+ final Loadable l = loadable;
+ Future f = l.loader.pool.submit(new Runnable() {
+
+ public void run() {
+ ThreadConnection c = ThreadConnection.getInstance();
+ try {
+ c.prepareStatement(l.getClearStatement());
+ c.executeUpdate();
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, l.loader.name + ": " +
+ e.getMessage(), e);
+ increaseErrorCount();
+ }
+ }
+ });
+ while (!f.isDone() || f.isCancelled()) {
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ logger.log(Level.WARNING, l.loader.name + ": Interrupted " +
+ "while waiting to clear table.", e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Loads the loadable into the database. Note that the loading is done
+ * asynchronously and is divided into two phases: 1) The preparation
+ * phase where all field values are generated and 2) Loading phase. These
+ * may be performed by different threads. The waitProcessing method
+ * will gracefully shut down the processing infrastructure and wait until
+ * all preparation is done. Shutdown will wait until all data loading
+ * is done.
+ * @param clazz The loadable class
+ * @param occurrences The number of load iterations
+ */
+ public static void load(Class<? extends Loadable> clazz, int occurrences) {
+
+ final Class<? extends Loadable> c = clazz;
+ final int occ = occurrences;
+ Thread mainLoader = new Thread() {
+
+ @Override
+ public void run() {
+ for (int i = 0; i < occ; i++) {
+ Loadable loadable = null;
+ try {
+ loadable = getLoadablePool(c).getLoadable();
+ } catch (Exception ex) {
+ logger.log(Level.SEVERE, "Error obtaining loadable", ex);
+ increaseErrorCount();
+ }
+ if (loadable != null) {
+ final Loadable l = loadable;
+ processor.execute(new Runnable() {
+
+ public void run() {
+ try {
+ l.prepare();
+ l.loader.add(l);
+ } catch (Exception e) {
+ logger.log(Level.WARNING, e.getMessage(), e);
+ Loader.increaseErrorCount();
+ }
+ }
+ });
+ }
+ }
+ }
+ };
+ mainLoaders.add(mainLoader);
+ mainLoader.start();
+ }
+
+ public static void exec(Runnable r) {
+ processor.execute(r);
+ }
+
+ /**
+ * Execute the post loads provided by the loadable.
+ * @param clazz The loadable class
+ */
+ public static void postLoad(Class<? extends Loadable> clazz) {
+ Loadable loadable = null;
+ try {
+ loadable = clazz.newInstance();
+ } catch (Exception ex) {
+ logger.log(Level.SEVERE, "Error instantiating loader class.", ex);
+ increaseErrorCount();
+ }
+
+ if (loadable != null) {
+
+ final Loadable l = loadable;
+ l.loader.pool.submit(new Runnable() {
+
+ public void run() {
+ try {
+ l.postLoad();
+ } catch (Exception e) {
+ logger.log(Level.WARNING, l.loader.name + ": " +
+ e.getMessage(), e);
+ increaseErrorCount();
+ }
+ }
+ });
+ }
+ }
+
+
+ private void add(Loadable l) {
+ queue.add(l);
+ int c = loadCount.incrementAndGet();
+ if (c % BATCHSIZE == 0)
+ flush(c);
+ }
+
+ private void flush(final int batchCount) {
+ pool.submit(new Runnable() {
+ public void run() {
+ ThreadConnection c = ThreadConnection.getInstance();
+ c.processBatch(name, batchCount, queue);
+ }
+ });
+ }
+
+ /**
+ * Terminates the preparation infrastructure and waits until all data
+ * preparation is done.
+ */
+ public static void waitProcessing() {
+ // Wait for the main loaders
+ for (Thread mainLoader : mainLoaders) {
+ for (;;)
+ try {
+ mainLoader.join();
+ break;
+ } catch (InterruptedException e) {
+ logger.log(Level.WARNING, e.getMessage(), e);
+ }
+ }
+ // We ensure the process pool is cleared, first.
+ if (processor != null) {
+ processor.shutdown();
+ boolean terminated = false;
+ while (!terminated)
+ try {
+ terminated = processor.awaitTermination(1, TimeUnit.HOURS);
+ } catch (InterruptedException e) {
+ }
+ processor = null;
+ }
+ }
+
+ /**
+ * Terminates the preparation infrastructure (if still alive) and
+ * then the loading infrastructure. Will return only after all the
+ * loadables in the queue are loaded.
+ */
+ public static void shutdown() {
+ waitProcessing();
+ for (Loader entry : typeMap.values())
+ entry.flush(0);
+ for (Loader entry : typeMap.values())
+ entry.pool.shutdown();
+ for (Loader entry : typeMap.values()) {
+ while (!entry.pool.isTerminated())
+ try {
+ entry.pool.awaitTermination(1, TimeUnit.HOURS);
+ } catch (InterruptedException e) {
+ }
+ }
+ typeMap.clear();
+ ThreadConnection.closeConnections();
+ }
+
+ /**
+ * Increments the global error count. If the count is beyond the threshold,
+ * the loader will terminate.
+ */
+ public static void increaseErrorCount() {
+ if (errorCount.incrementAndGet() > ERROR_THRESHOLD)
+ logger.severe("Error count exceeded threshold of " +
+ ERROR_THRESHOLD + "! Exiting.");
+ System.err.println("Error count exceeded threshold of " +
+ ERROR_THRESHOLD + "! Exiting.");
+ System.exit(2);
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: ThreadConnection.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Wraps a java.sql.connection on a per-thread basis.
+ */
+public class ThreadConnection {
+
+ private static Logger logger =
+ Logger.getLogger(ThreadConnection.class.getName());
+
+ private static ThreadLocal<ThreadConnection> resource =
+ new ThreadLocal<ThreadConnection>() {
+ public ThreadConnection initialValue() {
+ return new ThreadConnection();
+ }
+ };
+
+ private static boolean COMMIT_TX = Boolean.parseBoolean(
+ System.getProperty("commit.tx", "true"));
+ private static final List<ThreadConnection> CONNECTIONLIST =
+ Collections.synchronizedList(new ArrayList<ThreadConnection>());
+
+ public static String connectionURL;
+
+ private Connection conn;
+ private String statementText;
+ private PreparedStatement statement;
+ private int currentBatch;
+ private boolean closed = false;
+
+ /**
+ * The batch buffer buffers the loadables to be added in a batch.
+ * These have to be per-thread and the only reason they are
+ * maintained by the ThreadConnection. Otherwise we need to keep
+ * allocating and collecting. And since we already have the threadlocal,
+ * there is no more overhead getting to it.
+ */
+ Loadable[] batchBuffer;
+
+ private ThreadConnection() {
+ CONNECTIONLIST.add(this);
+ }
+
+ public static ThreadConnection getInstance() {
+ return resource.get();
+ }
+
+ boolean ensureConnection() {
+ if (closed) {
+ logger.severe("Connection used after closure!");
+ Loader.increaseErrorCount();
+ return false;
+ }
+
+ try {
+ if (conn == null || conn.isClosed()) {
+ conn = DriverManager.getConnection(connectionURL);
+ statement = null;
+ statementText = null;
+ }
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, "Error connecting to DB", e);
+ Loader.increaseErrorCount();
+ return false;
+ }
+ return true;
+ }
+
+ boolean resetConnection() {
+ if (closed) {
+ logger.severe("Connection used after closure!");
+ Loader.increaseErrorCount();
+ return false;
+ }
+
+ try {
+ conn = DriverManager.getConnection(connectionURL);
+ statement = null;
+ statementText = null;
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, "Error connecting to DB", e);
+ Loader.increaseErrorCount();
+ return false;
+ }
+ return true;
+ }
+
+ public PreparedStatement prepareStatement(String statementText)
+ throws SQLException {
+ if (conn == null)
+ ensureConnection();
+ if (!statementText.equals(this.statementText)) {
+ this.statementText = statementText;
+ statement = conn.prepareStatement(statementText);
+ }
+ return statement;
+ }
+
+ public void addBatch() throws SQLException {
+ statement.addBatch();
+ ++currentBatch;
+ }
+
+ public void executeUpdate() throws SQLException {
+ if (ensureConnection()) {
+ statement.executeUpdate();
+ if (COMMIT_TX)
+ conn.commit();
+ }
+ }
+
+ void processBatch(String name, int batchCount,
+ Queue<? extends Loadable> queue) {
+ // First we need to save the load objects from the queue
+ // so we do not loose them in case we need to retry.
+ if (batchBuffer == null) {
+ batchBuffer = new Loadable[Loader.BATCHSIZE];
+ }
+ int count = 0;
+ for (; count < Loader.BATCHSIZE; count++) {
+ Loadable l = queue.poll();
+ if (l == null)
+ break;
+ batchBuffer[count] = l;
+ }
+
+ if (count == 0) // Nothing to load.
+ return;
+
+ // Then we load our objects into the DB, retrying the whole
+ // saved ones in case we run into a closed connection.
+ if (!ensureConnection())
+ return;
+
+ String batchName;
+ if (batchCount > 0)
+ batchName = "object batch " + (batchCount - count + 1) + " - " +
+ batchCount + '.';
+ else
+ batchName = "final " + count + " object batch.";
+
+ int flushed = 0;
+ for (int retry = 0; retry < 2; retry++) {
+ try {
+ for (int i = flushed; i < count; i++) {
+ batchBuffer[i].load();
+
+ // Each Loadable object may load more than 1 record.
+ // So we need to check for the number of records
+ // in the batch. If it is more than batchsize, we
+ // need to flush the records, too.
+ if (currentBatch >= Loader.BATCHSIZE) {
+ flush();
+ flushed += currentBatch;
+ currentBatch = 0;
+ logger.finer(name + ": Flushed " + flushed +
+ " records in " + batchName);
+ }
+ }
+ if (currentBatch > 0) {
+ flush();
+ flushed += currentBatch;
+ currentBatch = 0;
+ logger.finer(name + ": Flushed final " + flushed +
+ " records in " + batchName);
+ }
+ logger.fine(name + ": Loaded " + batchName);
+ break; // We won't retry if everything is OK.
+ } catch (BatchUpdateException e) {
+ if (retry == 0) {
+ resetConnection();
+ logger.log(Level.WARNING, name +
+ ": Retry loading.", e);
+ } else {
+ int[] stats = e.getUpdateCounts();
+ int successes = 0;
+ for (int stat : stats) {
+ if (stat != Statement.EXECUTE_FAILED)
+ ++successes;
+ }
+ if (successes == 0) {
+ logger.log(Level.WARNING, name +
+ ": Failed to update.", e);
+ Loader.increaseErrorCount();
+ }
+ }
+ } catch (SQLException e) {
+ if (retry == 0) {
+ resetConnection();
+ logger.log(Level.WARNING, name + ": Retry loading.",
+ e);
+ } else {
+ logger.log(Level.WARNING, e.getMessage(), e);
+ Loader.increaseErrorCount();
+ }
+ }
+ }
+
+ // Once we're done with this buffer, don't hold on to the objects.
+ // Return them to the pool so we don't bloat memory.
+ for (int i = 0; i < batchBuffer.length; i++) {
+ batchBuffer[i].pool.putLoader(batchBuffer[i]);
+ batchBuffer[i] = null;
+ }
+ }
+
+ void flush() throws SQLException {
+ statement.executeBatch();
+ if (COMMIT_TX)
+ conn.commit();
+ }
+
+
+ void close() throws SQLException {
+ closed = true;
+ if (statement != null)
+ statement.close();
+ if (conn != null)
+ conn.close();
+ }
+
+ static void closeConnections() {
+ synchronized (CONNECTIONLIST) {
+ for (ThreadConnection c : CONNECTIONLIST)
+ try {
+ c.close();
+ } catch (SQLException e) {
+ logger.log(Level.SEVERE, e.getMessage(), e);
+ Loader.increaseErrorCount();
+ }
+ CONNECTIONLIST.clear();
+ }
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: ThreadResource.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+import com.sun.faban.driver.util.Random;
+
+import java.util.logging.Logger;
+import java.text.SimpleDateFormat;
+import java.text.DateFormat;
+
+public class ThreadResource {
+
+ Logger logger = Logger.getLogger(ThreadResource.class.getName());
+
+ private static ThreadLocal<ThreadResource> resource =
+ new ThreadLocal<ThreadResource>() {
+ public ThreadResource initialValue() {
+ return new ThreadResource();
+ }
+ };
+
+ Random random;
+ StringBuilder buffer;
+ SimpleDateFormat dateFormat;
+
+ private ThreadResource() {
+ buffer = new StringBuilder(256);
+ random = new Random();
+ }
+
+ public StringBuilder getBuffer() {
+ buffer.setLength(0); // Make sure we clear it
+ return buffer;
+ }
+
+ public Random getRandom() {
+ return random;
+ }
+
+ /**
+ * DateFormat is not thread safe. We need to include it into the
+ * ThreadResource.
+ * @return The thread instance of DateFormat.
+ */
+ public DateFormat getDateFormat() {
+ if (dateFormat == null)
+ dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ return dateFormat;
+ }
+
+ public static ThreadResource getInstance() {
+ return resource.get();
+ }
+}
Propchange: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/
------------------------------------------------------------------------------
svn:mergeinfo =
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: GrowthTest.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.util;
+
+/**
+ * Code to test the growth function and match it to existing research.
+ */
+public class GrowthTest {
+
+ // We use on average of 3.5 tags per event. Random 1..6 tags.
+ // Once we know the tag count, we have to select tags.
+
+ /*
+
+ http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/
+
+ As of this writing, a little over 700 users have tagged it, with 450+
+ unique tags, roughly two-thirds of which tags were (of course) used by
+ one and only one user.
+
+ It took only 10 users (not quite 1.5% of the current total) before the
+ top 3 tags were tagging ontology folksonomy, conveying much the same
+ sense, with only the use of tagging instead of tags making this
+ different from the current set of 3.
+ */
+
+
+ public static double cumuLogistic(
+ double x, double mean, double scale) {
+ return 0.5d + Math.tanh((x - mean) / (2 * scale)) / 2d;
+ }
+
+ public static double cumuHalfLogistic(double x, double scale) {
+ return (1d - Math.pow(Math.E, -x/scale)) / (1d + Math.pow(Math.E, -x/scale));
+ }
+
+ public static double sigmoid(double x, double mean, double scale) {
+ return 1d / (1d + Math.pow(Math.E, -((x / scale) - mean)));
+ }
+
+ public static void main(String[] args) {
+
+ int limit = 5000;
+ int mean = 5000;
+ int scale = 500;
+ for (int x = 0; x < 10000; x += 100) {
+ int y = (int) Math.round(limit * cumuLogistic(x, 5000, 1000));
+ int y2 = (int) Math.round(limit * cumuHalfLogistic(x, 10000)); // Done
+ int y3 = (int) Math.round(limit * sigmoid(x, 6, 1000));
+ System.out.println("-> " + x + ',' + y + ',' + y2 + ',' + y3);
+ }
+ }
+
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: MemCacheUtility.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.util;
+
+import com.danga.MemCached.MemCachedClient;
+import com.danga.MemCached.SockIOPool;
+import com.sun.faban.common.NameValuePair;
+import com.sun.faban.common.TextTable;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+
+
+/**
+ *
+ * @author Kim LiChong
+ */
+public class MemCacheUtility {
+
+ private static MemCachedClient cache = null;
+ private String[] serverList = null;
+ static Logger logger = Logger.getLogger(
+ MemCacheUtility.class.getName());
+ private long baseTime = Long.MIN_VALUE;
+
+
+ /** This constructor creates a new instance of MemCacheUtility
+ A memcache client is created with a pool of servers.
+ *
+ * @param servers ArrayList of NameValuePair<Integer> of servers:port numbers.
+ */
+ public MemCacheUtility(ArrayList<NameValuePair<Integer>> servers) {
+ if (cache == null) {
+ // the env memcachedInstances is in the
+ // form host1:port1, host2:port2, etc.
+ // in an ArrayList
+ //String servers = locator.getString("memcachedInstances");
+
+
+ serverList = new String[servers.size()];
+
+ serverList = convertNameValueToStringArray(servers);
+
+ //logger.info("size of the array is " + serverList.length);
+ //String[] serverList = servers.split(",?[ *]");
+ SockIOPool pool = SockIOPool.getInstance("livePool");
+ pool.setServers(serverList);
+ pool.initialize();
+
+ cache = new MemCachedClient();
+ cache.setPoolName("livePool");
+ }
+ }
+
+ /** This constructor creates a new instance of MemCacheUtility
+ A memcache client is created with a pool of servers.
+ *
+ * @param servers String [] servers:port.
+ */
+ public MemCacheUtility(String[] servers) {
+ if (cache == null) {
+ SockIOPool pool = SockIOPool.getInstance("livePool");
+ pool.setServers(servers);
+ pool.initialize();
+
+ cache = new MemCachedClient();
+ cache.setPoolName("livePool");
+ }
+ }
+
+ /*
+ * This method is a convenience method to convert ArrayList<NameValuePair<Integer> to
+ * a String array of server:port server2:port.
+ * @param servers ArrayList<NameValuePair<Integer>>
+ * @return String []
+ */
+
+ public static String[] convertNameValueToStringArray (
+ ArrayList<NameValuePair<Integer>> servers) {
+ String [] serverArr = new String[servers.size()];
+ int index = 0;
+ for (NameValuePair<Integer> serverEntry : servers) {
+ serverArr[index++] = serverEntry.name + ":" + serverEntry.value;
+ }
+ return serverArr;
+ }
+
+ /** Retrieves memcache stats for each instance of MemCacheUtility.
+ * A TextTable will be produced for each server used to create an
+ * instance of MemCacheUtility. Used to persist stats file for each server.
+ * Returning Map whose key is the servername, the value is a TextTable of statistics
+ * @return Map
+ */
+
+ public Map<String, TextTable> getStats() {
+
+
+ Map memcacheStats = cache.stats();
+ //cache.stats() will return a Map whose key is the name of the memcache server
+ //and whose value is a Map with the memcache statistics
+
+
+ //logger.info("Map size returning is " + memcacheStats.size());
+
+ //produce a TextTable for each server listed
+ Map<String, TextTable> returnMap = new HashMap<String, TextTable>();
+
+ TextTable outputTextTable = null;
+
+ Set<Map.Entry> statEntries = memcacheStats.entrySet();
+
+ //set counter to allow to set number of columns to output
+ for (Map.Entry statEntry : statEntries) {
+ String key = (String) statEntry.getKey();
+ Map statsMap = (Map) statEntry.getValue();
+ //is this case, it is a Map with the statistics
+ //get size so we know how big to make TextTable
+ outputTextTable = new TextTable(statsMap.size(), 2);
+ //set Header
+ outputTextTable.setHeader(0, "Parameter");
+ outputTextTable.setHeader(1, "Value");
+ //outputTextTable.setHeader(2, "for " + key);
+ //get this value's iterator
+ Set<Map.Entry> statsMapEntries = statsMap.entrySet();
+ int counter=0;
+ for (Map.Entry statsMapEntry : statsMapEntries) {
+ outputTextTable.setField(counter, 0,
+ (CharSequence) statsMapEntry.getKey());
+ outputTextTable.setField(counter++, 1,
+ (CharSequence) statsMapEntry.getValue());
+ }
+ //add each TextTable for each server listed to return Map.
+ returnMap.put(key, outputTextTable);
+
+ }
+ return returnMap;
+ }
+
+ /* This method is used for dynamic memcache stats gathering.
+ * The TextTable will contain all memcache server instances in columns
+ * and the server parameters in rows
+ * @return TextTable
+ *@see com.sun.faban.common.TextTable
+ */
+
+ public TextTable getTemporaryStats() {
+ Long time = System.currentTimeMillis();
+ int elapsed = 0;
+ if (baseTime == Long.MIN_VALUE)
+ baseTime = time;
+ else
+ elapsed = (int) (time - baseTime);
+
+ String elapsedSecs = String.format("%.3f", elapsed/1000d);
+
+ Map memcacheStats = cache.stats();
+ //cache.stats() will return a Map whose key is the name of the memcache server
+ //and whose value is a Map with the memcache statistics
+ TextTable outputTextTable = null;
+ Set<Map.Entry> serverEntries = memcacheStats.entrySet();
+
+ //set counter to allow to set number of columns to output
+ int counter = 0;
+ int columnIndex = 0;
+
+ //reset the iterator
+ for (Map.Entry serverEntry : serverEntries) {
+ String key = (String) serverEntry.getKey();
+ Map statsMap = (Map) serverEntry.getValue();
+ if (outputTextTable == null) {
+ // One extra row for elapsed time, one extra header column.
+ outputTextTable = new TextTable(statsMap.size(),
+ serverEntries.size() + 2);
+ }
+ //is this case, it is a Map with the statistics
+ //get size so we know how big to make TextTable
+ // the number of rows is the number of stats
+ // the number of columns is how many server instances there are
+ //set Header
+ outputTextTable.setHeader(0, "Elapsed (sec)");
+ outputTextTable.setHeader(1, "Parameter");
+ outputTextTable.setHeader(columnIndex + 2, key);
+
+ //get this value's iterator
+ Set<Map.Entry> statsMapEntries = statsMap.entrySet();
+ counter=0; //reset counter
+
+ // Populate the rest of the table.
+ for (Map.Entry statsMapEntry : statsMapEntries) {
+ outputTextTable.setField(counter, 0, elapsedSecs);
+ outputTextTable.setField(counter, 1,
+ (CharSequence) statsMapEntry.getKey());
+ outputTextTable.setField(counter++, columnIndex + 2,
+ (CharSequence) statsMapEntry.getValue());
+ }
+ ++columnIndex;
+ }
+ return outputTextTable;
+ }
+
+ /*
+ This main method is used to gather dynamic statistics on memcache server instances.
+ * It expects at least 4 arguments:
+ *
+ * host:server host:server (additional server instances can be designated as host1:port1 host1:port2 OR host2:port etc.
+ * -s start time: the ramp up time, in seconds. (status collection does not take place during the ramp up)
+ * -e end time: the steady state, in seconds. (time to do the statistics data collection)
+ * -i interval time: the snapshot period to collect the stats, in seconds.
+ *
+ * Usage: java com.sun.web20.MemCacheUtility server:port [server2:port server3:port] -s startTime -e endTime -i interval
+ * eg. java com.sun.web20.util.MemCacheUtility server1:12100 server2:12100 -s 300 -e 600 -i 3
+ * This will sleep for 300 seconds during ramp up, collect for 600 seconds with an interval of 3 seconds between
+ * each snapshot.
+ * @param args String []
+ *
+ */
+ public static void main (String[] args) {
+
+ if (args==null || args.length < 4) {//minimum amount of args - one server, -s, -e, -i
+ System.out.println("Usage: java com.sun.web20.MemCacheUtility server:port [server2:port server3:port] -s startTime -e endTime -i interval");
+ System.out.println(" where startTime = ramp up time in seconds. Statistics collection will NOT occur during ramp up time and will sleep for startTime period");
+ System.out.println(" endTime = steady State time in seconds. Statistics collection will only occur during the steady state period");
+ System.out.println(" interval = time between statistics collection snapshots, in seconds.");
+ }
+
+
+ int startTime = 0;
+ int endTime = 0;
+ int intervalTime = 0;
+ LinkedHashSet<String> serverSet = new LinkedHashSet<String>();
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].startsWith("-s")) {
+ if (args[i].length() > 2) // -sarg
+ startTime = Integer.parseInt(args[i].substring(2)) * 1000;
+ else // -s arg
+ startTime = Integer.parseInt(args[++i]) * 1000;
+ } else if (args[i].startsWith("-e")) {
+ if (args[i].length() > 2) // -earg
+ endTime = Integer.parseInt(args[i].substring(2)) * 1000;
+ else // -e arg
+ endTime = Integer.parseInt(args[++i]) * 1000;
+ } else if (args[i].startsWith("-i")) {
+ if (args[i].length() > 2) // -iarg
+ intervalTime =
+ Integer.parseInt(args[i].substring(2)) * 1000;
+ else // -i arg
+ intervalTime = Integer.parseInt(args[++i])* 1000;
+ } else if (args[i].contains(":")) {// host:port pair
+ serverSet.add(args[i]);
+ } else { // host only. Append default port 11211.
+ serverSet.add(args[i] + ":11211");
+ }
+ }
+
+ //finished processing all of the args. populate server list
+ String memCacheServers[] = new String[serverSet.size()];
+ memCacheServers = serverSet.toArray(memCacheServers);
+
+ logger.info("Starting memcache stats");
+
+ //collect only during steady state
+ MemCacheUtility memCacheUtil = new MemCacheUtility(memCacheServers);
+
+ try {
+ Timer timer = new Timer();
+ MemCacheTask task = new MemCacheTask(memCacheUtil);
+ timer.scheduleAtFixedRate(task, startTime, intervalTime);
+ //only print stats for steady state period
+ Thread.sleep(endTime);
+ //wake up and stop printing stats
+ timer.cancel();
+ } catch (InterruptedException ex) {
+ ex.printStackTrace();
+ return;
+ }
+ }
+
+ /* class for TimerTask */
+
+ private static class MemCacheTask extends TimerTask {
+
+ private MemCacheUtility memCacheUtility;
+
+ public MemCacheTask(MemCacheUtility memCacheUtil) {
+ memCacheUtility = memCacheUtil;
+
+ }
+
+ public void run() {
+
+ System.out.println(memCacheUtility.getTemporaryStats());
+
+ }
+
+ }
+
+
+
+
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/RandomUtil.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/RandomUtil.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/RandomUtil.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/RandomUtil.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.olio.workload.util;
+
+import com.sun.faban.driver.util.Random;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.logging.Logger;
+
+public class RandomUtil {
+
+ public static final String[] TIMEZONES = { "ACT", "AET", "AGT", "ART",
+ "AST", "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa",
+ "Africa/Algiers", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui",
+ "Africa/Banjul", "Africa/Bissau", "Africa/Blantyre",
+ "Africa/Brazzaville", "Africa/Bujumbura", "Africa/Cairo",
+ "Africa/Casablanca", "Africa/Ceuta", "Africa/Conakry", "Africa/Dakar",
+ "Africa/Dar_es_Salaam", "Africa/Djibouti", "Africa/Douala",
+ "Africa/El_Aaiun", "Africa/Freetown", "Africa/Gaborone",
+ "Africa/Harare", "Africa/Johannesburg", "Africa/Kampala",
+ "Africa/Khartoum", "Africa/Kigali", "Africa/Kinshasa", "Africa/Lagos",
+ "Africa/Libreville", "Africa/Lome", "Africa/Luanda",
+ "Africa/Lubumbashi", "Africa/Lusaka", "Africa/Malabo", "Africa/Maputo",
+ "Africa/Maseru", "Africa/Mbabane", "Africa/Mogadishu",
+ "Africa/Monrovia", "Africa/Nairobi", "Africa/Ndjamena", "Africa/Niamey",
+ "Africa/Nouakchott", "Africa/Ouagadougou", "Africa/Porto-Novo",
+ "Africa/Sao_Tome", "Africa/Timbuktu", "Africa/Tripoli", "Africa/Tunis",
+ "Africa/Windhoek", "America/Adak", "America/Anchorage",
+ "America/Anguilla", "America/Antigua", "America/Araguaina",
+ "America/Argentina/Buenos_Aires", "America/Argentina/Catamarca",
+ "America/Argentina/ComodRivadavia", "America/Argentina/Cordoba",
+ "America/Argentina/Jujuy", "America/Argentina/La_Rioja",
+ "America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos",
+ "America/Argentina/San_Juan", "America/Argentina/Tucuman",
+ "America/Argentina/Ushuaia", "America/Aruba", "America/Asuncion",
+ "America/Atikokan", "America/Atka", "America/Bahia", "America/Barbados",
+ "America/Belem", "America/Belize", "America/Blanc-Sablon",
+ "America/Boa_Vista", "America/Bogota", "America/Boise",
+ "America/Buenos_Aires", "America/Cambridge_Bay", "America/Campo_Grande",
+ "America/Cancun", "America/Caracas", "America/Catamarca",
+ "America/Cayenne", "America/Cayman", "America/Chicago",
+ "America/Chihuahua", "America/Coral_Harbour", "America/Cordoba",
+ "America/Costa_Rica", "America/Cuiaba", "America/Curacao",
+ "America/Danmarkshavn", "America/Dawson", "America/Dawson_Creek",
+ "America/Denver", "America/Detroit", "America/Dominica",
+ "America/Edmonton", "America/Eirunepe", "America/El_Salvador",
+ "America/Ensenada", "America/Fort_Wayne", "America/Fortaleza",
+ "America/Glace_Bay", "America/Godthab", "America/Goose_Bay",
+ "America/Grand_Turk", "America/Grenada", "America/Guadeloupe",
+ "America/Guatemala", "America/Guayaquil", "America/Guyana",
+ "America/Halifax", "America/Havana", "America/Hermosillo",
+ "America/Indiana/Indianapolis", "America/Indiana/Knox",
+ "America/Indiana/Marengo", "America/Indiana/Petersburg",
+ "America/Indiana/Vevay", "America/Indiana/Vincennes",
+ "America/Indianapolis", "America/Inuvik", "America/Iqaluit",
+ "America/Jamaica", "America/Jujuy", "America/Juneau",
+ "America/Kentucky/Louisville", "America/Kentucky/Monticello",
+ "America/Knox_IN", "America/La_Paz", "America/Lima",
+ "America/Los_Angeles", "America/Louisville", "America/Maceio",
+ "America/Managua", "America/Manaus", "America/Martinique",
+ "America/Mazatlan", "America/Mendoza", "America/Menominee",
+ "America/Merida", "America/Mexico_City", "America/Miquelon",
+ "America/Moncton", "America/Monterrey", "America/Montevideo",
+ "America/Montreal", "America/Montserrat", "America/Nassau",
+ "America/New_York", "America/Nipigon", "America/Nome",
+ "America/Noronha", "America/North_Dakota/Center",
+ "America/North_Dakota/New_Salem", "America/Panama",
+ "America/Pangnirtung", "America/Paramaribo", "America/Phoenix",
+ "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Acre",
+ "America/Porto_Velho", "America/Puerto_Rico", "America/Rainy_River",
+ "America/Rankin_Inlet", "America/Recife", "America/Regina",
+ "America/Rio_Branco", "America/Rosario", "America/Santiago",
+ "America/Santo_Domingo", "America/Sao_Paulo", "America/Scoresbysund",
+ "America/Shiprock", "America/St_Johns", "America/St_Kitts",
+ "America/St_Lucia", "America/St_Thomas", "America/St_Vincent",
+ "America/Swift_Current", "America/Tegucigalpa", "America/Thule",
+ "America/Thunder_Bay", "America/Tijuana", "America/Toronto",
+ "America/Tortola", "America/Vancouver", "America/Virgin",
+ "America/Whitehorse", "America/Winnipeg", "America/Yakutat",
+ "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis",
+ "Antarctica/DumontDUrville", "Antarctica/Mawson", "Antarctica/McMurdo",
+ "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole",
+ "Antarctica/Syowa", "Antarctica/Vostok", "Arctic/Longyearbyen",
+ "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", "Asia/Aqtau",
+ "Asia/Aqtobe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Baghdad",
+ "Asia/Bahrain", "Asia/Baku", "Asia/Bangkok", "Asia/Beirut",
+ "Asia/Bishkek", "Asia/Brunei", "Asia/Calcutta", "Asia/Choibalsan",
+ "Asia/Chongqing", "Asia/Chungking", "Asia/Colombo", "Asia/Dacca",
+ "Asia/Damascus", "Asia/Dhaka", "Asia/Dili", "Asia/Dubai",
+ "Asia/Dushanbe", "Asia/Gaza", "Asia/Harbin", "Asia/Hong_Kong",
+ "Asia/Hovd", "Asia/Irkutsk", "Asia/Istanbul", "Asia/Jakarta",
+ "Asia/Jayapura", "Asia/Jerusalem", "Asia/Kabul", "Asia/Kamchatka",
+ "Asia/Karachi", "Asia/Kashgar", "Asia/Katmandu", "Asia/Krasnoyarsk",
+ "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Kuwait", "Asia/Macao",
+ "Asia/Macau", "Asia/Magadan", "Asia/Makassar", "Asia/Manila",
+ "Asia/Muscat", "Asia/Nicosia", "Asia/Novosibirsk", "Asia/Omsk",
+ "Asia/Oral", "Asia/Phnom_Penh", "Asia/Pontianak", "Asia/Pyongyang",
+ "Asia/Qatar", "Asia/Qyzylorda", "Asia/Rangoon", "Asia/Riyadh",
+ "Asia/Riyadh87", "Asia/Riyadh88", "Asia/Riyadh89", "Asia/Saigon",
+ "Asia/Sakhalin", "Asia/Samarkand", "Asia/Seoul", "Asia/Shanghai",
+ "Asia/Singapore", "Asia/Taipei", "Asia/Tashkent", "Asia/Tbilisi",
+ "Asia/Tehran", "Asia/Tel_Aviv", "Asia/Thimbu", "Asia/Thimphu",
+ "Asia/Tokyo", "Asia/Ujung_Pandang", "Asia/Ulaanbaatar",
+ "Asia/Ulan_Bator", "Asia/Urumqi", "Asia/Vientiane", "Asia/Vladivostok",
+ "Asia/Yakutsk", "Asia/Yekaterinburg", "Asia/Yerevan", "Atlantic/Azores",
+ "Atlantic/Bermuda", "Atlantic/Canary", "Atlantic/Cape_Verde",
+ "Atlantic/Faeroe", "Atlantic/Jan_Mayen", "Atlantic/Madeira",
+ "Atlantic/Reykjavik", "Atlantic/South_Georgia", "Atlantic/St_Helena",
+ "Atlantic/Stanley", "Australia/ACT", "Australia/Adelaide",
+ "Australia/Brisbane", "Australia/Broken_Hill", "Australia/Canberra",
+ "Australia/Currie", "Australia/Darwin", "Australia/Hobart",
+ "Australia/LHI", "Australia/Lindeman", "Australia/Lord_Howe",
+ "Australia/Melbourne", "Australia/NSW", "Australia/North",
+ "Australia/Perth", "Australia/Queensland", "Australia/South",
+ "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria",
+ "Australia/West", "Australia/Yancowinna", "BET", "BST", "Brazil/Acre",
+ "Brazil/DeNoronha", "Brazil/East", "Brazil/West", "CAT", "CET", "CNT",
+ "CST", "CST6CDT", "CTT", "Canada/Atlantic", "Canada/Central",
+ "Canada/East-Saskatchewan", "Canada/Eastern", "Canada/Mountain",
+ "Canada/Newfoundland", "Canada/Pacific", "Canada/Saskatchewan",
+ "Canada/Yukon", "Chile/Continental", "Chile/EasterIsland", "Cuba",
+ "EAT", "ECT", "EET", "EST", "EST5EDT", "Egypt", "Eire", "Etc/GMT",
+ "Etc/GMT+0", "Etc/GMT+1", "Etc/GMT+10", "Etc/GMT+11", "Etc/GMT+12",
+ "Etc/GMT+2", "Etc/GMT+3", "Etc/GMT+4", "Etc/GMT+5", "Etc/GMT+6",
+ "Etc/GMT+7", "Etc/GMT+8", "Etc/GMT+9", "Etc/GMT-0", "Etc/GMT-1",
+ "Etc/GMT-10", "Etc/GMT-11", "Etc/GMT-12", "Etc/GMT-13", "Etc/GMT-14",
+ "Etc/GMT-2", "Etc/GMT-3", "Etc/GMT-4", "Etc/GMT-5", "Etc/GMT-6",
+ "Etc/GMT-7", "Etc/GMT-8", "Etc/GMT-9", "Etc/GMT0", "Etc/Greenwich",
+ "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", "Europe/Amsterdam",
+ "Europe/Andorra", "Europe/Athens", "Europe/Belfast", "Europe/Belgrade",
+ "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels",
+ "Europe/Bucharest", "Europe/Budapest", "Europe/Chisinau",
+ "Europe/Copenhagen", "Europe/Dublin", "Europe/Gibraltar",
+ "Europe/Guernsey", "Europe/Helsinki", "Europe/Isle_of_Man",
+ "Europe/Istanbul", "Europe/Jersey", "Europe/Kaliningrad", "Europe/Kiev",
+ "Europe/Lisbon", "Europe/Ljubljana", "Europe/London",
+ "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta",
+ "Europe/Mariehamn", "Europe/Minsk", "Europe/Monaco", "Europe/Moscow",
+ "Europe/Nicosia", "Europe/Oslo", "Europe/Paris", "Europe/Prague",
+ "Europe/Riga", "Europe/Rome", "Europe/Samara", "Europe/San_Marino",
+ "Europe/Sarajevo", "Europe/Simferopol", "Europe/Skopje", "Europe/Sofia",
+ "Europe/Stockholm", "Europe/Tallinn", "Europe/Tirane",
+ "Europe/Tiraspol", "Europe/Uzhgorod", "Europe/Vaduz", "Europe/Vatican",
+ "Europe/Vienna", "Europe/Vilnius", "Europe/Volgograd", "Europe/Warsaw",
+ "Europe/Zagreb", "Europe/Zaporozhye", "Europe/Zurich", "GB", "GB-Eire",
+ "GMT", "GMT0", "Greenwich", "HST", "Hongkong", "IET", "IST", "Iceland",
+ "Indian/Antananarivo", "Indian/Chagos", "Indian/Christmas",
+ "Indian/Cocos", "Indian/Comoro", "Indian/Kerguelen", "Indian/Mahe",
+ "Indian/Maldives", "Indian/Mauritius", "Indian/Mayotte",
+ "Indian/Reunion", "Iran", "Israel", "JST", "Jamaica", "Japan",
+ "Kwajalein", "Libya", "MET", "MIT", "MST", "MST7MDT",
+ "Mexico/BajaNorte", "Mexico/BajaSur", "Mexico/General",
+ "Mideast/Riyadh87", "Mideast/Riyadh88", "Mideast/Riyadh89", "NET",
+ "NST", "NZ", "NZ-CHAT", "Navajo", "PLT", "PNT", "PRC", "PRT", "PST",
+ "PST8PDT", "Pacific/Apia", "Pacific/Auckland", "Pacific/Chatham",
+ "Pacific/Easter", "Pacific/Efate", "Pacific/Enderbury",
+ "Pacific/Fakaofo", "Pacific/Fiji", "Pacific/Funafuti",
+ "Pacific/Galapagos", "Pacific/Gambier", "Pacific/Guadalcanal",
+ "Pacific/Guam", "Pacific/Honolulu", "Pacific/Johnston",
+ "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein",
+ "Pacific/Majuro", "Pacific/Marquesas", "Pacific/Midway",
+ "Pacific/Nauru", "Pacific/Niue", "Pacific/Norfolk", "Pacific/Noumea",
+ "Pacific/Pago_Pago", "Pacific/Palau", "Pacific/Pitcairn",
+ "Pacific/Ponape", "Pacific/Port_Moresby", "Pacific/Rarotonga",
+ "Pacific/Saipan", "Pacific/Samoa", "Pacific/Tahiti", "Pacific/Tarawa",
+ "Pacific/Tongatapu", "Pacific/Truk", "Pacific/Wake", "Pacific/Wallis",
+ "Pacific/Yap", "Poland", "Portugal", "ROK", "SST", "Singapore",
+ "SystemV/AST4", "SystemV/AST4ADT", "SystemV/CST6", "SystemV/CST6CDT",
+ "SystemV/EST5", "SystemV/EST5EDT", "SystemV/HST10", "SystemV/MST7",
+ "SystemV/MST7MDT", "SystemV/PST8", "SystemV/PST8PDT", "SystemV/YST9",
+ "SystemV/YST9YDT", "Turkey", "UCT", "US/Alaska", "US/Aleutian",
+ "US/Arizona", "US/Central", "US/East-Indiana", "US/Eastern",
+ "US/Hawaii", "US/Indiana-Starke", "US/Michigan", "US/Mountain",
+ "US/Pacific", "US/Pacific-New", "US/Samoa", "UTC", "Universal", "VST",
+ "W-SU", "WET", "Zulu"};
+
+
+ // Phone comes as +1 888 999 0000 or +77 66 999 0000 for non-US.
+ // 50% of the time, do US, 50% non-us.
+ public static String randomPhone(Random r, StringBuilder b) {
+ String v = r.makeNString(1, 2);
+ if (v.length() == 1) {
+ b.append("+1 ");
+ v = r.makeNString(3, 3);
+ b.append(v).append(' ');
+ } else {
+ b.append("+").append(v);
+ v = r.makeNString(2, 2);
+ b.append(' ').append(v).append(' ');
+ }
+ v = r.makeNString(3, 3);
+ b.append(v).append(' ');
+ v = r.makeNString(4, 4);
+ b.append(v);
+ return b.toString();
+ }
+
+ public static String randomTimeZone(Random r) {
+ return TIMEZONES[r.random(0, TIMEZONES.length - 1)];
+ }
+
+ public static StringBuilder randomName(Random r, StringBuilder b,
+ int minLength, int maxLength) {
+ if (minLength < 1 || maxLength < minLength)
+ throw new IllegalArgumentException();
+ b.append(r.makeCString(1, 1).toUpperCase());
+ b.append(r.makeCString(minLength - 1, maxLength - 1).toLowerCase());
+ return b;
+ }
+
+ /**
+ * From http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/
+
+ * "It took only 10 users (not quite 1.5% of the current total) before
+ * the top 3 tags were tagging ontology folksonomy, conveying much
+ * the same sense, with only the use of tagging instead of tags
+ * making this different from the current set of 3."<br>
+ *
+ * In order to achieve a high concentration on the first tags added
+ * to the system, we use a negative exponential distribution to select
+ * the tags.
+ *
+ * @param r The random value generator
+ * @param meanRatio The point of mean in the range from 0 to 1
+ * @return The randomly selected tag id starting with 1, with a
+ * high probability of the first tags being selected.
+ */
+ public static int randomTagId(Random r, double meanRatio) {
+ Logger logger = Logger.getLogger(RandomUtil.class.getName());
+ double mean = ScaleFactors.tagCount * meanRatio;
+
+ int selected;
+ int loops = 0;
+
+ do {
+ double x = r.drandom(0.0, 1.0);
+ if (x == 0)
+ x = 0.05;
+
+ // We use a negative exponential distribution to select the tags.
+ // The first tags tend to be the most often used.
+ selected = (int)(mean * -Math.log(x));
+
+ // if (selected >= ScaleFactors.tagCount)
+ // logger.warning("Selected: " + selected);
+
+ // However, if we exceed the limit, we do not select the last one.
+ // We redo the selection instead.
+ } while (selected >= ScaleFactors.tagCount && loops++ < 10);
+
+ if (loops >= 10)
+ logger.severe("Exceeded loop limit. Selected:" +
+ selected + " TagCount: " + ScaleFactors.tagCount);
+
+ // We use the user name mechanism to create the tag names
+ return ++selected;
+ }
+
+ /**
+ * Returns a random tag name. Implicitly calls randomTagId.
+ * @param r The random value generator
+ * @return The randomly selected tag name.
+ */
+ public static String randomTagName(Random r) {
+ return UserName.getUserName(r.random(1, ScaleFactors.tagCount));
+ //return UserName.getUserName(randomTagId(r, 0.1));
+ }
+
+ /**
+ * Randomly creates text between length x and y, inclusive that is
+ * separated by spaces. The words are between 1 and 12 characters long.
+ */
+ public static String randomText(Random r, int x, int y) {
+ int length = r.random(x, y);
+ StringBuilder buffer = new StringBuilder(length);
+ int leftover = length;
+ while (leftover > 0) {
+ String word = r.makeCString(1, leftover < 12 ? leftover : 12);
+ buffer.append(word).append(' ');
+ leftover -= word.length() + 1;
+ }
+ return buffer.toString();
+ }
+
+ public static String randomEventFromJson (Random r, StringBuilder jsonStr, String objectKey, String attribute) {
+ try {
+ JSONObject job = new JSONObject(jsonStr.toString());
+
+ JSONArray ja = job.getJSONArray(objectKey);
+
+ List<String> list = new ArrayList<String>();
+
+ for (int i = 0; i < ja.length(); i++) {
+ if (!ja.isNull(i)) {
+ JSONObject obj = ja.getJSONObject(i);
+ String value = obj.getString(attribute);
+ list.add(value);
+ }
+ }
+ if (list.size() == 0)
+ return null;
+
+ int rint = r.random(0, list.size()-1);
+ return list.get(rint);
+
+ } catch (JSONException ex) {
+ Logger logger = Logger.getLogger(RandomUtil.class.getName());
+ logger.severe("Could not create JSON Object - str = " + jsonStr.toString());
+ }
+ return null;
+ }
+ /**
+ * Randomly selects an event from the events page.
+ * @param r The random value generator
+ * @param eventListPage The page from the response buffer
+ * @return The selected event id, as a string
+ */
+ public static String randomEvent(Random r, StringBuilder eventListPage) {
+ String search1 = "<a href=\"events.";
+ String search2 = "?socialEventID=";
+ int idx = 0;
+ HashSet<String> eventIdSet = new HashSet<String>();
+ for (;;) {
+ idx = eventListPage.indexOf(search1, idx);
+ if (idx == -1)
+ break;
+ // We skip this the php or jsp, just knowing it is 3 chars
+ idx += search1.length() + 3;
+ // Check matching search2
+ if (eventListPage.indexOf(search2, idx) == idx) {
+ idx += search2.length();
+ int endIdx = eventListPage.indexOf("\"", idx);
+ if (endIdx == -1)
+ break;
+ eventIdSet.add(eventListPage.substring(idx, endIdx).trim());
+ idx = endIdx;
+ }
+ }
+ int size = eventIdSet.size();
+ if (size == 0)
+ return null;
+
+ String[] eventIds = new String[size];
+ eventIds = eventIdSet.toArray(eventIds);
+ return eventIds[r.random(0, size - 1)];
+ }
+
+ // Main for testing RandomEvent.
+ public static void main(String[] args) throws Exception {
+ FileReader reader = new FileReader(args[0]);
+ StringBuilder page = new StringBuilder();
+ char[] readBuffer = new char[8192];
+ int readSize;
+ while ((readSize = reader.read(readBuffer, 0, readBuffer.length))
+ != -1)
+ page.append(readBuffer, 0, readSize);
+ Random r = new Random();
+ String selected = randomEvent(r, page);
+ System.out.println("Selected: " + selected);
+ }
+}
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/ScaleFactors.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/ScaleFactors.java?rev=745169&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/ScaleFactors.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/ScaleFactors.java Tue Feb 17 17:43:25 2009
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id: ScaleFactors.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.util;
+
+import java.util.logging.Logger;
+
+/**
+ * This static class all the scale factors used for loading the data and
+ * load generation.
+ */
+public class ScaleFactors {
+
+ /** The ratio between loaded and active users. */
+ public static final int USERS_RATIO = 100;
+ public static int activeUsers = -1;
+
+ /** The total number of loaded users */
+ public static int users;
+ public static int events;
+ public static int tagCount;
+
+ private static Logger logger =
+ Logger.getLogger(ScaleFactors.class.getName());
+
+ /**
+ * Sets the number of users for the run/load.
+ * @param userCount
+ */
+ public static void setActiveUsers(int userCount) {
+ if (userCount < 25) {
+ logger.warning("Trying to load for " + userCount + " concurrent " +
+ "users which is below the minimum of 25 users. " +
+ "Adjusting to 25 users");
+ userCount = 25;
+ }
+ if (activeUsers == -1) {
+ activeUsers = userCount;
+ users = activeUsers * USERS_RATIO;
+ tagCount = getTagCount(users);
+ events = tagCount * 3;
+ }
+ }
+
+ /**
+ * From http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/
+ * "As of this writing, a little over 700 users have tagged it, with 450+
+ * with 450+ unique tags, roughly two-thirds of which tags were (of course)
+ * used by one and only one user."<br>
+ *
+ * This function uses a cumulative half logistic distribution to determine
+ * the tag growth. We have tested the distribution to be close to the
+ * quote above. I.e. 175 multi-user tags for 700 users. The quote above
+ * gives us one-third of 450 which is 150. Close enough.
+ *
+ * @param users The users of the data loaded.
+ * @return The tag count at this users
+ */
+ public static int getTagCount(int users) {
+ double prob = cumuHalfLogistic(users, 10000);
+ // We limit to 5000 tags
+ return (int) Math.round(5000 * prob);
+ }
+
+ /**
+ * The cumulative half logistic distribution itself, in the flesh!
+ * @param x The value on the x axis, in this case the number of users
+ * @param scale Determines the x-stretch of the curve how far it takes
+ * for the probability to converge to 1
+ * @return The resulting probability or y axis
+ */
+ private static double cumuHalfLogistic(double x, double scale) {
+ double power = -x / scale;
+ return (1d - Math.exp(power)) / (1d + Math.exp(power));
+ }
+}