You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rn...@apache.org on 2015/09/15 16:47:08 UTC
[3/3] bigtop git commit: BIGTOP-1991. Add BigTop Weatherman
BIGTOP-1991. Add BigTop Weatherman
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ad88e741
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ad88e741
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ad88e741
Branch: refs/heads/master
Commit: ad88e7414540d6631854f1a731a3c1c029a3c965
Parents: be796c9
Author: RJ Nowling <rn...@gmail.com>
Authored: Tue Sep 15 09:46:46 2015 -0500
Committer: RJ Nowling <rn...@gmail.com>
Committed: Tue Sep 15 09:46:46 2015 -0500
----------------------------------------------------------------------
bigtop-data-generators/README.md | 5 +
.../bigpetstore-data-generator/README.md | 34 +-
.../bigpetstore-data-generator/build.gradle | 60 +-
.../bigtop-location-data/README.md | 26 +-
.../bigtop-location-data/build.gradle | 49 +-
.../datagenerators/locations/Location.java | 136 +--
.../locations/LocationConstants.java | 10 +-
.../locations/LocationReader.java | 282 +++---
.../bigtop-name-generator/README.md | 26 +-
.../bigtop-name-generator/build.gradle | 50 +-
.../bigtop-samplers/README.md | 26 +-
.../bigtop-samplers/build.gradle | 51 +-
.../samplers/samplers/GammaSampler.java | 34 +
.../bigtop-weatherman/README.md | 36 +
.../bigtop-weatherman/build.gradle | 37 +
.../bigtop-weatherman/settings.gradle | 16 +
.../weatherman/WeatherGenerator.java | 67 ++
.../weatherman/WeatherRecord.java | 77 ++
.../weatherman/internal/Driver.java | 194 ++++
.../internal/PrecipitationSampler.java | 51 ++
.../weatherman/internal/TemperatureSampler.java | 71 ++
.../weatherman/internal/WeatherConstants.java | 53 ++
.../internal/WeatherParametersReader.java | 80 ++
.../internal/WeatherRecordBuilder.java | 163 ++++
.../weatherman/internal/WeatherSampler.java | 50 ++
.../internal/WeatherSamplerBuilder.java | 80 ++
.../internal/WeatherStationParameters.java | 110 +++
.../weatherman/internal/WindSpeedSampler.java | 74 ++
.../resources/input_data/weather_parameters.csv | 897 +++++++++++++++++++
bigtop-data-generators/build.gradle | 44 +
bigtop-data-generators/settings.gradle | 2 +-
31 files changed, 2405 insertions(+), 486 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/README.md
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/README.md b/bigtop-data-generators/README.md
index 4765662..e6fd5c7 100644
--- a/bigtop-data-generators/README.md
+++ b/bigtop-data-generators/README.md
@@ -24,6 +24,7 @@ The following data generators are included so far:
* BigPetStore -- generates customers, stores, products, and transactions for a fictional chain of pet stores
* BigTop Name Generator -- generates names by sampling from U.S. Census data
+* BigTop Weatherman -- weather simulator
We have the following libraries:
@@ -45,3 +46,7 @@ Jar files can be installed to a local Maven cache to simplify integration by ext
$ gradle install
You can then define the dependencies via Maven.
+
+Running
+-------
+Please see READMEs in individual project directories.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigpetstore-data-generator/README.md
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigpetstore-data-generator/README.md b/bigtop-data-generators/bigpetstore-data-generator/README.md
index 1acfe90..9e65db6 100644
--- a/bigtop-data-generators/bigpetstore-data-generator/README.md
+++ b/bigtop-data-generators/bigpetstore-data-generator/README.md
@@ -17,35 +17,11 @@
BigPetStore Data Generator
==========================
-BigPetStore ...
+Library for simulating customer purchasing behavior at a fictional chain of petstores for the purpose of generating synthetic transaction data.
-Data Generator ...
-
-=======
Building and Testing
--------------------
-We use the Gradle build system for the BPS data generator so you'll need
-to install Gradle on your system.
-Once that's done, you can use gradle to run the included unit tests
-and build the data generator jar.
-
-To build:
-
- $ gradle build
-
-This will create several directories and a jar located at:
-
- build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar
-
-Building automatically runs the included unit tests. If you would prefer
-to just run the unit tests, you can do so by:
-
- $ gradle test
-
-
-To clean up the build files, run:
-
- $ gradle clean
+The data generator is part of a Gradle multiproject build. Please see the README in the parent directory for build and test instructions.
Running the Data Generator
@@ -55,12 +31,12 @@ Hadoop or Spark applications) or using a command-line interface.
The data generator CLI requires several parameters. To get
descriptions:
- $ java -jar build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar
+ $ java -jar build/libs/bigpetstore-data-generator-1.1.0-SNAPSHOT.jar
Here is an example for generating 10 stores, 1000 customers, 100 purchasing models,
and a year of transactions:
- $ java -jar build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar generatedData/ 10 1000 100 365.0
+ $ java -jar build/libs/bigpetstore-data-generator-1.1.0-SNAPSHOT.jar generatedData/ 10 1000 100 365.0
Groovy Drivers for Scripting
@@ -75,5 +51,5 @@ To use Groovy scripts, you will need to have Groovy installed on your system. B
data generator as instructed above. Then run the scripts in the `groovy_example_drivers`
directory as so:
- $ groovy -classpath ../build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar MonteCarloExponentialSamplingExample.groovy
+ $ groovy -classpath ../build/libs/bigpetstore-data-generator-1.1.0-SNAPSHOT.jar MonteCarloExponentialSamplingExample.groovy
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigpetstore-data-generator/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigpetstore-data-generator/build.gradle b/bigtop-data-generators/bigpetstore-data-generator/build.gradle
index 9787512..2d5b4cd 100644
--- a/bigtop-data-generators/bigpetstore-data-generator/build.gradle
+++ b/bigtop-data-generators/bigpetstore-data-generator/build.gradle
@@ -13,55 +13,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-apply plugin: 'eclipse'
-apply plugin: 'groovy'
-apply plugin: 'java'
-apply plugin: 'maven'
-group = 'org.apache.bigtop'
-version = '1.1.0-SNAPSHOT'
jar {
-
- from {
- configurations.runtime.collect {
- it.isDirectory() ? it : zipTree(it)
- }
- }
-
- manifest {
- attributes 'Title': 'BigPetStore Data Generator', 'Version': version
- attributes 'Main-Class': 'org.apache.bigtop.datagenerators.bigpetstore.cli.Driver'
+ from {
+ configurations.runtime.collect {
+ it.isDirectory() ? it : zipTree(it)
}
-}
-
-repositories {
- mavenLocal()
- mavenCentral()
-}
-
-test {
- // show standard out and error on console
- testLogging.showStandardStreams = true
-
- // listen to events in the test execution lifecycle
- beforeTest { descriptor ->
- logger.lifecycle("Running test: " + descriptor)
- }
-
- // listen to standard out and standard error of the test JVM(s)
- onOutput { descriptor, event ->
- logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message )
- }
+ }
+ manifest {
+ attributes 'Title': 'BigPetStore Data Generator', 'Version': version
+ attributes 'Main-Class': 'org.apache.bigtop.datagenerators.bigpetstore.cli.Driver'
+ }
}
dependencies {
- compile 'com.google.guava:guava:18.0'
- compile 'com.google.code.gson:gson:2.3'
- compile 'org.apache.commons:commons-lang3:3.4'
- compile project(":bigtop-samplers")
- compile project(":bigtop-name-generator")
- compile project(":bigtop-location-data")
-
- testCompile 'junit:junit:4.+'
+ compile 'com.google.guava:guava:18.0'
+ compile 'com.google.code.gson:gson:2.3'
+ compile 'org.apache.commons:commons-lang3:3.4'
+ compile project(":bigtop-samplers")
+ compile project(":bigtop-name-generator")
+ compile project(":bigtop-location-data")
+
+ testCompile 'junit:junit:4.+'
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/README.md
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-location-data/README.md b/bigtop-data-generators/bigtop-location-data/README.md
index 9aff06f..b61eeb0 100644
--- a/bigtop-data-generators/bigtop-location-data/README.md
+++ b/bigtop-data-generators/bigtop-location-data/README.md
@@ -23,28 +23,4 @@ data model.
Building and Testing
--------------------
-We use the Gradle build system for the BPS data generator so you'll need
-to install Gradle on your system.
-Once that's done, you can use gradle to run the included unit tests
-and build the data generator jar.
-
-To build:
-
- $ gradle build
-
-This will create several directories and a jar located at:
-
- build/libs/bigtop-location-data-1.1.0-SNAPSHOT.jar
-
-Building automatically runs the included unit tests. If you would prefer
-to just run the unit tests, you can do so by:
-
- $ gradle test
-
-To clean up the build files, run:
-
- $ gradle clean
-
-To install a jar into your local maven repository:
-
- $ gradle install
+This project is part of a Gradle multi-project build -- please see the parent directory for build instructions.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-location-data/build.gradle b/bigtop-data-generators/bigtop-location-data/build.gradle
index 9eb91e3..bbced98 100644
--- a/bigtop-data-generators/bigtop-location-data/build.gradle
+++ b/bigtop-data-generators/bigtop-location-data/build.gradle
@@ -13,51 +13,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-apply plugin: 'eclipse'
-apply plugin: 'groovy'
-apply plugin: 'java'
-apply plugin: 'maven'
-group = 'org.apache.bigtop'
-version = '1.1.0-SNAPSHOT'
jar {
-
- from {
- configurations.runtime.collect {
- it.isDirectory() ? it : zipTree(it)
- }
- }
-
- manifest {
- attributes 'Title': 'BigTop Samplers', 'Version': version
+ from {
+ configurations.runtime.collect {
+ it.isDirectory() ? it : zipTree(it)
}
-}
-
-repositories {
- mavenLocal()
- mavenCentral()
-}
-
-test {
- // show standard out and error on console
- testLogging.showStandardStreams = true
-
- // listen to events in the test execution lifecycle
- beforeTest { descriptor ->
- logger.lifecycle("Running test: " + descriptor)
- }
-
- // listen to standard out and standard error of the test JVM(s)
- onOutput { descriptor, event ->
- logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message )
- }
+ }
+ manifest {
+ attributes 'Title': 'BigTop Samplers', 'Version': version
+ }
}
dependencies {
- compile 'com.google.guava:guava:18.0'
-
- compile 'org.apache.commons:commons-lang3:3.4'
+ compile 'com.google.guava:guava:18.0'
+ compile 'org.apache.commons:commons-lang3:3.4'
- testCompile 'junit:junit:4.+'
+ testCompile 'junit:junit:4.+'
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java
index 62afc0a..58f6cdf 100644
--- a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java
+++ b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java
@@ -19,72 +19,72 @@ import java.io.Serializable;
import org.apache.commons.lang3.tuple.Pair;
-public class Location implements Serializable
-{
- private static final long serialVersionUID = 1769986686070108470L;
-
- final String zipcode;
- final Pair<Double, Double> coordinates;
- final String city;
- final String state;
- final double medianHouseholdIncome;
- final long population;
-
- public Location(String zipcode, Pair<Double, Double> coordinates,
- String city, String state, double medianHouseholdIncome, long population)
- {
- this.city = city;
- this.state = state;
- this.zipcode = zipcode;
- this.coordinates = coordinates;
- this.medianHouseholdIncome = medianHouseholdIncome;
- this.population = population;
- }
-
- public String getZipcode()
- {
- return zipcode;
- }
-
- public Pair<Double, Double> getCoordinates()
- {
- return coordinates;
- }
-
- public double getMedianHouseholdIncome()
- {
- return medianHouseholdIncome;
- }
-
- public long getPopulation()
- {
- return population;
- }
-
- public double distance(Location other)
- {
- if(other.getZipcode().equals(zipcode))
- return 0.0;
-
- Pair<Double, Double> otherCoords = other.getCoordinates();
-
- double dist = Math.sin(Math.toRadians(coordinates.getLeft())) *
- Math.sin(Math.toRadians(otherCoords.getLeft())) +
- Math.cos(Math.toRadians(coordinates.getLeft())) *
- Math.cos(Math.toRadians(otherCoords.getLeft())) *
- Math.cos(Math.toRadians(coordinates.getRight() - otherCoords.getRight()));
- dist = Math.toDegrees(Math.acos(dist)) * 69.09;
-
- return dist;
- }
-
- public String getCity()
- {
- return city;
- }
-
- public String getState()
- {
- return state;
- }
+public class Location implements Serializable {
+ private static final long serialVersionUID = 1769986686070108470L;
+
+ final String zipcode;
+ final Pair<Double, Double> coordinates;
+ final String city;
+ final String state;
+ final double medianHouseholdIncome;
+ final long population;
+
+ public Location(String zipcode, Pair<Double, Double> coordinates, String city,
+ String state, double medianHouseholdIncome, long population) {
+ this.city = city;
+ this.state = state;
+ this.zipcode = zipcode;
+ this.coordinates = coordinates;
+ this.medianHouseholdIncome = medianHouseholdIncome;
+ this.population = population;
+ }
+
+ public String getZipcode() {
+ return zipcode;
+ }
+
+ public Pair<Double, Double> getCoordinates() {
+ return coordinates;
+ }
+
+ public double getMedianHouseholdIncome() {
+ return medianHouseholdIncome;
+ }
+
+ public long getPopulation() {
+ return population;
+ }
+
+ public double distance(Pair<Double, Double> otherCoords) {
+ if (Math.abs(coordinates.getLeft() - otherCoords.getLeft()) < 1e-5
+ || Math.abs(coordinates.getRight() - otherCoords.getRight()) < 1e-5)
+ return 0.0;
+
+ double dist = Math.sin(Math.toRadians(coordinates.getLeft()))
+ * Math.sin(Math.toRadians(otherCoords.getLeft()))
+ + Math.cos(Math.toRadians(coordinates.getLeft()))
+ * Math.cos(Math.toRadians(otherCoords.getLeft()))
+ * Math.cos(Math.toRadians(
+ coordinates.getRight() - otherCoords.getRight()));
+ dist = Math.toDegrees(Math.acos(dist)) * 69.09;
+
+ return dist;
+ }
+
+ public double distance(Location other) {
+ if (other.getZipcode().equals(zipcode))
+ return 0.0;
+
+ Pair<Double, Double> otherCoords = other.getCoordinates();
+
+ return distance(otherCoords);
+ }
+
+ public String getCity() {
+ return city;
+ }
+
+ public String getState() {
+ return state;
+ }
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java
index 4140bfd..e6ea242 100644
--- a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java
+++ b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java
@@ -17,9 +17,9 @@ package org.apache.bigtop.datagenerators.locations;
import java.io.File;
-public class LocationConstants
-{
- public static final File COORDINATES_FILE = new File("zips.csv");
- public static final File INCOMES_FILE = new File("ACS_12_5YR_S1903/ACS_12_5YR_S1903_with_ann.csv");
- public static final File POPULATION_FILE = new File("population_data.csv");
+public class LocationConstants {
+ public static final File COORDINATES_FILE = new File("zips.csv");
+ public static final File INCOMES_FILE = new File(
+ "ACS_12_5YR_S1903/ACS_12_5YR_S1903_with_ann.csv");
+ public static final File POPULATION_FILE = new File("population_data.csv");
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java
index 5be4d59..6d5b29e 100644
--- a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java
+++ b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java
@@ -32,151 +32,141 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-public class LocationReader
-{
- private static class ZipcodeLocationRecord
- {
- public final Pair<Double, Double> coordinates;
- public final String state;
- public final String city;
-
- public ZipcodeLocationRecord(Pair<Double, Double> coordinates,
- String city, String state)
- {
- this.coordinates = coordinates;
- this.city = city;
- this.state = state;
- }
- }
-
- private InputStream getResource(File filename)
- {
- InputStream stream = getClass().getResourceAsStream("/input_data/" + filename);
- return new BufferedInputStream(stream);
- }
-
- private ImmutableMap<String, Double> readIncomeData(InputStream path) throws FileNotFoundException
- {
- Scanner scanner = new Scanner(path);
-
- // skip headers
- scanner.nextLine();
- scanner.nextLine();
-
- Map<String, Double> entries = Maps.newHashMap();
- while(scanner.hasNextLine())
- {
- String line = scanner.nextLine().trim();
- String[] cols = line.split(",");
- // zipcodes are in the form "ZCTA5 XXXXX"
- String zipcode = cols[2].split(" ")[1].trim();
- try
- {
- double medianHouseholdIncome = Integer.parseInt(cols[5].trim());
- entries.put(zipcode, medianHouseholdIncome);
- }
- catch(NumberFormatException e)
- {
-
- }
- }
-
- scanner.close();
-
- return ImmutableMap.copyOf(entries);
- }
-
- private ImmutableMap<String, Long> readPopulationData(InputStream path) throws FileNotFoundException
- {
- Scanner scanner = new Scanner(path);
-
- // skip header
- scanner.nextLine();
-
- Map<String, Long> entries = Maps.newHashMap();
- while(scanner.hasNextLine())
- {
- String line = scanner.nextLine().trim();
-
- if(line.length() == 0)
- continue;
-
- String[] cols = line.split(",");
-
- String zipcode = cols[0].trim();
- Long population = Long.parseLong(cols[1].trim());
-
- if(entries.containsKey(zipcode))
- {
- entries.put(zipcode, Math.max(entries.get(zipcode), population));
- }
- else
- {
- entries.put(zipcode, population);
- }
- }
-
- scanner.close();
-
- return ImmutableMap.copyOf(entries);
- }
-
- private ImmutableMap<String, ZipcodeLocationRecord> readCoordinates(InputStream path) throws FileNotFoundException
- {
- Scanner scanner = new Scanner(path);
-
- // skip header
- scanner.nextLine();
-
- Map<String, ZipcodeLocationRecord> entries = Maps.newHashMap();
- while(scanner.hasNextLine())
- {
- String line = scanner.nextLine().trim();
-
- String[] cols = line.split(", ");
-
- // remove quote marks
- String zipcode = cols[0].substring(1, cols[0].length() - 1);
- String state = cols[1].substring(1, cols[1].length() - 1);
- Double latitude = Double.parseDouble(cols[2].substring(1, cols[2].length() - 1));
- Double longitude = Double.parseDouble(cols[3].substring(1, cols[3].length() - 1));
- String city = cols[4].substring(1, cols[4].length() - 1);
-
- Pair<Double, Double> coords = Pair.of(latitude, longitude);
-
- ZipcodeLocationRecord record = new ZipcodeLocationRecord(coords, city, state);
-
- entries.put(zipcode, record);
- }
-
- scanner.close();
-
- return ImmutableMap.copyOf(entries);
- }
-
- public ImmutableList<Location> readData() throws FileNotFoundException
- {
-
- ImmutableMap<String, Double> incomes = readIncomeData(getResource(LocationConstants.INCOMES_FILE));
- ImmutableMap<String, Long> populations = readPopulationData(getResource(LocationConstants.POPULATION_FILE));
- ImmutableMap<String, ZipcodeLocationRecord> coordinates = readCoordinates(getResource(LocationConstants.COORDINATES_FILE));
-
- Set<String> zipcodeSubset = new HashSet<String>(incomes.keySet());
- zipcodeSubset.retainAll(populations.keySet());
- zipcodeSubset.retainAll(coordinates.keySet());
-
- List<Location> table = new Vector<Location>();
- for(String zipcode : zipcodeSubset)
- {
- Location record = new Location(zipcode,
- coordinates.get(zipcode).coordinates,
- coordinates.get(zipcode).city,
- coordinates.get(zipcode).state,
- incomes.get(zipcode),
- populations.get(zipcode));
- table.add(record);
- }
+public class LocationReader {
+ private static class ZipcodeLocationRecord {
+ public final Pair<Double, Double> coordinates;
+ public final String state;
+ public final String city;
+
+ public ZipcodeLocationRecord(Pair<Double, Double> coordinates, String city,
+ String state) {
+ this.coordinates = coordinates;
+ this.city = city;
+ this.state = state;
+ }
+ }
- return ImmutableList.copyOf(table);
- }
+ private InputStream getResource(File filename) {
+ InputStream stream = getClass()
+ .getResourceAsStream("/input_data/" + filename);
+ return new BufferedInputStream(stream);
+ }
+
+ private ImmutableMap<String, Double> readIncomeData(InputStream path)
+ throws FileNotFoundException {
+ Scanner scanner = new Scanner(path);
+
+ // skip headers
+ scanner.nextLine();
+ scanner.nextLine();
+
+ Map<String, Double> entries = Maps.newHashMap();
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine().trim();
+ String[] cols = line.split(",");
+ // zipcodes are in the form "ZCTA5 XXXXX"
+ String zipcode = cols[2].split(" ")[1].trim();
+ try {
+ double medianHouseholdIncome = Integer.parseInt(cols[5].trim());
+ entries.put(zipcode, medianHouseholdIncome);
+ } catch (NumberFormatException e) {
+
+ }
+ }
+
+ scanner.close();
+
+ return ImmutableMap.copyOf(entries);
+ }
+
+ private ImmutableMap<String, Long> readPopulationData(InputStream path)
+ throws FileNotFoundException {
+ Scanner scanner = new Scanner(path);
+
+ // skip header
+ scanner.nextLine();
+
+ Map<String, Long> entries = Maps.newHashMap();
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine().trim();
+
+ if (line.length() == 0)
+ continue;
+
+ String[] cols = line.split(",");
+
+ String zipcode = cols[0].trim();
+ Long population = Long.parseLong(cols[1].trim());
+
+ if (entries.containsKey(zipcode)) {
+ entries.put(zipcode, Math.max(entries.get(zipcode), population));
+ } else {
+ entries.put(zipcode, population);
+ }
+ }
+
+ scanner.close();
+
+ return ImmutableMap.copyOf(entries);
+ }
+
+ private ImmutableMap<String, ZipcodeLocationRecord> readCoordinates(
+ InputStream path) throws FileNotFoundException {
+ Scanner scanner = new Scanner(path);
+
+ // skip header
+ scanner.nextLine();
+
+ Map<String, ZipcodeLocationRecord> entries = Maps.newHashMap();
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine().trim();
+
+ String[] cols = line.split(", ");
+
+ // remove quote marks
+ String zipcode = cols[0].substring(1, cols[0].length() - 1);
+ String state = cols[1].substring(1, cols[1].length() - 1);
+ Double latitude = Double
+ .parseDouble(cols[2].substring(1, cols[2].length() - 1));
+ Double longitude = Double
+ .parseDouble(cols[3].substring(1, cols[3].length() - 1));
+ String city = cols[4].substring(1, cols[4].length() - 1);
+
+ Pair<Double, Double> coords = Pair.of(latitude, longitude);
+
+ ZipcodeLocationRecord record = new ZipcodeLocationRecord(coords, city,
+ state);
+
+ entries.put(zipcode, record);
+ }
+
+ scanner.close();
+
+ return ImmutableMap.copyOf(entries);
+ }
+
+ public ImmutableList<Location> readData() throws FileNotFoundException {
+
+ ImmutableMap<String, Double> incomes = readIncomeData(
+ getResource(LocationConstants.INCOMES_FILE));
+ ImmutableMap<String, Long> populations = readPopulationData(
+ getResource(LocationConstants.POPULATION_FILE));
+ ImmutableMap<String, ZipcodeLocationRecord> coordinates = readCoordinates(
+ getResource(LocationConstants.COORDINATES_FILE));
+
+ Set<String> zipcodeSubset = new HashSet<String>(incomes.keySet());
+ zipcodeSubset.retainAll(populations.keySet());
+ zipcodeSubset.retainAll(coordinates.keySet());
+
+ List<Location> table = new Vector<Location>();
+ for (String zipcode : zipcodeSubset) {
+ Location record = new Location(zipcode,
+ coordinates.get(zipcode).coordinates,
+ coordinates.get(zipcode).city, coordinates.get(zipcode).state,
+ incomes.get(zipcode), populations.get(zipcode));
+ table.add(record);
+ }
+
+ return ImmutableList.copyOf(table);
+ }
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-name-generator/README.md
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-name-generator/README.md b/bigtop-data-generators/bigtop-name-generator/README.md
index aa204b2..93e23a5 100644
--- a/bigtop-data-generators/bigtop-name-generator/README.md
+++ b/bigtop-data-generators/bigtop-name-generator/README.md
@@ -24,28 +24,4 @@ their frequencies according to the U.S. Census.
Building and Testing
--------------------
-We use the Gradle build system for the BPS data generator so you'll need
-to install Gradle on your system.
-Once that's done, you can use gradle to run the included unit tests
-and build the data generator jar.
-
-To build:
-
- $ gradle build
-
-This will create several directories and a jar located at:
-
- build/libs/bigtop-name-generator-1.1.0-SNAPSHOT.jar
-
-Building automatically runs the included unit tests. If you would prefer
-to just run the unit tests, you can do so by:
-
- $ gradle test
-
-To clean up the build files, run:
-
- $ gradle clean
-
-To install a jar into your local maven repository:
-
- $ gradle install
+This project is part of a Gradle multiproject build. Please see the README in the parent directory for build instructions.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-name-generator/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-name-generator/build.gradle b/bigtop-data-generators/bigtop-name-generator/build.gradle
index 0bff824..133dc93 100644
--- a/bigtop-data-generators/bigtop-name-generator/build.gradle
+++ b/bigtop-data-generators/bigtop-name-generator/build.gradle
@@ -13,51 +13,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-apply plugin: 'eclipse'
-apply plugin: 'groovy'
-apply plugin: 'java'
-apply plugin: 'maven'
-group = 'org.apache.bigtop'
-version = '1.1.0-SNAPSHOT'
jar {
-
- from {
- configurations.runtime.collect {
- it.isDirectory() ? it : zipTree(it)
- }
- }
-
- manifest {
- attributes 'Title': 'BigTop Name Generator', 'Version': version
+ from {
+ configurations.runtime.collect {
+ it.isDirectory() ? it : zipTree(it)
}
-}
-
-repositories {
- mavenLocal()
- mavenCentral()
-}
-
-test {
- // show standard out and error on console
- testLogging.showStandardStreams = true
-
- // listen to events in the test execution lifecycle
- beforeTest { descriptor ->
- logger.lifecycle("Running test: " + descriptor)
- }
-
- // listen to standard out and standard error of the test JVM(s)
- onOutput { descriptor, event ->
- logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message )
- }
+ }
+ manifest {
+ attributes 'Title': 'BigTop Name Generator', 'Version': version
+ }
}
dependencies {
- compile 'com.google.guava:guava:18.0'
- compile 'org.apache.commons:commons-lang3:3.4'
- compile project(":bigtop-samplers")
+ compile 'com.google.guava:guava:18.0'
+ compile 'org.apache.commons:commons-lang3:3.4'
+ compile project(":bigtop-samplers")
- testCompile 'junit:junit:4.+'
+ testCompile 'junit:junit:4.+'
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-samplers/README.md
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-samplers/README.md b/bigtop-data-generators/bigtop-samplers/README.md
index 3a91c1b..5fb4a55 100644
--- a/bigtop-data-generators/bigtop-samplers/README.md
+++ b/bigtop-data-generators/bigtop-samplers/README.md
@@ -22,28 +22,4 @@ functions, probability mass functions, weight functions and samplers.
Building and Testing
--------------------
-We use the Gradle build system for the BPS data generator so you'll need
-to install Gradle on your system.
-Once that's done, you can use gradle to run the included unit tests
-and build the data generator jar.
-
-To build:
-
- $ gradle build
-
-This will create several directories and a jar located at:
-
- build/libs/bigtop-samplers-0.9.0-SNAPSHOT.jar
-
-Building automatically runs the included unit tests. If you would prefer
-to just run the unit tests, you can do so by:
-
- $ gradle test
-
-To clean up the build files, run:
-
- $ gradle clean
-
-To install a jar into your local maven repository:
-
- $ gradle install
+This project is part of a multiproject Gradle build. Please see the README in the parent directory for instructions on building.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-samplers/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-samplers/build.gradle b/bigtop-data-generators/bigtop-samplers/build.gradle
index 84b5fe0..1d9ced0 100644
--- a/bigtop-data-generators/bigtop-samplers/build.gradle
+++ b/bigtop-data-generators/bigtop-samplers/build.gradle
@@ -13,52 +13,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-apply plugin: 'eclipse'
-apply plugin: 'groovy'
-apply plugin: 'java'
-apply plugin: 'maven'
-group = 'org.apache.bigtop'
-version = '1.1.0-SNAPSHOT'
jar {
-
- from {
- configurations.runtime.collect {
- it.isDirectory() ? it : zipTree(it)
- }
- }
-
- manifest {
- attributes 'Title': 'BigTop Samplers', 'Version': version
+ from {
+ configurations.runtime.collect {
+ it.isDirectory() ? it : zipTree(it)
}
-}
-
-repositories {
- mavenLocal()
- mavenCentral()
-}
-
-test {
- // show standard out and error on console
- testLogging.showStandardStreams = true
-
- // listen to events in the test execution lifecycle
- beforeTest { descriptor ->
- logger.lifecycle("Running test: " + descriptor)
- }
-
- // listen to standard out and standard error of the test JVM(s)
- onOutput { descriptor, event ->
- logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message )
- }
+ }
+ manifest {
+ attributes 'Title': 'BigTop Samplers', 'Version': version
+ }
}
dependencies {
- compile 'com.google.guava:guava:18.0'
-
- compile 'org.apache.commons:commons-lang3:3.4'
- compile 'org.apache.commons:commons-math3:3.5'
+ compile 'com.google.guava:guava:18.0'
+ compile 'org.apache.commons:commons-lang3:3.4'
+ compile 'org.apache.commons:commons-math3:3.5'
- testCompile 'junit:junit:4.+'
+ testCompile 'junit:junit:4.+'
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java b/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java
new file mode 100644
index 0000000..fb08b48
--- /dev/null
+++ b/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.samplers.samplers;
+
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.commons.math3.distribution.GammaDistribution;
+
+public class GammaSampler implements Sampler<Double> {
+ private final GammaDistribution gamma;
+
+ public GammaSampler(double k, double theta, SeedFactory seedFactory) {
+ gamma = new GammaDistribution(k, theta);
+ gamma.reseedRandomGenerator(seedFactory.getNextSeed());
+ }
+
+ @Override
+ public Double sample() throws Exception {
+ return gamma.sample();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/README.md
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/README.md b/bigtop-data-generators/bigtop-weatherman/README.md
new file mode 100644
index 0000000..608496c
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/README.md
@@ -0,0 +1,36 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+BigTop Weatherman
+=================
+
+Library for generating weather on a zipcode-by-zipcode basis.
+
+Building and Testing
+--------------------
+This project is part of a multiproject Gradle build. Please see directions in the parent directory for build instructions.
+
+Running
+-------
+BigTop Weatherman is primarily designed for consumption as a library but it also provides a CLI interface in the jar. The jar can be executed as follows:
+
+ $ java -jar build/libs/bigtop-weatherman-1.1.0-SNAPSHOT.jar outputDir zipcode simulationLength startDate seed
+
+For example, simulating the weather in South Bend, IN 46617:
+
+ $ java -jar build/libs/bigtop-weatherman-1.1.0-SNAPSHOT.jar output/ 46617 365 2014-04-05 1234
+
+will produce a file `output/46617.txt` containing simulated daily temperature, wind chill, wind speed, total precipitation, rainfall, and snowfall readings.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/build.gradle b/bigtop-data-generators/bigtop-weatherman/build.gradle
new file mode 100644
index 0000000..a2835f7
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/build.gradle
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+jar {
+ from {
+ configurations.runtime.collect {
+ it.isDirectory() ? it : zipTree(it)
+ }
+ }
+
+ manifest {
+ attributes 'Title': 'BigTop Weatherman', 'Version': version
+ attributes 'Main-Class': 'org.apache.bigtop.datagenerators.weatherman.internal.Driver'
+ }
+}
+
+dependencies {
+ compile 'com.google.guava:guava:18.0'
+ compile 'org.apache.commons:commons-lang3:3.4'
+ compile 'joda-time:joda-time:2.8.2'
+ compile project(':bigtop-samplers')
+ compile project(':bigtop-location-data')
+
+ testCompile 'junit:junit:4.+'
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/settings.gradle
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/settings.gradle b/bigtop-data-generators/bigtop-weatherman/settings.gradle
new file mode 100644
index 0000000..dbc292c
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/settings.gradle
@@ -0,0 +1,16 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+rootProject.name = "bigtop-weatherman"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java
new file mode 100644
index 0000000..0601c14
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman;
+
+import java.util.List;
+
+import org.apache.bigtop.datagenerators.locations.Location;
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.bigtop.datagenerators.samplers.samplers.Sampler;
+import org.apache.bigtop.datagenerators.weatherman.internal.WeatherParametersReader;
+import org.apache.bigtop.datagenerators.weatherman.internal.WeatherSamplerBuilder;
+import org.apache.bigtop.datagenerators.weatherman.internal.WeatherStationParameters;
+import org.joda.time.LocalDate;
+
+/**
+ * Generates daily weather records for a given location.
+ *
+ * @author rnowling
+ *
+ */
+public class WeatherGenerator implements Sampler<WeatherRecord> {
+ private final Sampler<WeatherRecord> weatherSampler;
+
+ /**
+ * Initializes the generator.
+ *
+ * @param startDate
+ * - first day of simulation
+ * @param location
+ * - location to simulate weather for
+ * @param seedFactory
+ * - for initializing seeds
+ * @throws Exception
+ * - if weather parameter data cannot be read
+ */
+ public WeatherGenerator(LocalDate startDate, Location location,
+ SeedFactory seedFactory) throws Exception {
+ List<WeatherStationParameters> parameters = new WeatherParametersReader()
+ .readParameters();
+ WeatherSamplerBuilder builder = new WeatherSamplerBuilder(parameters,
+ location, startDate, seedFactory);
+ weatherSampler = builder.build();
+ }
+
+ /**
+ * Generates a daily WeatherRecord for consecutive days, one day per call.
+ *
+ * @return Weather data for a single day
+ * @throws Exception
+ */
+ public WeatherRecord sample() throws Exception {
+ return weatherSampler.sample();
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java
new file mode 100644
index 0000000..04d2eaa
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman;
+
+import org.joda.time.LocalDate;
+
+/**
+ * Weather data for a single day and location
+ *
+ * @author rnowling
+ *
+ */
+public interface WeatherRecord {
+
+ /**
+ * Day
+ *
+ * @return date
+ */
+ public abstract LocalDate getDate();
+
+ /**
+ * Average daily temperature in units of Fahrenheit
+ *
+ * @return temperature
+ */
+ public abstract double getTemperature();
+
+ /**
+ * Daily total precipitation measured in millimeters
+ *
+ * @return total precipitation
+ */
+ public abstract double getPrecipitation();
+
+ /**
+ * Daily average wind speed in meters per second.
+ *
+ * @return average wind speed
+ */
+ public abstract double getWindSpeed();
+
+ /**
+ * Daily average wind chill in Fahrenheit
+ *
+ * @return average wind chill
+ */
+ public abstract double getWindChill();
+
+ /**
+ * Daily total rainfall in millimeters
+ *
+ * @return total rainfall
+ */
+ public abstract double getRainFall();
+
+ /**
+ * Daily total snowfall in centimeters
+ *
+ * @return total snowfall
+ */
+ public abstract double getSnowFall();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java
new file mode 100644
index 0000000..0bd8c41
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.bigtop.datagenerators.locations.Location;
+import org.apache.bigtop.datagenerators.locations.LocationReader;
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.bigtop.datagenerators.weatherman.WeatherGenerator;
+import org.apache.bigtop.datagenerators.weatherman.WeatherRecord;
+import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import com.google.common.collect.Lists;
+
+public class Driver {
+ List<Location> locations;
+ Location location;
+ int simulationLength;
+ LocalDate startDate;
+ SeedFactory seedFactory;
+ File outputDir;
+
+ static final int NPARAMS = 5;
+
+ private void printUsage() {
+ String usage = "BigPetStore Data Generator\n" + "\n"
+ + "Usage: java -jar bigpetstore-weather-generator.java outputDir zipcode simulationLength startDate [seed]\n"
+ + "\n" + "outputDir - (string) directory to write files\n"
+ + "zipcode - (string) location zipcode\n"
+ + "simulationLength - (int) number of days to simulate\n"
+ + "startDate - (string) simulation start date in YYYY-MM-DD format\n"
+ + "seed - (long) seed for RNG. If not given, one is randomly generated.\n";
+
+ System.out.println(usage);
+ }
+
+ private Location findLocation(String zipcode) {
+ for (Location location : locations) {
+ if (location.getZipcode().equals(zipcode))
+ return location;
+ }
+
+ return null;
+ }
+
+ /**
+ * Returns local date from string in YYYY-MM-DD format
+ *
+ * @param text
+ * @return
+ */
+ private LocalDate parseDate(String text) {
+ DateTimeFormatter dtFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+ DateTime dt = dtFormatter.parseDateTime(text);
+
+ return dt.toLocalDate();
+ }
+
+ public void parseArgs(String[] args) {
+ if (args.length != NPARAMS && args.length != (NPARAMS - 1)) {
+ printUsage();
+ System.exit(1);
+ }
+
+ int i = -1;
+
+ outputDir = new File(args[++i]);
+ if (!outputDir.exists()) {
+ System.err.println("Given path (" + args[i] + ") does not exist.\n");
+ printUsage();
+ System.exit(1);
+ }
+
+ if (!outputDir.isDirectory()) {
+ System.err.println("Given path (" + args[i] + ") is not a directory.\n");
+ printUsage();
+ System.exit(1);
+ }
+
+ location = findLocation(args[++i]);
+ if (location == null) {
+ System.err.println(
+ "No location found for given zipcode \"" + args[i] + "\"");
+ printUsage();
+ System.exit(1);
+ }
+
+ try {
+ simulationLength = Integer.parseInt(args[++i]);
+ } catch (Exception e) {
+ System.err.println("Unable to parse '" + args[i]
+ + "' as an int for simulationLength.\n");
+ printUsage();
+ System.exit(1);
+ }
+
+ try {
+ startDate = parseDate(args[++i]);
+ } catch (Exception e) {
+ System.err.println("Unable to parse '" + args[i]
+ + "'. Expected string in 'YYYY-MM-DD' format.\n");
+ printUsage();
+ System.exit(1);
+ }
+
+ long seed = (new Random()).nextLong();
+ if (args.length == NPARAMS) {
+ try {
+ seed = Long.parseLong(args[++i]);
+ } catch (Exception e) {
+ System.err.println(
+ "Unable to parse '" + args[i] + "' as a long for the seed.\n");
+ printUsage();
+ System.exit(1);
+ }
+ }
+
+ seedFactory = new SeedFactory(seed);
+ }
+
+ private void writeWeather(Location location, List<WeatherRecord> trajectory)
+ throws Exception {
+ File outputFile = new File(outputDir.toString() + File.separator
+ + location.getZipcode() + ".txt");
+ Writer output = new BufferedWriter(new FileWriter(outputFile));
+
+ output.write(
+ "date,city,state,zipcode,temperature,windchill,windspeed,precipitation,rainfall,snowfall\n");
+ for (WeatherRecord weather : trajectory) {
+ String record = weather.getDate() + ",";
+ record += location.getCity() + ",";
+ record += location.getState() + ",";
+ record += location.getZipcode() + ",";
+ record += weather.getTemperature() + ",";
+ record += weather.getWindChill() + ",";
+ record += weather.getWindSpeed() + ",";
+ record += weather.getPrecipitation() + ",";
+ record += weather.getRainFall() + ",";
+ record += weather.getSnowFall() + "\n";
+
+ output.write(record);
+ }
+
+ output.close();
+ }
+
+ public void run(String[] args) throws Exception {
+ locations = new LocationReader().readData();
+
+ parseArgs(args);
+
+ WeatherGenerator generator = new WeatherGenerator(startDate, location,
+ seedFactory);
+
+ LocalDate date = startDate;
+ LocalDate endDate = startDate.plusDays(simulationLength);
+ List<WeatherRecord> trajectory = Lists.newArrayList();
+
+ while (date.isBefore(endDate)) {
+ WeatherRecord weather = generator.sample();
+ trajectory.add(weather);
+ date = weather.getDate();
+ }
+
+ writeWeather(location, trajectory);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Driver driver = new Driver();
+ driver.run(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java
new file mode 100644
index 0000000..118e245
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.ExponentialSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.Sampler;
+
+public class PrecipitationSampler implements
+ ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> {
+ private final Sampler<Double> precipitationSampler;
+
+ public PrecipitationSampler(double averagePrecipitation,
+ SeedFactory seedFactory) {
+ precipitationSampler = new ExponentialSampler(1.0 / averagePrecipitation,
+ seedFactory);
+ }
+
+ public WeatherRecordBuilder sample(WeatherRecordBuilder record)
+ throws Exception {
+ double temp = record.getTemperature();
+ double precipitation = precipitationSampler.sample();
+ record.setPrecipitation(precipitation);
+
+ double fractionRain = 1.0
+ / (1.0 + Math.exp(-WeatherConstants.PRECIPITATION_A
+ * (temp - WeatherConstants.PRECIPITATION_B)));
+
+ double rainfall = fractionRain * precipitation;
+ double snowfall = WeatherConstants.PRECIPITATION_TO_SNOWFALL
+ * (1.0 - fractionRain) * precipitation;
+ record.setRainFall(rainfall);
+ record.setSnowFall(snowfall);
+
+ return record;
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java
new file mode 100644
index 0000000..b64b6e0
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.GaussianSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.Sampler;
+import org.joda.time.LocalDate;
+
+public class TemperatureSampler implements
+ ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> {
+ final private Sampler<Double> R;
+ final private double average;
+ final private double coeffReal;
+ final private double coeffImag;
+
+ private LocalDate date;
+ private double noise;
+
+ public TemperatureSampler(LocalDate startDate, double tempAverage,
+ double tempRealCoeff, double tempImagCoeff, double tempSigma,
+ SeedFactory seedFactory) {
+ R = new GaussianSampler(0.0, tempSigma, seedFactory);
+
+ this.average = tempAverage;
+ this.coeffReal = tempRealCoeff;
+ this.coeffImag = tempImagCoeff;
+
+ date = startDate;
+ noise = 0.0;
+ }
+
+ public WeatherRecordBuilder sample(WeatherRecordBuilder weatherRecord)
+ throws Exception {
+ double temp = 0.0;
+ while (date.isEqual(weatherRecord.getDate())
+ || date.isBefore(weatherRecord.getDate())) {
+ double dayOfYear = date.getDayOfYear();
+ temp = average
+ + coeffReal * Math.cos(-2.0 * Math.PI * dayOfYear
+ / WeatherConstants.TEMPERATURE_PERIOD)
+ + coeffImag * Math.sin(-2.0 * Math.PI * dayOfYear
+ / WeatherConstants.TEMPERATURE_PERIOD)
+ + noise;
+
+ noise += -1.0 * noise * WeatherConstants.TEMPERATURE_GAMMA
+ * WeatherConstants.WEATHER_TIMESTEP
+ + Math.sqrt(WeatherConstants.WEATHER_TIMESTEP) * R.sample();
+
+ date = date.plusDays(WeatherConstants.WEATHER_TIMESTEP);
+ }
+
+ weatherRecord.setTemperature(temp);
+
+ return weatherRecord;
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java
new file mode 100644
index 0000000..8346160
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import java.io.File;
+
+public class WeatherConstants {
+ public static final File WEATHER_PARAMETERS_FILE = new File(
+ "weather_parameters.csv");
+
+ public static final double TEMPERATURE_GAMMA = 0.5; // 2 / day
+ public static final int WEATHER_TIMESTEP = 1; // days
+ public static final double TEMPERATURE_PERIOD = 365.0; // days
+
+ public static final double PRECIPITATION_A = 0.2;
+ public static final double PRECIPITATION_B = 27.0;
+ public static final double PRECIPITATION_TO_SNOWFALL = 10.0;
+
+ public static final double WIND_CHILL_PROBABILITY_A = 0.8;
+ public static final double WIND_CHILL_PROBABILITY_B = 0.5;
+ public static final double WIND_CHILL_PROBABILITY_C = 10.0; // F
+ public static final double WIND_CHILL_PROBABILITY_D = 0.2;
+
+ public static final double WIND_SPEED_PROBABILITY_A = -0.5;
+ public static final double WIND_SPEED_PROBABILITY_B = 0.8;
+ public static final double WIND_SPEED_PROBABILITY_C = 17.5; // mph
+ public static final double WIND_SPEED_PROBABILITY_D = 1.0;
+
+ public static final double SNOWFALL_PROBABILITY_A = -0.8;
+ public static final double SNOWFALL_PROBABILITY_B = 10.0;
+ public static final double SNOWFALL_PROBABILITY_C = 0.75; // in
+ public static final double SNOWFALL_PROBABILITY_D = 1.0;
+
+ public static final double RAINFALL_PROBABILITY_A = -0.6;
+ public static final double RAINFALL_PROBABILITY_B = 7.5;
+ public static final double RAINFALL_PROBABILITY_C = 0.75; // in
+ public static final double RAINFALL_PROBABILITY_D = 1.0;
+
+ public static final double M_PER_S_TO_MPH = 2.23694;
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java
new file mode 100644
index 0000000..e8097d6
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Scanner;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import com.google.common.collect.Lists;
+
+public class WeatherParametersReader {
+ private InputStream getResource(File filename) throws Exception {
+ InputStream stream = getClass()
+ .getResourceAsStream("/input_data/" + filename);
+ return new BufferedInputStream(stream);
+ }
+
+ private WeatherStationParameters parseLine(String line) {
+ line = line.trim();
+ String[] cols = line.split(",");
+
+ String WBAN = cols[0];
+ String city = cols[1];
+ String state = cols[2];
+ double latitude = Double.parseDouble(cols[3]);
+ double longitude = Double.parseDouble(cols[4]);
+ Pair<Double, Double> coordinates = Pair.of(latitude, longitude);
+ double temperatureAverage = Double.parseDouble(cols[5]);
+ double temperatureRealCoeff = Double.parseDouble(cols[6]);
+ double temperatureImagCoeff = Double.parseDouble(cols[7]);
+ double temperatureDerivStd = Double.parseDouble(cols[8]);
+ double precipitationAverage = Double.parseDouble(cols[9]);
+ double windSpeedRealCoeff = Double.parseDouble(cols[10]);
+ double windSpeedImagCoeff = Double.parseDouble(cols[11]);
+ double windSpeedK = Double.parseDouble(cols[12]);
+ double windSpeedTheta = Double.parseDouble(cols[13]);
+
+ return new WeatherStationParameters(WBAN, city, state, coordinates,
+ temperatureAverage, temperatureRealCoeff, temperatureImagCoeff,
+ temperatureDerivStd, precipitationAverage, windSpeedRealCoeff,
+ windSpeedImagCoeff, windSpeedK, windSpeedTheta);
+ }
+
+ public List<WeatherStationParameters> readParameters() throws Exception {
+ InputStream inputStream = getResource(
+ WeatherConstants.WEATHER_PARAMETERS_FILE);
+ Scanner scanner = new Scanner(inputStream);
+
+ // skip header
+ scanner.nextLine();
+
+ List<WeatherStationParameters> parameterList = Lists.newArrayList();
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ WeatherStationParameters parameters = parseLine(line);
+ parameterList.add(parameters);
+ }
+
+ scanner.close();
+
+ return parameterList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java
new file mode 100644
index 0000000..393414e
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import java.io.Serializable;
+
+import org.apache.bigtop.datagenerators.weatherman.WeatherRecord;
+import org.joda.time.LocalDate;
+
+public class WeatherRecordBuilder implements Serializable, WeatherRecord {
+ private static final long serialVersionUID = -6397575341624071821L;
+
+ private LocalDate date;
+ private Double temperature;
+ private Double precipitation;
+ private Double windSpeed;
+ private Double windChill;
+ private Double rainFall;
+ private Double snowFall;
+
+ public WeatherRecordBuilder(LocalDate date) {
+ this.date = date;
+ }
+
+ public WeatherRecordBuilder(LocalDate date, double temperature,
+ double precipitation, double windSpeed, double windChill,
+ double rainFall, double snowFall) {
+ this.date = date;
+ this.temperature = temperature;
+ this.precipitation = precipitation;
+ this.windSpeed = windSpeed;
+ this.windChill = windChill;
+ this.rainFall = rainFall;
+ this.snowFall = snowFall;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getDate()
+ */
+ @Override
+ public LocalDate getDate() {
+ return date;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getTemperature()
+ */
+ @Override
+ public double getTemperature() {
+ return temperature;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getPrecipitation()
+ */
+ @Override
+ public double getPrecipitation() {
+ return precipitation;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getWindSpeed()
+ */
+ @Override
+ public double getWindSpeed() {
+ return windSpeed;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getWindChill()
+ */
+ @Override
+ public double getWindChill() {
+ return windChill;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getRainFall()
+ */
+ @Override
+ public double getRainFall() {
+ return rainFall;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather.
+ * WeatherRecord#getSnowFall()
+ */
+ @Override
+ public double getSnowFall() {
+ return snowFall;
+ }
+
+ public WeatherRecord build() {
+ if (temperature == null || precipitation == null || windSpeed == null
+ || windChill == null || rainFall == null || snowFall == null) {
+ throw new IllegalStateException("Not all fields have been initialized.");
+ }
+
+ return this;
+ }
+
+ public void setDate(LocalDate date) {
+ this.date = date;
+ }
+
+ public void setTemperature(Double temperature) {
+ this.temperature = temperature;
+ }
+
+ public void setPrecipitation(Double precipitation) {
+ this.precipitation = precipitation;
+ }
+
+ public void setWindSpeed(Double windSpeed) {
+ this.windSpeed = windSpeed;
+ }
+
+ public void setWindChill(Double windChill) {
+ this.windChill = windChill;
+ }
+
+ public void setRainFall(Double rainFall) {
+ this.rainFall = rainFall;
+ }
+
+ public void setSnowFall(Double snowFall) {
+ this.snowFall = snowFall;
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java
new file mode 100644
index 0000000..3885caa
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.Sampler;
+import org.apache.bigtop.datagenerators.weatherman.WeatherRecord;
+import org.joda.time.LocalDate;
+
+public class WeatherSampler implements Sampler<WeatherRecord> {
+ private final ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> tempSampler;
+ private final ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> windSpeedSampler;
+ private final ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> precipitationSampler;
+ private LocalDate date;
+
+ public WeatherSampler(LocalDate startDate,
+ ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> tempSampler,
+ ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> windSpeedSampler,
+ ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> precipitationSampler) {
+ this.tempSampler = tempSampler;
+ this.windSpeedSampler = windSpeedSampler;
+ this.precipitationSampler = precipitationSampler;
+ date = startDate;
+ }
+
+ public WeatherRecord sample() throws Exception {
+ WeatherRecordBuilder weatherRecord = new WeatherRecordBuilder(date);
+
+ weatherRecord = tempSampler.sample(weatherRecord);
+ weatherRecord = windSpeedSampler.sample(weatherRecord);
+ weatherRecord = precipitationSampler.sample(weatherRecord);
+
+ date = date.plusDays(WeatherConstants.WEATHER_TIMESTEP);
+
+ return weatherRecord.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java
new file mode 100644
index 0000000..40128ae
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import java.util.Collection;
+
+import org.apache.bigtop.datagenerators.locations.Location;
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.Sampler;
+import org.apache.bigtop.datagenerators.weatherman.WeatherRecord;
+import org.joda.time.LocalDate;
+
+public class WeatherSamplerBuilder {
+
+ private final WeatherStationParameters parameters;
+ private final SeedFactory seedFactory;
+ private final LocalDate startDate;
+
+ public WeatherSamplerBuilder(
+ Collection<WeatherStationParameters> weatherParameters,
+ Location location, LocalDate startDate, SeedFactory seedFactory) {
+ parameters = findClosest(weatherParameters, location);
+ this.seedFactory = seedFactory;
+ this.startDate = startDate;
+ }
+
+ private WeatherStationParameters findClosest(
+ Collection<WeatherStationParameters> weatherParameters,
+ Location location) {
+ WeatherStationParameters closestStation = null;
+ double minDist = Double.MAX_VALUE;
+
+ for (WeatherStationParameters parameters : weatherParameters) {
+ double dist = location.distance(parameters.getCoordinates());
+ if (dist < minDist) {
+ minDist = dist;
+ closestStation = parameters;
+ }
+ }
+
+ return closestStation;
+ }
+
+ private ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> buildTempSampler() {
+ return new TemperatureSampler(startDate, parameters.getTemperatureAverage(),
+ parameters.getTemperatureRealCoeff(),
+ parameters.getTemperatureImagCoeff(),
+ parameters.getTemperatureDerivStd(), seedFactory);
+ }
+
+ private ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> buildWindSpeedSampler() {
+ return new WindSpeedSampler(parameters.getWindSpeedRealCoeff(),
+ parameters.getWindSpeedImagCoeff(), parameters.getWindSpeedK(),
+ parameters.getWindSpeedTheta(), seedFactory);
+ }
+
+ private ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> buildPrecipitationSampler() {
+ return new PrecipitationSampler(parameters.getPrecipitationAverage(),
+ seedFactory);
+ }
+
+ public Sampler<WeatherRecord> build() {
+ return new WeatherSampler(startDate, buildTempSampler(),
+ buildWindSpeedSampler(), buildPrecipitationSampler());
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java
new file mode 100644
index 0000000..7edba34
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+public class WeatherStationParameters implements Serializable {
+ private static final long serialVersionUID = -7268791467819627718L;
+
+ private String WBAN;
+ private String city;
+ private String state;
+ private Pair<Double, Double> coordinates;
+ private double temperatureAverage;
+ private double temperatureRealCoeff;
+ private double temperatureImagCoeff;
+ private double temperatureDerivStd;
+ private double precipitationAverage;
+ private double windSpeedRealCoeff;
+ private double windSpeedImagCoeff;
+ private double windSpeedK;
+ private double windSpeedTheta;
+
+ public WeatherStationParameters(String WBAN, String city, String state,
+ Pair<Double, Double> coordinates, double temperatureAverage,
+ double temperatureRealCoeff, double temperatureImagCoeff,
+ double temperatureDerivStd, double precipitationAverage,
+ double windSpeedRealCoeff, double windSpeedImagCoeff,
+ double windSpeedK, double windSpeedTheta) {
+ this.city = city;
+ this.state = state;
+ this.coordinates = coordinates;
+ this.temperatureAverage = temperatureAverage;
+ this.temperatureRealCoeff = temperatureRealCoeff;
+ this.temperatureImagCoeff = temperatureImagCoeff;
+ this.temperatureDerivStd = temperatureDerivStd;
+ this.precipitationAverage = precipitationAverage;
+ this.windSpeedRealCoeff = windSpeedRealCoeff;
+ this.windSpeedImagCoeff = windSpeedImagCoeff;
+ this.windSpeedK = windSpeedK;
+ this.windSpeedTheta = windSpeedTheta;
+ }
+
+ public String getWBAN() {
+ return WBAN;
+ }
+
+ public String getCity() {
+ return city;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public Pair<Double, Double> getCoordinates() {
+ return coordinates;
+ }
+
+ public double getTemperatureAverage() {
+ return temperatureAverage;
+ }
+
+ public double getTemperatureRealCoeff() {
+ return temperatureRealCoeff;
+ }
+
+ public double getTemperatureImagCoeff() {
+ return temperatureImagCoeff;
+ }
+
+ public double getTemperatureDerivStd() {
+ return temperatureDerivStd;
+ }
+
+ public double getPrecipitationAverage() {
+ return precipitationAverage;
+ }
+
+ public double getWindSpeedRealCoeff() {
+ return windSpeedRealCoeff;
+ }
+
+ public double getWindSpeedImagCoeff() {
+ return windSpeedImagCoeff;
+ }
+
+ public double getWindSpeedK() {
+ return windSpeedK;
+ }
+
+ public double getWindSpeedTheta() {
+ return windSpeedTheta;
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java
----------------------------------------------------------------------
diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java
new file mode 100644
index 0000000..c445432
--- /dev/null
+++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.datagenerators.weatherman.internal;
+
+import org.apache.bigtop.datagenerators.samplers.SeedFactory;
+import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.GammaSampler;
+import org.apache.bigtop.datagenerators.samplers.samplers.Sampler;
+
+public class WindSpeedSampler implements
+ ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> {
+ private final double coeffReal;
+ private final double coeffImag;
+ private final Sampler<Double> gamma;
+
+ public WindSpeedSampler(double windSpeedRealCoeff, double windSpeedImagCoeff,
+ double windSpeedK, double windSpeedTheta, SeedFactory seedFactory) {
+ coeffReal = windSpeedRealCoeff;
+ coeffImag = windSpeedImagCoeff;
+
+ gamma = new GammaSampler(windSpeedK, windSpeedTheta, seedFactory);
+
+ }
+
+ /**
+ *
+ * @param temp
+ * - given in Fahrenheit
+ * @param windSpeed
+ * - given in MPH
+ * @return
+ */
+ private Double windChill(double temp, double windSpeed) {
+ double v_16 = Math.pow(windSpeed, 0.16);
+ double windChill = 35.74 + 0.6215 * temp - 35.74 * v_16
+ + 0.4275 * temp * v_16;
+
+ return windChill;
+ }
+
+ public WeatherRecordBuilder sample(WeatherRecordBuilder weatherRecord)
+ throws Exception {
+
+ double dayOfYear = weatherRecord.getDate().getDayOfYear();
+ // meters/second
+ double windSpeed = Math.max(0.0,
+ coeffReal
+ * Math.cos(-2.0 * Math.PI * dayOfYear
+ / WeatherConstants.TEMPERATURE_PERIOD)
+ + coeffImag * Math.sin(-2.0 * Math.PI * dayOfYear
+ / WeatherConstants.TEMPERATURE_PERIOD) + gamma.sample());
+
+ double windChill = windChill(weatherRecord.getTemperature(),
+ windSpeed * WeatherConstants.M_PER_S_TO_MPH);
+
+ weatherRecord.setWindSpeed(windSpeed);
+ weatherRecord.setWindChill(windChill);
+
+ return weatherRecord;
+ }
+}