You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2018/11/15 23:50:54 UTC
[incubator-pinot] branch master updated: [TE] dataframe - tolerate
complex column names (#3491)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e2dface [TE] dataframe - tolerate complex column names (#3491)
e2dface is described below
commit e2dface26b0232eefdac19415d7e64a8dcdcf853
Author: Alexander Pucher <ap...@linkedin.com>
AuthorDate: Thu Nov 15 15:50:49 2018 -0800
[TE] dataframe - tolerate complex column names (#3491)
---
.../com/linkedin/thirdeye/dataframe/DataFrame.java | 19 +++++++++++++++----
.../linkedin/thirdeye/dataframe/DataFrameTest.java | 14 ++++++++++++++
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/DataFrame.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/DataFrame.java
index 432319f..081f6cb 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/DataFrame.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/DataFrame.java
@@ -37,6 +37,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Period;
@@ -91,11 +92,13 @@ public class DataFrame {
for(int i=0; i<seriesNames.size(); i++) {
String rawName = seriesNames.get(i);
- String[] parts = rawName.split(":", 2);
- if(parts.length == 2) {
+ String[] parts = rawName.split(":");
+ String typeString = parts[parts.length - 1];
+
+ if(parts.length > 1 && getValidTypes().contains(typeString)) {
// user specified type
- String name = parts[0];
- Series.SeriesType type = Series.SeriesType.valueOf(parts[1].toUpperCase());
+ String name = StringUtils.join(Arrays.copyOf(parts, parts.length - 1), ":");
+ Series.SeriesType type = Series.SeriesType.valueOf(typeString);
Series series = buildSeries(type, i);
df.addSeries(name, series);
@@ -2560,6 +2563,14 @@ public class DataFrame {
return DataFrame.toSeries(values);
}
+ private static Set<String> getValidTypes() {
+ Set<String> values = new HashSet<>();
+ for (Series.SeriesType type : Series.SeriesType.values()) {
+ values.add(type.name());
+ }
+ return values;
+ }
+
public static class Tuple implements Comparable<Tuple> {
private final Object[] values;
diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/dataframe/DataFrameTest.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/dataframe/DataFrameTest.java
index 617a760..0c96a17 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/dataframe/DataFrameTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/dataframe/DataFrameTest.java
@@ -347,6 +347,20 @@ public class DataFrameTest {
assertEquals(df.getObjects("object"), 1, 2, 3, 4);
}
+ @Test
+ public void testDataFrameBuilderStaticTypingMultiple() {
+ DataFrame df = DataFrame.builder("double:string:LONG").append(2.5d).build();
+ Assert.assertTrue(df.contains("double:string"));
+ Assert.assertEquals(df.get("double:string").type(), Series.SeriesType.LONG);
+ }
+
+ @Test
+ public void testDataFrameBuilderStaticTypingUnknown() {
+ DataFrame df = DataFrame.builder("double:1:2:string").append(1.1d).build();
+ Assert.assertTrue(df.contains("double:1:2:string"));
+ Assert.assertEquals(df.get("double:1:2:string").type(), Series.SeriesType.DOUBLE);
+ }
+
@Test(expectedExceptions = NumberFormatException.class)
public void testDataFrameBuilderStaticTypingFailDouble() {
DataFrame.builder("double:DOUBLE").append("true").build();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org