You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by GitBox <gi...@apache.org> on 2018/09/05 20:53:14 UTC

[GitHub] drcrallen closed pull request #3118: Allow `/` in dimension names

drcrallen closed pull request #3118: Allow `/` in dimension names
URL: https://github.com/apache/incubator-druid/pull/3118
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java
index e90fbdb4952..9f3edcf732a 100644
--- a/processing/src/main/java/io/druid/segment/IndexIO.java
+++ b/processing/src/main/java/io/druid/segment/IndexIO.java
@@ -42,6 +42,7 @@
 import com.metamx.collections.spatial.ImmutableRTree;
 import com.metamx.common.IAE;
 import com.metamx.common.ISE;
+import com.metamx.common.StringUtils;
 import com.metamx.common.io.smoosh.FileSmoosher;
 import com.metamx.common.io.smoosh.Smoosh;
 import com.metamx.common.io.smoosh.SmooshedFileMapper;
@@ -79,6 +80,7 @@
 import io.druid.segment.serde.LongGenericColumnPartSerde;
 import io.druid.segment.serde.LongGenericColumnSupplier;
 import io.druid.segment.serde.SpatialIndexColumnPartSupplier;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.joda.time.Interval;
 
 import java.io.ByteArrayOutputStream;
@@ -86,6 +88,9 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.AbstractList;
@@ -120,17 +125,17 @@ public IndexIO(ObjectMapper mapper, ColumnConfig columnConfig)
     this.columnConfig = Preconditions.checkNotNull(columnConfig, "null ColumnConfig");
     defaultIndexIOHandler = new DefaultIndexIOHandler(mapper);
     indexLoaders = ImmutableMap.<Integer, IndexLoader>builder()
-                               .put(0, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(1, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(2, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(3, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(4, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(5, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(6, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(7, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(8, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
-                               .put(9, new V9IndexLoader(columnConfig))
-                               .build();
+        .put(0, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(1, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(2, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(3, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(4, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(5, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(6, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(7, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(8, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig))
+        .put(9, new V9IndexLoader(columnConfig))
+        .build();
 
 
   }
@@ -589,7 +594,7 @@ public void convertV8toV9(File v8Dir, File v9Dir, IndexSpec indexSpec)
 
           ByteBuffer dimBuffer = v8SmooshedFiles.mapFile(filename);
           String dimension = serializerUtils.readString(dimBuffer);
-          if (!filename.equals(String.format("dim_%s.drd", dimension))) {
+          if (!filename.equals(makeDimFileName(dimension))) {
             throw new ISE("loaded dimension[%s] from file[%s]", dimension, filename);
           }
 
@@ -1086,18 +1091,72 @@ private Column deserializeColumn(ObjectMapper mapper, ByteBuffer byteBuffer) thr
     }
   }
 
+  public static String makeDimFileName(String dimension)
+  {
+    return String.format("dim_%s.drd", sanitizeFileName(dimension));
+  }
+
   public static File makeDimFile(File dir, String dimension)
   {
-    return new File(dir, String.format("dim_%s.drd", dimension));
+    return new File(dir, makeDimFileName(dimension));
+  }
+
+  public static String makeTimeFileName(ByteOrder order)
+  {
+    return String.format("time_%s.drd", order);
   }
 
   public static File makeTimeFile(File dir, ByteOrder order)
   {
-    return new File(dir, String.format("time_%s.drd", order));
+    return new File(dir, makeTimeFileName(order));
+  }
+
+  public static String makeMetricFileName(String metricName, ByteOrder order)
+  {
+    return String.format("met_%s_%s.drd", sanitizeFileName(metricName), order);
   }
 
   public static File makeMetricFile(File dir, String metricName, ByteOrder order)
   {
-    return new File(dir, String.format("met_%s_%s.drd", metricName, order));
+    return new File(dir, makeMetricFileName(metricName, order));
+  }
+
+  public static String sanitizeFileName(String fileName)
+  {
+    String encoding = System.getProperty("file.encoding", "UTF-8");
+
+    final String sha1 = DigestUtils.sha1Hex(fileName).substring(0, 6);
+    String fnameBase = fileName.replace(File.separator, "_");
+
+    try {
+      fnameBase = new URI(String.format("file:/%s", fnameBase)).toASCIIString().substring("file:/".length());
+    }
+    catch (URISyntaxException e) {
+      log.debug(e, "Unable to encode base name [%s]. Trying as hex of UTF-8.", fnameBase);
+      fnameBase = javax.xml.bind.DatatypeConverter.printHexBinary(StringUtils.toUtf8(fnameBase));
+    }
+
+    // For REALLY long names we truncate
+    byte[] fnameBytes;
+    do {
+      try {
+        fnameBytes = fnameBase.getBytes(encoding);
+      }
+      catch (UnsupportedEncodingException e) {
+        log.debug(e, "Error coding [%s] in %s, defaulting to UTF-8", fnameBase, encoding);
+        fnameBytes = StringUtils.toUtf8(fnameBase);
+      }
+      if (fnameBytes.length > 100) {
+        if (fnameBase.length() > 1) {
+          // We loop around until the bytes are fewer than 100
+          fnameBase = fnameBase.substring(0, fnameBase.length() / 2);
+        } else {
+          // length of 1 but byte encoding is > 100... wtf kind of encoding allows that?
+          fnameBase = "";
+        }
+      }
+    } while (fnameBytes.length > 100 && !fnameBase.isEmpty());
+
+    return fnameBase + '.' + sha1;
   }
 }
diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java
index 1831e1a1f67..2b5e2338eae 100644
--- a/processing/src/main/java/io/druid/segment/IndexMerger.java
+++ b/processing/src/main/java/io/druid/segment/IndexMerger.java
@@ -55,7 +55,6 @@
 import com.metamx.common.logger.Logger;
 import io.druid.collections.CombiningIterable;
 import io.druid.common.guava.FileOutputSupplier;
-import io.druid.common.guava.GuavaUtils;
 import io.druid.common.utils.JodaUtils;
 import io.druid.common.utils.SerializerUtils;
 import io.druid.query.aggregation.AggregatorFactory;
@@ -998,12 +997,26 @@ public void close() throws IOException
       final ArrayList<String> expectedFiles = Lists.newArrayList(
           Iterables.concat(
               Arrays.asList(
-                  "index.drd", "inverted.drd", "spatial.drd", String.format("time_%s.drd", IndexIO.BYTE_ORDER)
+                  "index.drd", "inverted.drd", "spatial.drd", IndexIO.makeTimeFileName(IndexIO.BYTE_ORDER)
               ),
-              Iterables.transform(mergedDimensions, GuavaUtils.formatFunction("dim_%s.drd")),
+              Iterables.transform(mergedDimensions, new Function<String, String>()
+              {
+                @Override
+                public String apply(String input)
+                {
+                  return IndexIO.makeDimFileName(input);
+                }
+              }),
               Iterables.transform(
-                  mergedMetrics, GuavaUtils.formatFunction(String.format("met_%%s_%s.drd", IndexIO.BYTE_ORDER))
-              )
+                  mergedMetrics, new Function<String, String>()
+                  {
+                    @Nullable
+                    @Override
+                    public String apply(@Nullable String input)
+                    {
+                      return IndexIO.makeMetricFileName(input, IndexIO.BYTE_ORDER);
+                    }
+                  })
           )
       );
 
diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java
index f8c42d7bfbd..f348e785098 100644
--- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java
+++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java
@@ -26,8 +26,10 @@
 import com.google.common.io.CountingOutputStream;
 import com.google.common.io.InputSupplier;
 import com.google.common.primitives.Ints;
+import io.druid.segment.IndexIO;
 
 import java.io.Closeable;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -88,7 +90,7 @@ public void write(T objectToWrite) throws IOException
 
   private String makeFilename(String suffix)
   {
-    return String.format("%s.%s", filenameBase, suffix);
+    return String.format("%s.%s", IndexIO.sanitizeFileName(filenameBase), suffix);
   }
 
   @Override
diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java
index 4ff7389031d..07113c8da93 100644
--- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java
+++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java
@@ -22,7 +22,9 @@
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingOutputStream;
 import com.google.common.primitives.Ints;
+import io.druid.segment.IndexIO;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
@@ -49,7 +51,7 @@ public VSizeIndexedIntsWriter(
   )
   {
     this.ioPeon = ioPeon;
-    this.valueFileName = String.format("%s.values", filenameBase);
+    this.valueFileName = String.format("%s.values", IndexIO.sanitizeFileName(filenameBase));
     this.numBytes = VSizeIndexedInts.getNumBytesForMax(maxValue);
   }
 
diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java
index 3a78bb34c08..8e9c7803ce6 100644
--- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java
+++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java
@@ -27,6 +27,7 @@
 import com.google.common.io.CountingOutputStream;
 import com.google.common.io.InputSupplier;
 import com.google.common.primitives.Ints;
+import io.druid.segment.IndexIO;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -63,9 +64,9 @@ public VSizeIndexedWriter(
   )
   {
     this.ioPeon = ioPeon;
-    this.metaFileName = String.format("%s.meta", filenameBase);
-    this.headerFileName = String.format("%s.header", filenameBase);
-    this.valuesFileName = String.format("%s.values", filenameBase);
+    this.metaFileName = String.format("%s.meta", IndexIO.sanitizeFileName(filenameBase));
+    this.headerFileName = String.format("%s.header", IndexIO.sanitizeFileName(filenameBase));
+    this.valuesFileName = String.format("%s.values", IndexIO.sanitizeFileName(filenameBase));
     this.maxId = maxId;
   }
 
@@ -132,7 +133,8 @@ public void close() throws IOException
     return ByteStreams.join(
         Iterables.transform(
             Arrays.asList(metaFileName, headerFileName, valuesFileName),
-            new Function<String,InputSupplier<InputStream>>() {
+            new Function<String, InputSupplier<InputStream>>()
+            {
 
               @Override
               public InputSupplier<InputStream> apply(final String input)
diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTest.java b/processing/src/test/java/io/druid/segment/IndexMergerTest.java
index 3ebbb30e00f..11ca56214b7 100644
--- a/processing/src/test/java/io/druid/segment/IndexMergerTest.java
+++ b/processing/src/test/java/io/druid/segment/IndexMergerTest.java
@@ -50,9 +50,11 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.IndexSizeExceededException;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
+import org.apache.commons.lang.RandomStringUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -70,6 +72,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 @RunWith(Parameterized.class)
 public class IndexMergerTest
@@ -189,6 +192,182 @@ public void testPersist() throws Exception
     );
   }
 
+  @Test
+  public void testPersistDimensionWithSlash() throws Exception
+  {
+    final long timestamp = System.currentTimeMillis();
+
+    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    toPersist.add(new MapBasedInputRow(
+        timestamp,
+        Arrays.asList("dim1/sub1", "dim2"),
+        ImmutableMap.<String, Object>of("dim1/sub1", "1", "dim2", "2")
+    ));
+
+    final File tempDir = temporaryFolder.newFolder();
+    QueryableIndex index = closer.closeLater(
+        INDEX_IO.loadIndex(
+            INDEX_MERGER.persist(
+                toPersist,
+                tempDir,
+                indexSpec
+            )
+        )
+    );
+
+    Assert.assertEquals(3, index.getColumn(Column.TIME_COLUMN_NAME).getLength());
+    Assert.assertEquals(Arrays.asList("dim1", "dim2", "dim1/sub1"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(4, index.getColumnNames().size());
+
+    assertDimCompression(index, indexSpec.getDimensionCompressionStrategy());
+
+    Assert.assertArrayEquals(
+        IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
+        index.getMetadata().getAggregators()
+    );
+
+    Assert.assertEquals(
+        QueryGranularities.NONE,
+        index.getMetadata().getQueryGranularity()
+    );
+  }
+
+  @Ignore // breaks meta.smoosh
+  @Test
+  public void testPersistDimensionWithComma() throws Exception
+  {
+    final long timestamp = System.currentTimeMillis();
+
+    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    toPersist.add(new MapBasedInputRow(
+        timestamp,
+        Arrays.asList("dim1,sub1", "dim2"),
+        ImmutableMap.<String, Object>of("dim1,sub1", "1", "dim2", "2")
+    ));
+
+    final File tempDir = temporaryFolder.newFolder();
+    QueryableIndex index = closer.closeLater(
+        INDEX_IO.loadIndex(
+            INDEX_MERGER.persist(
+                toPersist,
+                tempDir,
+                indexSpec
+            )
+        )
+    );
+
+    Assert.assertEquals(3, index.getColumn(Column.TIME_COLUMN_NAME).getLength());
+    Assert.assertEquals(Arrays.asList("dim1", "dim2", "dim1,sub1"), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(4, index.getColumnNames().size());
+
+    assertDimCompression(index, indexSpec.getDimensionCompressionStrategy());
+
+    Assert.assertArrayEquals(
+        IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
+        index.getMetadata().getAggregators()
+    );
+
+    Assert.assertEquals(
+        QueryGranularities.NONE,
+        index.getMetadata().getQueryGranularity()
+    );
+  }
+
+
+  @Test
+  public void testDimensionWithSillyLongName() throws Exception
+  {
+    final StringBuilder sb = new StringBuilder(512);
+    for (int i = 0; i < 512; ++i) {
+      sb.append('a');
+    }
+    final String superLong = sb.toString();
+    final long timestamp = System.currentTimeMillis();
+
+    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    toPersist.add(new MapBasedInputRow(
+        timestamp,
+        Arrays.asList(superLong, "dim2"),
+        ImmutableMap.<String, Object>of(superLong, "1", "dim2", "2")
+    ));
+
+    final File tempDir = temporaryFolder.newFolder();
+    QueryableIndex index = closer.closeLater(
+        INDEX_IO.loadIndex(
+            INDEX_MERGER.persist(
+                toPersist,
+                tempDir,
+                indexSpec
+            )
+        )
+    );
+
+    Assert.assertEquals(3, index.getColumn(Column.TIME_COLUMN_NAME).getLength());
+    Assert.assertEquals(Arrays.asList("dim1", "dim2", superLong), Lists.newArrayList(index.getAvailableDimensions()));
+    Assert.assertEquals(4, index.getColumnNames().size());
+
+    assertDimCompression(index, indexSpec.getDimensionCompressionStrategy());
+
+    Assert.assertArrayEquals(
+        IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
+        index.getMetadata().getAggregators()
+    );
+
+    Assert.assertEquals(
+        QueryGranularities.NONE,
+        index.getMetadata().getQueryGranularity()
+    );
+  }
+
+  @Test
+  public void testDimensionWithRandomName() throws Exception
+  {
+    final Random random = new Random(675431536791L);
+    final String randomString = RandomStringUtils.random(1024, 0, Integer.MAX_VALUE, false, false, null, random);
+    final long timestamp = System.currentTimeMillis();
+
+    IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null);
+    IncrementalIndexTest.populateIndex(timestamp, toPersist);
+    toPersist.add(new MapBasedInputRow(
+        timestamp,
+        Arrays.asList(randomString, "dim2"),
+        ImmutableMap.<String, Object>of(randomString, "1", "dim2", "2")
+    ));
+
+    final File tempDir = temporaryFolder.newFolder();
+    QueryableIndex index = closer.closeLater(
+        INDEX_IO.loadIndex(
+            INDEX_MERGER.persist(
+                toPersist,
+                tempDir,
+                indexSpec
+            )
+        )
+    );
+
+    Assert.assertEquals(3, index.getColumn(Column.TIME_COLUMN_NAME).getLength());
+    Assert.assertEquals(
+        Arrays.asList("dim1", "dim2", randomString),
+        Lists.newArrayList(index.getAvailableDimensions())
+    );
+    Assert.assertEquals(4, index.getColumnNames().size());
+
+    assertDimCompression(index, indexSpec.getDimensionCompressionStrategy());
+
+    Assert.assertArrayEquals(
+        IncrementalIndexTest.getDefaultCombiningAggregatorFactories(),
+        index.getMetadata().getAggregators()
+    );
+
+    Assert.assertEquals(
+        QueryGranularities.NONE,
+        index.getMetadata().getQueryGranularity()
+    );
+  }
+
   @Test
   public void testPersistWithDifferentDims() throws Exception
   {


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org