You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by gi...@apache.org on 2022/09/08 03:42:30 UTC
[druid] branch 24.0.0-revert-glob-filter created (now 2d90f29042)
This is an automated email from the ASF dual-hosted git repository.
gian pushed a change to branch 24.0.0-revert-glob-filter
in repository https://gitbox.apache.org/repos/asf/druid.git
at 2d90f29042 In branch: lower coverage requirement for azure-extensions/pom.xml, add subset of CloudObjectInputSourceTest.
This branch includes the following new commits:
new 455a0d61e6 Merge branch '24.0.0' into remove_unused_count
new 2d90f29042 In branch: lower coverage requirement for azure-extensions/pom.xml, add subset of CloudObjectInputSourceTest.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
[druid] 01/02: Merge branch '24.0.0' into remove_unused_count
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch 24.0.0-revert-glob-filter
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 455a0d61e6b974aaa93d41aeede7245d7f201681
Merge: 9e41de1e37 20035022a8
Author: Gian Merlino <gi...@gmail.com>
AuthorDate: Wed Sep 7 20:32:08 2022 -0700
Merge branch '24.0.0' into remove_unused_count
.gitignore | 2 +-
.../druid/benchmark/IPv4AddressBenchmark.java | 3 +-
.../org/apache/druid/math/expr/ApplyFunction.java | 62 +--
.../apache/druid/math/expr/BinaryOperatorExpr.java | 12 +-
.../org/apache/druid/math/expr/ConstantExpr.java | 2 +-
.../java/org/apache/druid/math/expr/ExprEval.java | 4 +-
.../org/apache/druid/math/expr/ExprMacroTable.java | 4 +-
.../math/expr/ExpressionProcessingException.java | 36 ++
.../druid/math/expr/ExpressionTypeFactory.java | 2 +-
.../math/expr/ExpressionValidationException.java | 36 ++
.../java/org/apache/druid/math/expr/Function.java | 251 ++++-----
.../org/apache/druid/math/expr/FunctionalExpr.java | 2 +-
.../org/apache/druid/math/expr/NamedFunction.java | 175 ++++++
.../druid/math/expr/vector/VectorProcessors.java | 8 +-
.../apache/druid/math/expr/ApplyFunctionTest.java | 57 +-
.../org/apache/druid/math/expr/FunctionTest.java | 86 +--
.../org/apache/druid/math/expr/ParserTest.java | 4 +-
docs/assets/nested-combined-json.png | Bin 0 -> 272858 bytes
docs/assets/nested-display-data-types.png | Bin 0 -> 254224 bytes
docs/assets/nested-examine-schema.png | Bin 0 -> 251706 bytes
docs/assets/nested-extract-as-type.png | Bin 0 -> 288918 bytes
docs/assets/nested-extract-elements.png | Bin 0 -> 317905 bytes
docs/assets/nested-group-aggregate.png | Bin 0 -> 249178 bytes
docs/assets/nested-msq-ingestion-transform.png | Bin 0 -> 370419 bytes
docs/assets/nested-msq-ingestion.png | Bin 0 -> 302033 bytes
docs/assets/nested-parse-deserialize.png | Bin 0 -> 213791 bytes
docs/assets/nested-retrieve-json.png | Bin 0 -> 295566 bytes
docs/assets/nested-return-json.png | Bin 0 -> 349144 bytes
docs/ingestion/data-formats.md | 4 +-
docs/ingestion/ingestion-spec.md | 2 +-
docs/ingestion/schema-design.md | 8 +-
docs/misc/math-expr.md | 6 +-
docs/querying/nested-columns.md | 597 +++++++++++++++++++++
docs/querying/sql-data-types.md | 24 +-
docs/querying/sql-json-functions.md | 5 +-
docs/querying/sql-translation.md | 154 +++++-
docs/querying/virtual-columns.md | 2 +-
examples/bin/greet | 32 ++
examples/bin/supervise | 21 +-
examples/conf/supervise/single-server/large.conf | 1 +
examples/conf/supervise/single-server/medium.conf | 1 +
.../supervise/single-server/micro-quickstart.conf | 1 +
.../supervise/single-server/nano-quickstart.conf | 1 +
examples/conf/supervise/single-server/small.conf | 1 +
examples/conf/supervise/single-server/xlarge.conf | 1 +
.../query/expressions/BloomFilterExpressions.java | 27 +-
.../expressions/BloomFilterExpressionsTest.java | 6 +-
.../druid/msq/querykit/BaseLeafFrameProcessor.java | 14 +-
.../querykit/BaseLeafFrameProcessorFactory.java | 3 +-
.../groupby/GroupByPreShuffleFrameProcessor.java | 25 +-
.../GroupByPreShuffleFrameProcessorFactory.java | 22 +-
.../msq/querykit/scan/ScanQueryFrameProcessor.java | 37 +-
.../scan/ScanQueryFrameProcessorFactory.java | 22 +-
.../querykit/scan/ScanQueryFrameProcessorTest.java | 240 +++++++++
.../druid/query/expressions/SleepExprMacro.java | 8 +-
.../apache/druid/testing/tools/SleepExprMacro.java | 8 +-
.../CaseInsensitiveContainsExprMacro.java | 5 +-
.../druid/query/expression/ContainsExprMacro.java | 5 +-
.../apache/druid/query/expression/ExprUtils.java | 12 -
.../query/expression/HyperUniqueExpressions.java | 38 +-
.../expression/IPv4AddressMatchExprMacro.java | 11 +-
.../expression/IPv4AddressParseExprMacro.java | 5 +-
.../expression/IPv4AddressStringifyExprMacro.java | 5 +-
.../druid/query/expression/LikeExprMacro.java | 12 +-
.../druid/query/expression/LookupExprMacro.java | 10 +-
.../query/expression/NestedDataExpressions.java | 47 +-
.../query/expression/RegexpExtractExprMacro.java | 9 +-
.../query/expression/RegexpLikeExprMacro.java | 7 +-
.../query/expression/TimestampCeilExprMacro.java | 5 +-
.../expression/TimestampExtractExprMacro.java | 14 +-
.../query/expression/TimestampFloorExprMacro.java | 5 +-
.../query/expression/TimestampFormatExprMacro.java | 8 +-
.../query/expression/TimestampParseExprMacro.java | 5 +-
.../query/expression/TimestampShiftExprMacro.java | 5 +-
.../druid/query/expression/TrimExprMacro.java | 5 +-
.../segment/transform/ExpressionTransform.java | 30 +-
.../expression/CaseInsensitiveExprMacroTest.java | 4 +-
.../query/expression/ContainsExprMacroTest.java | 4 +-
.../expression/HyperUniqueExpressionsTest.java | 14 +-
.../expression/IPv4AddressMatchExprMacroTest.java | 7 +-
.../expression/IPv4AddressParseExprMacroTest.java | 4 +-
.../IPv4AddressStringifyExprMacroTest.java | 4 +-
.../expression/NestedDataExpressionsTest.java | 6 +-
.../expression/RegexpExtractExprMacroTest.java | 4 +-
.../query/expression/RegexpLikeExprMacroTest.java | 10 +-
.../query/expression/TimestampShiftMacroTest.java | 8 +-
.../server/coordinator/duty/CompactSegments.java | 3 +-
.../sql/calcite/expression/ExpressionsTest.java | 40 +-
.../expression/IPv4AddressMatchExpressionTest.java | 7 +-
.../expression/IPv4AddressParseExpressionTest.java | 5 +-
.../IPv4AddressStringifyExpressionTest.java | 5 +-
.../ingestion-progress-dialog.tsx | 4 +-
website/sidebars.json | 2 +
93 files changed, 1770 insertions(+), 623 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
[druid] 02/02: In branch: lower coverage requirement for azure-extensions/pom.xml, add subset of CloudObjectInputSourceTest.
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch 24.0.0-revert-glob-filter
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 2d90f2904283461179dcdf8b2a944acf7db88b5b
Author: Gian Merlino <gi...@gmail.com>
AuthorDate: Wed Sep 7 20:41:18 2022 -0700
In branch: lower coverage requirement for azure-extensions/pom.xml, add subset of CloudObjectInputSourceTest.
---
.../input/impl/CloudObjectInputSourceTest.java | 135 +++++++++++++++++++++
extensions-core/azure-extensions/pom.xml | 12 +-
2 files changed, 141 insertions(+), 6 deletions(-)
diff --git a/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectInputSourceTest.java b/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectInputSourceTest.java
new file mode 100644
index 0000000000..291b1806d5
--- /dev/null
+++ b/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectInputSourceTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.data.input.impl;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.data.input.InputSplit;
+import org.apache.druid.data.input.MaxSizeSplitHintSpec;
+import org.apache.druid.java.util.common.parsers.JSONPathSpec;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class CloudObjectInputSourceTest
+{
+ private static final String SCHEME = "s3";
+
+ private static final List<URI> URIS = Collections.singletonList(
+ URI.create("s3://foo/bar/file.csv")
+ );
+
+ private static final List<URI> URIS2 = Arrays.asList(
+ URI.create("s3://foo/bar/file.csv"),
+ URI.create("s3://bar/foo/file2.parquet")
+ );
+
+ private static final List<URI> PREFIXES = Arrays.asList(
+ URI.create("s3://foo/bar/"),
+ URI.create("s3://bar/foo/")
+ );
+
+ private static final List<CloudObjectLocation> OBJECTS = Collections.singletonList(
+ new CloudObjectLocation(URI.create("s3://foo/bar/file.csv"))
+ );
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void testGetUris()
+ {
+ CloudObjectInputSource inputSource = Mockito.mock(CloudObjectInputSource.class, Mockito.withSettings()
+ .useConstructor(SCHEME, URIS, null, null)
+ .defaultAnswer(Mockito.CALLS_REAL_METHODS)
+ );
+
+ Assert.assertEquals(
+ URIS,
+ inputSource.getUris()
+ );
+ }
+
+ @Test
+ public void testGetPrefixes()
+ {
+ CloudObjectInputSource inputSource = Mockito.mock(CloudObjectInputSource.class, Mockito.withSettings()
+ .useConstructor(SCHEME, null, PREFIXES, null)
+ .defaultAnswer(Mockito.CALLS_REAL_METHODS)
+ );
+
+ Assert.assertEquals(
+ PREFIXES,
+ inputSource.getPrefixes()
+ );
+ }
+
+ @Test
+ public void testWithUris()
+ {
+ CloudObjectInputSource inputSource = Mockito.mock(CloudObjectInputSource.class, Mockito.withSettings()
+ .useConstructor(SCHEME, URIS, null, null)
+ .defaultAnswer(Mockito.CALLS_REAL_METHODS)
+ );
+
+ Stream<InputSplit<List<CloudObjectLocation>>> splits = inputSource.createSplits(
+ new JsonInputFormat(JSONPathSpec.DEFAULT, null, null),
+ new MaxSizeSplitHintSpec(null, 1)
+ );
+
+ List<CloudObjectLocation> returnedLocations = splits.map(InputSplit::get).collect(Collectors.toList()).get(0);
+
+ List<URI> returnedLocationUris = returnedLocations.stream().map(object -> object.toUri(SCHEME)).collect(Collectors.toList());
+
+ Assert.assertEquals(URIS, returnedLocationUris);
+ }
+
+ @Test
+ public void testWithObjects()
+ {
+ CloudObjectInputSource inputSource = Mockito.mock(CloudObjectInputSource.class, Mockito.withSettings()
+ .useConstructor(SCHEME, null, null, OBJECTS)
+ .defaultAnswer(Mockito.CALLS_REAL_METHODS)
+ );
+
+ Stream<InputSplit<List<CloudObjectLocation>>> splits = inputSource.createSplits(
+ new JsonInputFormat(JSONPathSpec.DEFAULT, null, null),
+ new MaxSizeSplitHintSpec(null, 1)
+ );
+
+ List<CloudObjectLocation> returnedLocations = splits.map(InputSplit::get).collect(Collectors.toList()).get(0);
+
+ Assert.assertEquals(OBJECTS, returnedLocations);
+ }
+
+ @Test
+ public void testEquals()
+ {
+ EqualsVerifier.forClass(CloudObjectInputSource.class).usingGetClass().verify();
+ }
+}
diff --git a/extensions-core/azure-extensions/pom.xml b/extensions-core/azure-extensions/pom.xml
index 68cf0cb983..24e8618c8e 100644
--- a/extensions-core/azure-extensions/pom.xml
+++ b/extensions-core/azure-extensions/pom.xml
@@ -178,32 +178,32 @@
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
- <minimum>0.86</minimum>
+ <minimum>0.70</minimum>
</limit>
<limit>
<counter>LINE</counter>
<value>COVEREDRATIO</value>
- <minimum>0.85</minimum>
+ <minimum>0.70</minimum>
</limit>
<limit>
<counter>BRANCH</counter>
<value>COVEREDRATIO</value>
- <minimum>0.88</minimum>
+ <minimum>0.70</minimum>
</limit>
<limit>
<counter>COMPLEXITY</counter>
<value>COVEREDRATIO</value>
- <minimum>0.80</minimum>
+ <minimum>0.70</minimum>
</limit>
<limit>
<counter>METHOD</counter>
<value>COVEREDRATIO</value>
- <minimum>0.78</minimum>
+ <minimum>0.70</minimum>
</limit>
<limit>
<counter>CLASS</counter>
<value>COVEREDRATIO</value>
- <minimum>0.90</minimum>
+ <minimum>0.70</minimum>
</limit>
</limits>
</rule>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org