You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2023/01/12 15:45:48 UTC
[asterixdb] branch master updated (cbdf6fdb48 -> 6ff8eec85d)
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from cbdf6fdb48 [ASTERIXDB-3096][COMP] Array-index recognition bug w/ multiple subplans
new d7e54cf144 [NO ISSUE] Process shadowed projects w/ license plugin
new 023e28c15d Merge branch 'gerrit/mad-hatter'
new faae0fb4a9 Merge branch 'gerrit/cheshire-cat'
new 667a908755 [NO ISSUE] Exclude licenses that are exclusively shadowed
new cac1e4ab32 [ASTERIXDB-3092][COMP] Consider only field-access functions in LoadRecordFieldsRule
new 2538c159d3 [NO ISSUE][OTH] Log node and node address on negative heartbeat ack
new ab2fbf432a Merge branch 'neo'
new 6ff8eec85d Merge "Merge branch 'neo'"
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../optimizer/rules/LoadRecordFieldsRule.java | 14 +--
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 +
.../load-record-fields.1.ddl.sqlpp | 30 +++++
.../load-record-fields.2.update.sqlpp | 25 +++++
.../load-record-fields.3.query.sqlpp | 30 +++++
.../load-record-fields.4.query.sqlpp | 30 +++++
.../load-record-fields/load-record-fields.3.adm | 2 +
.../load-record-fields/load-record-fields.4.plan | 26 +++++
.../asterix/lang/common/util/FunctionUtil.java | 13 +++
.../main/licenses/templates/asterix-license.ftl | 5 +-
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../hyracks/control/cc/work/NodeHeartbeatWork.java | 5 +
.../maven/license/DownloadLicensesMojo.java | 2 +-
.../hyracks/maven/license/GenerateFileMojo.java | 124 ++++++++++++++++++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +++++---
.../maven/license/project/LicensedProjects.java | 6 +
.../hyracks/maven/license/project/Project.java | 12 +-
18 files changed, 344 insertions(+), 38 deletions(-)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan
[asterixdb] 01/08: [NO ISSUE] Process shadowed projects w/ license plugin
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d7e54cf14441d895ef4fd26e228d9857a2439d2e
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Nov 19 18:59:28 2022 -0500
[NO ISSUE] Process shadowed projects w/ license plugin
Change-Id: Id21dcd3dc61bd4766e71c5d509e7ef6802e423be
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17290
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../main/licenses/templates/asterix-license.ftl | 3 +
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../maven/license/DownloadLicensesMojo.java | 2 +-
.../hyracks/maven/license/GenerateFileMojo.java | 120 ++++++++++++++++++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +++++---
.../hyracks/maven/license/project/Project.java | 12 ++-
7 files changed, 159 insertions(+), 31 deletions(-)
diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl b/asterixdb/src/main/licenses/templates/asterix-license.ftl
index 381d400a90..434ce447f6 100644
--- a/asterixdb/src/main/licenses/templates/asterix-license.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl
@@ -44,6 +44,9 @@ ${license.content}
<#assign isare = "are"/>
</#if>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
* ${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/asterixdb/src/main/licenses/templates/asterix-notice.ftl b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
index c825397c1a..5090cb0868 100644
--- a/asterixdb/src/main/licenses/templates/asterix-notice.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
@@ -34,6 +34,9 @@ AsterixDB utilizes many libraries, which come with the following applicable NOTI
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
index 8a2eb05d29..0cf9378659 100644
--- a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
+++ b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
@@ -30,6 +30,9 @@ Hyracks and Algebricks utilize many libraries, which come with the following app
<#assign noticeText = e.getKey()/>
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
index 293c08c6cf..c8980cd3da 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
@@ -63,7 +63,7 @@ public class DownloadLicensesMojo extends LicenseMojo {
String fileName = entry.getLicense().getContentFile(false);
doDownload(timeoutMillis, i, url, fileName);
});
- } catch (ProjectBuildingException e) {
+ } catch (ProjectBuildingException | IOException e) {
throw new MojoExecutionException("Unexpected exception: " + e, e);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index e69887c289..b3a0bb299c 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -39,6 +39,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -52,15 +53,22 @@ import java.util.jar.JarFile;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.maven.license.freemarker.IndentDirective;
import org.apache.hyracks.maven.license.freemarker.LoadFileDirective;
import org.apache.hyracks.maven.license.project.LicensedProjects;
import org.apache.hyracks.maven.license.project.Project;
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.DefaultArtifact;
+import org.apache.maven.artifact.handler.ArtifactHandler;
+import org.apache.maven.artifact.handler.DefaultArtifactHandler;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.apache.maven.project.MavenProject;
import org.apache.maven.project.ProjectBuildingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -106,6 +114,9 @@ public class GenerateFileMojo extends LicenseMojo {
@Parameter
private boolean stripFoundationAssertionFromNotices = false;
+ @Parameter
+ private boolean validateShadowLicenses = false;
+
private SortedMap<String, SortedSet<Project>> noticeMap;
@java.lang.Override
@@ -139,11 +150,12 @@ public class GenerateFileMojo extends LicenseMojo {
}
licenseSpecs.addAll(urlToLicenseMap.values());
for (LicenseSpec license : licenseSpecs) {
- resolveArtifactContent(license, true);
+ resolveArtifactContent(license, true, false);
}
}
- private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort) throws IOException {
+ private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort, boolean suppressWarning)
+ throws IOException {
if (artifact.getContent() == null) {
getLog().debug("Resolving content for " + artifact.getUrl() + " (" + artifact.getContentFile() + ")");
File cFile = new File(artifact.getContentFile());
@@ -157,7 +169,9 @@ public class GenerateFileMojo extends LicenseMojo {
}
if (!cFile.exists()) {
if (!bestEffort) {
- getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl());
+ if (!suppressWarning) {
+ getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl());
+ }
artifact.setContent("MISSING: " + artifact.getContentFile() + " (" + artifact.getUrl() + ")");
}
} else {
@@ -264,10 +278,12 @@ public class GenerateFileMojo extends LicenseMojo {
for (Project p : lps.getProjects()) {
String licenseText = p.getLicenseText();
if (licenseText == null) {
- warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE,
- "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense()
- + ")");
- licenseText = resolveArtifactContent(lps.getLicense(), false);
+ if (validateProjectLicense(p)) {
+ warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE,
+ "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense()
+ + ")");
+ }
+ licenseText = resolveArtifactContent(lps.getLicense(), false, !validateProjectLicense(p));
}
LicenseSpec spec = lps.getLicense();
if (spec.getDisplayName() == null) {
@@ -293,6 +309,10 @@ public class GenerateFileMojo extends LicenseMojo {
licenseMap = licenseMap2;
}
+ private boolean validateProjectLicense(Project p) {
+ return !p.isShadowed() || validateShadowLicenses;
+ }
+
private Set<Project> getProjects() {
Set<Project> projects = new HashSet<>();
licenseMap.values().forEach(p -> projects.addAll(p.getProjects()));
@@ -305,9 +325,11 @@ public class GenerateFileMojo extends LicenseMojo {
String noticeText = p.getNoticeText();
if (noticeText == null && noticeOverrides.containsKey(p.gav())) {
String noticeUrl = noticeOverrides.get(p.gav());
- warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE,
- "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")");
- p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false));
+ if (validateProjectLicense(p)) {
+ warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE,
+ "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")");
+ }
+ p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false, p.isShadowed()));
} else if (noticeText == null && !noticeOverrides.containsKey(p.gav())
&& Boolean.TRUE.equals(getProjectFlag(p.gav(), IGNORE_NOTICE_OVERRIDE))) {
getLog().warn(p + " has IGNORE_NOTICE_OVERRIDE flag set, but no override defined...");
@@ -338,20 +360,20 @@ public class GenerateFileMojo extends LicenseMojo {
// TODO(mblow): this will match *any* NOTICE[.txt] file located within the artifact- this seems way too liberal
resolveArtifactFiles("NOTICE", IGNORE_MISSING_EMBEDDED_NOTICE, ALTERNATE_NOTICE_FILE,
entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.txt)?"), Project::setNoticeText,
- text -> stripFoundationAssertionFromNotices ? FOUNDATION_PATTERN.matcher(text).replaceAll("") : text);
+ getNoticeFileContentTransformer(), !validateShadowLicenses);
}
private void resolveLicenseFiles() throws MojoExecutionException, IOException {
// TODO(mblow): this will match *any* LICENSE[.txt] file located within the artifact- this seems way too liberal
resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE, ALTERNATE_LICENSE_FILE,
entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.txt)?"), Project::setLicenseText,
- UnaryOperator.identity());
+ UnaryOperator.identity(), !validateShadowLicenses);
}
private void resolveArtifactFiles(final String name, final ProjectFlag ignoreFlag,
final ProjectFlag alternateFilenameFlag, final Predicate<JarEntry> filter,
- final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer)
- throws MojoExecutionException, IOException {
+ final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer,
+ boolean skipShadowed) throws MojoExecutionException, IOException {
for (Project p : getProjects()) {
File artifactFile = new File(p.getArtifactPath());
if (!artifactFile.exists()) {
@@ -359,6 +381,9 @@ public class GenerateFileMojo extends LicenseMojo {
} else if (!artifactFile.getName().endsWith(".jar")) {
getLog().info("Skipping unknown artifact file type: " + artifactFile);
continue;
+ } else if (skipShadowed && p.isShadowed()) {
+ getLog().info("Skipping shadowed project: " + p.gav());
+ continue;
}
String alternateFilename = (String) getProjectFlag(p.gav(), alternateFilenameFlag);
Predicate<JarEntry> finalFilter =
@@ -404,4 +429,71 @@ public class GenerateFileMojo extends LicenseMojo {
}
return matches;
}
+
+ private UnaryOperator<String> getNoticeFileContentTransformer() {
+ UnaryOperator<String> transformer;
+ if (stripFoundationAssertionFromNotices) {
+ transformer = text -> FOUNDATION_PATTERN.matcher(text).replaceAll("");
+ } else {
+ transformer = UnaryOperator.identity();
+ }
+ return transformer;
+ }
+
+ @java.lang.Override
+ protected void gatherProjectDependencies(MavenProject project,
+ Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ @java.lang.Override
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ private void gatherShadowedDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws MojoExecutionException, ProjectBuildingException {
+ Set<MavenProject> projects = new TreeSet<>(Comparator.comparing(MavenProject::getId));
+ projects.addAll(dependencyLicenseMap.keySet());
+ for (MavenProject p : projects) {
+ boolean finished = false;
+ File artifactFile = p.getArtifact().getFile();
+ if (!artifactFile.exists()) {
+ throw new MojoExecutionException("Artifact file " + artifactFile + " does not exist!");
+ } else if (!artifactFile.getName().endsWith(".jar")) {
+ getLog().info("Skipping unknown artifact file type: " + artifactFile);
+ finished = true;
+ }
+ if (!finished) {
+ try (JarFile jarFile = new JarFile(artifactFile)) {
+ SortedMap<String, JarEntry> matches = gatherMatchingEntries(jarFile,
+ entry -> entry.getName().matches("(.*/|^)" + "pom\\.properties"));
+ if (!matches.isEmpty()) {
+ for (JarEntry entry : matches.values()) {
+ Properties props = new Properties();
+ props.load(jarFile.getInputStream(entry));
+ String groupId = props.getProperty("groupId");
+ String artifactId = props.getProperty("artifactId");
+ String version = props.getProperty("version");
+ String gav = groupId + ":" + artifactId + ":" + version;
+ if (!dependencyGavMap.containsKey(gav)) {
+ getLog().info("adding " + gav + " (shadowed from " + p.getId() + ")");
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ String[] gavParts = StringUtils.split(gav, ':');
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2],
+ Artifact.SCOPE_COMPILE, "jar", null, handler);
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, true);
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException(e);
+ }
+ }
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
index 2206621fb7..73cc69559b 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
@@ -70,6 +70,7 @@ import org.apache.maven.project.inheritance.ModelInheritanceAssembler;
public abstract class LicenseMojo extends AbstractMojo {
+ private static final String SHADOWED_KEY = LicenseMojo.class.getName() + "_shadowed";
@Parameter
protected List<Override> overrides = new ArrayList<>();
@@ -249,7 +250,7 @@ public abstract class LicenseMojo extends AbstractMojo {
});
}
- protected void addDependenciesToLicenseMap() throws ProjectBuildingException {
+ protected void addDependenciesToLicenseMap() throws ProjectBuildingException, MojoExecutionException, IOException {
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = gatherDependencies();
dependencyLicenseMap.forEach((depProject, value) -> {
Set<String> locations = dependencySets.isEmpty() ? Collections.singleton(location)
@@ -310,7 +311,9 @@ public abstract class LicenseMojo extends AbstractMojo {
licenseUrl = fakeLicenseUrl;
}
}
- addProject(new Project(depProject, depLocation, depProject.getArtifact().getFile()),
+ addProject(
+ new Project(depProject, depLocation, depProject.getArtifact().getFile(),
+ Boolean.parseBoolean(String.valueOf(depProject.getContextValue(SHADOWED_KEY)))),
new LicenseSpec(licenseUrl, displayName), true);
}
@@ -363,11 +366,13 @@ public abstract class LicenseMojo extends AbstractMojo {
}
}
- protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies() throws ProjectBuildingException {
+ protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies()
+ throws ProjectBuildingException, MojoExecutionException, IOException {
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = new HashMap<>();
Map<String, MavenProject> dependencyGavMap = new HashMap<>();
gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
for (Override override : overrides) {
// Collect both <gav></gav> and <gavs><gav></gav><gav></gav>...</gavs>
@@ -399,27 +404,40 @@ public abstract class LicenseMojo extends AbstractMojo {
return dependencyLicenseMap;
}
- private void gatherProjectDependencies(MavenProject project,
+ protected void gatherProjectDependencies(MavenProject project,
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
- Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException {
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
final Set<Artifact> dependencyArtifacts = project.getArtifacts();
if (dependencyArtifacts != null) {
for (Artifact depArtifact : dependencyArtifacts) {
- processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap);
+ processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap, false);
}
}
- for (String gav : extraDependencies) {
- ArtifactHandler handler = new DefaultArtifactHandler("jar");
- String[] gavParts = StringUtils.split(gav, ':');
- Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE,
- "jar", null, handler);
- processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap);
+ }
+
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ for (String extraDependency : extraDependencies) {
+ String[] gavParts = StringUtils.split(extraDependency, ':');
+ String gav = gavParts[0] + ":" + gavParts[1] + ":" + gavParts[2];
+ if (!dependencyGavMap.containsKey(gav)) {
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE,
+ "jar", null, handler);
+ boolean shadowed = false;
+ if (gavParts.length > 3) {
+ shadowed = "shadowed".equals(gavParts[3]);
+ }
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, shadowed);
+ } else {
+ getLog().warn("not adding extra dependency " + gav + " as it is already added");
+ }
}
}
- private void processArtifact(Artifact depArtifact,
+ protected void processArtifact(Artifact depArtifact,
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
- Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException {
+ Map<String, MavenProject> dependencyGavMap, boolean shadowed) throws ProjectBuildingException {
if (!excludedScopes.contains(depArtifact.getScope())) {
MavenProject dep = resolveDependency(depArtifact);
if (!depArtifact.isResolved()) {
@@ -442,6 +460,7 @@ public abstract class LicenseMojo extends AbstractMojo {
: (license1.getName() != null ? license1.getName() : "LICENSE_EMPTY_NAME_URL");
licenseUrls.add(new ImmutablePair<>(url, license1.getName()));
}
+ dep.setContextValue(SHADOWED_KEY, shadowed);
dependencyLicenseMap.put(dep, licenseUrls);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
index e44914d7d4..a755c05ab5 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
@@ -41,6 +41,7 @@ public class Project {
private String licenseText;
private String sourcePointer;
private String classifier;
+ private final boolean shadowed;
@JsonIgnore
private MavenProject mavenProject;
@@ -48,7 +49,7 @@ public class Project {
public static final Comparator<Project> PROJECT_COMPARATOR =
(o1, o2) -> o1.compareToken().compareTo(o2.compareToken());
- public Project(MavenProject project, String location, File artifactPath) {
+ public Project(MavenProject project, String location, File artifactPath, boolean shadowed) {
mavenProject = project;
name = project.getName();
groupId = project.getGroupId();
@@ -57,6 +58,7 @@ public class Project {
url = project.getUrl();
classifier = project.getArtifact().getClassifier();
this.artifactPath = artifactPath.getPath();
+ this.shadowed = shadowed;
setLocation(location);
}
@@ -65,7 +67,8 @@ public class Project {
@JsonProperty("artifactId") String artifactId, @JsonProperty("url") String url,
@JsonProperty("version") String version, @JsonProperty("location") String location,
@JsonProperty("artifactPath") String artifactPath, @JsonProperty("noticeText") String noticeText,
- @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier) {
+ @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier,
+ @JsonProperty("shadowed") boolean shadowed) {
this.name = name;
this.groupId = groupId;
this.artifactId = artifactId;
@@ -76,6 +79,7 @@ public class Project {
this.noticeText = noticeText;
this.licenseText = licenseText;
this.classifier = classifier;
+ this.shadowed = shadowed;
}
public String getName() {
@@ -185,6 +189,10 @@ public class Project {
this.sourcePointer = sourcePointer;
}
+ public boolean isShadowed() {
+ return shadowed;
+ }
+
@Override
public String toString() {
return "Project [" + gav() + "]";
[asterixdb] 05/08: [ASTERIXDB-3092][COMP] Consider only field-access functions in LoadRecordFieldsRule
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit cac1e4ab32b24ae35c657c72d9979f41029c45d7
Author: Wail Alkowaileet <wa...@couchbase.com>
AuthorDate: Fri Dec 9 12:03:40 2022 -0800
[ASTERIXDB-3092][COMP] Consider only field-access functions in LoadRecordFieldsRule
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
The compiler rule LoadRecordFieldsRule fails when it encounters
a non-field-access function. This patch fixes this issue by ensuring
that only field accesses are considered for this rule.
Change-Id: I88f72fd51716dd8152e709c841489e87af0a5137
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17298
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wa...@gmail.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../optimizer/rules/LoadRecordFieldsRule.java | 14 +++++-----
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 ++++
.../load-record-fields.1.ddl.sqlpp | 30 ++++++++++++++++++++++
.../load-record-fields.2.update.sqlpp | 25 ++++++++++++++++++
.../load-record-fields.3.query.sqlpp | 30 ++++++++++++++++++++++
.../load-record-fields.4.query.sqlpp | 30 ++++++++++++++++++++++
.../load-record-fields/load-record-fields.3.adm | 2 ++
.../load-record-fields/load-record-fields.4.plan | 26 +++++++++++++++++++
.../asterix/lang/common/util/FunctionUtil.java | 13 ++++++++++
9 files changed, 168 insertions(+), 7 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
index b9d512bb8e..42cce5233e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.asterix.algebra.base.OperatorAnnotation;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.constants.AsterixConstantValue;
@@ -64,7 +65,7 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
- private ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
+ private final ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
@@ -98,13 +99,13 @@ public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
// checking if we can annotate a Selection as using just one field
// access
SelectOperator sigma = (SelectOperator) op1;
- LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
+ List<LogicalVariable> vars = new ArrayList<>();
VariableUtilities.getUsedVariables(sigma, vars);
if (vars.size() == 1) {
// we can annotate Selection
AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
- AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
- if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ ILogicalExpression expr1 = getFirstExpr(assign1);
+ if (FunctionUtil.isFieldAccessFunction(expr1)) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
// f should be a call to a field/data access kind of
// function
@@ -141,7 +142,7 @@ public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
}
// create an assign
LogicalVariable v = context.newVar();
- AssignOperator a2 = new AssignOperator(v, new MutableObject<ILogicalExpression>(f));
+ AssignOperator a2 = new AssignOperator(v, new MutableObject<>(f));
a2.setSourceLocation(expr.getSourceLocation());
pushFieldAssign(a2, topOp, context);
context.computeAndSetTypeEnvironmentForOperator(a2);
@@ -151,7 +152,7 @@ public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
LogicalVariable var = ref.getVariableReference();
List<LogicalVariable> keys = context.findPrimaryKey(var);
if (keys != null) {
- List<LogicalVariable> tail = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> tail = new ArrayList<>();
tail.add(v);
FunctionalDependency pk = new FunctionalDependency(keys, tail);
context.addPrimaryKey(pk);
@@ -408,5 +409,4 @@ public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
private static ILogicalExpression getFirstExpr(AssignOperator assign) {
return assign.getExpressions().get(0).getValue();
}
-
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index 623237b550..8532fe9034 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -244,4 +244,9 @@
<expected-warn>Duplicate field name 'fname1' (in line 25, at column 45)</expected-warn>
</compilation-unit>
</test-case>
+ <test-case FilePath="objects">
+ <compilation-unit name="load-record-fields">
+ <output-dir compare="Text">load-record-fields</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp
new file mode 100644
index 0000000000..99898f9b28
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+ id: int
+};
+
+CREATE DATASET MyDataset(OpenType)
+PRIMARY KEY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp
new file mode 100644
index 0000000000..1f80ae12c8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO MyDataset (
+ {"id": 1, "name": "Alice"},
+ {"id": 2, "name": "Bob"}
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp
new file mode 100644
index 0000000000..278f5d2b1e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+-- Disabled for a simpler plan
+SET `compiler.sort.parallel` "false";
+
+
+SELECT VALUE md.name
+FROM MyDataset md
+LET currentData = {"myDate": current_date()}
+WHERE currentData.myDate = current_date()
+ORDER BY md.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp
new file mode 100644
index 0000000000..f44921f99b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+-- Disabled for a simpler plan
+SET `compiler.sort.parallel` "false";
+
+EXPLAIN
+SELECT VALUE md.name
+FROM MyDataset md
+LET currentData = {"myDate": current_date()}
+WHERE currentData.myDate = current_date()
+ORDER BY md.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm
new file mode 100644
index 0000000000..ac2dc970eb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm
@@ -0,0 +1,2 @@
+"Alice"
+"Bob"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan
new file mode 100644
index 0000000000..5ac69f34ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan
@@ -0,0 +1,26 @@
+distribute result [$$28]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$28])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$30(ASC) ] |PARTITIONED|
+ project ([$$28, $$30])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (eq($$31, current-date()))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$31] <- [current-date()]
+ -- ASSIGN |PARTITIONED|
+ project ([$$30, $$28])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$28] <- [$$md.getField("name")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$30, $$md] <- test.MyDataset
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index 42a35f3406..1cf88c2c35 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -58,6 +58,7 @@ import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -303,4 +304,16 @@ public class FunctionUtil {
function.getSignature(), e.getMessage());
}
}
+
+ public static boolean isFieldAccessFunction(ILogicalExpression expression) {
+ if (expression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ return BuiltinFunctions.FIELD_ACCESS_BY_INDEX.equals(fid) || BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid)
+ || BuiltinFunctions.FIELD_ACCESS_NESTED.equals(fid);
+ }
}
[asterixdb] 08/08: Merge "Merge branch 'neo'"
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6ff8eec85d2b433f654a1d8a84707d398dedced1
Merge: cbdf6fdb48 ab2fbf432a
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Thu Jan 12 15:33:29 2023 +0000
Merge "Merge branch 'neo'"
.../optimizer/rules/LoadRecordFieldsRule.java | 14 +--
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 +
.../load-record-fields.1.ddl.sqlpp | 30 +++++
.../load-record-fields.2.update.sqlpp | 25 +++++
.../load-record-fields.3.query.sqlpp | 30 +++++
.../load-record-fields.4.query.sqlpp | 30 +++++
.../load-record-fields/load-record-fields.3.adm | 2 +
.../load-record-fields/load-record-fields.4.plan | 26 +++++
.../asterix/lang/common/util/FunctionUtil.java | 13 +++
.../main/licenses/templates/asterix-license.ftl | 5 +-
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../hyracks/control/cc/work/NodeHeartbeatWork.java | 5 +
.../maven/license/DownloadLicensesMojo.java | 2 +-
.../hyracks/maven/license/GenerateFileMojo.java | 124 ++++++++++++++++++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +++++---
.../maven/license/project/LicensedProjects.java | 6 +
.../hyracks/maven/license/project/Project.java | 12 +-
18 files changed, 344 insertions(+), 38 deletions(-)
[asterixdb] 02/08: Merge branch 'gerrit/mad-hatter'
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 023e28c15d6d856419d7c595105818a76d3e3511
Merge: 4af9e73bb6 d7e54cf144
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Nov 20 21:41:37 2022 -0500
Merge branch 'gerrit/mad-hatter'
Change-Id: I300a5e2910bde9e7bcd5f187efca26e4c7689ffb
.../main/licenses/templates/asterix-license.ftl | 3 +
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../maven/license/DownloadLicensesMojo.java | 2 +-
.../hyracks/maven/license/GenerateFileMojo.java | 120 ++++++++++++++++++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +++++---
.../hyracks/maven/license/project/Project.java | 12 ++-
7 files changed, 159 insertions(+), 31 deletions(-)
diff --cc hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 6bc92b8539,b3a0bb299c..6d30a691fb
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@@ -335,19 -357,17 +357,19 @@@ public class GenerateFileMojo extends L
}
private void resolveNoticeFiles() throws MojoExecutionException, IOException {
- // TODO(mblow): this will match *any* NOTICE[.txt] file located within the artifact- this seems way too liberal
+ // TODO(mblow): this will match *any* NOTICE[.(txt|md)] file located within the artifact-
+ // this seems way too liberal
resolveArtifactFiles("NOTICE", IGNORE_MISSING_EMBEDDED_NOTICE, ALTERNATE_NOTICE_FILE,
- entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.txt)?"), Project::setNoticeText,
+ entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.(txt|md))?"), Project::setNoticeText,
- text -> stripFoundationAssertionFromNotices ? FOUNDATION_PATTERN.matcher(text).replaceAll("") : text);
+ getNoticeFileContentTransformer(), !validateShadowLicenses);
}
private void resolveLicenseFiles() throws MojoExecutionException, IOException {
- // TODO(mblow): this will match *any* LICENSE[.txt] file located within the artifact- this seems way too liberal
+ // TODO(mblow): this will match *any* LICENSE[.(txt|md)] file located within the artifact-
+ // this seems way too liberal
resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE, ALTERNATE_LICENSE_FILE,
- entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.txt)?"), Project::setLicenseText,
+ entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.(txt|md))?"), Project::setLicenseText,
- UnaryOperator.identity());
+ UnaryOperator.identity(), !validateShadowLicenses);
}
private void resolveArtifactFiles(final String name, final ProjectFlag ignoreFlag,
[asterixdb] 04/08: [NO ISSUE] Exclude licenses that are exclusively shadowed
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 667a908755391fcf07957ca2c01d9f42cb89b291
Author: Michael Blow <mb...@apache.org>
AuthorDate: Fri Dec 9 15:56:28 2022 -0500
[NO ISSUE] Exclude licenses that are exclusively shadowed
Change-Id: I19eb0000e2977e85293fd7c2e67768a218062fe9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17302
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
asterixdb/src/main/licenses/templates/asterix-license.ftl | 8 ++++----
.../apache/hyracks/maven/license/project/LicensedProjects.java | 6 ++++++
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl b/asterixdb/src/main/licenses/templates/asterix-license.ftl
index 434ce447f6..f9c969e27c 100644
--- a/asterixdb/src/main/licenses/templates/asterix-license.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl
@@ -34,8 +34,11 @@ ${license.content}
<#list licenseMap as e>
<#assign licenseUrl = e.getKey()/>
<#assign entry = e.getValue()/>
- <#assign projects = entry.projects/>
+ <#assign projects = entry.nonShadowedProjects/>
<#assign license = entry.getLicense()/>
+ <#if projects?size == 0>
+ <#continue/>
+ </#if>
<#if projects?size == 1>
Component:
<#assign isare = "is"/>
@@ -44,9 +47,6 @@ ${license.content}
<#assign isare = "are"/>
</#if>
<#list projects as p>
- <#if p.shadowed>
- <#continue/>
- </#if>
* ${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
index f6f9f32444..31b4a86061 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.maven.license.project;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.function.Predicate;
import org.apache.hyracks.maven.license.LicenseSpec;
@@ -51,6 +52,11 @@ public class LicensedProjects {
return projects;
}
+ public SortedSet<Project> getNonShadowedProjects() {
+ return projects.stream().filter(Predicate.not(Project::isShadowed))
+ .collect(() -> new TreeSet<>(Project.PROJECT_COMPARATOR), TreeSet::add, TreeSet::addAll);
+ }
+
public void addProject(Project project) {
projects.add(project);
}
[asterixdb] 07/08: Merge branch 'neo'
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ab2fbf432a4a59b6171fb0d5e02cd59746d082fd
Merge: 9f04aefb57 2538c159d3
Author: Murtadha Hubail <mu...@couchbase.com>
AuthorDate: Tue Jan 10 21:09:06 2023 +0300
Merge branch 'neo'
Change-Id: I1334d543bc86fffa849dd38e60fdf98c706444ac
.../optimizer/rules/LoadRecordFieldsRule.java | 14 +--
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 +
.../load-record-fields.1.ddl.sqlpp | 30 +++++
.../load-record-fields.2.update.sqlpp | 25 +++++
.../load-record-fields.3.query.sqlpp | 30 +++++
.../load-record-fields.4.query.sqlpp | 30 +++++
.../load-record-fields/load-record-fields.3.adm | 2 +
.../load-record-fields/load-record-fields.4.plan | 26 +++++
.../asterix/lang/common/util/FunctionUtil.java | 13 +++
.../main/licenses/templates/asterix-license.ftl | 5 +-
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../hyracks/control/cc/work/NodeHeartbeatWork.java | 5 +
.../maven/license/DownloadLicensesMojo.java | 2 +-
.../hyracks/maven/license/GenerateFileMojo.java | 124 ++++++++++++++++++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +++++---
.../maven/license/project/LicensedProjects.java | 6 +
.../hyracks/maven/license/project/Project.java | 12 +-
18 files changed, 344 insertions(+), 38 deletions(-)
diff --cc hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 347c19ad3c,8edc1b212f..2c7a23b0f5
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@@ -110,6 -118,9 +118,11 @@@ public class GenerateFileMojo extends L
@Parameter
private boolean stripFoundationAssertionFromNotices = false;
++ @Parameter
++ private boolean includeShadowedDependencies = true;
+ @Parameter
+ private boolean validateShadowLicenses = false;
+
private SortedMap<String, SortedSet<Project>> noticeMap;
@java.lang.Override
@@@ -472,4 -496,71 +498,76 @@@
}
return matches;
}
+
+ private UnaryOperator<String> getNoticeFileContentTransformer() {
+ UnaryOperator<String> transformer;
+ if (stripFoundationAssertionFromNotices) {
+ transformer = text -> FOUNDATION_PATTERN.matcher(text).replaceAll("");
+ } else {
+ transformer = UnaryOperator.identity();
+ }
+ return transformer;
+ }
+
+ @java.lang.Override
+ protected void gatherProjectDependencies(MavenProject project,
+ Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ @java.lang.Override
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ private void gatherShadowedDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws MojoExecutionException, ProjectBuildingException {
+ Set<MavenProject> projects = new TreeSet<>(Comparator.comparing(MavenProject::getId));
++ if (!includeShadowedDependencies) {
++ getLog().info("Not gathering shadowed dependencies as 'includeShadowedDependencies' is set to "
++ + includeShadowedDependencies);
++ return;
++ }
+ projects.addAll(dependencyLicenseMap.keySet());
+ for (MavenProject p : projects) {
+ boolean finished = false;
+ File artifactFile = p.getArtifact().getFile();
+ if (!artifactFile.exists()) {
+ throw new MojoExecutionException("Artifact file " + artifactFile + " does not exist!");
+ } else if (!artifactFile.getName().endsWith(".jar")) {
+ getLog().info("Skipping unknown artifact file type: " + artifactFile);
+ finished = true;
+ }
+ if (!finished) {
+ try (JarFile jarFile = new JarFile(artifactFile)) {
+ SortedMap<String, JarEntry> matches = gatherMatchingEntries(jarFile,
+ entry -> entry.getName().matches("(.*/|^)" + "pom\\.properties"));
+ if (!matches.isEmpty()) {
+ for (JarEntry entry : matches.values()) {
+ Properties props = new Properties();
+ props.load(jarFile.getInputStream(entry));
+ String groupId = props.getProperty("groupId");
+ String artifactId = props.getProperty("artifactId");
+ String version = props.getProperty("version");
+ String gav = groupId + ":" + artifactId + ":" + version;
+ if (!dependencyGavMap.containsKey(gav)) {
+ getLog().info("adding " + gav + " (shadowed from " + p.getId() + ")");
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ String[] gavParts = StringUtils.split(gav, ':');
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2],
+ Artifact.SCOPE_COMPILE, "jar", null, handler);
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, true);
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException(e);
+ }
+ }
+ }
+ }
}
[asterixdb] 03/08: Merge branch 'gerrit/cheshire-cat'
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit faae0fb4a91754aea4c1cc3fe0304ea8158d68d4
Merge: b69c90dc17 023e28c15d
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun Nov 20 23:11:52 2022 -0500
Merge branch 'gerrit/cheshire-cat'
Change-Id: Id677655399af06ca4f1703a64482daf281a8c482
.../main/licenses/templates/asterix-license.ftl | 3 +
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../maven/license/DownloadLicensesMojo.java | 2 +-
.../hyracks/maven/license/GenerateFileMojo.java | 117 ++++++++++++++++++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 ++++++---
.../hyracks/maven/license/project/Project.java | 12 ++-
7 files changed, 157 insertions(+), 30 deletions(-)
diff --cc hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 347c19ad3c,6d30a691fb..8edc1b212f
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@@ -348,45 -365,17 +370,44 @@@ public class GenerateFileMojo extends L
}
private void resolveLicenseFiles() throws MojoExecutionException, IOException {
- // TODO(mblow): this will match *any* LICENSE[.(txt|md)] file located within the artifact-
- // this seems way too liberal
- resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE, ALTERNATE_LICENSE_FILE,
- entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.(txt|md))?"), Project::setLicenseText,
- UnaryOperator.identity(), !validateShadowLicenses);
+ resolveArtifactFiles(LICENSE);
}
- private void resolveArtifactFiles(final String name, final ProjectFlag ignoreFlag,
- final ProjectFlag alternateFilenameFlag, final Predicate<JarEntry> filter,
- final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer,
- boolean skipShadowed) throws MojoExecutionException, IOException {
+ private void resolveArtifactFiles(final EmbeddedArtifact artifact) throws MojoExecutionException, IOException {
+ final String name;
+ final ProjectFlag ignoreFlag;
+ final ProjectFlag alternateFilenameFlag;
+ final ProjectFlag onMultipleFlag;
+ final Predicate<JarEntry> filter;
+ final BiConsumer<Project, String> consumer;
+ final UnaryOperator<String> contentTransformer;
+
+ switch (artifact) {
+ case NOTICE:
+ name = "NOTICE";
+ ignoreFlag = IGNORE_MISSING_EMBEDDED_NOTICE;
+ alternateFilenameFlag = ALTERNATE_NOTICE_FILE;
+ onMultipleFlag = ON_MULTIPLE_EMBEDDED_NOTICE;
+ // TODO(mblow): this will match *any* NOTICE[.(txt|md)] file located within the artifact-
+ // this seems way too liberal
+ filter = entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.(txt|md))?");
+ consumer = Project::setNoticeText;
- contentTransformer = UnaryOperator.identity();
++ contentTransformer = getNoticeFileContentTransformer();
+ break;
+ case LICENSE:
+ name = "LICENSE";
+ ignoreFlag = IGNORE_MISSING_EMBEDDED_LICENSE;
+ alternateFilenameFlag = ALTERNATE_LICENSE_FILE;
+ onMultipleFlag = ON_MULTIPLE_EMBEDDED_LICENSE;
+ // TODO(mblow): this will match *any* LICENSE[.(txt|md)] file located within the artifact-
+ // this seems way too liberal
+ filter = entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.(txt|md))?");
+ consumer = Project::setLicenseText;
- contentTransformer = stripFoundationAssertionFromNotices
- ? text -> FOUNDATION_PATTERN.matcher(text).replaceAll("") : UnaryOperator.identity();
++ contentTransformer = UnaryOperator.identity();
+ break;
+ default:
+ throw new IllegalStateException("NYI: " + artifact);
+ }
for (Project p : getProjects()) {
File artifactFile = new File(p.getArtifactPath());
if (!artifactFile.exists()) {
@@@ -394,6 -383,9 +415,9 @@@
} else if (!artifactFile.getName().endsWith(".jar")) {
getLog().info("Skipping unknown artifact file type: " + artifactFile);
continue;
- } else if (skipShadowed && p.isShadowed()) {
++ } else if (!validateShadowLicenses && p.isShadowed()) {
+ getLog().info("Skipping shadowed project: " + p.gav());
+ continue;
}
String alternateFilename = (String) getProjectFlag(p.gav(), alternateFilenameFlag);
Predicate<JarEntry> finalFilter =
[asterixdb] 06/08: [NO ISSUE][OTH] Log node and node address on negative heartbeat ack
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2538c159d3e1c67b1c3cab15e626d2d4fcd7b79d
Author: Murtadha Hubail <mu...@couchbase.com>
AuthorDate: Wed Jan 4 20:14:08 2023 +0300
[NO ISSUE][OTH] Log node and node address on negative heartbeat ack
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When sending a negative heartbeat ack to an unregistered node,
log the node id as well as its address.
Change-Id: I001b98ef1faa953d92332d36c2d4985b34ed57dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17305
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
index b772ef93da..6fa6224f3d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
@@ -28,9 +28,12 @@ import org.apache.hyracks.control.cc.cluster.INodeManager;
import org.apache.hyracks.control.common.heartbeat.HeartbeatData;
import org.apache.hyracks.control.common.ipc.NodeControllerRemoteProxy;
import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NodeHeartbeatWork extends AbstractHeartbeatWork {
+ private static final Logger LOGGER = LogManager.getLogger();
private final InetSocketAddress ncAddress;
public NodeHeartbeatWork(ClusterControllerService ccs, String nodeId, HeartbeatData hbData,
@@ -47,6 +50,8 @@ public class NodeHeartbeatWork extends AbstractHeartbeatWork {
ncState.getNodeController().heartbeatAck(ccs.getCcId(), null);
} else {
// unregistered nc- let him know
+ LOGGER.info("received a heartbeat from unregistered node {}; sending negative ack to node address {}",
+ nodeId, ncAddress);
NodeControllerRemoteProxy nc =
new NodeControllerRemoteProxy(ccs.getCcId(), ccs.getClusterIPC().getReconnectingHandle(ncAddress));
nc.heartbeatAck(ccs.getCcId(), HyracksDataException.create(ErrorCode.NO_SUCH_NODE, nodeId));