You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2023/09/11 07:06:28 UTC
[streampipes] branch dev updated: Refactor streampipes-maven-plugin to auto-generate docs (#1907)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 556c4d89b Refactor streampipes-maven-plugin to auto-generate docs (#1907)
556c4d89b is described below
commit 556c4d89b55c83c1d36a103dd9ba868dab8ce2fa
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Mon Sep 11 09:06:17 2023 +0200
Refactor streampipes-maven-plugin to auto-generate docs (#1907)
* Refactor streampipes-maven-plugin to auto-generate docs
* Add additional assets to image folder
* Assign proper pipeline element type
* Add github action to create docs
* Upload multiple doc artifacts
* Fix copy-paste leftover
---
.github/workflows/extensions-docs.yml | 68 +++++++
.../streampipes-extensions-all-jvm/pom.xml | 23 +++
streampipes-maven-plugin/README.md | 88 +++++++++
streampipes-maven-plugin/pom.xml | 18 +-
.../apache/streampipes/smp/CreateAssetMojo.java | 106 ----------
.../streampipes/smp/ExtractDocumentationMojo.java | 220 +++++++++------------
.../smp/extractor/AssetModelItemExtractor.java | 90 ---------
.../smp/extractor/ControllerExtractor.java | 57 ------
.../smp/extractor/ControllerFileFinder.java | 63 ------
.../smp/extractor/DockerImageExtractor.java | 77 --------
.../streampipes/smp/extractor/ElementFinder.java | 39 ----
.../smp/extractor/ExtensionsFinder.java | 77 ++++++++
.../streampipes/smp/extractor/FieldReplacer.java | 58 ------
.../smp/extractor/LocalesExtractor.java | 40 ++--
.../extractor/PipelineElementTypeExtractor.java | 43 ----
.../extractor/ResourceDirectoryElementFinder.java | 73 -------
.../AdditionalAssetsResourceGenerator.java | 88 +++++++++
.../streampipes/smp/generator/AssetGenerator.java | 59 ------
.../smp/generator/DataJsonGenerator.java | 75 -------
.../generator/DocumentationResourceGenerator.java | 63 ++++++
.../smp/generator/IconResourceGenerator.java | 56 ++++++
.../smp/generator/ImagePathReplacer.java | 11 +-
.../streampipes/smp/generator/OutputGenerator.java | 36 ----
.../generator/PipelineElementGridGenerator.java | 157 ---------------
.../PipelineElementOverviewGenerator.java | 53 -----
.../smp/generator/ResourceGenerator.java} | 34 ++--
.../smp/generator/SidebarConfigGenerator.java | 72 ++++++-
.../apache/streampipes/smp/model/AssetModel.java | 39 +---
.../streampipes/smp/util/DuplicateRemover.java | 45 -----
.../smp/extractor/TestAssetModelExtractor.java | 60 ------
30 files changed, 674 insertions(+), 1314 deletions(-)
diff --git a/.github/workflows/extensions-docs.yml b/.github/workflows/extensions-docs.yml
new file mode 100644
index 000000000..f43f8aef4
--- /dev/null
+++ b/.github/workflows/extensions-docs.yml
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "build-extensions-docs-and-share-as-artifact"
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 3 * * 6" # runs every saturday at 03:00:00
+
+jobs:
+ extensions-docs-artifact-building:
+ runs-on: ubuntu-latest
+ steps:
+ - name: clone
+ uses: actions/checkout@v4
+
+ - name: create working branch & set GitHub config
+ run: |
+ git checkout -b build-extensions-docs-${{ github.run_id }}
+ git config user.name 'github-actions[bot]'
+ git config user.email 'github-actions[bot]@users.noreply.github.com'
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: '17'
+ cache: 'maven'
+
+ - name: Build with Maven
+ run: mvn clean package
+
+ - name: Publish Extensions docs as artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: streampipes-extensions-docs
+ path: |
+ streampipes-extensions/streampipes-extensions-all-jvm/target/docs/pe/
+ retention-days: 5
+
+ - name: Publish Extensions image assets as artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: streampipes-extensions-docs-image-assets
+ path: |
+ streampipes-extensions/streampipes-extensions-all-jvm/target/docs/img/
+ retention-days: 5
+
+ - name: Publish Extensions sidebar
+ uses: actions/upload-artifact@v3
+ with:
+ name: streampipes-extensions-docs-sidebar
+ path: |
+ streampipes-extensions/streampipes-extensions-all-jvm/target/docs/sidebars.json
+ retention-days: 5
diff --git a/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml b/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml
index 3a057237c..3b03c5940 100644
--- a/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml
+++ b/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml
@@ -78,6 +78,29 @@
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-maven-plugin</artifactId>
+ <version>0.93.0-SNAPSHOT</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client-java</artifactId>
+ <version>5.0.2</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <initClass>org.apache.streampipes.extensions.all.jvm.AllExtensionsInit</initClass>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>extract-docs</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
diff --git a/streampipes-maven-plugin/README.md b/streampipes-maven-plugin/README.md
new file mode 100644
index 000000000..666f46c81
--- /dev/null
+++ b/streampipes-maven-plugin/README.md
@@ -0,0 +1,88 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+# Apache StreamPipes Maven Plugin
+
+## Description
+
+The Apache StreamPipes is an internal developer plugin which eases the generation of documentation for
+adapters and pipeline elements.
+
+The plugin generates documentation files which can be integrated into the UI.
+
+## Usage
+
+The streampipes-maven-plugin can either be started from the command line or embedded into a project's pom.
+
+### Prerequisites
+
+The plugin must be started from a module which contains an `Init` class which inherits `ExtensionsModelSubmitter`.
+By default, the goal runs in the `package` phase.
+
+### Command line
+
+```bash
+# Switch to a directory containing StreamPipes extensions and an Init class, e.g., streampipes-extensions-all-jvm
+
+mvn streampipes:extract-docs -DinitClass=org.apache.streampipes.extensions.all.jvm.AllExtensionsInit
+```
+
+### Inclusion in pom file
+
+```xml
+
+<plugin>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-maven-plugin</artifactId>
+ <version>0.93.0</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client-java</artifactId>
+ <version>5.0.2</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <initClass>org.apache.streampipes.extensions.all.jvm.AllExtensionsInit</initClass>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>extract-docs</goal>
+ </goals>
+ </execution>
+ </executions>
+</plugin>
+```
+
+* Replace the `version` with the current development version.
+* Replace the `initClass` with the module's init class.
+* The dependency to RocketMQ is only needed for modules containing the RocketMQ sink and can be omitted otherwise.
+
+## Output
+
+The plugin creates a new folder `docs` in the module's `target` directory.
+
+The folder includes:
+
+* An `img` folder which has a subdirectory for each extension (named by the `appId`) containing the icon.
+* A `pe` folder which has a subdirectory for each extension (named by the `appId`) containing the `documentation.md`
+ file, which has been rewritten to match the requirements of the Docusaurus Markdown parser.
+* An updated `sidebar.json` file containing the sidebar, which is downloaded from the `streampipes-website` repo on
+ branch `dev` and updated with the current set of extensions.
diff --git a/streampipes-maven-plugin/pom.xml b/streampipes-maven-plugin/pom.xml
index 17d70dae0..e994f0192 100644
--- a/streampipes-maven-plugin/pom.xml
+++ b/streampipes-maven-plugin/pom.xml
@@ -118,27 +118,19 @@
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</dependency>
- <dependency>
- <groupId>org.jboss.forge.roaster</groupId>
- <artifactId>roaster-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jboss.forge.roaster</groupId>
- <artifactId>roaster-jdt</artifactId>
- <scope>runtime</scope>
- </dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
- <dependency>
- <groupId>com.j2html</groupId>
- <artifactId>j2html</artifactId>
- </dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-service-extensions</artifactId>
+ <version>0.93.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
<build>
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java
deleted file mode 100644
index d5567453d..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp;
-
-
-import org.apache.streampipes.smp.extractor.ControllerFileFinder;
-import org.apache.streampipes.smp.generator.AssetGenerator;
-import org.apache.streampipes.smp.model.AssetModel;
-import org.apache.streampipes.smp.util.DuplicateRemover;
-
-import org.apache.maven.execution.MavenSession;
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.codehaus.plexus.components.interactivity.Prompter;
-import org.codehaus.plexus.components.interactivity.PrompterException;
-
-import java.io.File;
-import java.util.List;
-
-/**
- * Goal which creates an asset directory.
- */
-@Mojo(name = "create-assets", defaultPhase = LifecyclePhase.NONE)
-public class CreateAssetMojo
- extends AbstractMojo {
-
- private static final String APP_ID_PROPERTY = "appId";
- private static final String GENERATE_ALL_PROPERTY = "all";
- @Parameter(defaultValue = "${session}")
- private MavenSession session;
- @Parameter(defaultValue = "${project}", readonly = true)
- private MavenProject project;
- @Component
- private Prompter prompter;
-
- public void execute() throws MojoExecutionException {
-
- String appId = this.session.getExecutionProperties().getProperty(APP_ID_PROPERTY);
- String addAll = this.session.getExecutionProperties().getProperty(GENERATE_ALL_PROPERTY);
- File baseDir = this.project.getBasedir();
-
- if (appId != null) {
- getLog().info("Generating asset directory for element " + appId);
- AssetModel assetModel = new AssetModel(appId, "Pipeline Element Name", "Pipeline Element "
- + "Description");
- new AssetGenerator(baseDir.getAbsolutePath(), assetModel);
- }
-
- if (addAll != null) {
- List<String> sourceRoots = this.project.getCompileSourceRoots();
- getLog().info("Generating asset directories for source root " + sourceRoots.get(0));
-
- getLog().info("Finding controllers...");
- List<AssetModel> allAssetModels = new ControllerFileFinder(getLog(), baseDir.getAbsolutePath(),
- sourceRoots.get(0),
- "**/*Controller.java").makeAssetModels();
-
- getLog().info("Checking for already existing asset directories...");
-
- List<AssetModel> filteredModels =
- new DuplicateRemover(baseDir.getAbsolutePath(), allAssetModels).removeAlreadyExisting();
-
- try {
- String proceed = prompter.prompt(makeProceedText(filteredModels));
- if (proceed.equals("Y")) {
- filteredModels.forEach(am -> new AssetGenerator(baseDir.getAbsolutePath(), am)
- .genreateAssetDirectoryAndContents());
- }
- } catch (PrompterException e) {
- e.printStackTrace();
- }
- }
- }
-
- private String makeProceedText(List<AssetModel> assetModels) {
- StringBuilder builder = new StringBuilder();
- builder.append("The following asset directories will be created: ");
-
- assetModels.forEach(f -> builder.append(f.toString()));
-
- builder.append("Proceed? [Y][N]");
- return builder.toString();
- }
-
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java
index 1942bec18..a5b68bfaa 100644
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java
@@ -18,49 +18,49 @@
package org.apache.streampipes.smp;
-import org.apache.streampipes.smp.extractor.ControllerFileFinder;
-import org.apache.streampipes.smp.extractor.ResourceDirectoryElementFinder;
-import org.apache.streampipes.smp.generator.DataJsonGenerator;
-import org.apache.streampipes.smp.generator.ImagePathReplacer;
-import org.apache.streampipes.smp.generator.MarkdownHeaderGenerator;
-import org.apache.streampipes.smp.generator.MarkdownTitleRemover;
-import org.apache.streampipes.smp.generator.PipelineElementOverviewGenerator;
+import org.apache.streampipes.smp.extractor.ExtensionsFinder;
+import org.apache.streampipes.smp.extractor.LocalesExtractor;
+import org.apache.streampipes.smp.generator.AdditionalAssetsResourceGenerator;
+import org.apache.streampipes.smp.generator.DocumentationResourceGenerator;
+import org.apache.streampipes.smp.generator.IconResourceGenerator;
import org.apache.streampipes.smp.generator.SidebarConfigGenerator;
import org.apache.streampipes.smp.model.AssetModel;
-import org.apache.streampipes.smp.util.DirectoryManager;
-import org.apache.streampipes.smp.util.Utils;
+import com.google.common.base.Charsets;
import org.apache.commons.io.FileUtils;
+import org.apache.maven.artifact.DependencyResolutionRequiredException;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.plugins.annotations.ResolutionScope;
import org.apache.maven.project.MavenProject;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import java.util.stream.Collectors;
/**
* goal which extracts pipeline element documentations for the StreamPipes website + documentation
*/
-@Mojo(name = "extract-docs", defaultPhase = LifecyclePhase.NONE, aggregator = true)
+@Mojo(
+ name = "extract-docs",
+ defaultPhase = LifecyclePhase.PACKAGE,
+ requiresDependencyResolution = ResolutionScope.TEST,
+ requiresDependencyCollection = ResolutionScope.TEST)
public class ExtractDocumentationMojo extends AbstractMojo {
private static final String DOCS_ROOT_FOLDER = "docs";
- private static final String DOCS_FOLDER = "docs-documentation";
- private static final String DOCS_WEBSITE_FOLDER = "docs-website";
+ private static final String DOCS_PE_FOLDER = "pe";
private static final String IMG_FOLDER = "img";
private static final String SP_IGNORE_FILENAME = ".spignore";
@@ -70,127 +70,97 @@ public class ExtractDocumentationMojo extends AbstractMojo {
@Parameter(defaultValue = "${project}", readonly = true)
private MavenProject project;
+ @Parameter(name = "initClass", required = true)
+ private String initClass;
+
@Override
- public void execute() throws MojoExecutionException, MojoFailureException {
- String targetDir = this.session.getExecutionRootDirectory() + File.separator + "target";
- String spIgnoreFile =
+ public void execute() throws MojoFailureException {
+ var log = getLog();
+ var targetDir = this.session.getExecutionRootDirectory() + File.separator + "target";
+ var spIgnoreFile =
this.session.getExecutionRootDirectory() + File.separator + SP_IGNORE_FILENAME;
- Path docsBasePath = Paths.get(targetDir, DOCS_ROOT_FOLDER);
-
- List<MavenProject> projects = session.getProjects();
- List<AssetModel> documentedPipelineElements = new ArrayList<>();
- List<String> pipelineElementsToExclude = new ArrayList<>();
-
- if (Files.exists(Paths.get(spIgnoreFile))) {
- try {
- pipelineElementsToExclude = Files.readAllLines(Paths.get(spIgnoreFile));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
+ var docsBasePath = Paths.get(targetDir, DOCS_ROOT_FOLDER);
- for (MavenProject currentModule : projects) {
- File baseDir = currentModule.getBasedir();
- List<String> sourceRoots = currentModule.getCompileSourceRoots();
- List<AssetModel> allAssetModels = new ArrayList<>();
- //String targetDir = this.project.getModel().getBuild().getDirectory();
-
- //String dockerImageName =
- // new DockerImageExtractor(baseDir.getAbsolutePath()).extractImageName();
-
- if (currentModule.getName().equals("streampipes-connect-adapters")) {
- System.out.println("Opening adapters");
- allAssetModels = new ResourceDirectoryElementFinder(sourceRoots.get(0), getLog(), baseDir.getAbsolutePath())
- .makeAssetModels();
- allAssetModels.forEach(am -> System.out.println(am.getAppId()));
- } else {
- allAssetModels = new ControllerFileFinder(getLog(),
- baseDir.getAbsolutePath(), sourceRoots.get(0),
- "**/*Controller.java").makeAssetModels();
- }
-
- allAssetModels.forEach(am -> {
- am.setBaseDir(baseDir.getAbsolutePath());
- am.setModuleName(currentModule.getName());
- //am.setContainerName(dockerImageName);
- });
-
- List<String> finalPipelineElementsToExclude = pipelineElementsToExclude;
- documentedPipelineElements.addAll(
- allAssetModels
- .stream()
- .filter(am -> finalPipelineElementsToExclude.stream().noneMatch(pe -> pe.equals(am.getAppId())))
- .collect(Collectors.toList()));
- }
try {
- Collections.sort(documentedPipelineElements);
- documentedPipelineElements.forEach(am -> System.out.println(
- am.getAppId() + ", " + am.getPipelineElementName() + ", " + am.getPipelineElementDescription()));
-
- for (AssetModel assetModel : documentedPipelineElements) {
- Path docsPath = Paths.get(targetDir, DOCS_ROOT_FOLDER, DOCS_FOLDER,
- "pe");
- Path docsWebsitePath = Paths.get(targetDir, DOCS_ROOT_FOLDER, DOCS_WEBSITE_FOLDER,
- assetModel.getAppId());
- Path imgPath = Paths.get(targetDir, DOCS_ROOT_FOLDER, IMG_FOLDER,
- assetModel.getAppId());
- DirectoryManager.createIfNotExists(docsPath);
- DirectoryManager.createIfNotExists(imgPath);
-
- Boolean iconExists = Files.exists(Utils.makeResourcePath(assetModel.getBaseDir(),
- assetModel.getAppId()).resolve("icon.png"));
-
- if (iconExists) {
- Files.copy(Utils.makeResourcePath(assetModel.getBaseDir(),
- assetModel.getAppId()).resolve("icon.png")
- , imgPath.resolve("icon.png"));
- } else {
- ClassLoader classLoader = this.getClass().getClassLoader();
- InputStream inputStream = classLoader.getResourceAsStream("placeholder-icon.png");
- Files.copy(inputStream, imgPath.resolve("icon.png"));
+ List<String> pipelineElementsToExclude = new ArrayList<>();
+
+ var path = Paths.get(spIgnoreFile);
+ if (Files.exists(path)) {
+ try {
+ pipelineElementsToExclude = Files.readAllLines(path);
+ } catch (IOException e) {
+ log.error(String.format("Could not open .spignore file at path %s", path));
}
-
- String originalDocumentationFileContents =
- FileUtils.readFileToString(Utils.makeResourcePath(assetModel.getBaseDir(),
- assetModel.getAppId()).resolve("documentation.md").toFile());
-
- // modify docs for documentation page
- String documentationFileContents =
- new MarkdownTitleRemover(originalDocumentationFileContents).removeTitle();
-
- documentationFileContents =
- new MarkdownHeaderGenerator(assetModel, documentationFileContents).createHeaders();
-
- documentationFileContents = new ImagePathReplacer(documentationFileContents,
- assetModel.getAppId()).replaceContentForDocs();
-
- FileUtils.writeStringToFile(docsPath.resolve(assetModel.getAppId() + ".md").toFile(),
- documentationFileContents);
-
}
- Boolean existsOverviewFile = Files.exists(docsBasePath.resolve("pipeline-elements.md"));
- if (!existsOverviewFile) {
- AssetModel assetModel = new AssetModel("pipeline-elements", "Overview", "");
- String header = new MarkdownHeaderGenerator(assetModel, "").createHeaders();
- FileUtils.writeStringToFile(docsBasePath.resolve("pipeline-elements.md").toFile(), header);
+ var loader = getClassLoader(project);
+ var finalPipelineElementsToExclude = pipelineElementsToExclude;
+ var localesExtractor = new LocalesExtractor(log, loader);
+ var extensionsElements = new ExtensionsFinder(loader, initClass)
+ .findExtensions()
+ .stream()
+ .filter(am -> !finalPipelineElementsToExclude.contains(am.getAppId()))
+ .peek(localesExtractor::applyLocales)
+ .sorted(AssetModel::compareTo)
+ .toList();
+
+ log.info(String.format("Will generate documentation for %s resources", extensionsElements.size()));
+
+ for (AssetModel extensionsElement : extensionsElements) {
+ try {
+ var imgPath = Paths.get(targetDir, DOCS_ROOT_FOLDER, IMG_FOLDER, extensionsElement.getAppId());
+ new IconResourceGenerator(loader, extensionsElement, imgPath).generate();
+
+ var docsPath = Paths.get(
+ targetDir,
+ DOCS_ROOT_FOLDER,
+ DOCS_PE_FOLDER,
+ extensionsElement.getAppId());
+ new DocumentationResourceGenerator(loader, extensionsElement, docsPath).generate();
+
+ log.info(
+ String.format(
+ "Generated documentation for %s (%s, %s)",
+ extensionsElement.getAppId(),
+ extensionsElement.getPipelineElementName(),
+ extensionsElement.getPipelineElementDescription()));
+
+ new AdditionalAssetsResourceGenerator(log, loader, extensionsElement, imgPath).generate();
+ } catch (IOException e) {
+ log.warn(
+ String.format(
+ "Could not generate documentation for %s (%s, %s)",
+ extensionsElement.getAppId(),
+ extensionsElement.getPipelineElementName(),
+ extensionsElement.getPipelineElementDescription()), e);
+ }
}
- String pipelineElementOverviewContent =
- new PipelineElementOverviewGenerator(documentedPipelineElements).generate();
- FileUtils.writeStringToFile(docsBasePath.resolve("pipeline-elements.md").toFile(),
- pipelineElementOverviewContent, true);
-
-
+ var sidebarConfig = new SidebarConfigGenerator(log, extensionsElements).generate();
+ log.info("Writing sidebar config");
FileUtils.writeStringToFile(docsBasePath.resolve("sidebars.json").toFile(),
- new SidebarConfigGenerator(documentedPipelineElements).generate());
+ sidebarConfig, Charsets.UTF_8);
- FileUtils.writeStringToFile(docsBasePath.resolve("_data.json").toFile(),
- new DataJsonGenerator(documentedPipelineElements).generate());
+ } catch (IOException | DependencyResolutionRequiredException | ClassNotFoundException
+ | IllegalAccessException | InstantiationException e) {
+ throw new MojoFailureException(
+ "Could not generate documentation - please check the initClass for available extensions", e);
+ }
+ }
+ private ClassLoader getClassLoader(MavenProject project)
+ throws DependencyResolutionRequiredException, MalformedURLException {
+ List<URL> runtimeUrls = new ArrayList<>();
+ List<String> runtimeClasspathElements = project.getCompileClasspathElements();
- } catch (IOException e) {
- e.printStackTrace();
+ for (String element : runtimeClasspathElements) {
+ runtimeUrls.add(new File(element).toURI().toURL());
}
+
+ URL[] array = new URL[runtimeUrls.size()];
+ array = runtimeUrls.toArray(array);
+
+ return new URLClassLoader(array,
+ Thread.currentThread().getContextClassLoader());
}
}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java
deleted file mode 100644
index 74c6873dd..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.constants.PeType;
-import org.apache.streampipes.smp.model.AssetModel;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class AssetModelItemExtractor {
-
- private static final String REGEX = "(DataSinkBuilder|ProcessingElementBuilder)\\.create\\"
- + "((\".*?\"),.*?(\".*?\"),.*?(\".*?\")\\)";
-
- private static final String ONLY_APP_ID_REGEX = "(DataSinkBuilder|ProcessingElementBuilder)\\"
- + ".create\\"
- + "((\".*?\")\\)";
-
- private static final Pattern pattern = Pattern.compile(REGEX);
- private static final Pattern onlyAppIdPattern = Pattern.compile(ONLY_APP_ID_REGEX);
-
- private String declareModelContent;
-
- public AssetModelItemExtractor(String declareModelContent) {
- this.declareModelContent = declareModelContent
- .replaceAll("\n", "");
-
- }
-
-
- public AssetModel extractAssetItem() throws IllegalArgumentException {
- AssetModel assetModel = new AssetModel();
- Matcher matcher = pattern.matcher(this.declareModelContent);
- Boolean matches = false;
-
-// while (matcher.find()) {
-// matches = true;
-// PeType peType = new PipelineElementTypeExtractor(matcher.group(1)).extractType();
-// String appId = removeQuotes(matcher.group(2));
-// String pipelineElementName = removeQuotes(matcher.group(3));
-// String pipelineElementDescription = removeQuotes(matcher.group(4));
-//
-// assetModel.setPeType(peType);
-// assetModel.setAppId(appId);
-// assetModel.setPipelineElementName(pipelineElementName);
-// assetModel.setPipelineElementDescription(pipelineElementDescription);
-//
-// }
-
- if (matches) {
- return assetModel;
- } else {
- Matcher appIdMatcher = onlyAppIdPattern.matcher(this.declareModelContent);
- while (appIdMatcher.find()) {
- matches = true;
- PeType peType = new PipelineElementTypeExtractor(appIdMatcher.group(1)).extractType();
- assetModel.setAppId(removeQuotes(appIdMatcher.group(2)));
- assetModel.setPeType(peType);
- }
- if (!matches) {
- throw new IllegalArgumentException("Could not extract appId");
- } else {
- return assetModel;
- }
- }
- }
-
- private String removeQuotes(String text) {
- return text
- .replaceAll("\" \\+ \"", "")
- .replaceAll("\"", "");
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java
deleted file mode 100644
index 7f2c0436a..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.model.AssetModel;
-
-import org.jboss.forge.roaster.Roaster;
-import org.jboss.forge.roaster.model.source.JavaClassSource;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-public class ControllerExtractor {
-
- private static String declareModelMethod = "declareModel";
- private String filename;
- private String baseDir;
-
- public ControllerExtractor(String baseDir, String filename) {
- this.filename = filename;
- this.baseDir = baseDir;
- }
-
- public AssetModel extractControllerDetails() throws Exception {
- String fileContents = new String(Files.readAllBytes(Paths.get(filename)));
- JavaClassSource clazz = Roaster.parse(JavaClassSource.class, fileContents);
- if (clazz.hasMethodSignature(declareModelMethod)) {
- String declareMethodContent = (clazz.getMethod(declareModelMethod).getBody());
- declareMethodContent =
- new FieldReplacer(clazz, declareMethodContent).replaceDeclareModelContent();
- AssetModel assetModel = new AssetModelItemExtractor(declareMethodContent).extractAssetItem();
- if (assetModel.getPipelineElementName() == null) {
- assetModel = new LocalesExtractor(baseDir, assetModel).extract();
- }
- return assetModel;
-
- } else {
- throw new Exception("Not a valid controller class or could not read id elements");
- }
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java
deleted file mode 100644
index d6d5dae10..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.model.AssetModel;
-
-import org.apache.maven.plugin.logging.Log;
-import org.apache.maven.shared.model.fileset.FileSet;
-import org.apache.maven.shared.model.fileset.util.FileSetManager;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ControllerFileFinder extends ElementFinder {
-
- private String filterPattern;
-
- public ControllerFileFinder(Log log, String baseDir, String sourceRoot, String filterPattern) {
- super(sourceRoot, log, baseDir);
- this.filterPattern = filterPattern;
- }
-
- public String[] findFiles() {
- FileSet fileSet = new FileSet();
- fileSet.setDirectory(sourceRoot);
- fileSet.addInclude(filterPattern);
- FileSetManager fileSetManager = new FileSetManager();
-
- return fileSetManager.getIncludedFiles(fileSet);
- }
-
- @Override
- public List<AssetModel> makeAssetModels() {
- List<AssetModel> allAssetModels = new ArrayList<>();
- for (String file : findFiles()) {
- try {
- allAssetModels.add(
- new ControllerExtractor(baseDir, sourceRoot + File.separator + file).extractControllerDetails());
- } catch (Exception e) {
- log.error(e.getMessage());
- log.info("Could not parse file " + file + ", skipping...");
- }
- }
- return allAssetModels;
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java
deleted file mode 100644
index a62d70795..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
-
-public class DockerImageExtractor {
-
- private static final String Deployment = "deployment";
- private static final String DockerCompose = "docker-compose.yml";
- private static final String Image = "image";
- private static final String Services = "services";
- private static final String Colon = ":";
- private static final String ImagePrefix = "${SP_PE_DOCKER_REGISTRY}/streampipes"
- + "/streampipes"
- + "-pipeline-elements/";
-
- private String baseDir;
-
- public DockerImageExtractor(String baseDir) {
- this.baseDir = baseDir;
- }
-
- public String extractImageName() {
- Path dockerComposePath = Paths.get(baseDir, Deployment, DockerCompose);
-
- Yaml yaml = new Yaml();
- try {
- Map<String, Object> fileContents =
- yaml.load(new String(Files.readAllBytes(dockerComposePath)));
-
- return extractNameFromYaml(fileContents);
-
- } catch (IOException e) {
- e.printStackTrace();
- return "";
- }
- }
-
- public String extractNameFromYaml(Map<String, Object> fileContents) {
- Map<String, Object> services = (Map<String, Object>) fileContents.get(Services);
- AtomicReference<String> imageName = new AtomicReference<>("");
- if (services.size() > 0) {
- services.forEach((key, value) -> imageName.set(parseValue(((Map<String,
- Object>) value).get(Image).toString())));
- }
-
- return imageName.get();
- }
-
- private String parseValue(String imageName) {
- return imageName.replace(ImagePrefix, "").split(Colon)[0];
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java
deleted file mode 100644
index 55818c8d1..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.model.AssetModel;
-
-import org.apache.maven.plugin.logging.Log;
-
-import java.util.List;
-
-public abstract class ElementFinder {
-
- protected String sourceRoot;
- protected Log log;
- protected String baseDir;
-
- public ElementFinder(String sourceRoot, Log log, String baseDir) {
- this.sourceRoot = sourceRoot;
- this.log = log;
- this.baseDir = baseDir;
- }
-
- public abstract List<AssetModel> makeAssetModels();
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java
new file mode 100644
index 000000000..b36abb767
--- /dev/null
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.smp.extractor;
+
+import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
+import org.apache.streampipes.extensions.api.pe.config.IDataProcessorConfiguration;
+import org.apache.streampipes.extensions.api.pe.config.IDataSinkConfiguration;
+import org.apache.streampipes.extensions.api.pe.config.IPipelineElementConfiguration;
+import org.apache.streampipes.extensions.management.model.SpServiceDefinition;
+import org.apache.streampipes.service.extensions.ExtensionsModelSubmitter;
+import org.apache.streampipes.smp.constants.PeType;
+import org.apache.streampipes.smp.model.AssetModel;
+
+import org.apache.maven.artifact.DependencyResolutionRequiredException;
+
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class ExtensionsFinder {
+
+ private final ClassLoader loader;
+ private final String initClass;
+
+ public ExtensionsFinder(ClassLoader loader,
+ String initClass) {
+ this.loader = loader;
+ this.initClass = initClass;
+ }
+
+ public List<AssetModel> findExtensions()
+ throws MalformedURLException, DependencyResolutionRequiredException, ClassNotFoundException,
+ InstantiationException, IllegalAccessException {
+ var extensions = new ArrayList<AssetModel>();
+ var serviceDef = ((ExtensionsModelSubmitter) loader.loadClass(initClass).newInstance()).provideServiceDefinition();
+
+ extensions.addAll(findAdapters(serviceDef));
+ extensions.addAll(findPipelineElements(serviceDef, IDataProcessorConfiguration.class, PeType.PROCESSOR));
+ extensions.addAll(findPipelineElements(serviceDef, IDataSinkConfiguration.class, PeType.SINK));
+
+ return extensions;
+ }
+
+ private List<AssetModel> findPipelineElements(SpServiceDefinition serviceDef,
+ Class<? extends IPipelineElementConfiguration<?, ?>> configType,
+ PeType peType) {
+ return serviceDef.getDeclarers()
+ .stream()
+ .map(IStreamPipesPipelineElement::declareConfig)
+ .filter(configType::isInstance)
+ .map(config -> new AssetModel(config.getDescription().getAppId(), peType)).toList();
+ }
+
+ private Collection<? extends AssetModel> findAdapters(SpServiceDefinition serviceDef) {
+ return serviceDef.getAdapters().stream().map(adapter -> {
+ var config = adapter.declareConfig();
+ return new AssetModel(config.getAdapterDescription().getAppId(), PeType.ADAPTER);
+ }).toList();
+ }
+}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java
deleted file mode 100644
index a66182032..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.jboss.forge.roaster.model.source.JavaClassSource;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class FieldReplacer {
-
- private static final String CHECK_FIELD_REGEX = "(DataSinkBuilder|ProcessingElementBuilder)\\"
- + ".create\\"
- + "((.*?),(.*?)\\)";
- private static final Pattern checkFieldPattern = Pattern.compile(CHECK_FIELD_REGEX);
- private String declareModelContent;
- private JavaClassSource source;
-
-
- public FieldReplacer(JavaClassSource source, String declareModelContent) {
- this.source = source;
- this.declareModelContent = declareModelContent;
- }
-
- public String replaceDeclareModelContent() {
- Matcher matcher = checkFieldPattern.matcher(this.declareModelContent);
- while (matcher.find()) {
- String match = matcher.group(2);
- if (!match.startsWith("\"")) {
- declareModelContent = declareModelContent.replaceFirst(match,
- getFieldValue(match));
- }
- }
- return declareModelContent;
- }
-
- private String getFieldValue(String fieldName) {
- return "\""
- + source.getField(fieldName).getStringInitializer()
- + "\"";
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java
index ec50554f7..7688c2e64 100644
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java
@@ -19,41 +19,47 @@
package org.apache.streampipes.smp.extractor;
import org.apache.streampipes.smp.model.AssetModel;
-import org.apache.streampipes.smp.util.Utils;
-import java.io.FileReader;
+import org.apache.maven.plugin.logging.Log;
+
import java.io.IOException;
+import java.io.InputStream;
import java.util.Properties;
public class LocalesExtractor {
+ public static final String LOCALES_FILE_EN = "strings.en";
+
private static final String TITLE = ".title";
private static final String DESCRIPTION = ".description";
- private AssetModel assetModel;
- private String baseDir;
- public LocalesExtractor(String baseDir, AssetModel assetModel) {
- this.baseDir = baseDir;
- this.assetModel = assetModel;
- }
+ private final Log log;
+ private final ClassLoader loader;
- public AssetModel extract() throws IOException {
- String localesPath = Utils.makeLocalesPath(baseDir, assetModel.getAppId());
- Properties props = loadProperties(localesPath);
-
- assetModel.setPipelineElementName(extractKey(props, assetModel, TITLE));
- assetModel.setPipelineElementDescription(extractKey(props, assetModel, DESCRIPTION));
+ public LocalesExtractor(Log log,
+ ClassLoader loader) {
+ this.log = log;
+ this.loader = loader;
+ }
- return assetModel;
+ public void applyLocales(AssetModel assetModel) {
+ var localeFile = loader.getResourceAsStream(assetModel.getAppId() + "/" + LOCALES_FILE_EN);
+ try {
+ var props = loadProperties(localeFile);
+ assetModel.setPipelineElementName(extractKey(props, assetModel, TITLE));
+ assetModel.setPipelineElementDescription(extractKey(props, assetModel, DESCRIPTION));
+ } catch (IOException e) {
+ log.warn(String.format("Could not load properties file %s", assetModel.getAppId()), e);
+ }
}
private String extractKey(Properties props, AssetModel assetModel, String appendix) {
return props.getProperty(assetModel.getAppId() + appendix);
}
- private Properties loadProperties(String localesPath) throws IOException {
+ private Properties loadProperties(InputStream stream) throws IOException {
Properties props = new Properties();
- props.load(new FileReader(localesPath));
+ props.load(stream);
return props;
}
}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java
deleted file mode 100644
index df5322fba..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.constants.PeType;
-
-public class PipelineElementTypeExtractor {
-
- private static final String ProcessorString = "ProcessingElementBuilder";
- private static final String SinkString = "DataSinkBuilder";
-
- private String typeString;
-
- public PipelineElementTypeExtractor(String typeString) {
- this.typeString = typeString;
- }
-
- public PeType extractType() throws IllegalArgumentException {
- if (typeString.equals(ProcessorString)) {
- return PeType.PROCESSOR;
- } else if (typeString.equals(SinkString)) {
- return PeType.SINK;
- } else {
- throw new IllegalArgumentException("Unknown pipeline element type " + typeString);
- }
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java
deleted file mode 100644
index 1151b78f5..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.constants.PeType;
-import org.apache.streampipes.smp.model.AssetModel;
-import org.apache.streampipes.smp.util.Utils;
-
-import org.apache.maven.plugin.logging.Log;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class ResourceDirectoryElementFinder extends ElementFinder {
-
-
- public ResourceDirectoryElementFinder(String sourceRoot, Log log, String baseDir) {
- super(sourceRoot, log, baseDir);
- System.out.println(sourceRoot);
- System.out.println(baseDir);
- }
-
- @Override
- public List<AssetModel> makeAssetModels() {
- List<AssetModel> adapterModels = new ArrayList<>();
- Path resourceDirPath = Utils.makeResourcePath(baseDir);
- File[] resourceDirs = resourceDirPath.toFile().listFiles(File::isDirectory);
- if (resourceDirs != null) {
- List<File> appDirs = Arrays.stream(resourceDirs)
- .filter(rd -> rd.getName().startsWith("org.apache.streampipes"))
- .collect(Collectors.toList());
-
- appDirs.forEach(ap -> {
- try {
- adapterModels.add(extractModel(baseDir, ap));
- } catch (IOException e) {
- e.printStackTrace();
- }
- });
- }
-
- return adapterModels;
- }
-
- private AssetModel extractModel(String sourceRoot, File resourceDir) throws IOException {
- AssetModel model = new AssetModel();
-
- model.setPeType(PeType.ADAPTER);
- model.setAppId(resourceDir.getName());
- return new LocalesExtractor(sourceRoot, model).extract();
- }
-
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java
new file mode 100644
index 000000000..0122c4233
--- /dev/null
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.smp.generator;
+
+import org.apache.streampipes.smp.extractor.LocalesExtractor;
+import org.apache.streampipes.smp.model.AssetModel;
+
+import org.apache.maven.plugin.logging.Log;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+public class AdditionalAssetsResourceGenerator extends ResourceGenerator {
+
+ private static final String Slash = "/";
+
+ private final Log log;
+
+ public AdditionalAssetsResourceGenerator(Log log,
+ ClassLoader loader,
+ AssetModel extensionsElement,
+ Path targetPath) {
+ super(loader, extensionsElement, targetPath);
+ this.log = log;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ var resourceName = extensionsElement.getAppId() + Slash;
+ var resources = loader.getResources(resourceName);
+ while (resources.hasMoreElements()) {
+ URL resourceUrl = resources.nextElement();
+ if (resourceUrl.getProtocol().equals("jar")) {
+ JarURLConnection jarConnection = (JarURLConnection) resourceUrl.openConnection();
+ try (JarFile jarFile = jarConnection.getJarFile()) {
+ Enumeration<JarEntry> entries = jarFile.entries();
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ if (!entry.isDirectory() && entry.getName().startsWith(resourceName)) {
+ var filename = entry.getName();
+ if (!isDefaultResource(filename)) {
+ try (InputStream inputStream = jarFile.getInputStream(entry)) {
+ Files.createDirectories(targetPath);
+ filename = filename.substring(filename.lastIndexOf(Slash) + 1);
+ Files.copy(inputStream, targetPath.resolve(filename), StandardCopyOption.REPLACE_EXISTING);
+ log.info(String.format("Additional resource file %s copied to %s", filename, targetPath));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private boolean isDefaultResource(String resourceName) {
+ var defaultResources = List.of(
+ IconResourceGenerator.ICON_NAME,
+ DocumentationResourceGenerator.DOCUMENTATION_FILE_NAME,
+ LocalesExtractor.LOCALES_FILE_EN);
+ return defaultResources.stream().anyMatch(resourceName::contains);
+ }
+}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java
deleted file mode 100644
index 7f6bd8c1b..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.generator;
-
-import org.apache.streampipes.smp.model.AssetModel;
-import org.apache.streampipes.smp.parser.DocumentationParser;
-import org.apache.streampipes.smp.util.Utils;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-
-public class AssetGenerator {
-
- private static final String DOCUMENTATION_FILE = "documentation.md";
- private AssetModel assetModel;
- private String baseDir;
-
- public AssetGenerator(String baseDir, AssetModel assetModel) {
- this.assetModel = assetModel;
- this.baseDir = baseDir;
- }
-
- public void genreateAssetDirectoryAndContents() {
- ClassLoader classLoader = this.getClass().getClassLoader();
- String resourcePath = Utils.makePath(baseDir, assetModel.getAppId());
-
- Boolean dir = new File(resourcePath).mkdirs();
- InputStream inputStream = classLoader.getResourceAsStream(DOCUMENTATION_FILE);
- try {
- String content =
- new DocumentationParser(assetModel)
- .parseAndStoreDocumentation(IOUtils.toString(inputStream));
- FileUtils.writeStringToFile(new File(resourcePath + File.separator + DOCUMENTATION_FILE),
- content);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java
deleted file mode 100644
index f99239841..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.generator;
-
-import org.apache.streampipes.smp.constants.PeType;
-import org.apache.streampipes.smp.model.AssetModel;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-
-import java.util.List;
-
-public class DataJsonGenerator extends OutputGenerator {
-
- public DataJsonGenerator(List<AssetModel> assetModels) {
- super(assetModels);
- }
-
- @Override
- public String generate() {
- JsonObject jsonObject = new JsonObject();
- jsonObject.add("pipelineElements", makePipelineElementList());
-
- return jsonObject.toString();
- }
-
- private JsonArray makePipelineElementList() {
- JsonArray elementList = new JsonArray();
-
- assetModels.forEach(am -> {
- elementList.add(makePipelineElementEntry(am));
- });
-
- return elementList;
- }
-
- private JsonObject makePipelineElementEntry(AssetModel am) {
- JsonObject entry = new JsonObject();
- entry.addProperty("appId", am.getAppId());
- entry.addProperty("type", am.getPeType().getFriendly());
- entry.addProperty("name", am.getPipelineElementName());
- entry.addProperty("description", am.getPipelineElementDescription());
- entry.addProperty("dockerHubLink", makeDockerHubLink(am.getPeType(), am.getModuleName()));
- entry.addProperty("githubLink", makeGithubLink(am.getModuleName()));
-
- return entry;
- }
-
- private String makeDockerHubLink(PeType peType, String moduleName) {
- String dockerHubRepoName = peType == PeType.ADAPTER ? "connect-worker" : moduleName.replace("streampipes-", "");
- return "https://hub.docker.com/r/apachestreampipes/" + dockerHubRepoName;
- }
-
- private String makeGithubLink(String moduleName) {
- return "https://github.com/apache/incubator-streampipes-extensions/tree/dev/" + moduleName;
- }
-
-
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java
new file mode 100644
index 000000000..7b83aa767
--- /dev/null
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.smp.generator;
+
+import org.apache.streampipes.smp.model.AssetModel;
+import org.apache.streampipes.smp.util.DirectoryManager;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public class DocumentationResourceGenerator extends ResourceGenerator {
+
+ public static final String DOCUMENTATION_FILE_NAME = "documentation.md";
+
+ public DocumentationResourceGenerator(ClassLoader loader,
+ AssetModel extensionsElement,
+ Path targetPath) {
+ super(loader, extensionsElement, targetPath);
+ }
+
+ @Override
+ public void generate() throws IOException {
+ DirectoryManager.createIfNotExists(targetPath);
+ try (var inputStream = getResourceInputStream(DOCUMENTATION_FILE_NAME)) {
+ if (inputStream != null) {
+ var originalDocumentationFileContents = new String(inputStream.readAllBytes());
+ // modify docs for documentation page
+ String documentationFileContents =
+ new MarkdownTitleRemover(originalDocumentationFileContents).removeTitle();
+
+ documentationFileContents =
+ new MarkdownHeaderGenerator(extensionsElement, documentationFileContents).createHeaders();
+
+ documentationFileContents = new ImagePathReplacer(documentationFileContents,
+ extensionsElement.getAppId()).replaceContentForDocs();
+
+ FileUtils.writeStringToFile(
+ targetPath.resolve(extensionsElement.getAppId() + ".md").toFile(),
+ documentationFileContents,
+ Charsets.UTF_8);
+ }
+ }
+ }
+}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java
new file mode 100644
index 000000000..b64581d42
--- /dev/null
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.smp.generator;
+
+import org.apache.streampipes.smp.model.AssetModel;
+import org.apache.streampipes.smp.util.DirectoryManager;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+public class IconResourceGenerator extends ResourceGenerator {
+
+ public static final String ICON_NAME = "icon.png";
+ private static final String PlaceholderIconName = "placeholder-icon.png";
+
+ public IconResourceGenerator(ClassLoader loader,
+ AssetModel extensionsElement,
+ Path targetPath) {
+ super(loader, extensionsElement, targetPath);
+ }
+
+ @Override
+ public void generate() throws IOException {
+ DirectoryManager.createIfNotExists(targetPath);
+ try (var inputStream = getResourceInputStream(ICON_NAME)) {
+ if (inputStream != null) {
+ Files.copy(inputStream, targetPath.resolve(ICON_NAME), StandardCopyOption.REPLACE_EXISTING);
+ } else {
+ ClassLoader classLoader = this.getClass().getClassLoader();
+ try (var placeHolderIconStream = classLoader.getResourceAsStream(PlaceholderIconName)) {
+ if (placeHolderIconStream != null) {
+ Files.copy(placeHolderIconStream, targetPath.resolve(ICON_NAME), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java
index cda528f0c..85bd0bc63 100644
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java
@@ -26,11 +26,10 @@ public class ImagePathReplacer {
private static final String IMAGE_REGEX = "(<img.*src=\")(.*)(\")";
private static final Pattern pattern = Pattern.compile(IMAGE_REGEX);
- private static final String DOCS_PREFIX = "/docs/img/pipeline-elements/";
- private static final String WEBSITE_PREFIX = "/img/pipeline-elements/";
+ private static final String DOCS_PREFIX = "/img/pipeline-elements/";
- private String originalContent;
- private String appId;
+ private final String originalContent;
+ private final String appId;
public ImagePathReplacer(String originalContent, String appId) {
this.originalContent = originalContent;
@@ -41,10 +40,6 @@ public class ImagePathReplacer {
return replaceContent(DOCS_PREFIX);
}
- public String replaceContentForWebsite() {
- return replaceContent(WEBSITE_PREFIX);
- }
-
public String replaceContent(String prefix) {
String newContent;
Matcher matcher = pattern.matcher(originalContent);
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/OutputGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/OutputGenerator.java
deleted file mode 100644
index 75186a33d..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/OutputGenerator.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.generator;
-
-import org.apache.streampipes.smp.model.AssetModel;
-
-import java.util.List;
-
-public abstract class OutputGenerator {
-
- protected List<AssetModel> assetModels;
-
- public OutputGenerator(List<AssetModel> assetModels) {
- this.assetModels = assetModels;
- }
-
- public abstract String generate();
-
-
-}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementGridGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementGridGenerator.java
deleted file mode 100644
index 42a08b50c..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementGridGenerator.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.generator;
-
-import org.apache.streampipes.smp.constants.PeGridConst;
-import org.apache.streampipes.smp.constants.PeType;
-import org.apache.streampipes.smp.model.AssetModel;
-
-import j2html.tags.DomContent;
-
-import static j2html.TagCreator.a;
-import static j2html.TagCreator.div;
-import static j2html.TagCreator.i;
-import static j2html.TagCreator.img;
-import static j2html.TagCreator.text;
-
-public class PipelineElementGridGenerator {
-
- private static final String IMG_PATH_PREFIX = "/docs/img/pipeline-elements/";
- private static final String IMG_PATH_APPENDIX = "/icon.png";
-
- private static final String DOCUMENTATION = "Documentation";
- private static final String GITHUB = "Code on Github";
- private static final String DOCKER_HUB = "Docker Container";
-
- private static final String STREAMPIPES_GITHUB_URL = "https://www.github"
- + ".com/apache/incubator-streampipes-extensions/tree/dev/";
-
- private static final String DOCKER_HUB_URL = "https://hub.docker.com/r/streampipes/";
-
- private AssetModel assetModel;
-
- public PipelineElementGridGenerator(AssetModel assetModel) {
- this.assetModel = assetModel;
- }
-
- public DomContent makeGrid() {
- return makeContainerItem();
- }
-
- private DomContent makeContainerItem() {
- return div(makeContainerItemHeader(),
- makeContainerItemBody(),
- makeContainerItemFooter()).withClasses(PeGridConst.PE_CONTAINER_ITEM,
- getContainerItemType());
- }
-
- private DomContent makeContainerItemFooter() {
- return div(makeFile(), makeDocker(), makeGithub()).withClass(PeGridConst.PE_CONTAINER_ITEM_FOOTER);
- }
-
- private DomContent makeGithub() {
- return div(makeGithubIcon(), makeBlank(), makeGithubLink());
- }
-
- private DomContent makeGithubLink() {
- return a(GITHUB).withHref(STREAMPIPES_GITHUB_URL + assetModel.getModuleName());
- }
-
- private DomContent makeGithubIcon() {
- return i().withClasses(PeGridConst.FAB, PeGridConst.FA_GITHUB);
- }
-
- private DomContent makeDocker() {
- return div(makeDockerIcon(), makeBlank(), makeDockerLink());
- }
-
- private DomContent makeDockerLink() {
- return a(DOCKER_HUB).withHref(DOCKER_HUB_URL + assetModel.getModuleName().replace("streampipes-", ""));
- }
-
- private DomContent makeDockerIcon() {
- return i().withClasses(PeGridConst.FAB, PeGridConst.FA_DOCKER);
- }
-
- private DomContent makeFile() {
- return div(makeFileIcon(), makeBlank(), makeDocumentationLink());
- }
-
- private DomContent makeDocumentationLink() {
- return a(DOCUMENTATION).withHref("/docs/docs/pe/" + assetModel.getAppId());
- }
-
- private DomContent makeBlank() {
- return text(" ");
- }
-
- private DomContent makeFileIcon() {
- return i().withClasses(PeGridConst.FAS, PeGridConst.FA_FILE);
- }
-
- private DomContent makeContainerItemBody() {
- return div(assetModel.getPipelineElementDescription()).withClass(PeGridConst.PE_CONTAINER_ITEM_BODY);
- }
-
- private DomContent makeContainerItemHeader() {
- return div(makeContainerItemIconWrapper(),
- makeContainerItemHeaderPe()).withClass(PeGridConst.PE_CONTAINER_ITEM_HEADER);
- }
-
- private DomContent makeContainerItemHeaderPe() {
- return div(makeContainerItemLabel(), makeContainerItemLabelName()).withClass(
- PeGridConst.PE_CONTAINER_ITEM_HEADER_PE);
- }
-
- private DomContent makeContainerItemLabelName() {
- return div(assetModel.getPipelineElementName()).withClass(PeGridConst.PE_CONTAINER_ITEM_LABEL_NAME);
- }
-
- private DomContent makeContainerItemLabel() {
- return div(makeLabelText()).withClasses(PeGridConst.PE_CONTAINER_ITEM_LABEL, makeLabelType());
- }
-
- private String makeLabelText() {
- return assetModel.getPeType().equals(PeType.PROCESSOR) ? "Data Processor" : "Data Sink";
- }
-
- private String makeLabelType() {
- return assetModel.getPeType().equals(PeType.PROCESSOR) ? PeGridConst.PE_CONTAINER_ITEM_LABEL_PROCESSOR :
- PeGridConst.PE_CONTAINER_ITEM_LABEL_SINK;
- }
-
- private DomContent makeContainerItemIconWrapper() {
- return div(makeIconImg()).withClasses(PeGridConst.PE_CONTAINER_ITEM_ICON, makeIconType());
- }
-
- private DomContent makeIconImg() {
- return img().withClass(PeGridConst.PE_ICON).withSrc(IMG_PATH_PREFIX + assetModel.getAppId() + IMG_PATH_APPENDIX);
- }
-
- private String makeIconType() {
- return assetModel.getPeType().equals(PeType.PROCESSOR) ? PeGridConst.PE_ITEM_ICON_PROCESSOR :
- PeGridConst.PE_ITEM_ICON_SINK;
- }
-
- private String getContainerItemType() {
- return assetModel.getPeType().equals(PeType.PROCESSOR) ? PeGridConst.PE_CONTAINER_ITEM_PROCESSOR :
- PeGridConst.PE_CONTAINER_ITEM_SINK;
- }
-
-}
\ No newline at end of file
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementOverviewGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementOverviewGenerator.java
deleted file mode 100644
index 75277ffbe..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementOverviewGenerator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.generator;
-
-import org.apache.streampipes.smp.constants.PeGridConst;
-import org.apache.streampipes.smp.model.AssetModel;
-
-import j2html.tags.ContainerTag;
-import j2html.tags.DomContent;
-
-import java.util.List;
-
-import static j2html.TagCreator.div;
-
-public class PipelineElementOverviewGenerator extends OutputGenerator {
-
- public PipelineElementOverviewGenerator(List<AssetModel> assetModels) {
- super(assetModels);
- }
-
- public String generate() {
-
- ContainerTag gridContent = div().withClass(PeGridConst.PE_GRID_CONTAINER);
- for (AssetModel am : assetModels) {
- gridContent.with(makeGrid(am));
- }
- String markdownContent = gridContent.render();
- return markdownContent;
- //return new MarkdownHeaderGenerator(assetModel, markdownContent).createHeaders();
- }
-
- private DomContent makeGrid(AssetModel assetModel) {
- return new PipelineElementGridGenerator(assetModel).makeGrid();
- }
-
-
-}
diff --git a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java
similarity index 55%
rename from streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java
rename to streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java
index 87e60b975..9816ed785 100644
--- a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java
@@ -18,27 +18,29 @@
package org.apache.streampipes.smp.generator;
-import org.apache.streampipes.smp.constants.PeType;
import org.apache.streampipes.smp.model.AssetModel;
-import org.junit.Test;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
-import java.util.Arrays;
+public abstract class ResourceGenerator {
-import static org.junit.Assert.assertTrue;
+ protected final ClassLoader loader;
+ protected final AssetModel extensionsElement;
+ protected final Path targetPath;
-public class TestPipelineElementOverviewGenerator {
-
- private AssetModel am = new AssetModel("a", "B", "C");
-
- @Test
- public void testHtmlGenerator() {
- am.setPeType(PeType.PROCESSOR);
- String html =
- new PipelineElementOverviewGenerator(Arrays.asList(am)).generate();
-
- System.out.println(html);
+ public ResourceGenerator(ClassLoader loader,
+ AssetModel extensionsElement,
+ Path targetPath) {
+ this.loader = loader;
+ this.extensionsElement = extensionsElement;
+ this.targetPath = targetPath;
+ }
- assertTrue(true);
+ protected InputStream getResourceInputStream(String resourceName) {
+ return loader.getResourceAsStream(extensionsElement.getAppId() + "/" + resourceName);
}
+
+ public abstract void generate() throws IOException;
}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java
index 3c492e66d..5f21291e4 100644
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java
@@ -18,29 +18,81 @@
package org.apache.streampipes.smp.generator;
+import org.apache.streampipes.smp.constants.PeType;
import org.apache.streampipes.smp.model.AssetModel;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import org.apache.http.client.fluent.Request;
+import org.apache.maven.plugin.logging.Log;
+import java.io.IOException;
import java.util.List;
-public class SidebarConfigGenerator extends OutputGenerator {
+public class SidebarConfigGenerator {
+ private static final String ExistingSidebarsUrl =
+ "https://raw.githubusercontent.com/apache/streampipes-website/dev/website-v2/sidebars.json";
+ private static final String DocumentationSection = "documentation";
+ private static final String PipelineElementSection = "\uD83D\uDCDA Pipeline Elements";
- public SidebarConfigGenerator(List<AssetModel> assetModels) {
- super(assetModels);
+ private final List<AssetModel> assetModels;
+ private final Log log;
+
+ public SidebarConfigGenerator(Log log,
+ List<AssetModel> assetModels) {
+ this.log = log;
+ this.assetModels = assetModels;
}
- public String generate() {
+ public String generate() throws IOException {
+ log.info("Downloading existing sidebar from Git repo");
+ var existingSidebar = loadExistingSidebar();
+ log.info("Sidebar download successful");
+ var existingSidebarJson = JsonParser.parseString(existingSidebar);
+
JsonArray pipelineElements = new JsonArray();
assetModels.forEach(pe -> pipelineElements.add("pe/" + pe.getAppId()));
- JsonObject pipelineElementObject = new JsonObject();
- pipelineElementObject.add("Pipeline Elements", pipelineElements);
- JsonObject jsonObject = new JsonObject();
- jsonObject.add("pipeline-elements", pipelineElementObject);
+ pipelineElements.add(makeItems(PeType.ADAPTER, "Adapters"));
+ pipelineElements.add(makeItems(PeType.PROCESSOR, "Data Processors"));
+ pipelineElements.add(makeItems(PeType.SINK, "Data Sinks"));
+
+ var section = existingSidebarJson
+ .getAsJsonObject()
+ .get(DocumentationSection)
+ .getAsJsonObject();
+
+ section.add(PipelineElementSection, pipelineElements);
+
+ return existingSidebarJson.toString();
+ }
+
+ private JsonObject makeItems(PeType type,
+ String typeName) {
+ var obj = new JsonObject();
+ var peJsonArray = new JsonArray();
+ obj.add("type", new JsonPrimitive("category"));
+ obj.add("label", new JsonPrimitive(typeName));
+
+ assetModels
+ .stream()
+ .filter(am -> am.getPeType() == type)
+ .forEach(am -> {
+ peJsonArray.add(new JsonPrimitive("pe/" + am.getAppId()));
+ });
+
+ obj.add("items", peJsonArray);
+ return obj;
+ }
- return jsonObject.toString();
+ private String loadExistingSidebar() throws IOException {
+ return Request
+ .Get(ExistingSidebarsUrl)
+ .execute()
+ .returnContent()
+ .asString();
}
-}
\ No newline at end of file
+}
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java
index 60447217d..ecb77de77 100644
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java
+++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java
@@ -26,14 +26,16 @@ public class AssetModel implements Comparable<AssetModel> {
private String pipelineElementName;
private String pipelineElementDescription;
private PeType peType = PeType.PROCESSOR;
- private String moduleName = "";
- private String containerName = "";
- private String baseDir;
public AssetModel() {
}
+ public AssetModel(String appId, PeType peType) {
+ this.appId = appId;
+ this.peType = peType;
+ }
+
public AssetModel(String appId, String pipelineElementName, String pipelineElementDescription) {
this.appId = appId;
this.pipelineElementName = pipelineElementName;
@@ -44,9 +46,6 @@ public class AssetModel implements Comparable<AssetModel> {
return appId;
}
- public void setAppId(String appId) {
- this.appId = appId;
- }
public String getPipelineElementName() {
return pipelineElementName;
@@ -68,34 +67,6 @@ public class AssetModel implements Comparable<AssetModel> {
return peType;
}
- public void setPeType(PeType peType) {
- this.peType = peType;
- }
-
- public String getModuleName() {
- return moduleName;
- }
-
- public void setModuleName(String moduleName) {
- this.moduleName = moduleName;
- }
-
- public String getContainerName() {
- return containerName;
- }
-
- public void setContainerName(String containerName) {
- this.containerName = containerName;
- }
-
- public String getBaseDir() {
- return baseDir;
- }
-
- public void setBaseDir(String baseDir) {
- this.baseDir = baseDir;
- }
-
@Override
public String toString() {
return pipelineElementName + " (" + peType.toString() + ", " + appId + "): "
diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java
deleted file mode 100644
index 5dc380eff..000000000
--- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.util;
-
-import org.apache.streampipes.smp.model.AssetModel;
-
-import java.io.File;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class DuplicateRemover {
-
- private List<AssetModel> assetModels;
- private String sourceDir;
-
- public DuplicateRemover(String sourceDir, List<AssetModel> assetModels) {
- this.assetModels = assetModels;
- this.sourceDir = sourceDir;
- }
-
- public List<AssetModel> removeAlreadyExisting() {
-
- return assetModels
- .stream()
- .filter(am -> !(new File(Utils.makePath(sourceDir, am.getAppId() + File.separator
- + "documentation.md")).exists()))
- .collect(Collectors.toList());
- }
-}
diff --git a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java b/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java
deleted file mode 100644
index 0dccd4ed1..000000000
--- a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.smp.extractor;
-
-import org.apache.streampipes.smp.constants.PeType;
-import org.apache.streampipes.smp.model.AssetModel;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-public class TestAssetModelExtractor {
-
- private String getGetDeclareModelContent4 =
- "return ProcessingElementBuilder.create(\"org.apache.streampipes.processors.filters.jvm.numericalfilter\")\n"
- + " .category(DataProcessorType.FILTER)\n"
- + " .withAssets(Assets.DOCUMENTATION, Assets.ICON)\n"
- + " .withLocales(Locales.EN)\n"
- + " .requiredStream(StreamRequirementsBuilder\n"
- + " .create()\n"
- + " .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(),\n"
- + " Labels.withId(NUMBER_MAPPING),\n"
- + " PropertyScope.NONE).build())\n"
- + " .outputStrategy(OutputStrategies.keep())\n"
- + " .requiredSingleValueSelection(Labels.withId(OPERATION), Options.from(\"<\", \"<=\", \">\",\n"
- + " \">=\", \"==\", \"!=\"))\n"
- + " .requiredFloatParameter(Labels.withId(VALUE), NUMBER_MAPPING)\n"
- + " .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())\n"
- + " .supportedFormats(SupportedFormats.jsonFormat())\n"
- + " .build();";
-
-
- @Test
- public void testAssetExtraction4() {
- AssetModel model = new AssetModelItemExtractor(getGetDeclareModelContent4).extractAssetItem();
-
- assertEquals("org.apache.streampipes.processors.filters.jvm.numericalfilter", model.getAppId());
- assertEquals(PeType.PROCESSOR, model.getPeType());
- assertNull(model.getPipelineElementName());
- assertNull(model.getPipelineElementDescription());
-
- }
-}