You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by ji...@apache.org on 2023/02/14 16:12:52 UTC
[daffodil] branch main updated: Merge runtime2's two redundant processors
This is an automated email from the ASF dual-hosted git repository.
jinterrante pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/main by this push:
new 04944afce Merge runtime2's two redundant processors
04944afce is described below
commit 04944afce95aefaea61cc74c714661f26d82f59d
Author: John Interrante <in...@research.ge.com>
AuthorDate: Mon Feb 13 09:34:03 2023 -0800
Merge runtime2's two redundant processors
Simplify runtime2 by removing Runtime2DataProcessor and merging its
code into Runtime2TDMLDFDLProcessor. Also add Runtime2 to some names
to make their names different than other code generators' classes.
Merge Steve's code for extracting C resources from jar (PR #959).
DAFFODIL-2790, DAFFODIL-2751
build.sbt: Add Runtime2 to genExamples & genExamplesSettings. Change
genRuntime2Examples' mainClass to Runtime2ExamplesGenerator.
Compiler.scala: Change C code generator's className to
Runtime2CodeGenerator.
DFDLParserUnparser.scala: Remove trait DataProcessorBase and move its
methods back into trait DataProcessor since we no longer need to
define Runtime2DataProcessor extending DataProcessorBase's methods.
CodeGenerator.scala: Rename object CodeGenerator to
Runtime2ExamplesGenerator and move to its own file. Rename rest of
file to Runtime2CodeGenerator.scala.
Runtime2CodeGenerator.scala: Rename class CodeGenerator to
Runtime2CodeGenerator. Merge Steve's code for extracting C resources
from jar file to avoid FileSystemAlreadyExistsException in CI tests
(PR #959). Improve code for reporting compilation warnings/errors.
Runtime2DataProcessor.scala: Remove file after moving its code into
Runtime2TDMLDFDLProcessor.scala.
Runtime2ExamplesGenerator.scala: Move object Runtime2ExamplesGenerator
to its own file.
TestRuntime2ExamplesGenerator.scala: Move Runtime2ExamplesGenerator
test to its own file.
Runtime2TDMLDFDLProcessor.scala: Merge Runtime2DataProcessor's code
into Runtime2TDMLDFDLProcessor to avoid having two redundant
processors.
TestCodeGenerator.scala/TestDaffodilC.scala: Rename file to
TestDaffodilC.scala. Keep only tests to run Runtime2CodeGenerator &
Runtime2TDMLDFDLProcessor and add a Runtime2TDMLDFDLProcessorFactory
test as well. Move Runtime2ExamplesGenerator test to its own file.
---
build.sbt | 25 +-
.../apache/daffodil/core/compiler/Compiler.scala | 2 +-
.../daffodil/runtime1/api/DFDLParserUnparser.scala | 26 +-
.../apache/daffodil/runtime2/CodeGenerator.scala | 274 -----------------
.../daffodil/runtime2/Runtime2CodeGenerator.scala | 189 +++++++++++-
.../daffodil/runtime2/Runtime2DataProcessor.scala | 207 -------------
.../runtime2/Runtime2ExamplesGenerator.scala | 90 ++++++
.../runtime2/TestRuntime2ExamplesGenerator.scala | 39 +++
.../processor/tdml/Runtime2TDMLDFDLProcessor.scala | 327 ++++++++++++++-------
.../daffodil/processor/tdml/TestDaffodilC.scala | 126 +++++---
10 files changed, 623 insertions(+), 682 deletions(-)
diff --git a/build.sbt b/build.sbt
index c0b1886bf..7cae367ce 100644
--- a/build.sbt
+++ b/build.sbt
@@ -32,7 +32,7 @@ Global / excludeLintKeys ++= Set(
lazy val genManaged = taskKey[Seq[File]]("Generate managed sources and resources")
lazy val genProps = taskKey[Seq[File]]("Generate properties scala source")
lazy val genSchemas = taskKey[Seq[File]]("Generated DFDL schemas")
-lazy val genExamples = taskKey[Seq[File]]("Generate runtime2 example files")
+lazy val genRuntime2Examples = taskKey[Seq[File]]("Generate runtime2 example files")
lazy val daffodil = project.in(file(".")).configs(IntegrationTest)
.enablePlugins(JavaUnidocPlugin, ScalaUnidocPlugin)
@@ -59,7 +59,7 @@ lazy val daffodil = project.in(file(".")).configs(IntegrationTest)
tutorials,
udf,
)
- .settings(commonSettings, nopublish, ratSettings, unidocSettings, genExamplesSettings)
+ .settings(commonSettings, nopublish, ratSettings, unidocSettings, genRuntime2ExamplesSettings)
lazy val macroLib = Project("daffodil-macro-lib", file("daffodil-macro-lib")).configs(IntegrationTest)
.settings(commonSettings, nopublish)
@@ -155,7 +155,7 @@ lazy val test = Project("daffodil-test", file("daffodil-test")).conf
.dependsOn(tdmlProc % "test", runtime2 % "test->test", udf % "test->test")
.settings(commonSettings, nopublish)
//
- // Uncomment the following line to run these tests
+ // Uncomment the following line to run these tests
// against IBM DFDL using the Cross Tester
//
//.settings(IBMDFDLCrossTesterPlugin.settings)
@@ -164,7 +164,7 @@ lazy val testIBM1 = Project("daffodil-test-ibm1", file("daffodil-test-ib
.dependsOn(tdmlProc % "test")
.settings(commonSettings, nopublish)
//
- // Uncomment the following line to run these tests
+ // Uncomment the following line to run these tests
// against IBM DFDL using the Cross Tester
//
//.settings(IBMDFDLCrossTesterPlugin.settings)
@@ -210,7 +210,6 @@ lazy val commonSettings = Seq(
testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "--verbosity=1"),
) ++ Defaults.itSettings
-
def buildScalacOptions(scalaVersion: String) = {
val commonOptions = Seq(
"-target:jvm-1.8",
@@ -336,7 +335,7 @@ lazy val libManagedSettings = Seq(
val files = forkCaptureLogger.stdout.map { f =>
new File(f)
}.toSet
- stream.log.info(s"generated ${files.size} Scala sources to ${outdir}")
+ stream.log.info(s"generated ${files.size} Scala sources to $outdir")
files
}
cachedFun(filesToWatch).toSeq
@@ -352,7 +351,7 @@ lazy val libManagedSettings = Seq(
IO.copyFile(schema, out)
out
}
- stream.log.info(s"generated ${files.size} XML schemas to ${outdir}")
+ stream.log.info(s"generated ${files.size} XML schemas to $outdir")
files
}
cachedFun(filesToWatch).toSeq
@@ -393,19 +392,19 @@ lazy val unidocSettings = Seq(
},
)
-lazy val genExamplesSettings = Seq(
- Compile / genExamples := {
+lazy val genRuntime2ExamplesSettings = Seq(
+ Compile / genRuntime2Examples := {
val cp = (runtime2 / Test / dependencyClasspath).value
val inSrc = (runtime2 / Compile / sources).value
val inRSrc = (runtime2 / Test / resources).value
val stream = (runtime2 / streams).value
val filesToWatch = (inSrc ++ inRSrc).toSet
- val cachedFun = FileFunction.cached(stream.cacheDirectory / "genExamples") { _ =>
+ val cachedFun = FileFunction.cached(stream.cacheDirectory / "genRuntime2Examples") { _ =>
val forkCaptureLogger = ForkCaptureLogger()
val forkOpts = ForkOptions()
.withOutputStrategy(Some(LoggedOutput(forkCaptureLogger)))
.withBootJars(cp.files.toVector)
- val mainClass = "org.apache.daffodil.runtime2.CodeGenerator"
+ val mainClass = "org.apache.daffodil.runtime2.Runtime2ExamplesGenerator"
val outdir = (runtime2 / Test / sourceDirectory).value / "c" / "examples"
val args = Seq(mainClass, outdir.toString)
val ret = Fork.java(forkOpts, args)
@@ -416,14 +415,14 @@ lazy val genExamplesSettings = Seq(
val files = forkCaptureLogger.stdout.filterNot(_.startsWith("WARNING")).map { f =>
new File(f)
}.toSet
- stream.log.info(s"generated ${files.size} runtime2 example files to ${outdir}")
+ stream.log.info(s"generated ${files.size} runtime2 examples to $outdir")
files
}
cachedFun(filesToWatch).toSeq
},
Compile / compile := {
val res = (Compile / compile).value
- (Compile / genExamples).value
+ (Compile / genRuntime2Examples).value
res
}
)
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/compiler/Compiler.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/compiler/Compiler.scala
index 791841bf2..5fd81e601 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/compiler/Compiler.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/compiler/Compiler.scala
@@ -104,7 +104,7 @@ final class ProcessorFactory private(
// it after we observe how the validator SPI evolves and wait for our
// requirements to become clearer
val className = language match {
- case "c" => "org.apache.daffodil.runtime2.CodeGenerator"
+ case "c" => "org.apache.daffodil.runtime2.Runtime2CodeGenerator"
case _ => throw new InvalidParserException(s"code generator; source language $language is not supported")
}
import scala.language.existentials // Needed to make next line compile
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/api/DFDLParserUnparser.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/api/DFDLParserUnparser.scala
index 07dc34768..9f2e4bb21 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/api/DFDLParserUnparser.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/api/DFDLParserUnparser.scala
@@ -20,15 +20,14 @@ package org.apache.daffodil.runtime1.api
import org.apache.daffodil.lib.api._
import java.io.File
-
import org.xml.sax.SAXException
import org.xml.sax.XMLReader
import org.xml.sax.ContentHandler
-
import org.apache.daffodil.lib.externalvars.Binding
import org.apache.daffodil.runtime1.infoset.InfosetInputter
import org.apache.daffodil.runtime1.infoset.InfosetOutputter
import org.apache.daffodil.io.InputSourceDataInputStream
+import org.apache.daffodil.lib.api.WithDiagnostics
import org.apache.daffodil.runtime1.processors.Failure
import org.apache.daffodil.runtime1.processors.ProcessorResult
import org.apache.daffodil.runtime1.processors.Success
@@ -123,25 +122,6 @@ object DFDL {
* and/or generation of source code to process data matching the compiled schema
*/
trait ProcessorFactory extends WithDiagnostics {
-
- /**
- * If you didn't set a root on the compiler, then get another
- * chance to specify one here.
- *
- * If you don't set a root at all it uses the first element in the
- * first schema document as the root.
- *
- * If you don't specify a namespace, or pass null, then it searches, and if
- * it is unambiguous, it will use the unique global element with
- * that name.
- *
- * Note: null used specifically here not an Option type, because this API
- * will shine through to a Java API.
- *
- * To explicitly specify that there is no-namespace, pass "" as
- * the namespace argument.
- */
-
/**
* Returns a [[DataProcessor]] to process data matching a compiled XPath expression
* @param xpath XPath expression in DFDL schema that data should match (you can use only "/" at this time)
@@ -175,7 +155,7 @@ object DFDL {
def compileCode(codeDir: os.Path): os.Path
}
- trait DataProcessorBase {
+ trait DataProcessor extends WithDiagnostics {
/**
* Returns a data processor with all the same state, but the validation mode changed to that of the argument.
*
@@ -195,9 +175,7 @@ object DFDL {
def tunables: DaffodilTunables
def variableMap: VariableMap
def validationMode: ValidationMode.Type
-}
- trait DataProcessor extends DataProcessorBase with WithDiagnostics {
/**
* Creates a new instance of XMLReader for SAX Parsing
*/
diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/CodeGenerator.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/CodeGenerator.scala
deleted file mode 100644
index f0b7642b6..000000000
--- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/CodeGenerator.scala
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.daffodil.runtime2
-
-import java.io.File
-import java.nio.file.FileSystems
-import java.nio.file.Files
-import java.nio.file.Paths
-import java.util.Collections
-import org.apache.daffodil.runtime1.api.DFDL
-import org.apache.daffodil.lib.api.Diagnostic
-import org.apache.daffodil.core.compiler.Compiler
-import org.apache.daffodil.core.dsom.Root
-import org.apache.daffodil.runtime1.dsom.SchemaDefinitionError
-import org.apache.daffodil.runtime2.generators.CodeGeneratorState
-import org.apache.daffodil.lib.util.Misc
-
-import scala.util.Properties.isWin
-
-/**
- * We need a mutux object for exclusive access to a code block
- */
-private object mutex {}
-
-/**
- * Generates and compiles C source files from a DFDL schema encapsulated in the parameter.
- * Implements the DFDL.CodeGenerator trait to allow it to be called by Daffodil code.
- * Note: Also implements WithDiagnostics trait with mutable state which means you need
- * to create a mew CodeGenerator each time you generate code.
- * @param root Provides the DFDL schema for code generation
- */
-class CodeGenerator(root: Root) extends DFDL.CodeGenerator {
- // CodeGenerator is not thread-safe due to these variables
- // which are needed to implement our WithDiagnostics trait
- private var diagnostics: Seq[Diagnostic] = Nil
- private var errorStatus: Boolean = false
-
- /**
- * Writes C source files into a "c" subdirectory of the given output directory.
- * Removes the "c" subdirectory if it existed before. Returns the newly created
- * "c" subdirectory.
- */
- override def generateCode(outputDirArg: String): os.Path = {
- // Get the paths of the C resources, the output directory, and its code subdirectory
- val resources = "/org/apache/daffodil/runtime2/c"
- val outputDir = os.Path(Paths.get(outputDirArg).toAbsolutePath)
- val codeDir = outputDir/"c"
-
- // Ensure our output directory exists while our code subdirectory does not
- os.makeDir.all(outputDir)
- os.remove.all(codeDir)
-
- // Copy all the C source files from our resources to our code subdirectory
- // (using synchronized to avoid calling FileSystems.newFileSystem concurrently)
- val resourceUri = Misc.getRequiredResource(resources)
- mutex.synchronized {
- val fileSystem = if (resourceUri.getScheme == "jar") {
- val env: java.util.Map[String, String] = Collections.emptyMap()
- FileSystems.newFileSystem(resourceUri, env)
- } else {
- null
- }
- try {
- val resourceDir = os.Path(if (fileSystem != null) fileSystem.getPath(resources) else Paths.get(resourceUri))
- os.copy(resourceDir, codeDir)
- }
- finally
- if (fileSystem != null) fileSystem.close()
- }
-
- // Generate C code from the DFDL schema, appending any warnings to our diagnostics
- val codeGeneratorState = new CodeGeneratorState(root)
- Runtime2CodeGenerator.generateCode(root.document, codeGeneratorState)
- diagnostics = diagnostics ++ root.warnings
- val versionHeaderText = codeGeneratorState.generateVersionHeader
- val codeHeaderText = codeGeneratorState.generateCodeHeader
- val codeFileText = codeGeneratorState.generateCodeFile
-
- // Write the generated C code into our code subdirectory
- val generatedVersionHeader = codeDir/"libcli"/"daffodil_version.h"
- val generatedCodeHeader = codeDir/"libruntime"/"generated_code.h"
- val generatedCodeFile = codeDir/"libruntime"/"generated_code.c"
- os.write.over(generatedVersionHeader, versionHeaderText)
- os.write(generatedCodeHeader, codeHeaderText)
- os.write(generatedCodeFile, codeFileText)
-
- // Return our code directory in case caller wants to call compileCode next
- codeDir
- }
-
- /**
- * Compiles any C source files inside the given code directory. Returns the path
- * of the newly built executable in order to run it in a TDML test.
- */
- override def compileCode(codeDir: os.Path): os.Path = {
- // Get the path of the executable we will build
- val exe = if (isWin) codeDir/"daffodil.exe" else codeDir/"daffodil"
-
- try {
- // Assemble the compiler's command line arguments
- val compiler = pickCompiler
- val cFlags = Seq("-std=gnu11")
- val includes = Seq("-Ilibcli", "-Ilibruntime")
- val absFiles = os.walk(codeDir, skip = _.last == "tests").filter(_.ext == "c")
- val relFiles = Seq("libcli/*.c", "libruntime/*.c")
- val libs = Seq("-lmxml")
-
- // Run the compiler in the code directory (if we found "zig cc"
- // as a compiler, it will cache previously built files in zig's
- // global cache directory, not a local zig_cache directory)
- if (compiler.nonEmpty) {
- val result = os
- .proc(compiler, cFlags, includes, if (isWin) relFiles else absFiles, libs, "-o", exe)
- .call(cwd = codeDir, stderr = os.Pipe)
-
- // Report any compiler output as a warning
- if (result.out.text.nonEmpty || result.err.text.nonEmpty) {
- warning("Unexpected compiler output on stdout: %s on stderr: %s", result.out.text, result.err.text)
- }
- }
- } catch {
- // Report any subprocess termination error as an error
- case e: os.SubprocessException =>
- error("Error compiling generated code: %s wd: %s", Misc.getSomeMessage(e).get, codeDir.toString)
- }
-
- // Report any failure to build the executable as an error
- if (!os.exists(exe)) error("No executable was built: %s", exe.toString)
-
- // Return our executable in case caller wants to run it next
- exe
- }
-
- /**
- * Searches for any available C compiler on the system. Tries to find the
- * compiler given by `CC` if `CC` exists in the environment, then tries to
- * find any compiler from the following list:
- *
- * - zig cc
- * - cc
- * - clang
- * - gcc
- *
- * Returns the first compiler found as a sequence of strings in case the
- * compiler is a program with a subcommand argument. Returns the empty
- * sequence if no compiler could be found in the user's PATH.
- */
- lazy val pickCompiler: Seq[String] = {
- val ccEnv = sys.env.getOrElse("CC", "zig cc")
- val compilers = Seq(ccEnv, "zig cc", "cc", "clang", "gcc")
- val path = sys.env.getOrElse("PATH", ".").split(File.pathSeparatorChar)
- def inPath(compiler: String): Boolean = {
- (compiler != null) && {
- val exec = compiler.takeWhile(_ != ' ')
- val exec2 = exec + ".exe"
- path.exists(dir => Files.isExecutable(Paths.get(dir, exec))
- || (isWin && Files.isExecutable(Paths.get(dir, exec2))))
- }
- }
- val compiler = compilers.find(inPath)
- if (compiler.isDefined)
- compiler.get.split(' ').toSeq
- else
- Seq.empty[String]
- }
-
- /**
- * Adds a warning message to the diagnostics
- */
- def warning(formatString: String, args: Any*): Unit = {
- val sde = new SchemaDefinitionError(None, None, formatString, args: _*)
- diagnostics :+= sde
- }
-
- /**
- * Adds an error message to the diagnostics and sets isError true
- */
- def error(formatString: String, args: Any*): Unit = {
- val sde = new SchemaDefinitionError(None, None, formatString, args: _*)
- diagnostics :+= sde
- errorStatus = true
- }
-
- // Implements the WithDiagnostics methods
- override def getDiagnostics: Seq[Diagnostic] = diagnostics
- override def isError: Boolean = errorStatus
-}
-
-/** Runs from "sbt compile" to keep all example generated code files up to date */
-object CodeGenerator {
- // Update one set of example generated code files from an example schema
- private def updateGeneratedCodeExample(schemaFile: os.Path, rootName: Option[String],
- exampleCodeHeader: os.Path, exampleCodeFile: os.Path): Unit = {
- // Generate code from the example schema file
- val pf = Compiler().compileFile(schemaFile.toIO, rootName)
- assert(!pf.isError, pf.getDiagnostics.map(_.getMessage()).mkString("\n"))
- val cg = pf.forLanguage("c")
- val tempDir = os.temp.dir(dir = null, prefix = "daffodil-runtime2-")
- val codeDir = cg.generateCode(tempDir.toString)
- assert(!cg.isError, cg.getDiagnostics.map(_.getMessage()).mkString("\n"))
-
- // Replace the example generated files with the newly generated files
- val generatedCodeHeader = codeDir/"libruntime"/"generated_code.h"
- val generatedCodeFile = codeDir/"libruntime"/"generated_code.c"
- os.copy(generatedCodeHeader, exampleCodeHeader, replaceExisting = true, createFolders = true)
- os.copy(generatedCodeFile, exampleCodeFile, replaceExisting = true, createFolders = true)
-
- // Print the example generated files' names so "sbt 'show genExamples'" can list them
- System.out.println(exampleCodeHeader)
- System.out.println(exampleCodeFile)
-
- // tempDir should be removed automatically after main exits; this is just in case
- os.remove.all(tempDir)
- }
-
- // Make sure "sbt compile" calls this main method
- def main(args: Array[String]): Unit = {
- // We expect one mandatory parameter, the examples directory's absolute location.
- if (args.length != 1) {
- System.err.println(s"Usage: $CodeGenerator <examples directory location>")
- System.exit(1)
- }
-
- // Get paths to our example schemas and example generated code files
- val rootDir = if (os.exists(os.pwd/"src")) os.pwd/os.up else os.pwd
-
- val schemaDir = rootDir/"daffodil-runtime2"/"src"/"test"/"resources"/"org"/"apache"/"daffodil"/"runtime2"
- val exNumsSchema = schemaDir/"ex_nums.dfdl.xsd"
- val exNumsRootName = None
- val nestedSchema = schemaDir/"nested.dfdl.xsd"
- val nestedRootName = Some("NestedUnion")
- val padTestSchema = schemaDir/"padtest.dfdl.xsd"
- val padTestRootName = None
- val variableLenSchema = schemaDir/"variablelen.dfdl.xsd"
- val variableLenRootName = Some("expressionElement")
-
- val examplesDir = os.Path(args(0))
- val exNumsCodeHeader = examplesDir/"ex_nums"/"generated_code.h"
- val exNumsCodeFile = examplesDir/"ex_nums"/"generated_code.c"
- val nestedCodeHeader = examplesDir/"NestedUnion"/"generated_code.h"
- val nestedCodeFile = examplesDir/"NestedUnion"/"generated_code.c"
- val padTestCodeHeader = examplesDir/"padtest"/"generated_code.h"
- val padTestCodeFile = examplesDir/"padtest"/"generated_code.c"
- val variableLenCodeHeader = examplesDir/"variablelen"/"generated_code.h"
- val variableLenCodeFile = examplesDir/"variablelen"/"generated_code.c"
-
- // Update each set of example generated code files
- try {
- updateGeneratedCodeExample(exNumsSchema, exNumsRootName, exNumsCodeHeader, exNumsCodeFile)
- updateGeneratedCodeExample(nestedSchema, nestedRootName, nestedCodeHeader, nestedCodeFile)
- updateGeneratedCodeExample(padTestSchema, padTestRootName, padTestCodeHeader, padTestCodeFile)
- updateGeneratedCodeExample(variableLenSchema, variableLenRootName, variableLenCodeHeader, variableLenCodeFile)
- } catch {
- case e: Throwable =>
- System.err.println(s"Error generating example code files: $e")
- System.exit(1);
- }
- }
-}
diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2CodeGenerator.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2CodeGenerator.scala
index 8ca143041..389543794 100644
--- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2CodeGenerator.scala
+++ b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2CodeGenerator.scala
@@ -17,6 +17,11 @@
package org.apache.daffodil.runtime2
+import java.io.File
+import java.net.JarURLConnection
+import java.nio.file.Files
+import java.nio.file.Paths
+import org.apache.daffodil.core.dsom.Root
import org.apache.daffodil.core.grammar.Gram
import org.apache.daffodil.core.grammar.Prod
import org.apache.daffodil.core.grammar.SeqComp
@@ -42,14 +47,191 @@ import org.apache.daffodil.core.grammar.primitives.RightFill
import org.apache.daffodil.core.grammar.primitives.ScalarOrderedSequenceChild
import org.apache.daffodil.core.grammar.primitives.SpecifiedLengthExplicit
import org.apache.daffodil.core.grammar.primitives.SpecifiedLengthImplicit
+import org.apache.daffodil.lib.api.Diagnostic
+import org.apache.daffodil.lib.util.Misc
+import org.apache.daffodil.runtime1.api.DFDL
+import org.apache.daffodil.runtime1.dsom.SchemaDefinitionError
+import org.apache.daffodil.runtime1.dsom.SchemaDefinitionWarning
import org.apache.daffodil.runtime2.generators.AlignmentFillCodeGenerator
import org.apache.daffodil.runtime2.generators.BinaryBooleanCodeGenerator
import org.apache.daffodil.runtime2.generators.BinaryFloatCodeGenerator
import org.apache.daffodil.runtime2.generators.BinaryIntegerKnownLengthCodeGenerator
import org.apache.daffodil.runtime2.generators.CodeGeneratorState
import org.apache.daffodil.runtime2.generators.HexBinaryCodeGenerator
-import org.apache.daffodil.lib.util.Misc
+import scala.collection.JavaConverters._
+import scala.util.Properties.isWin
+
+/**
+ * Generates C source files from a DFDL schema. Implements
+ * DFDL.CodeGenerator trait in order to be called by Daffodil's
+ * command line interface. Contains mutable state for implementing
+ * WithDiagnostics trait, so you need to instantiate a new
+ * Runtime2CodeGenerator each time you generate code for any schema.
+ *
+ * @param root Passes DFDL schema from which to generate code
+ */
+class Runtime2CodeGenerator(root: Root) extends DFDL.CodeGenerator {
+ // Note this class is not thread-safe due to mutable state needed to
+ // implement WithDiagnostics trait
+ private var diagnostics: Seq[Diagnostic] = Nil
+ private var errorStatus: Boolean = false
+
+ /**
+ * Writes C source files generated from the schema into a "c"
+ * subdirectory of the given output directory. Removes the "c"
+ * subdirectory if it existed before. Returns the newly created "c"
+ * subdirectory.
+ */
+ override def generateCode(outputDirArg: String): os.Path = {
+ // Get the paths of the output directory and its code subdirectory and
+ // recreate the code subdirectory to ensure it has no old files in it
+ val outputDir = os.Path(Paths.get(outputDirArg).toAbsolutePath)
+ val codeDir = outputDir/"c"
+ os.makeDir.all(outputDir)
+ os.remove.all(codeDir)
+
+ // Copy all the C resources to the code subdirectory
+ val resources = "/org/apache/daffodil/runtime2/c"
+ val resourceUri = Misc.getRequiredResource(resources)
+ if (resourceUri.getScheme == "jar") {
+ val jarConnection = resourceUri.toURL.openConnection().asInstanceOf[JarURLConnection]
+ val jarFile = jarConnection.getJarFile()
+ jarFile.entries.asScala
+ .filter { entry => ("/" + entry.getName).startsWith(resources) }
+ .filterNot { entry => entry.isDirectory }
+ .foreach { entry =>
+ val entryPath = "/" + entry.getName
+ val subPath = os.SubPath(entryPath.stripPrefix(resources + "/"))
+ val dstPath = codeDir / subPath
+ val stream = jarFile.getInputStream(entry)
+ os.write(dstPath, stream, createFolders = true)
+ }
+ } else {
+ val resourceDir = os.Path(Paths.get(resourceUri))
+ os.copy(resourceDir, codeDir)
+ }
+
+ // Generate C code from the given root element of the DFDL schema,
+ // while appending any warnings to our diagnostics
+ val cgState = new CodeGeneratorState(root)
+ Runtime2CodeGenerator.generateCode(root.document, cgState)
+ diagnostics = diagnostics ++ root.warnings
+ val versionHeaderText = cgState.generateVersionHeader
+ val codeHeaderText = cgState.generateCodeHeader
+ val codeFileText = cgState.generateCodeFile
+
+ // Write the generated C code into our code subdirectory
+ val generatedVersionHeader = codeDir/"libcli"/"daffodil_version.h"
+ val generatedCodeHeader = codeDir/"libruntime"/"generated_code.h"
+ val generatedCodeFile = codeDir/"libruntime"/"generated_code.c"
+ os.write.over(generatedVersionHeader, versionHeaderText)
+ os.write(generatedCodeHeader, codeHeaderText)
+ os.write(generatedCodeFile, codeFileText)
+
+ // Return our code directory in case caller wants to call compileCode next
+ codeDir
+ }
+
+ /**
+ * Compiles any C files inside the given code directory. Returns the path
+ * of the newly built executable in order to run it in a TDML test.
+ */
+ override def compileCode(codeDir: os.Path): os.Path = {
+ // Get the path of the executable we will build
+ val exe = if (isWin) codeDir/"daffodil.exe" else codeDir/"daffodil"
+
+ try {
+ // Assemble the compiler's command line arguments
+ val compiler = pickCompiler
+ val cFlags = Seq("-std=gnu11")
+ val includes = Seq("-Ilibcli", "-Ilibruntime")
+ val absFiles = os.walk(codeDir, skip = _.last == "tests").filter(_.ext == "c")
+ val relFiles = Seq("libcli/*.c", "libruntime/*.c")
+ val libs = Seq("-lmxml")
+
+ // Run the compiler within the code directory
+ if (compiler.nonEmpty) {
+ val result = os.proc(compiler, cFlags, includes, if (isWin) relFiles else absFiles, libs, "-o", exe).call(cwd = codeDir, stderr = os.Pipe)
+ if (result.chunks.nonEmpty) {
+ // Report any compiler output as a warning
+ warning(result.toString())
+ }
+ }
+ } catch {
+ // Report any subprocess termination error as an error
+ case e: os.SubprocessException =>
+ error("Error compiling C files: %s wd: %s", Misc.getSomeMessage(e).get, codeDir.toString)
+ }
+
+ // Report any failure to build the executable as an error
+ if (!os.exists(exe)) error("No executable was built: %s", exe.toString)
+
+ // Return our executable in case caller wants to run it next
+ exe
+ }
+
+ /**
+ * Searches for any available C compiler on the system. Tries to find the
+ * compiler given by `CC` if `CC` exists in the environment, then tries to
+ * find any compiler from the following list:
+ *
+ * - zig cc
+ * - cc
+ * - clang
+ * - gcc
+ *
+ * Returns the first compiler found as a sequence of strings in case the
+ * compiler is a program with a subcommand argument. Returns the empty
+ * sequence if no compiler could be found in the user's PATH.
+ */
+ private lazy val pickCompiler: Seq[String] = {
+ val ccEnv = sys.env.getOrElse("CC", "zig cc")
+ val compilers = Seq(ccEnv, "zig cc", "cc", "clang", "gcc")
+ val path = sys.env.getOrElse("PATH", ".").split(File.pathSeparatorChar)
+ def inPath(compiler: String): Boolean = {
+ (compiler != null) && {
+ val exec = compiler.takeWhile(_ != ' ')
+ val exec2 = exec + ".exe"
+ path.exists(dir => Files.isExecutable(Paths.get(dir, exec))
+ || (isWin && Files.isExecutable(Paths.get(dir, exec2))))
+ }
+ }
+ val compiler = compilers.find(inPath)
+ if (compiler.isDefined)
+ compiler.get.split(' ').toSeq
+ else
+ Seq.empty[String]
+ }
+
+ /**
+ * Adds a warning message to the diagnostics
+ */
+ private def warning(formatString: String, args: Any*): Unit = {
+ val sde = new SchemaDefinitionWarning(None, None, formatString, args: _*)
+ diagnostics :+= sde
+ }
+
+ /**
+ * Adds an error message to the diagnostics and sets isError true
+ */
+ private def error(formatString: String, args: Any*): Unit = {
+ val sde = new SchemaDefinitionError(None, None, formatString, args: _*)
+ diagnostics :+= sde
+ errorStatus = true
+ }
+
+ // Implements the WithDiagnostics trait
+ override def getDiagnostics: Seq[Diagnostic] = diagnostics
+ override def isError: Boolean = errorStatus
+}
+
+/**
+ * Performs recursive pattern matching starting from a root document
+ * to generate code for selected components recognized by the pattern
+ * matcher. Handles more complicated code generation situations by
+ * delegating them to trait methods defined in other classes.
+ */
object Runtime2CodeGenerator
extends AlignmentFillCodeGenerator
with BinaryBooleanCodeGenerator
@@ -57,6 +239,11 @@ object Runtime2CodeGenerator
with BinaryFloatCodeGenerator
with HexBinaryCodeGenerator {
+ /**
+ * Starting from a root document, performs recursive pattern
+ * matching to generate code for components recognized by the
+ * pattern matching code.
+ */
def generateCode(gram: Gram, cgState: CodeGeneratorState): Unit = {
gram match {
case g: AlignmentFill => alignmentFillGenerateCode(g, cgState)
diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala
deleted file mode 100644
index 236e765a3..000000000
--- a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2DataProcessor.scala
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.daffodil.runtime2
-
-import java.io.File
-import java.io.InputStream
-import java.io.OutputStream
-import org.apache.daffodil.runtime1.api.DFDL
-import org.apache.daffodil.lib.api.DaffodilTunables
-import org.apache.daffodil.lib.api.DataLocation
-import org.apache.daffodil.lib.api.ValidationMode
-import org.apache.daffodil.lib.api.ValidationResult
-import org.apache.daffodil.lib.api.ValidationWarning
-import org.apache.daffodil.runtime1.dsom.ValidationError
-import org.apache.daffodil.lib.externalvars.Binding
-import org.apache.daffodil.runtime1.processors.Failure
-import org.apache.daffodil.runtime1.processors.ProcessorResult
-import org.apache.daffodil.runtime1.processors.Success
-import org.apache.daffodil.runtime1.processors.VariableMap
-import org.apache.daffodil.runtime1.processors.WithDiagnosticsImpl
-import org.apache.daffodil.runtime1.processors.parsers.ParseError
-import org.apache.daffodil.runtime1.processors.unparsers.UnparseError
-import org.apache.daffodil.lib.util.Maybe
-import org.apache.daffodil.lib.util.Maybe.Nope
-
-/**
- * Effectively a scala proxy object that does its work via the underlying C-code
- * to get infoset, walk infoset, and generate XML for use by TDML tests.
- */
-class Runtime2DataProcessor(executableFile: os.Path) extends DFDL.DataProcessorBase {
-
- //$COVERAGE-OFF$
- override def withValidationMode(mode: ValidationMode.Type): DFDL.DataProcessor = ???
- override def withTunable(name: String, value: String): DFDL.DataProcessor = ???
- override def withTunables(tunables: Map[String, String]): DFDL.DataProcessor = ???
- override def withExternalVariables(extVars: Map[String, String]): DFDL.DataProcessor = ???
- override def withExternalVariables(extVars: File): DFDL.DataProcessor = ???
- override def withExternalVariables(extVars: Seq[Binding]): DFDL.DataProcessor = ???
- override def withDebugger(dbg:AnyRef): DFDL.DataProcessor = ???
- override def withDebugging(flag: Boolean): DFDL.DataProcessor = ???
- override def save(output: DFDL.Output): Unit = ???
- override def tunables: DaffodilTunables = ???
- override def variableMap: VariableMap = ???
- override def validationMode: ValidationMode.Type = ???
- //$COVERAGE-ON$
-
- /**
- * Returns an object which contains the result, and/or diagnostic information.
- */
- def parse(input: InputStream): ParseResult = {
- val tempDir = os.temp.dir(dir = null, prefix = "daffodil-runtime2-")
- val infile = tempDir/"infile"
- val outfile = tempDir/"outfile"
- try {
- os.write(infile, input)
- val result = os.proc(executableFile, "-o", outfile, "parse", infile).call(cwd = tempDir, stderr = os.Pipe)
- if (result.out.text.isEmpty && result.err.text.isEmpty) {
- val parseResult = new ParseResult(infile, outfile, Success)
- parseResult
- } else {
- val msg = s"stdout: ${result.out.text} stderr: ${result.err.text}"
- val warning = new ValidationWarning { override def getMessage: String = msg }
- val validationResult = ValidationResult(Seq(warning), Seq.empty)
- val parseResult = new ParseResult(infile, outfile, Success, Option(validationResult))
- parseResult.addDiagnostic(new ValidationError(msg))
- parseResult
- }
- } catch {
- case e: os.SubprocessException =>
- val parseError = if (e.result.out.text.isEmpty && e.result.err.text.isEmpty) {
- new ParseError(Nope, Nope, Maybe(e), Nope)
- } else {
- val msg = s"${e.getMessage} stdout: ${e.result.out.text} stderr: ${e.result.err.text}"
- new ParseError(Nope, Nope, Nope, Maybe(msg))
- }
- val parseResult = new ParseResult(infile, outfile, Failure(parseError))
- parseResult.addDiagnostic(parseError)
- parseResult
- } finally {
- os.remove.all(tempDir)
- }
- }
-
- /**
- * Unparses (that is, serializes) data to the output, returns an object which contains any diagnostics.
- */
- def unparse(input: InputStream, output: OutputStream): UnparseResult = {
- val tempDir = os.temp.dir(dir = null, prefix = "daffodil-runtime2-")
- val infile = tempDir/"infile"
- val outfile = tempDir/"outfile"
- try {
- os.write(infile, input)
- val result = os.proc(executableFile, "-o", outfile, "unparse", infile).call(cwd = tempDir, stderr = os.Pipe)
- val finalBitPos0b = os.size(outfile) * 8 // File sizes are bytes, so must multiply to get final position in bits
- os.read.stream(outfile).writeBytesTo(output)
- if (result.out.text.isEmpty && result.err.text.isEmpty) {
- val unparseResult = new UnparseResult(finalBitPos0b, Success)
- unparseResult
- } else {
- val msg = s"stdout: ${result.out.text} stderr: ${result.err.text}"
- val unparseError = new UnparseError(Nope, Nope, Nope, Maybe(msg))
- val unparseResult = new UnparseResult(finalBitPos0b, Failure(unparseError))
- unparseResult.addDiagnostic(unparseError)
- unparseResult
- }
- } catch {
- case e: os.SubprocessException =>
- val unparseError = if (e.result.out.text.isEmpty && e.result.err.text.isEmpty) {
- new UnparseError(Nope, Nope, Maybe(e), Nope)
- } else {
- val msg = s"${e.getMessage} stdout: ${e.result.out.text} stderr: ${e.result.err.text}"
- new UnparseError(Nope, Nope, Nope, Maybe(msg))
- }
- val finalBitPos0b = 0L
- val unparseResult = new UnparseResult(finalBitPos0b, Failure(unparseError))
- unparseResult.addDiagnostic(unparseError)
- unparseResult
- } finally {
- os.remove.all(tempDir)
- }
- }
-}
-
-object Runtime2DataLocation {
- class Runtime2DataLocation( _bitPos1b: Long,
- _bytePos1b: Long) extends DataLocation {
- override def bitPos1b: Long = _bitPos1b
- override def bytePos1b: Long = _bytePos1b
- }
-
- def apply(bitPos1b: Long,
- bytePos1b: Long): DataLocation = {
- new Runtime2DataLocation(bitPos1b, bytePos1b)
- }
-}
-
-final class ParseResult(infile: os.Path,
- outfile: os.Path,
- override val processorStatus: ProcessorResult,
- override val validationResult: Option[ValidationResult] = None)
- extends DFDL.ParseResult
- with DFDL.State
- with WithDiagnosticsImpl {
-
- val loc: DataLocation = {
- val infileLengthInBytes = infile.toIO.length()
- Runtime2DataLocation(infileLengthInBytes * 8, infileLengthInBytes)
- }
-
- override def resultState: DFDL.State = this
-
- override def validationStatus: Boolean = validationResult.isEmpty
-
- override def currentLocation: DataLocation = loc
-
- // We must read outFile right away (def or lazy val will not work) because the
- // parse method will delete outFile before returning ParseResult, but we must
- // prevent loadFile errors from interrupting ParseResult's construction.
- val infosetAsXML : scala.xml.Elem = {
- val elem = try {
- scala.xml.XML.loadFile(outfile.toIO)
- } catch {
- case _: org.xml.sax.SAXParseException =>
- <dummy></dummy>
- }
- elem
- }
-}
-
-final class UnparseResult(val finalBitPos0b: Long,
- override val processorStatus: ProcessorResult)
- extends DFDL.UnparseResult
- with DFDL.State
- with WithDiagnosticsImpl {
-
- // Note DataLocation uses 1-based bit/byte positions, so we have to add 1.
- val loc: DataLocation = Runtime2DataLocation(finalBitPos0b + 1, (finalBitPos0b + 1) / 8)
-
- /**
- * Data is 'scannable' if it consists entirely of textual data, and that data
- * is all in the same encoding.
- */
- override def isScannable: Boolean = false // Safest answer since we don't know for sure
-
- override def encodingName: String = ??? // We don't need encoding unless isScannable is true
-
- override def validationStatus: Boolean = processorStatus.isSuccess
-
- override def currentLocation: DataLocation = loc
-
- override def resultState: DFDL.State = this
-}
diff --git a/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2ExamplesGenerator.scala b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2ExamplesGenerator.scala
new file mode 100644
index 000000000..a37a9786b
--- /dev/null
+++ b/daffodil-runtime2/src/main/scala/org/apache/daffodil/runtime2/Runtime2ExamplesGenerator.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime2
+
+import org.apache.daffodil.core.compiler.Compiler
+import org.apache.daffodil.lib.api.TDMLImplementation
+
+/**
+ * Runs from "sbt compile" to keep all examples of generated C code up
+ * to date whenever C code generator is changed
+ */
+object Runtime2ExamplesGenerator {
+
+ // Update one example of generated C code from a sample schema
+ private def updateRuntime2Example(
+ schemaFile: os.Path,
+ rootName: Option[String],
+ exampleDir: os.Path,
+ ): Unit = {
+ // Generate example code from the sample schema
+ val pf = Compiler().compileFile(schemaFile.toIO, rootName)
+ assert(!pf.isError, pf.getDiagnostics.map(_.getMessage()).mkString("\n"))
+ val cg = pf.forLanguage("c")
+ val tempDir = os.temp.dir(dir = null, prefix = TDMLImplementation.DaffodilC.toString)
+ val codeDir = cg.generateCode(tempDir.toString)
+ assert(!cg.isError, cg.getDiagnostics.map(_.getMessage()).mkString("\n"))
+
+ // Replace the example generated files with the newly generated files
+ val generatedCodeHeader = codeDir/"libruntime"/"generated_code.h"
+ val generatedCodeFile = codeDir/"libruntime"/"generated_code.c"
+ val exampleCodeHeader = exampleDir/"generated_code.h"
+ val exampleCodeFile = exampleDir/"generated_code.c"
+ os.copy(generatedCodeHeader, exampleCodeHeader, replaceExisting = true, createFolders = true)
+ os.copy(generatedCodeFile, exampleCodeFile, replaceExisting = true, createFolders = true)
+
+ // Print the example directory so "sbt 'show genRuntime2Examples'" can list it
+ println(exampleDir)
+
+ // JVM will remove tempDir automatically when it exits; this is just in case
+ os.remove.all(tempDir)
+ }
+
+ /**
+ * Make sure "sbt compile" calls this main method
+ */
+ def main(args: Array[String]): Unit = {
+ // We expect one mandatory parameter, the absolute location of the examples directory
+ assert(args.length == 1, s"Usage: $Runtime2ExamplesGenerator <examples directory location>")
+
+ // Get paths to our sample schemas and their corresponding example directories
+ val rootDir = if (os.exists(os.pwd/"src")) os.pwd/os.up else os.pwd
+
+ val schemaDir = rootDir/"daffodil-runtime2"/"src"/"test"/"resources"/"org"/"apache"/"daffodil"/"runtime2"
+ val exNumsSchema = schemaDir/"ex_nums.dfdl.xsd"
+ val exNumsRootName = None
+ val nestedSchema = schemaDir/"nested.dfdl.xsd"
+ val nestedRootName = Some("NestedUnion")
+ val padTestSchema = schemaDir/"padtest.dfdl.xsd"
+ val padTestRootName = None
+ val variableLenSchema = schemaDir/"variablelen.dfdl.xsd"
+ val variableLenRootName = Some("expressionElement")
+
+ val examplesDir = os.Path(args(0))
+ val exNumsExampleDir = examplesDir/"ex_nums"
+ val nestedExampleDir = examplesDir/"NestedUnion"
+ val padTestExampleDir = examplesDir/"padtest"
+ val variableLenExampleDir = examplesDir/"variablelen"
+
+ // Update each example of generated C code
+ updateRuntime2Example(exNumsSchema, exNumsRootName, exNumsExampleDir)
+ updateRuntime2Example(nestedSchema, nestedRootName, nestedExampleDir)
+ updateRuntime2Example(padTestSchema, padTestRootName, padTestExampleDir)
+ updateRuntime2Example(variableLenSchema, variableLenRootName, variableLenExampleDir)
+ }
+}
diff --git a/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2ExamplesGenerator.scala b/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2ExamplesGenerator.scala
new file mode 100644
index 000000000..d159566bb
--- /dev/null
+++ b/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestRuntime2ExamplesGenerator.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime2
+
+import org.junit.Test
+
+/**
+ * Runs Runtime2ExamplesGenerator in a test since "sbt coverage
+ * compile" doesn't capture call of genRuntime2Examples
+ */
+class TestRuntime2ExamplesGenerator {
+ // Test added for code coverage and debugging
+ @Test def test_Runtime2ExamplesGenerator_main(): Unit = {
+ // Generate the C examples in a safe place (target/examples)
+ val rootDir = if (os.exists(os.pwd/"src")) os.pwd/os.up else os.pwd
+ val examplesDir = rootDir/"daffodil-runtime2"/"target"/"examples"
+ val args = Array(examplesDir.toString)
+ Runtime2ExamplesGenerator.main(args)
+
+ // Verify the C examples were generated
+ val generatedCode = examplesDir/"variablelen"/"generated_code.c"
+ assert(os.exists(generatedCode))
+ }
+}
diff --git a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/Runtime2TDMLDFDLProcessor.scala b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/Runtime2TDMLDFDLProcessor.scala
index db7c07129..d59ad52ed 100644
--- a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/Runtime2TDMLDFDLProcessor.scala
+++ b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/Runtime2TDMLDFDLProcessor.scala
@@ -17,179 +17,280 @@
package org.apache.daffodil.processor.tdml
-import org.apache.daffodil.tdml.processor._
-
-import java.io.ByteArrayInputStream
-
-import org.apache.daffodil.lib.api._
import org.apache.daffodil.core.compiler.Compiler
+import org.apache.daffodil.lib.api.DaffodilSchemaSource
+import org.apache.daffodil.lib.api.DataLocation
+import org.apache.daffodil.lib.api.Diagnostic
+import org.apache.daffodil.lib.api.TDMLImplementation
+import org.apache.daffodil.lib.api.ValidationMode
import org.apache.daffodil.lib.externalvars.Binding
-import org.apache.daffodil.runtime2.ParseResult
-import org.apache.daffodil.runtime2.Runtime2DataProcessor
-import org.apache.daffodil.runtime2.UnparseResult
-
+import org.apache.daffodil.lib.util.Maybe
+import org.apache.daffodil.lib.util.Maybe.Nope
+import org.apache.daffodil.runtime1.dsom.SchemaDefinitionDiagnosticBase
+import org.apache.daffodil.runtime1.processors.Failure
+import org.apache.daffodil.runtime1.processors.ProcessorResult
+import org.apache.daffodil.runtime1.processors.Success
+import org.apache.daffodil.runtime1.processors.parsers.ParseError
+import org.apache.daffodil.runtime1.processors.unparsers.UnparseError
+import org.apache.daffodil.tdml.processor.AbstractTDMLDFDLProcessorFactory
+import org.apache.daffodil.tdml.processor.TDML
+import org.apache.daffodil.tdml.processor.TDMLDFDLProcessor
+import org.apache.daffodil.tdml.processor.TDMLParseResult
+import org.apache.daffodil.tdml.processor.TDMLUnparseResult
+import org.xml.sax.SAXParseException
+import os.CommandResult
+
+import java.io.FileNotFoundException
+import java.io.InputStream
+import java.io.OutputStream
import scala.xml.Node
+import scala.xml.XML
-final class Runtime2TDMLDFDLProcessorFactory private(
- private var compiler: Compiler,
- private var checkAllTopLevel: Boolean,
- validateDFDLSchemasArg: Boolean)
+/**
+ * A factory called by the TDML runner to create a TDMLDFDL processor.
+ * The factory is immutable and can be shared across threads.
+ */
+final class Runtime2TDMLDFDLProcessorFactory(compiler: Compiler)
extends AbstractTDMLDFDLProcessorFactory {
- override def validateDFDLSchemas = validateDFDLSchemasArg
+ def this() = this(Compiler())
+ private def copy(compiler: Compiler = compiler) =
+ new Runtime2TDMLDFDLProcessorFactory(compiler)
override type R = Runtime2TDMLDFDLProcessorFactory
-
- override def implementationName = TDMLImplementation.DaffodilC.toString
-
- def this() = this(compiler = Compiler(validateDFDLSchemas = true),
- checkAllTopLevel = false,
- validateDFDLSchemasArg = true)
-
- private def copy(
- compiler: Compiler = compiler,
- checkAllTopLevel: Boolean = checkAllTopLevel,
- validateDFDLSchemas: Boolean = validateDFDLSchemas) =
- new Runtime2TDMLDFDLProcessorFactory(compiler, checkAllTopLevel, validateDFDLSchemas)
-
- override def withValidateDFDLSchemas(bool: Boolean): Runtime2TDMLDFDLProcessorFactory = {
- copy(compiler = compiler.withValidateDFDLSchemas(bool))
- }
-
- override def withCheckAllTopLevel(checkAllTopLevel: Boolean): Runtime2TDMLDFDLProcessorFactory = {
- copy(compiler = compiler.withCheckAllTopLevel(checkAllTopLevel))
- }
-
- override def withTunables(tunables: Map[String, String]): Runtime2TDMLDFDLProcessorFactory =
- copy(compiler = compiler.withTunables(tunables))
-
- // Return result is a TDML.CompileResult - so it's the result
- // of compiling the schema for the test.
+ override def implementationName: String = TDMLImplementation.DaffodilC.toString
+ override def validateDFDLSchemas: Boolean = compiler.validateDFDLSchemas
+ override def withValidateDFDLSchemas(validateDFDLSchemas: Boolean): R =
+ copy(compiler.withValidateDFDLSchemas(validateDFDLSchemas))
+ override def withCheckAllTopLevel(checkAllTopLevel: Boolean): R =
+ copy(compiler.withCheckAllTopLevel(checkAllTopLevel))
+ override def withTunables(tunables: Map[String, String]): R =
+ copy(compiler.withTunables(tunables))
+
+ // Compiles the given schema and returns the result of compiling the
+ // schema (only diagnostics on left, or processor too on right)
override def getProcessor(
schemaSource: DaffodilSchemaSource,
useSerializedProcessor: Boolean,
optRootName: Option[String] = None,
optRootNamespace: Option[String] = None,
- tunables: Map[String, String]): TDML.CompileResult = {
+ tunables: Map[String, String],
+ ): TDML.CompileResult = {
- // Compile the DFDL schema into a ProcessorFactory
+ // Compile the DFDL schema into diagnostics and/or a processor factory
val pf = compiler.compileSource(schemaSource, optRootName, optRootNamespace)
val res = if (pf.isError) {
Left(pf.getDiagnostics) // DFDL schema compilation diagnostics
} else {
- // Create a CodeGenerator from the DFDL schema for the C language
+ // Create a code generator for the C language
val generator = pf.forLanguage("c")
- // Generate the C source code in a temporary unique directory
- val tempDir = os.temp.dir(dir = null, prefix = "daffodil-runtime2-")
+ // Generate the C code in a temporary unique directory
+ val tempDir = os.temp.dir(dir = null, prefix = TDMLImplementation.DaffodilC.toString)
val codeDir = generator.generateCode(tempDir.toString)
- // Compile the generated code into an executable
+ // Compile the C code into an executable
val executable = generator.compileCode(codeDir)
- // There is is no way to clean up TDMLDFDLProcessors, they are just garbage
+ // There is no way to clean up TDMLDFDLProcessors - they are just garbage
// collected when no longer needed. So recursively mark all generated
// files as deleteOnExit so we at least clean them up when the JVM exits
os.walk.stream(tempDir).foreach { _.toIO.deleteOnExit() }
- // Summarize the result of compiling the schema for the test
+ // Summarize the result of compiling the C code
val compileResult = if (generator.isError) {
- Left(generator.getDiagnostics) // C code compilation diagnostics
+ Left(generator.getDiagnostics) // C compilation diagnostics
} else {
- // Create a processor for running the test using the executable, passing it
- // tempDir so its cleanUp function will delete tempDir for us
- val processor = new Runtime2TDMLDFDLProcessor(tempDir, executable)
- // Although we return generator diagnostics to TDMLRunner, TDMLRunner won't
- // do anything with them in its usual path
+ // Create a processor for running the executable
+ val processor = new Runtime2TDMLDFDLProcessor(executable)
+ // Although we return C compilation diagnostics to TDMLRunner, TDMLRunner
+ // won't do anything with them in its usual path
Right((generator.getDiagnostics, processor))
}
compileResult
}
res
}
-
}
/**
- * Delegates all execution, error gathering, error access to the Runtime2DataProcessor.
- * The responsibility of this class is just for TDML matching up. That is dealing with
- * TDML XML Infosets, feeding to the unparser, creating XML from the result created by
- * the Runtime2DataProcessor. All the "real work" is done by Runtime2DataProcessor.
+ * A TDMLDFDL processor which runs an executable built from C code for
+ * a given DFDL schema. Deals with TDML XML infosets, feeding to the
+ * executable, returning the output created by the executable, etc.
*/
-class Runtime2TDMLDFDLProcessor(tempDir: os.Path, executable: os.Path)
+final class Runtime2TDMLDFDLProcessor(executable: os.Path)
extends TDMLDFDLProcessor {
+ // We don't pass any options to the executable
override type R = Runtime2TDMLDFDLProcessor
-
- private val dataProcessor = new Runtime2DataProcessor(executable)
-
- override def withDebugging(b: Boolean): Runtime2TDMLDFDLProcessor = this
-
- override def withTracing(bool: Boolean): Runtime2TDMLDFDLProcessor = this
-
- override def withDebugger(db: AnyRef): Runtime2TDMLDFDLProcessor = this
-
- override def withValidationMode(validationMode: ValidationMode.Type): Runtime2TDMLDFDLProcessor = this
-
- override def withExternalDFDLVariables(externalVarBindings: Seq[Binding]): Runtime2TDMLDFDLProcessor = this
-
- // No need to report errors from this class itself
+ override def withDebugging(onOff: Boolean): R = this
+ override def withTracing(onOff: Boolean): R = this
+ override def withDebugger(db: AnyRef): R = this
+ override def withValidationMode(validationMode: ValidationMode.Type): R = this
+ override def withExternalDFDLVariables(externalVarBindings: Seq[Binding]): R = this
+ // We return errors and diagnostics in TDMLParseResult and TDMLUnparseResult
override def isError: Boolean = false
override def getDiagnostics: Seq[Diagnostic] = Seq.empty
- // Run the C code, collect and save the infoset with any errors and
- // diagnostics, and return a [[TDMLParseResult]] summarizing the result.
- // The C code will run in a subprocess, parse the input stream, write
- // an XML infoset on its standard output, and write any error messages
- // on its standard output (all done in [[Runtime2DataProcessor.parse]]).
- override def parse(is: java.io.InputStream, lengthLimitInBits: Long): TDMLParseResult = {
- // TODO: pass lengthLimitInBits to the C program to tell it how big the data is
- val pr = dataProcessor.parse(is)
- new Runtime2TDMLParseResult(pr)
+ // Parses the input stream to an infoset and returns a TDMLParseResult
+ // containing the infoset with any errors and diagnostics.
+ //
+ // We save the input stream to an input file and run the executable in a
+ // subprocess. The executable will parse data from the input file and write an
+ // infoset to an output file. We return the infoset with any errors and
+ // diagnostics messages written by the executable on its standard error and/or
+ // output.
+ override def parse(input: InputStream, lengthLimitInBits: Long): TDMLParseResult = {
+ // Write the input to an input file to let the executable parse it
+ val tempDir = os.temp.dir(dir = null, prefix = TDMLImplementation.DaffodilC.toString)
+ val infile = tempDir/"infile"
+ val outfile = tempDir/"outfile"
+ os.write(infile, input)
+
+ // Verify the input file has the correct size TDML runner said it would
+ val inputSizeInBits = os.size(infile) * 8
+ assert(inputSizeInBits == lengthLimitInBits, s"$infile has $inputSizeInBits bits, but needed $lengthLimitInBits bits")
+
+ // Parse the input file using the executable and capture its exit status
+ val parseResult = try {
+ // Darwin and MSYS2 support only "daffodil -o outfile parse infile" (all getopt options must come first)
+ val result = os.proc(executable, "-o", outfile, "parse", infile).call(cwd = tempDir, stderr = os.Pipe)
+ new Runtime2TDMLParseResult(result, lengthLimitInBits, outfile, Success)
+ } catch {
+ case e: os.SubprocessException =>
+ val result = e.result
+ val parseError = new ParseError(Nope, Nope, Maybe(e), Nope)
+ new Runtime2TDMLParseResult(result, lengthLimitInBits, outfile, Failure(parseError))
+ } finally {
+ os.remove.all(tempDir)
+ }
+
+ parseResult
}
- // Run the C code, collect and save the unparsed data with any errors and
- // diagnostics, and return a [[TDMLUnparseResult]] summarizing the result.
- // The C code will run in a subprocess, unparse the input stream, write
- // the unparsed data on its standard output, and write any error messages
- // on its standard output (all done in [[Runtime2DataProcessor.unparse]]).
- override def unparse(infosetXML: scala.xml.Node, outStream: java.io.OutputStream): TDMLUnparseResult = {
- val inStream = new ByteArrayInputStream(infosetXML.toString.getBytes())
- val upr = dataProcessor.unparse(inStream, outStream)
- new Runtime2TDMLUnparseResult(upr)
+ // Unparses the infoset and returns a TDMLUnparseResult containing the data with
+ // any errors and diagnostics.
+ //
+ // We save the infoset to an input file and run the executable in a subprocess.
+ // The executable will unparse the infoset and write data to an output file. We
+ // return the data with any errors and diagnostics messages written by the
+ // executable on its standard error and/or output.
+ override def unparse(infosetXML: Node, output: OutputStream): TDMLUnparseResult = {
+ // Write the infoset to an input file to let the executable parse it
+ val tempDir = os.temp.dir(dir = null, prefix = TDMLImplementation.DaffodilC.toString)
+ val infile = tempDir/"infile"
+ val outfile = tempDir/"outfile"
+ os.write(infile, infosetXML.toString)
+
+ // Unparse the infoset using the executable, capture its exit status, and return the data
+ val unparseResult = try {
+ // Darwin and MSYS2 support only "daffodil -o outfile parse infile" (all getopt options must come first)
+ val result = os.proc(executable, "-o", outfile, "unparse", infile).call(cwd = tempDir, stderr = os.Pipe)
+ os.read.stream(outfile).writeBytesTo(output)
+ val finalBitPos0b = os.size(outfile) * 8
+ new Runtime2TDMLUnparseResult(result, finalBitPos0b, Success)
+ } catch {
+ case e: os.SubprocessException =>
+ val result = e.result
+ val finalBitPos0b = os.size(outfile) * 8
+ val unparseError = new UnparseError(Nope, Nope, Maybe(e), Nope)
+ new Runtime2TDMLUnparseResult(result, finalBitPos0b, Failure(unparseError))
+ } finally {
+ os.remove.all(tempDir)
+ }
+
+ unparseResult
}
- def unparse(parseResult: TDMLParseResult, outStream: java.io.OutputStream): TDMLUnparseResult = {
- unparse(parseResult.getResult, outStream)
+ // Complete a round trip from data to infoset and back to data
+ def unparse(parseResult: TDMLParseResult, outStream: OutputStream): TDMLUnparseResult = {
+ val infosetXML = parseResult.getResult
+ val res = unparse(infosetXML, outStream)
+ res
}
}
-final class Runtime2TDMLParseResult(pr: ParseResult) extends TDMLParseResult {
- override def addDiagnostic(failure: Diagnostic): Unit = pr.addDiagnostic(failure)
-
- override def getResult: Node = pr.infosetAsXML
-
- override def currentLocation: DataLocation = pr.currentLocation
+/**
+ * A TDML parse result which captures the result of running the executable
+ */
+final class Runtime2TDMLParseResult(
+ result: CommandResult,
+ finalBitPos0b: Long,
+ outfile: os.Path,
+ processorResult: ProcessorResult,
+) extends TDMLParseResult {
+
+ private var diagnostics: Seq[Diagnostic] = processorResult match {
+ case Success =>
+ if (result.chunks.nonEmpty) List(Runtime2TDMLMessages(result.toString())) else Nil
+ case Failure(cause) => List(cause)
+ }
- override def isValidationError: Boolean = pr.isValidationError
+ override def addDiagnostic(diagnostic: Diagnostic): Unit = {
+ diagnostics = diagnostics :+ diagnostic
+ }
- override def isProcessingError: Boolean = pr.isProcessingError
+ // We must read outFile right away (def or lazy val will not work) because the parse
+ // method will delete outFile before returning the parse result, but we must prevent
+ // loadFile errors from interrupting the parse result's construction
+ override val getResult: Node = {
+ val elem = try {
+ XML.loadFile(outfile.toIO)
+ } catch {
+ case _: FileNotFoundException => <noFile></noFile>
+ case _: SAXParseException => <parseError></parseError>
+ }
+ elem
+ }
- override def getDiagnostics: Seq[Diagnostic] = pr.getDiagnostics
+ override def currentLocation: DataLocation = Runtime2TDMLDataLocation(finalBitPos0b)
+ override def isValidationError: Boolean = diagnostics.exists(_.isValidation)
+ override def isProcessingError: Boolean = processorResult.isFailure
+ override def getDiagnostics: Seq[Diagnostic] = diagnostics
}
-final class Runtime2TDMLUnparseResult(upr: UnparseResult) extends TDMLUnparseResult {
- override def bitPos0b: Long = upr.finalBitPos0b
-
- override def finalBitPos0b: Long = upr.finalBitPos0b
-
- override def isScannable: Boolean = upr.isScannable
+/**
+ * A TDML diagnostic which captures the executable's standard output/error
+ */
+final case class Runtime2TDMLMessages(messages: String)
+ extends SchemaDefinitionDiagnosticBase(Nope, Nope, None, Nope, Maybe(messages)) {
- override def encodingName: String = upr.encodingName
+ override def isValidation: Boolean = true
+ override def isError: Boolean = false
+ override protected def modeName: String = TDMLImplementation.DaffodilC.toString
+}
- override def isValidationError: Boolean = upr.isValidationError
+/**
+ * A TDML data location which captures the executable's final position
+ */
+object Runtime2TDMLDataLocation {
+ // DataLocation stores 1-based bit/byte positions
+ private class Runtime2TDMLDataLocation(val bitPos1b: Long, val bytePos1b: Long)
+ extends DataLocation
+ // We pass a 0-based bit position, so we have to add 1 to it
+ def apply(finalBitPos0b: Long): DataLocation =
+ new Runtime2TDMLDataLocation(finalBitPos0b + 1, (finalBitPos0b + 1) / 8)
+}
- override def isProcessingError: Boolean = upr.isProcessingError
+/**
+ * A TDML unparse result which captures the result of running the executable
+ */
+final class Runtime2TDMLUnparseResult(
+ result: CommandResult,
+ override val finalBitPos0b: Long,
+ processorResult: ProcessorResult,
+) extends TDMLUnparseResult {
+
+ private val diagnostics: Seq[Diagnostic] = processorResult match {
+ case Success =>
+ if (result.chunks.nonEmpty) List(Runtime2TDMLMessages(result.toString())) else Nil
+ case Failure(cause) => List(cause)
+ }
- override def getDiagnostics: Seq[Diagnostic] = upr.getDiagnostics
+ override def bitPos0b: Long = finalBitPos0b
+ override def isScannable: Boolean = false // binary data is not scannable
+ override def encodingName: String = "" // encoding needed only if scannable
+ override def isValidationError: Boolean = diagnostics.exists(_.isValidation)
+ override def isProcessingError: Boolean = processorResult.isFailure
+ override def getDiagnostics: Seq[Diagnostic] = diagnostics
}
diff --git a/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestCodeGenerator.scala b/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala
similarity index 53%
rename from daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestCodeGenerator.scala
rename to daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala
index b820666c5..9160f834d 100644
--- a/daffodil-runtime2/src/test/scala/org/apache/daffodil/runtime2/TestCodeGenerator.scala
+++ b/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/processor/tdml/TestDaffodilC.scala
@@ -15,36 +15,44 @@
* limitations under the License.
*/
-package org.apache.daffodil.runtime2
+package org.apache.daffodil.processor.tdml
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
-import java.nio.channels.Channels
-import org.apache.daffodil.lib.Implicits.intercept
import org.apache.daffodil.core.compiler.Compiler
+import org.apache.daffodil.lib.Implicits.intercept
+import org.apache.daffodil.lib.api.TDMLImplementation
+import org.apache.daffodil.lib.api.UnitTestSchemaSource
import org.apache.daffodil.lib.util.Misc
import org.apache.daffodil.lib.util.SchemaUtils
-import org.apache.daffodil.core.util.TestUtils
+import org.apache.daffodil.lib.xml.XMLUtils
+import org.apache.daffodil.runtime2.Runtime2CodeGenerator
import org.junit.After
import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.fail
import org.junit.Test
/**
- * Checks that we can create a [[CodeGenerator]] and call its methods.
- * The value of this test is to debug the call path from [[Compiler]]
- * to [[CodeGenerator]] for a single test DFDL schema. Running TDML
- * tests with daffodil-runtime2 is a more effective way to check that
- * CodeGenerator can generate appropriate code for as many DFDL schemas
- * as you could want.
+ * Checks that we can generate C code from a DFDL schema, build an
+ * executable from the C code, and parse or unparse data or infosets
+ * with the executable.
+ *
+ * Those tests run Runtime2CodeGenerator, Runtime2TDMLDFDLProcessor,
+ * and Runtime2TDMLDFDLProcessorFactory on a very simple, single DFDL
+ * schema to debug the complete call path. You can test even more
+ * DFDL schemas by writing TDML tests and running them with DaffodilC.
*/
-class TestCodeGenerator {
- // Ensure all tests remove tempDir after creating it
- val tempDir: os.Path = os.temp.dir(dir = null, prefix = "daffodil-runtime2-")
+class TestDaffodilC {
+
+ // Defines temporary directory for all tests to use
+ val tempDir: os.Path = os.temp.dir(dir = null, prefix = TDMLImplementation.DaffodilC.toString)
+
+ // Ensures each test cleans up temporary directory afterwards
@After def after(): Unit = {
os.remove.all(tempDir)
}
- // Define a simple DFDL test schema for debugging our code path
+ // Defines a very simple DFDL test schema for all tests to use
private val testSchema = SchemaUtils.dfdlTestSchema(
<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>,
<dfdl:format representation="binary" ref="GeneralFormat"/>,
@@ -56,16 +64,19 @@ class TestCodeGenerator {
</xs:complexType>
</xs:element>)
- @Test def test_forLanguageC_success(): Unit = {
+ // Checks that processorFactory.forLanguage("c") succeeds
+ @Test def test_forLanguage_success(): Unit = {
// Create a ProcessorFactory from the test schema
val pf = Compiler().compileNode(testSchema)
assert(!pf.isError, pf.getDiagnostics.map(_.getMessage()).mkString("\n"))
// Create a CodeGenerator from the ProcessorFactory for a supported language
val cg = pf.forLanguage("c")
+ assert(cg.isInstanceOf[Runtime2CodeGenerator])
assert(!cg.isError, cg.getDiagnostics.map(_.getMessage()).mkString("\n"))
}
+ // Checks that processorFactory.forLanguage("hls") fails
@Test def test_forLanguage_error(): Unit = {
// Create a ProcessorFactory from the test schema
val pf = Compiler().compileNode(testSchema)
@@ -73,11 +84,12 @@ class TestCodeGenerator {
// Create a CodeGenerator from the ProcessorFactory for an unsupported language
val e = intercept[Exception] {
- pf.forLanguage("vhld")
+ pf.forLanguage("hls")
}
- assert(e.getMessage.contains("source language vhld is not supported"))
+ assert(e.getMessage.contains("source language hls is not supported"))
}
+ // Checks that codeGenerator.generateCode(tempDir) succeeds
@Test def test_generateCode_success(): Unit = {
// Create a ProcessorFactory and CodeGenerator from the test schema
val pf = Compiler().compileNode(testSchema)
@@ -95,6 +107,7 @@ class TestCodeGenerator {
assert(os.exists(generatedCodeFile))
}
+ // Checks that codeGenerator.compileCode(codeDir) succeeds
@Test def test_compileCode_success(): Unit = {
// Create a CodeGenerator and generate code from the test schema
val pf = Compiler().compileNode(testSchema)
@@ -107,23 +120,25 @@ class TestCodeGenerator {
assert(os.exists(executable))
}
+ // Checks that daffodilTDMLDFDLProcessor.parse(goodData) succeeds
@Test def test_parse_success(): Unit = {
- // Compile the test schema into a C executable
+ // Compile the test schema into an executable
val pf = Compiler().compileNode(testSchema)
val cg = pf.forLanguage("c")
val codeDir = cg.generateCode(tempDir.toString)
val executable = cg.compileCode(codeDir)
- // Create a Runtime2DataProcessor and parse a binary int32 number successfully
- val dp = new Runtime2DataProcessor(executable)
- val b = Misc.hex2Bytes("00000005")
+ // Create a Runtime2TDMLDFDLProcessor and parse a binary int32 number successfully
+ val tdp = new Runtime2TDMLDFDLProcessor(executable)
+ val b = Misc.hex2Bytes("00000500")
val input = new ByteArrayInputStream(b)
- val pr = dp.parse(input)
- assert(!pr.isError && pf.getDiagnostics.isEmpty, pr.getDiagnostics.map(_.getMessage()).mkString("\n"))
- val expected = <e1><x>5</x></e1>
- TestUtils.assertEqualsXMLElements(expected, pr.infosetAsXML)
+ val pr = tdp.parse(input, b.length * 8)
+ assert(!pr.isProcessingError && pf.getDiagnostics.isEmpty, pr.getDiagnostics.map(_.getMessage()).mkString("\n"))
+ val expected = <e1><x>1280</x></e1>
+ XMLUtils.compareAndReport(expected, pr.getResult)
}
+ // Checks that daffodilTDMLDFDLProcessor.parse(badData) fails
@Test def test_parse_error(): Unit = {
// Compile the test schema into a C executable
val pf = Compiler().compileNode(testSchema)
@@ -131,53 +146,66 @@ class TestCodeGenerator {
val codeDir = cg.generateCode(tempDir.toString)
val executable = cg.compileCode(codeDir)
- // Create a Runtime2DataProcessor and parse an empty file unsuccessfully
- val dp = new Runtime2DataProcessor(executable)
- val b = Misc.hex2Bytes("")
+ // Create a Runtime2TDMLDFDLProcessor and parse an empty file unsuccessfully
+ val tdp = new Runtime2TDMLDFDLProcessor(executable)
+ val b = Misc.hex2Bytes("50")
val input = new ByteArrayInputStream(b)
- val pr = dp.parse(input)
- assert(pr.isError, "expected pr.isError to be true")
+ val pr = tdp.parse(input, b.length * 8)
+ assert(pr.isProcessingError, "expected pr.isError to be true")
assert(pr.getDiagnostics.nonEmpty, "expected pr.getDiagnostics to be non-empty")
}
+ // Checks that daffodilTDMLDFDLProcessor.unparse(goodData) succeeds
@Test def test_unparse_success(): Unit = {
- // Compile the test schema into a C executable
+ // Compile the test schema into an executable
val pf = Compiler().compileNode(testSchema)
val cg = pf.forLanguage("c")
val codeDir = cg.generateCode(tempDir.toString)
val executable = cg.compileCode(codeDir)
- // Create a Runtime2DataProcessor and unparse a binary int32 number successfully
- val dp = new Runtime2DataProcessor(executable)
- val input = Channels.newInputStream(Misc.stringToReadableByteChannel("<e1><x>5</x></e1>"))
+ // Create a Runtime2TDMLDFDLProcessor and unparse a binary int32 number successfully
+ val tdp = new Runtime2TDMLDFDLProcessor(executable)
+ val infosetXML = <e1 xmlns="http://example.com"><x>1280</x></e1>
val output = new ByteArrayOutputStream()
- val ur = dp.unparse(input, output)
- assert(!ur.isError && pf.getDiagnostics.isEmpty, ur.getDiagnostics.map(_.getMessage()).mkString("\n"))
- val expected = Misc.hex2Bytes("00000005")
+ val ur = tdp.unparse(infosetXML, output)
+ assert(!ur.isProcessingError && pf.getDiagnostics.isEmpty, ur.getDiagnostics.map(_.getMessage()).mkString("\n"))
+ val expected = Misc.hex2Bytes("00000500")
assertArrayEquals(expected, output.toByteArray)
}
+ // Checks that daffodilTDMLDFDLProcessor.unparse(badData) fails
@Test def test_unparse_error(): Unit = {
- // Compile the test schema into a C executable
+ // Compile the test schema into an executable
val pf = Compiler().compileNode(testSchema)
val cg = pf.forLanguage("c")
val codeDir = cg.generateCode(tempDir.toString)
val executable = cg.compileCode(codeDir)
- // Create a Runtime2DataProcessor and unparse a binary int32 number unsuccessfully
- val dp = new Runtime2DataProcessor(executable)
- val input = Channels.newInputStream(Misc.stringToReadableByteChannel("<e1><x>FAIL</x></e1>"))
+ // Create a Runtime2TDMLDFDLProcessor and unparse a binary int32 number unsuccessfully
+ val dp = new Runtime2TDMLDFDLProcessor(executable)
+ val infosetXML = <e1 xmlns="http://example.com"><x>FAIL</x></e1>
val output = new ByteArrayOutputStream()
- val ur = dp.unparse(input, output)
- assert(ur.isError, "expected ur.isError to be true")
+ val ur = dp.unparse(infosetXML, output)
+ assert(ur.isProcessingError, "expected ur.isError to be true")
assert(ur.getDiagnostics.nonEmpty, "expected ur.getDiagnostics to be non-empty")
}
- // Test added for code coverage because "sbt coverage compile" doesn't include genExamples
- @Test def test_CodeGenerator_main(): Unit = {
- val rootDir = if (os.exists(os.pwd/"src")) os.pwd/os.up else os.pwd
- val examplesDir = rootDir/"daffodil-runtime2"/"target"/"test_CodeGenerator_main"
- val args = Array(examplesDir.toString)
- CodeGenerator.main(args)
+ // Checks that Runtime2TDMLDFDLProcessorFactory.getProcessor succeeds
+ @Test def test_getProcessor_success(): Unit = {
+ val processorFactory = new Runtime2TDMLDFDLProcessorFactory()
+ val schemaSource = UnitTestSchemaSource(testSchema, nameHint = "getProcessor", Some(tempDir.toIO))
+ val useSerializedProcessor = false
+ val optRootName = None
+ val optRootNamespace = None
+ val tunables = Map.empty[String, String]
+ val cr = processorFactory.getProcessor(schemaSource, useSerializedProcessor, optRootName, optRootNamespace, tunables)
+ cr match {
+ case Left(diagnostics) => fail(s"getProcessor failed: ${diagnostics.mkString}")
+ case Right((diagnostics, tdmlDFDLProcessor)) =>
+ assert(diagnostics.isEmpty)
+ assert(!tdmlDFDLProcessor.isError)
+ assert(tdmlDFDLProcessor.isInstanceOf[Runtime2TDMLDFDLProcessor])
+ }
}
+
}