You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by mb...@apache.org on 2022/08/22 13:29:16 UTC
[daffodil] branch main updated: Added TDML with precompiled binary schema feature.
This is an automated email from the ASF dual-hosted git repository.
mbeckerle pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/main by this push:
new 9c199033e Added TDML with precompiled binary schema feature.
9c199033e is described below
commit 9c199033e6ec043a470f814a6b82ba4c62192f46
Author: Michael Beckerle <mb...@apache.org>
AuthorDate: Fri Aug 19 16:06:38 2022 -0400
Added TDML with precompiled binary schema feature.
DAFFODIL-2725
---
.../org/apache/daffodil/compiler/Compiler.scala | 15 +++++--
.../apache/daffodil/api/DaffodilSchemaSource.scala | 50 ++++++++++++++-------
.../tdml/processor/DaffodilTDMLDFDLProcessor.scala | 16 ++++---
.../test/tdml/fake-precompiled-dfdl-schema.bin | Bin 0 -> 190 bytes
.../org/apache/daffodil/tdml/TestTDMLRunner.scala | 22 +++++++++
project/Rat.scala | 1 +
6 files changed, 79 insertions(+), 25 deletions(-)
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/compiler/Compiler.scala b/daffodil-core/src/main/scala/org/apache/daffodil/compiler/Compiler.scala
index 5c4753ccf..4afc5cbf5 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/compiler/Compiler.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/compiler/Compiler.scala
@@ -45,6 +45,7 @@ import org.apache.daffodil.processors.DataProcessor
import org.apache.daffodil.util.Logger
import org.apache.daffodil.util.Misc
import org.apache.daffodil.xml._
+import java.net.URI
/**
* Some grammar rules need to be conditional based on whether we're trying
@@ -281,12 +282,18 @@ class Compiler private (var validateDFDLSchemas: Boolean,
def withCheckAllTopLevel(flag: Boolean): Compiler =
copy(checkAllTopLevel = flag)
- def reload(savedParser: File) = reload(new FileInputStream(savedParser).getChannel())
+ def reload(savedParser: File) = reload(new FileInputStream(savedParser))
- def reload(savedParser: java.nio.channels.ReadableByteChannel): DFDL.DataProcessor = {
- try {
- val is = Channels.newInputStream(savedParser)
+ def reload(savedParser: java.nio.channels.ReadableByteChannel): DFDL.DataProcessor =
+ reload(Channels.newInputStream(savedParser))
+
+ def reload(schemaSource: DaffodilSchemaSource): DFDL.DataProcessor =
+ reload(schemaSource.uriForLoading)
+ def reload(uri: URI): DFDL.DataProcessor = reload(uri.toURL.openStream())
+
+ def reload(is: java.io.InputStream): DFDL.DataProcessor = {
+ try {
// Read the required prefix and version information for this saved parser
// directly from the input stream. This information is not compressed or
// serialized by a Java ObjectOutputStream
diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala
index 0e7bc3b8c..13067f271 100644
--- a/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala
+++ b/daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala
@@ -27,7 +27,6 @@ import java.io.File
import java.io.FileInputStream
import java.nio.file.Paths
import org.apache.daffodil.exceptions.Assert
-import org.apache.daffodil.equality._
import java.nio.file.FileSystemNotFoundException
@@ -54,6 +53,12 @@ sealed trait DaffodilSchemaSource {
* Use to get the URI that can be used to load the xml.
*/
def uriForLoading: URI
+
+ /**
+ * @return True if the schema source is for an XSD source file (extension ".xsd")
+ * False otherwise.
+ */
+ def isXSD: Boolean
}
object URISchemaSource {
@@ -73,17 +78,32 @@ class URISchemaSource protected (val fileOrResource: URI) extends DaffodilSchema
private lazy val url = fileOrResource.toURL
+ final def isXSD = isXSD_
+
/**
* Must be lazy so that it captures the file mod time when it is opened
* and the content used.
*/
- lazy val (isFile, file, fileModTime) = try {
- val path = Paths.get(fileOrResource)
- val f = path.toFile()
- (true, f, f.lastModified())
- } catch {
- case e: FileSystemNotFoundException => (false, null, 0L)
- case e: UnsupportedOperationException => (false, null, 0L)
+ lazy val (isFile, isXSD_, file, fileModTime) = {
+ val uriString = uriForLoading.toString
+ //
+ // doing this the manual way using split
+ // on string because FilenameUtils.getExtension
+ // does something complicated for the nested
+ // jar URIs we encounter here. It seems to not
+ // move past the nesting, and sees ".jar" not
+ // ".xsd" for files that are clearly XSD.
+ //
+ val ext = uriString.split("\\.").last
+ val isXSDExt = (ext == "xsd")
+ try {
+ val path = Paths.get(fileOrResource)
+ val f = path.toFile()
+ (true, isXSDExt, f, f.lastModified())
+ } catch {
+ case e: FileSystemNotFoundException => (false, isXSDExt, null, 0L)
+ case e: UnsupportedOperationException => (false, isXSDExt, null, 0L)
+ }
}
override def newInputSource() = {
@@ -96,16 +116,11 @@ class URISchemaSource protected (val fileOrResource: URI) extends DaffodilSchema
override def uriForLoading = fileOrResource
/**
- * True if this URI is for a file, other URI is for a file
- * (it is required that they're both the same URI. Usage error otherwise),
- * but the modification date
- * of the two is such that this is newer than the other at the time the
- * other was accessed via newInputSource()
- *
- * Otherwise false.
+ * @return True if this URI is for a file, other URI is for a file
+ * but the modification date of this is newer than the modification
+ * date of the other.
*/
def isNewerThan(other: URISchemaSource): Boolean = {
- Assert.usage(fileOrResource =:= other.fileOrResource)
if (this.isFile && other.isFile) {
val thisTime = fileModTime
val otherTime = other.fileModTime
@@ -127,6 +142,9 @@ class InputStreamSchemaSource(is: java.io.InputStream, tmpDir: Option[File], bla
xmlStream.close()
csName
}
+
+ def isXSD = Assert.usageError("isXSD not supported for InputStreamInputSource")
+
override def newInputSource() = {
val is = new FileInputStream(tempSchemaFile)
val inSrc = new InputSource(is)
diff --git a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala
index fe4b6254a..203c0043a 100644
--- a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala
+++ b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/DaffodilTDMLDFDLProcessor.scala
@@ -130,8 +130,7 @@ final class TDMLDFDLProcessorFactory private (
val output = Channels.newChannel(os)
p.save(output)
val is = new java.io.ByteArrayInputStream(os.toByteArray)
- val input = Channels.newChannel(is)
- compiler.reload(input)
+ compiler.reload(is)
} else p
}
val diags = p.getDiagnostics
@@ -139,6 +138,7 @@ final class TDMLDFDLProcessorFactory private (
}
}
+
private def compileProcessor(
schemaSource: DaffodilSchemaSource,
useSerializedProcessor: Boolean,
@@ -160,9 +160,15 @@ final class TDMLDFDLProcessorFactory private (
optRootName: Option[String],
optRootNamespace: Option[String],
tunables: Map[String, String]): TDML.CompileResult = {
-
- val res = compileProcessor(schemaSource, useSerializedProcessor, optRootName, optRootNamespace)
- res
+ if (schemaSource.isXSD) {
+ val res = compileProcessor(schemaSource, useSerializedProcessor, optRootName, optRootNamespace)
+ res
+ } else {
+ val dp = compiler.reload(schemaSource)
+ val diags = dp.getDiagnostics
+ Assert.invariant(diags.forall{! _.isError })
+ Right((diags, new DaffodilTDMLDFDLProcessor(dp)))
+ }
}
}
diff --git a/daffodil-tdml-processor/src/test/resources/test/tdml/fake-precompiled-dfdl-schema.bin b/daffodil-tdml-processor/src/test/resources/test/tdml/fake-precompiled-dfdl-schema.bin
new file mode 100644
index 000000000..d862b6e86
Binary files /dev/null and b/daffodil-tdml-processor/src/test/resources/test/tdml/fake-precompiled-dfdl-schema.bin differ
diff --git a/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala b/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala
index 8537e89ea..ad982648a 100644
--- a/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala
+++ b/daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala
@@ -912,4 +912,26 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
assertTrue(exc.getMessage().contains("Duplicate definitions found for defineConfig: dupConfig"))
}
+ @Test def testNonXSDModelNegativeBadCompiledFile(): Unit = {
+ val testSuite =
+ <tdml:testSuite suiteName="theSuiteName"
+ xmlns:tns={ tns }
+ xmlns:tdml={ tdml }
+ xmlns:dfdl={ dfdl }
+ xmlns:xsd={ xsd }
+ xmlns:xs={ xsd }
+ xmlns:xsi={ xsi }>
+ <parserTestCase xmlns={ tdml } name="testCase" root="data"
+ model="test/tdml/fake-precompiled-dfdl-schema.bin" >
+ <document/>
+ <infoset><dfdlInfoset><data/></dfdlInfoset></infoset>
+ </parserTestCase>
+ </tdml:testSuite>
+ val runner = new Runner(testSuite)
+ val exc = intercept[Exception] {
+ runner.runOneTest("testCase")
+ }
+ runner.reset()
+ assertTrue(exc.getMessage.contains("only compatible with Daffodil 300.400.500"))
+ }
}
diff --git a/project/Rat.scala b/project/Rat.scala
index 9eb7524f2..f5d522feb 100644
--- a/project/Rat.scala
+++ b/project/Rat.scala
@@ -186,6 +186,7 @@ object Rat {
file("daffodil-test/src/test/resources/org/apache/daffodil/usertests/Book2.csv"),
file("daffodil-test/src/test/resources/org/apache/daffodil/usertests/test_prefix_separator_as_variable"),
file("daffodil-test/src/test/resources/test space/A BTinyData.tdml.dat"),
+ file("daffodil-tdml-processor/src/test/resources/test/tdml/fake-precompiled-dfdl-schema.bin"),
file("test-stdLayout/src/test/resources/org1/test-outer-data1.txt"),
file("test-stdLayout/src/test/resources/org2/test-data1.txt"),
)