You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/07/21 13:45:47 UTC
[incubator-nlpcraft] branch NLPCRAFT-90 created (now 035bf7d)
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a change to branch NLPCRAFT-90
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git.
at 035bf7d Reading streams approach refactored.
This branch includes the following new commits:
new 035bf7d Reading streams approach refactored.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-nlpcraft] 01/01: Reading streams approach refactored.
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-90
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 035bf7df47e78d484ff4b214439bbdbab9df4701
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Tue Jul 21 16:45:35 2020 +0300
Reading streams approach refactored.
---
.../org/apache/nlpcraft/common/util/NCUtils.scala | 172 ++++++++-------------
.../server/geo/tools/NCGeoNamesGenerator.scala | 2 +-
.../geo/tools/unstats/NCUnsdStatsService.scala | 4 +-
3 files changed, 67 insertions(+), 111 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 620dfdc..1df5b55 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -29,8 +29,8 @@ import java.time.{Instant, ZoneId, ZonedDateTime}
import java.util.concurrent.{ExecutorService, TimeUnit}
import java.util.jar.JarFile
import java.util.stream.Collectors
-import java.util.zip.{ZipInputStream, GZIPInputStream => GIS, GZIPOutputStream ⇒ GOS}
-import java.util.{Locale, Properties, Random, Timer, TimerTask, Calendar ⇒ C}
+import java.util.zip.{ZipInputStream, GZIPInputStream => GIS, GZIPOutputStream => GOS}
+import java.util.{Locale, Properties, Random, Timer, TimerTask, Calendar => C}
import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.core.`type`.TypeReference
@@ -44,6 +44,7 @@ import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.IOUtils
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.blowfish.NCBlowfishHasher
+import org.apache.nlpcraft.common.util.NCUtils.getStream
import resource._
import scala.annotation.tailrec
@@ -289,7 +290,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readPath(path: String, enc: String, log: Logger = logger): List[String] =
readFile(new File(path), enc, log)
/**
@@ -300,8 +301,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
- readStream(getStream(res), enc, log)
+ def readResource(res: String, enc: String, log: Logger = logger): List[String] = readStream(getStream(res), enc, log)
/**
* Reads lines from given file.
@@ -311,7 +311,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readGzipPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readGzipPath(path: String, enc: String, log: Logger = logger): List[String] =
readGzipFile(new File(path), enc, log)
/**
@@ -322,22 +322,14 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readFile(f: File, enc: String, log: Logger = logger): Iterator[String] = {
- var src: Source = null
-
- try {
- src = Source.fromFile(f, enc)
-
- val data = src.getLines().map(p ⇒ p)
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromFile(f, enc)) acquireAndGet { src ⇒
+ getAndLog(src.getLines().map(p ⇒ p).toList, f, log)
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
- }
/**
* Reads lines from given stream.
@@ -347,9 +339,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readStream(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
+ def readStream(in: InputStream, enc: String, log: Logger = logger): List[String] =
try
- Source.fromInputStream(in, enc).getLines().map(p ⇒ p)
+ managed(Source.fromInputStream(in, enc)) acquireAndGet { src ⇒
+ src.getLines().map(p ⇒ p).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -363,22 +357,18 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextFile(f: File, enc: String, log: Logger = logger): Iterator[String] = {
- var src: Source = null
-
- try {
- src = Source.fromFile(f, enc)
-
- val data = src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readTextFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromFile(f, enc)) acquireAndGet { src ⇒
+ getAndLog(
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList,
+ f,
+ logger
+ )
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
- }
/**
* Reads lines from given stream converting to lower case, trimming, and filtering
@@ -389,11 +379,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextStream(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
- try {
- Source.fromInputStream(in, enc).getLines().map(_.toLowerCase.trim).
- filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
- }
+ def readTextStream(in: InputStream, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromInputStream(in, enc)) acquireAndGet { src ⇒
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -407,15 +397,15 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextGzipFile(f: File, enc: String, log: Logger = logger): Iterator[String] =
- try {
- val data = Source.fromInputStream(new GIS(new FileInputStream(f)), enc).getLines().map(_.toLowerCase.trim).
- filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readTextGzipFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromInputStream(new GIS(new FileInputStream(f)), enc)) acquireAndGet { src ⇒
+ getAndLog(
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList,
+ f,
+ log
+ )
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
@@ -429,10 +419,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextGzipResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextGzipResource(res: String, enc: String, log: Logger = logger): List[String] =
try
- Source.fromInputStream(new GIS(getStream(res)), enc).getLines().map(_.toLowerCase.trim).
- filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
+ managed(Source.fromInputStream(new GIS(getStream(res)), enc)) acquireAndGet { src ⇒
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -446,7 +437,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextPath(path: String, enc: String, log: Logger = logger): List[String] =
readTextFile(new File(path), enc, log)
/**
@@ -458,7 +449,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextResource(res: String, enc: String, log: Logger = logger): List[String] =
readTextStream(getStream(res), enc, log)
/**
@@ -470,21 +461,10 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextGzipPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextGzipPath(path: String, enc: String, log: Logger = logger): List[String] =
readTextGzipFile(new File(path), enc, log)
/**
- *
- * @param path Folder path to read from.
- * @param enc Encoding.
- * @param log Logger to use.
- */
- @throws[NCE]
- def readTextFolder(path: String, enc: String, log: Logger = logger): Iterator[String] =
- (for (file ← new File(path).listFiles()) yield
- readTextFile(file, enc, logger)).toIterator.flatten
-
- /**
* Converts given name into properly capitalized first and last name.
*
* @param name Full name.
@@ -566,14 +546,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readGzipFile(f: File, enc: String, log: Logger = logger): Iterator[String] =
- try {
- val data = Source.fromInputStream(new GIS(new FileInputStream(f)), enc).getLines().map(p ⇒ p)
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readGzipFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromInputStream(new GIS(new FileInputStream(f)), enc)) acquireAndGet { src ⇒
+ getAndLog(src.getLines().map(p ⇒ p).toList, f, log)
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
@@ -586,9 +563,11 @@ object NCUtils extends LazyLogging {
* @return
*/
@throws[NCE]
- def readGzipResource(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
+ def readGzipResource(in: InputStream, enc: String, log: Logger = logger): List[String] =
try
- Source.fromInputStream(new GIS(in), enc).getLines().map(p ⇒ p)
+ managed(Source.fromInputStream(new GIS(in), enc)) acquireAndGet { src ⇒
+ src.getLines().map(p ⇒ p).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -600,8 +579,7 @@ object NCUtils extends LazyLogging {
* @param log Logger.
*/
@throws[NCE]
- def readPathBytes(path: String, log: Logger = logger): Array[Byte] =
- readFileBytes(new File(path), log)
+ def readPathBytes(path: String, log: Logger = logger): Array[Byte] = readFileBytes(new File(path), log)
/**
* Reads bytes from given file.
@@ -618,9 +596,7 @@ object NCUtils extends LazyLogging {
in.read(arr)
}
- logger.trace(s"File read: $f")
-
- arr
+ getAndLog(arr, f, log)
}
catch {
case e: IOException ⇒ throw new NCE(s"Error reading file: $f", e)
@@ -628,40 +604,20 @@ object NCUtils extends LazyLogging {
}
/**
- * Reads bytes from given file.
*
- * @param f File to read from.
- * @param log Logger to use.
+ * @param data
+ * @param f
+ * @param log
+ * @tparam T
+ * @return
*/
- @throws[NCE]
- def readBinaryFile(f: File, log: Logger = logger): Array[Byte] = {
- var src: Source = null
+ private def getAndLog[T](data: T, f: File, log: Logger = logger): T = {
+ log.trace(s"Loaded file: ${f.getAbsolutePath}")
- try {
- src = Source.fromFile(f, "ISO-8859-1")
-
- val data = src.map(_.toByte).toArray
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
- catch {
- case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
- }
+ data
}
/**
- * Reads bytes from given file.
- *
- * @param path File path to read from.
- * @param log Logger to use.
- */
- @throws[NCE]
- def readBinaryPath(path: String, log: Logger = logger): Array[Byte] =
- readBinaryFile(new File(path), log)
-
- /**
* Gzip file.
*
* @param f File.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
index 17c8f09..5e92b50 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
@@ -74,7 +74,7 @@ object NCGeoNamesGenerator extends App {
// GEO name ID -> internal representation mapping.
private val ids = mutable.Map.empty[String, Location]
- private def read(path: String): Iterator[String] = U.readPath(path, "UTF8").filter(!_.startsWith("#"))
+ private def read(path: String): Seq[String] = U.readPath(path, "UTF8").filter(!_.startsWith("#"))
// Process country and continent information.
private def processCountries(unsdContinents: Seq[NCUnsdStatsContinent]): Set[Country] = {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
index 2a6ebe9..90942d3 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
@@ -72,7 +72,7 @@ object NCUnsdStatsService {
private val codesPath = s"$dir/codes.txt"
private def read(path: String): Seq[String] =
- U.readPath(path, "UTF-8").map(_.trim).filter(_.nonEmpty).filter(_.head != '#').toSeq
+ U.readPath(path, "UTF-8").map(_.trim).filter(_.nonEmpty).filter(_.head != '#')
def skip(iso: String): Boolean = SKIPPED_COUNTRIES_ISO3.contains(iso)
@@ -86,7 +86,7 @@ object NCUnsdStatsService {
}).toMap
val subContsCodes = CONTINENTS.flatMap(_._2).toSeq
- val contsCodes = CONTINENTS.unzip._1.toSeq
+ val contsCodes = CONTINENTS.keys.toSeq
val buf = mutable.Buffer.empty[NCUnsdStatsContinent]