You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/07/21 13:45:48 UTC
[incubator-nlpcraft] 01/01: Reading streams approach refactored.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch NLPCRAFT-90
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
commit 035bf7df47e78d484ff4b214439bbdbab9df4701
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Tue Jul 21 16:45:35 2020 +0300
Reading streams approach refactored.
---
.../org/apache/nlpcraft/common/util/NCUtils.scala | 172 ++++++++-------------
.../server/geo/tools/NCGeoNamesGenerator.scala | 2 +-
.../geo/tools/unstats/NCUnsdStatsService.scala | 4 +-
3 files changed, 67 insertions(+), 111 deletions(-)
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 620dfdc..1df5b55 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -29,8 +29,8 @@ import java.time.{Instant, ZoneId, ZonedDateTime}
import java.util.concurrent.{ExecutorService, TimeUnit}
import java.util.jar.JarFile
import java.util.stream.Collectors
-import java.util.zip.{ZipInputStream, GZIPInputStream => GIS, GZIPOutputStream ⇒ GOS}
-import java.util.{Locale, Properties, Random, Timer, TimerTask, Calendar ⇒ C}
+import java.util.zip.{ZipInputStream, GZIPInputStream => GIS, GZIPOutputStream => GOS}
+import java.util.{Locale, Properties, Random, Timer, TimerTask, Calendar => C}
import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.core.`type`.TypeReference
@@ -44,6 +44,7 @@ import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.IOUtils
import org.apache.nlpcraft.common._
import org.apache.nlpcraft.common.blowfish.NCBlowfishHasher
+import org.apache.nlpcraft.common.util.NCUtils.getStream
import resource._
import scala.annotation.tailrec
@@ -289,7 +290,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readPath(path: String, enc: String, log: Logger = logger): List[String] =
readFile(new File(path), enc, log)
/**
@@ -300,8 +301,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
- readStream(getStream(res), enc, log)
+ def readResource(res: String, enc: String, log: Logger = logger): List[String] = readStream(getStream(res), enc, log)
/**
* Reads lines from given file.
@@ -311,7 +311,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readGzipPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readGzipPath(path: String, enc: String, log: Logger = logger): List[String] =
readGzipFile(new File(path), enc, log)
/**
@@ -322,22 +322,14 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readFile(f: File, enc: String, log: Logger = logger): Iterator[String] = {
- var src: Source = null
-
- try {
- src = Source.fromFile(f, enc)
-
- val data = src.getLines().map(p ⇒ p)
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromFile(f, enc)) acquireAndGet { src ⇒
+ getAndLog(src.getLines().map(p ⇒ p).toList, f, log)
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
- }
/**
* Reads lines from given stream.
@@ -347,9 +339,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readStream(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
+ def readStream(in: InputStream, enc: String, log: Logger = logger): List[String] =
try
- Source.fromInputStream(in, enc).getLines().map(p ⇒ p)
+ managed(Source.fromInputStream(in, enc)) acquireAndGet { src ⇒
+ src.getLines().map(p ⇒ p).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -363,22 +357,18 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextFile(f: File, enc: String, log: Logger = logger): Iterator[String] = {
- var src: Source = null
-
- try {
- src = Source.fromFile(f, enc)
-
- val data = src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readTextFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromFile(f, enc)) acquireAndGet { src ⇒
+ getAndLog(
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList,
+ f,
+ logger
+ )
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
- }
/**
* Reads lines from given stream converting to lower case, trimming, and filtering
@@ -389,11 +379,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextStream(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
- try {
- Source.fromInputStream(in, enc).getLines().map(_.toLowerCase.trim).
- filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
- }
+ def readTextStream(in: InputStream, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromInputStream(in, enc)) acquireAndGet { src ⇒
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -407,15 +397,15 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextGzipFile(f: File, enc: String, log: Logger = logger): Iterator[String] =
- try {
- val data = Source.fromInputStream(new GIS(new FileInputStream(f)), enc).getLines().map(_.toLowerCase.trim).
- filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readTextGzipFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromInputStream(new GIS(new FileInputStream(f)), enc)) acquireAndGet { src ⇒
+ getAndLog(
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList,
+ f,
+ log
+ )
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
@@ -429,10 +419,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextGzipResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextGzipResource(res: String, enc: String, log: Logger = logger): List[String] =
try
- Source.fromInputStream(new GIS(getStream(res)), enc).getLines().map(_.toLowerCase.trim).
- filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
+ managed(Source.fromInputStream(new GIS(getStream(res)), enc)) acquireAndGet { src ⇒
+ src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -446,7 +437,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextPath(path: String, enc: String, log: Logger = logger): List[String] =
readTextFile(new File(path), enc, log)
/**
@@ -458,7 +449,7 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextResource(res: String, enc: String, log: Logger = logger): List[String] =
readTextStream(getStream(res), enc, log)
/**
@@ -470,21 +461,10 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readTextGzipPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+ def readTextGzipPath(path: String, enc: String, log: Logger = logger): List[String] =
readTextGzipFile(new File(path), enc, log)
/**
- *
- * @param path Folder path to read from.
- * @param enc Encoding.
- * @param log Logger to use.
- */
- @throws[NCE]
- def readTextFolder(path: String, enc: String, log: Logger = logger): Iterator[String] =
- (for (file ← new File(path).listFiles()) yield
- readTextFile(file, enc, logger)).toIterator.flatten
-
- /**
* Converts given name into properly capitalized first and last name.
*
* @param name Full name.
@@ -566,14 +546,11 @@ object NCUtils extends LazyLogging {
* @param log Logger to use.
*/
@throws[NCE]
- def readGzipFile(f: File, enc: String, log: Logger = logger): Iterator[String] =
- try {
- val data = Source.fromInputStream(new GIS(new FileInputStream(f)), enc).getLines().map(p ⇒ p)
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
+ def readGzipFile(f: File, enc: String, log: Logger = logger): List[String] =
+ try
+ managed(Source.fromInputStream(new GIS(new FileInputStream(f)), enc)) acquireAndGet { src ⇒
+ getAndLog(src.getLines().map(p ⇒ p).toList, f, log)
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
}
@@ -586,9 +563,11 @@ object NCUtils extends LazyLogging {
* @return
*/
@throws[NCE]
- def readGzipResource(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
+ def readGzipResource(in: InputStream, enc: String, log: Logger = logger): List[String] =
try
- Source.fromInputStream(new GIS(in), enc).getLines().map(p ⇒ p)
+ managed(Source.fromInputStream(new GIS(in), enc)) acquireAndGet { src ⇒
+ src.getLines().map(p ⇒ p).toList
+ }
catch {
case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
}
@@ -600,8 +579,7 @@ object NCUtils extends LazyLogging {
* @param log Logger.
*/
@throws[NCE]
- def readPathBytes(path: String, log: Logger = logger): Array[Byte] =
- readFileBytes(new File(path), log)
+ def readPathBytes(path: String, log: Logger = logger): Array[Byte] = readFileBytes(new File(path), log)
/**
* Reads bytes from given file.
@@ -618,9 +596,7 @@ object NCUtils extends LazyLogging {
in.read(arr)
}
- logger.trace(s"File read: $f")
-
- arr
+ getAndLog(arr, f, log)
}
catch {
case e: IOException ⇒ throw new NCE(s"Error reading file: $f", e)
@@ -628,40 +604,20 @@ object NCUtils extends LazyLogging {
}
/**
- * Reads bytes from given file.
*
- * @param f File to read from.
- * @param log Logger to use.
+ * @param data
+ * @param f
+ * @param log
+ * @tparam T
+ * @return
*/
- @throws[NCE]
- def readBinaryFile(f: File, log: Logger = logger): Array[Byte] = {
- var src: Source = null
+ private def getAndLog[T](data: T, f: File, log: Logger = logger): T = {
+ log.trace(s"Loaded file: ${f.getAbsolutePath}")
- try {
- src = Source.fromFile(f, "ISO-8859-1")
-
- val data = src.map(_.toByte).toArray
-
- log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
- data
- }
- catch {
- case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
- }
+ data
}
/**
- * Reads bytes from given file.
- *
- * @param path File path to read from.
- * @param log Logger to use.
- */
- @throws[NCE]
- def readBinaryPath(path: String, log: Logger = logger): Array[Byte] =
- readBinaryFile(new File(path), log)
-
- /**
* Gzip file.
*
* @param f File.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
index 17c8f09..5e92b50 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
@@ -74,7 +74,7 @@ object NCGeoNamesGenerator extends App {
// GEO name ID -> internal representation mapping.
private val ids = mutable.Map.empty[String, Location]
- private def read(path: String): Iterator[String] = U.readPath(path, "UTF8").filter(!_.startsWith("#"))
+ private def read(path: String): Seq[String] = U.readPath(path, "UTF8").filter(!_.startsWith("#"))
// Process country and continent information.
private def processCountries(unsdContinents: Seq[NCUnsdStatsContinent]): Set[Country] = {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
index 2a6ebe9..90942d3 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
@@ -72,7 +72,7 @@ object NCUnsdStatsService {
private val codesPath = s"$dir/codes.txt"
private def read(path: String): Seq[String] =
- U.readPath(path, "UTF-8").map(_.trim).filter(_.nonEmpty).filter(_.head != '#').toSeq
+ U.readPath(path, "UTF-8").map(_.trim).filter(_.nonEmpty).filter(_.head != '#')
def skip(iso: String): Boolean = SKIPPED_COUNTRIES_ISO3.contains(iso)
@@ -86,7 +86,7 @@ object NCUnsdStatsService {
}).toMap
val subContsCodes = CONTINENTS.flatMap(_._2).toSeq
- val contsCodes = CONTINENTS.unzip._1.toSeq
+ val contsCodes = CONTINENTS.keys.toSeq
val buf = mutable.Buffer.empty[NCUnsdStatsContinent]