You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@any23.apache.org by an...@apache.org on 2015/10/29 23:38:55 UTC
[1/4] any23 git commit: Replaced the static SimpleDateFormat field in
ItemPropValue with a ThreadLocal. The previous solution would yield broken
results when multiple documents were parsed concurrently. Added a unit test
that failed every time on my mach
Repository: any23
Updated Branches:
refs/heads/master 63ba2fc82 -> 73ce0c29e
Replaced the static SimpleDateFormat field in ItemPropValue with a
ThreadLocal. The previous solution would yield broken results when
multiple documents were parsed concurrently. Added a unit test that
failed every time on my machine with the old version and succeeds every
time with the new version.
Project: http://git-wip-us.apache.org/repos/asf/any23/repo
Commit: http://git-wip-us.apache.org/repos/asf/any23/commit/9afa87db
Tree: http://git-wip-us.apache.org/repos/asf/any23/tree/9afa87db
Diff: http://git-wip-us.apache.org/repos/asf/any23/diff/9afa87db
Branch: refs/heads/master
Commit: 9afa87db7efaad706e64e76a82e6a53f657a817f
Parents: 63ba2fc
Author: Antoni Mylka <an...@quantinum.com>
Authored: Thu Oct 29 15:11:26 2015 +0100
Committer: Antoni Mylka <an...@quantinum.com>
Committed: Thu Oct 29 15:11:26 2015 +0100
----------------------------------------------------------------------
.../microformats2/annotations/package-info.java | 2 +-
.../extractor/microdata/ItemPropValue.java | 31 ++++---
.../microdata/MicrodataParserTest.java | 85 ++++++++++++++++++--
3 files changed, 99 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/any23/blob/9afa87db/core/src/main/java/org/apache/any23/extractor/html/microformats2/annotations/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/any23/extractor/html/microformats2/annotations/package-info.java b/core/src/main/java/org/apache/any23/extractor/html/microformats2/annotations/package-info.java
index 3311c98..8e09321 100644
--- a/core/src/main/java/org/apache/any23/extractor/html/microformats2/annotations/package-info.java
+++ b/core/src/main/java/org/apache/any23/extractor/html/microformats2/annotations/package-info.java
@@ -21,4 +21,4 @@
*
* @see org.apache.any23.extractor.html.MicroformatExtractor
*/
-package org.apache.any23.extractor.microformats2.annotations;
\ No newline at end of file
+package org.apache.any23.extractor.html.microformats2.annotations;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/any23/blob/9afa87db/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java b/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
index 0dcad10..a74849d 100644
--- a/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
+++ b/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
@@ -17,13 +17,12 @@
package org.apache.any23.extractor.microdata;
-import org.apache.any23.util.StringUtils;
-
import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import org.apache.any23.util.StringUtils;
/**
* Describes a possible value for a <b>Microdata item property</b>.
@@ -32,7 +31,7 @@ import java.util.Date;
*/
public class ItemPropValue {
- private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ private static final ThreadLocal<SimpleDateFormat> sdf = new ThreadLocal<SimpleDateFormat>();
/**
* Supported types.
@@ -45,11 +44,20 @@ public class ItemPropValue {
}
public static Date parseDateTime(String dateStr) throws ParseException {
- return sdf.parse(dateStr);
+ return getSdf().parse(dateStr);
}
public static String formatDateTime(Date in) {
- return sdf.format(in);
+ return getSdf().format(in);
+ }
+
+ private static SimpleDateFormat getSdf() {
+ SimpleDateFormat simpleDateFormat = sdf.get();
+ if (simpleDateFormat == null) {
+ simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ sdf.set(simpleDateFormat);
+ }
+ return simpleDateFormat;
}
/**
@@ -88,7 +96,7 @@ public class ItemPropValue {
if(content instanceof String && ((String) content).trim().length() == 0) {
content = "Null";
// ANY23-115 Empty spans seem to break ANY23
- // instead of throwing the exception and in effect failing the entire
+ // instead of throwing the exception and in effect failing the entire
// parse job we wish to be lenient on web content publishers and add
// Null (String) as content.
//throw new IllegalArgumentException("Invalid content '" + content + "'");
@@ -143,7 +151,9 @@ public class ItemPropValue {
* @return <code>true</code> if type is an integer.
*/
public boolean isInteger() {
- if(type != Type.Plain) return false;
+ if(type != Type.Plain) {
+ return false;
+ }
try {
Integer.parseInt((String) content);
return true;
@@ -156,7 +166,9 @@ public class ItemPropValue {
* @return <code>true</code> if type is a float.
*/
public boolean isFloat() {
- if(type != Type.Plain) return false;
+ if(type != Type.Plain) {
+ return false;
+ }
try {
Float.parseFloat((String) content);
return true;
@@ -226,7 +238,7 @@ public class ItemPropValue {
if(content instanceof String) {
contentStr = "\"" + StringUtils.escapeAsJSONString((String) content) + "\"";
} else if(content instanceof Date) {
- contentStr = "\"" + sdf.format((Date) content) + "\"";
+ contentStr = "\"" + getSdf().format((Date) content) + "\"";
} else {
contentStr = content.toString();
}
@@ -258,5 +270,4 @@ public class ItemPropValue {
}
return false;
}
-
}
http://git-wip-us.apache.org/repos/asf/any23/blob/9afa87db/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java b/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
index 78d13a4..0db9fbc 100644
--- a/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
+++ b/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
@@ -17,21 +17,29 @@
package org.apache.any23.extractor.microdata;
-import org.apache.any23.extractor.html.TagSoupParser;
-import org.apache.any23.util.StreamUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
import java.util.GregorianCalendar;
+import java.util.List;
import java.util.Properties;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.any23.extractor.html.TagSoupParser;
+import org.apache.any23.util.StreamUtils;
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import static org.junit.Assert.assertFalse;
/**
* Test case for {@link MicrodataParser}.
@@ -88,7 +96,58 @@ public class MicrodataParserTest {
target.getProperties().get("birthday").get(0).getValue().getAsDate()
);
}
+
+ @Test
+ public void testGetDateConcurrent() throws IOException, ParseException {
+ final Date expectedDate = new GregorianCalendar(2009, Calendar.MAY, 10).getTime(); // 2009-05-10
+ final byte [] content = IOUtils.toByteArray(getClass().getResourceAsStream("/microdata/microdata-basic.html"));
+ final int threadCount = 10;
+ final int attemptCount = 100;
+ final List<Thread> threads = new ArrayList<Thread>();
+ final CyclicBarrier barrier = new CyclicBarrier(threadCount + 1);
+ final AtomicBoolean foundFailure = new AtomicBoolean(false);
+ for (int i = 0; i < threadCount; i++) {
+ threads.add(new Thread("Test-thread-" + i) {
+ @Override
+ public void run() {
+ await(barrier);
+ try {
+ int counter = 0;
+ while (counter++ < attemptCount && !foundFailure.get()) {
+ final Document document = getDom(content);
+ final MicrodataParserReport report = MicrodataParser.getMicrodata(document);
+ final ItemScope target = report.getDetectedItemScopes()[4];
+ Date actualDate = target.getProperties().get("birthday").get(0).getValue().getAsDate();
+ if (!expectedDate.equals(actualDate)) {
+ foundFailure.set(true);
+ }
+ }
+ }
+ catch (Exception ex) {
+ ex.printStackTrace();
+ foundFailure.set(true);
+ }
+ await(barrier);
+ }
+ });
+ }
+ for (Thread thread : threads) {
+ thread.start();
+ }
+ await(barrier);
+ await(barrier);
+ assertFalse(foundFailure.get());
+ }
+ private void await(CyclicBarrier barrier) {
+ try {
+ barrier.await();
+ }
+ catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
/**
* Test the main use case of {@link MicrodataParser#deferProperties(String...)}
*
@@ -156,6 +215,16 @@ public class MicrodataParserTest {
is.close();
}
}
+
+ private Document getDom(byte [] document) throws IOException {
+ final InputStream is = new ByteArrayInputStream(document);
+ try {
+ final TagSoupParser tagSoupParser = new TagSoupParser(is, "http://test-document");
+ return tagSoupParser.getDOM();
+ } finally {
+ is.close();
+ }
+ }
private Document getMicrodataDom(String htmlFile) throws IOException {
return getDom("/microdata/" + htmlFile + ".html");
[2/4] any23 git commit: Reverted two small changes automatically
introduced by my the java style settings of my IDE back to their original
state.
Posted by an...@apache.org.
Reverted two small changes automatically introduced by my the java style
settings of my IDE back to their original state.
Project: http://git-wip-us.apache.org/repos/asf/any23/repo
Commit: http://git-wip-us.apache.org/repos/asf/any23/commit/a8f1bd0a
Tree: http://git-wip-us.apache.org/repos/asf/any23/tree/a8f1bd0a
Diff: http://git-wip-us.apache.org/repos/asf/any23/diff/a8f1bd0a
Branch: refs/heads/master
Commit: a8f1bd0a3d8b5ea368e25529bcb959d79946c969
Parents: 9afa87d
Author: Antoni Mylka <an...@quantinum.com>
Authored: Thu Oct 29 15:17:29 2015 +0100
Committer: Antoni Mylka <an...@quantinum.com>
Committed: Thu Oct 29 15:17:29 2015 +0100
----------------------------------------------------------------------
.../org/apache/any23/extractor/microdata/ItemPropValue.java | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/any23/blob/a8f1bd0a/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java b/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
index a74849d..5ba34cd 100644
--- a/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
+++ b/core/src/main/java/org/apache/any23/extractor/microdata/ItemPropValue.java
@@ -151,9 +151,7 @@ public class ItemPropValue {
* @return <code>true</code> if type is an integer.
*/
public boolean isInteger() {
- if(type != Type.Plain) {
- return false;
- }
+ if(type != Type.Plain) return false;
try {
Integer.parseInt((String) content);
return true;
@@ -166,9 +164,7 @@ public class ItemPropValue {
* @return <code>true</code> if type is a float.
*/
public boolean isFloat() {
- if(type != Type.Plain) {
- return false;
- }
+ if(type != Type.Plain) return false;
try {
Float.parseFloat((String) content);
return true;
[3/4] any23 git commit: ANY23-265 : Convert test to use CountDownLatch
Posted by an...@apache.org.
ANY23-265 : Convert test to use CountDownLatch
Also add a JUnit Timeout rule to ensure completion, without having to
have specific timeouts on the await methods.
Project: http://git-wip-us.apache.org/repos/asf/any23/repo
Commit: http://git-wip-us.apache.org/repos/asf/any23/commit/2632bbf1
Tree: http://git-wip-us.apache.org/repos/asf/any23/tree/2632bbf1
Diff: http://git-wip-us.apache.org/repos/asf/any23/diff/2632bbf1
Branch: refs/heads/master
Commit: 2632bbf17de7f1b50b92f26c0bb1e5008c45ee04
Parents: a8f1bd0
Author: Peter Ansell <p_...@yahoo.com>
Authored: Fri Oct 30 09:29:38 2015 +1100
Committer: Peter Ansell <p_...@yahoo.com>
Committed: Fri Oct 30 09:29:38 2015 +1100
----------------------------------------------------------------------
.../microdata/MicrodataParserTest.java | 74 +++++++++++---------
1 file changed, 39 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/any23/blob/2632bbf1/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java b/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
index 0db9fbc..b6ee6dd 100644
--- a/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
+++ b/core/src/test/java/org/apache/any23/extractor/microdata/MicrodataParserTest.java
@@ -29,16 +29,21 @@ import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
+
import org.apache.any23.extractor.html.TagSoupParser;
import org.apache.any23.util.StreamUtils;
import org.apache.commons.io.IOUtils;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
+
import static org.junit.Assert.assertFalse;
/**
@@ -48,6 +53,9 @@ import static org.junit.Assert.assertFalse;
*/
public class MicrodataParserTest {
+ @Rule
+ public final Timeout timeout = new Timeout(100000);
+
private static final Logger logger = LoggerFactory.getLogger(MicrodataParserTest.class);
@Test
@@ -98,56 +106,52 @@ public class MicrodataParserTest {
}
@Test
- public void testGetDateConcurrent() throws IOException, ParseException {
+ public void testGetDateConcurrent() throws Exception {
final Date expectedDate = new GregorianCalendar(2009, Calendar.MAY, 10).getTime(); // 2009-05-10
- final byte [] content = IOUtils.toByteArray(getClass().getResourceAsStream("/microdata/microdata-basic.html"));
+ final byte[] content = IOUtils.toByteArray(getClass().getResourceAsStream("/microdata/microdata-basic.html"));
final int threadCount = 10;
final int attemptCount = 100;
final List<Thread> threads = new ArrayList<Thread>();
- final CyclicBarrier barrier = new CyclicBarrier(threadCount + 1);
+ final CountDownLatch beforeLatch = new CountDownLatch(1);
+ final CountDownLatch afterLatch = new CountDownLatch(threadCount);
final AtomicBoolean foundFailure = new AtomicBoolean(false);
for (int i = 0; i < threadCount; i++) {
threads.add(new Thread("Test-thread-" + i) {
- @Override
- public void run() {
- await(barrier);
- try {
- int counter = 0;
- while (counter++ < attemptCount && !foundFailure.get()) {
- final Document document = getDom(content);
- final MicrodataParserReport report = MicrodataParser.getMicrodata(document);
- final ItemScope target = report.getDetectedItemScopes()[4];
- Date actualDate = target.getProperties().get("birthday").get(0).getValue().getAsDate();
- if (!expectedDate.equals(actualDate)) {
- foundFailure.set(true);
- }
- }
- }
- catch (Exception ex) {
- ex.printStackTrace();
- foundFailure.set(true);
- }
- await(barrier);
- }
+ @Override
+ public void run() {
+ try {
+ beforeLatch.await();
+ int counter = 0;
+ while (counter++ < attemptCount && !foundFailure.get()) {
+ final Document document = getDom(content);
+ final MicrodataParserReport report = MicrodataParser.getMicrodata(document);
+ final ItemScope target = report.getDetectedItemScopes()[4];
+ Date actualDate = target.getProperties().get("birthday").get(0).getValue().getAsDate();
+ if (!expectedDate.equals(actualDate)) {
+ foundFailure.set(true);
+ }
+ }
+ }
+ catch (Exception ex) {
+ ex.printStackTrace();
+ foundFailure.set(true);
+ }
+ finally {
+ afterLatch.countDown();
+ }
+ }
});
}
for (Thread thread : threads) {
thread.start();
}
- await(barrier);
- await(barrier);
+ // Let threads start computation
+ beforeLatch.countDown();
+ // Wait for all threads to complete
+ afterLatch.await();
assertFalse(foundFailure.get());
}
- private void await(CyclicBarrier barrier) {
- try {
- barrier.await();
- }
- catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
-
/**
* Test the main use case of {@link MicrodataParser#deferProperties(String...)}
*
[4/4] any23 git commit: ANY23-265 : Fix ThreadSafety issue in
ItemPropValue
Posted by an...@apache.org.
ANY23-265 : Fix ThreadSafety issue in ItemPropValue
Project: http://git-wip-us.apache.org/repos/asf/any23/repo
Commit: http://git-wip-us.apache.org/repos/asf/any23/commit/73ce0c29
Tree: http://git-wip-us.apache.org/repos/asf/any23/tree/73ce0c29
Diff: http://git-wip-us.apache.org/repos/asf/any23/diff/73ce0c29
Branch: refs/heads/master
Commit: 73ce0c29eb0094d119c33f07a1ed0e0c6d3d7250
Parents: 63ba2fc 2632bbf
Author: Peter Ansell <p_...@yahoo.com>
Authored: Fri Oct 30 09:38:38 2015 +1100
Committer: Peter Ansell <p_...@yahoo.com>
Committed: Fri Oct 30 09:38:38 2015 +1100
----------------------------------------------------------------------
.../microformats2/annotations/package-info.java | 2 +-
.../extractor/microdata/ItemPropValue.java | 23 ++++--
.../microdata/MicrodataParserTest.java | 87 ++++++++++++++++++--
3 files changed, 96 insertions(+), 16 deletions(-)
----------------------------------------------------------------------