You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/10/09 03:25:41 UTC
[2/4] git commit: STREAMS-181 | Refactored RSSProvider to not be a
resource hog and added test and docs
STREAMS-181 | Refactored RSSProvider to not be a resource hog and added test and docs
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ac5ac99d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ac5ac99d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ac5ac99d
Branch: refs/heads/master
Commit: ac5ac99d10c379dd228766351c7114cddd13f87e
Parents: 507e679
Author: Ryan Ebanks <re...@Informations-MacBook-Pro-3.local>
Authored: Thu Sep 25 14:11:06 2014 -0500
Committer: Ryan Ebanks <re...@Informations-MacBook-Pro-3.local>
Committed: Thu Sep 25 14:11:06 2014 -0500
----------------------------------------------------------------------
pom.xml | 7 +
streams-contrib/streams-provider-rss/pom.xml | 10 +
.../streams/rss/provider/RssStreamProvider.java | 169 ++-
.../rss/provider/RssStreamProviderTask.java | 214 +++-
.../provider/perpetual/RssFeedScheduler.java | 112 ++
.../rss/provider/RssStreamProviderTaskTest.java | 148 +++
.../rss/provider/RssStreamProviderTest.java | 103 ++
.../perpetual/RssFeedSchedulerTest.java | 101 ++
.../test/resources/test_rss_xml/economist1.xml | 233 ++++
.../test/resources/test_rss_xml/economist2.xml | 1069 ++++++++++++++++++
.../test_rss_xml/economistCombined.xml | 221 ++++
11 files changed, 2268 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c2e1766..aa17f6a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
<json-path.version>0.9.1</json-path.version>
<build-helper.version>1.8</build-helper.version>
<facebook4j.version>2.1.0</facebook4j.version>
+ <mockito.version>1.9.5</mockito.version>
</properties>
<modules>
@@ -195,6 +196,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/pom.xml b/streams-contrib/streams-provider-rss/pom.xml
index c9f24d5..3c91277 100644
--- a/streams-contrib/streams-provider-rss/pom.xml
+++ b/streams-contrib/streams-provider-rss/pom.xml
@@ -67,6 +67,16 @@
<artifactId>rome</artifactId>
<version>1.0</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.healthmarketscience.common</groupId>
+ <artifactId>common-util</artifactId>
+ <version>1.0.2</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java
index 0cc3430..3d8eb05 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java
@@ -18,7 +18,10 @@
package org.apache.streams.rss.provider;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Queues;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.sun.syndication.feed.synd.SyndEntry;
@@ -33,6 +36,8 @@ import org.apache.streams.core.StreamsProvider;
import org.apache.streams.core.StreamsResultSet;
import org.apache.streams.rss.FeedDetails;
import org.apache.streams.rss.RssStreamConfiguration;
+import org.apache.streams.rss.provider.perpetual.RssFeedScheduler;
+import org.apache.streams.util.ComponentUtils;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,103 +47,88 @@ import java.io.Serializable;
import java.math.BigInteger;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.List;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
/**
- * Created by sblackmon on 12/10/13.
+ *
*/
public class RssStreamProvider implements StreamsProvider {
private final static Logger LOGGER = LoggerFactory.getLogger(RssStreamProvider.class);
+ private final static int MAX_SIZE = 1000;
private RssStreamConfiguration config;
+ private boolean perpetual;
+ private Set<String> urlFeeds;
+ private ExecutorService executor;
+ private BlockingQueue<StreamsDatum> dataQueue;
+ private AtomicBoolean isComplete;
+ private int consecutiveEmptyReads;
- private Class klass;
-
- public RssStreamConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(RssStreamConfiguration config) {
- this.config = config;
- }
-
- protected BlockingQueue inQueue = new LinkedBlockingQueue<SyndEntry>(10000);
-
- protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<StreamsDatum>();
-
- public BlockingQueue<Object> getInQueue() {
- return inQueue;
- }
-
- protected ListeningExecutorService executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(100, 100));
+ @VisibleForTesting
+ protected RssFeedScheduler scheduler;
- protected List<SyndFeed> feeds;
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
- return new ThreadPoolExecutor(nThreads, nThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<Runnable>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
+ public RssStreamProvider() {
+ this(RssStreamConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("rss")), false);
}
- public RssStreamProvider() {
- Config config = StreamsConfigurator.config.getConfig("rss");
- this.config = RssStreamConfigurator.detectConfiguration(config);
+ public RssStreamProvider(boolean perpetual) {
+ this(RssStreamConfigurator.detectConfiguration(StreamsConfigurator.config.getConfig("rss")), perpetual);
}
public RssStreamProvider(RssStreamConfiguration config) {
- this.config = config;
+ this(config, false);
}
- public RssStreamProvider(Class klass) {
- Config config = StreamsConfigurator.config.getConfig("rss");
- this.config = RssStreamConfigurator.detectConfiguration(config);
- this.klass = klass;
+ public RssStreamProvider(RssStreamConfiguration config, boolean perpetual) {
+ this.perpetual = perpetual;
+ this.config = config;
}
- public RssStreamProvider(RssStreamConfiguration config, Class klass) {
+ public void setConfig(RssStreamConfiguration config) {
this.config = config;
- this.klass = klass;
}
- @Override
- public void startStream() {
-
- Preconditions.checkNotNull(this.klass);
-
- Preconditions.checkNotNull(config.getFeeds());
-
- Preconditions.checkNotNull(config.getFeeds().get(0).getUrl());
-
- for( FeedDetails feedDetails : config.getFeeds()) {
-
- executor.submit(new RssFeedSetupTask(this, feedDetails));
-
- }
-
- for( int i = 0; i < ((config.getFeeds().size() / 5) + 1); i++ )
- executor.submit(new RssEventProcessor(inQueue, providerQueue, klass));
-
+ public void setRssFeeds(Set<String> urlFeeds) {
+ this.urlFeeds = urlFeeds;
}
- public void stop() {
- for (int i = 0; i < ((config.getFeeds().size() / 5) + 1); i++) {
- inQueue.add(RssEventProcessor.TERMINATE);
+ public void setRssFeeds(Map<String, Long> feeds) {
+ if(this.config == null) {
+ this.config = new RssStreamConfiguration();
}
+ List<FeedDetails> feedDetails = Lists.newLinkedList();
+ for(String feed : feeds.keySet()) {
+ Long delay = feeds.get(feed);
+ FeedDetails detail = new FeedDetails();
+ detail.setUrl(feed);
+ detail.setPollIntervalMillis(delay);
+ feedDetails.add(detail);
+ }
+ this.config.setFeeds(feedDetails);
}
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
+ @Override
+ public void startStream() {
+ LOGGER.trace("Starting Rss Scheduler");
+ this.executor.submit(this.scheduler);
}
@Override
public StreamsResultSet readCurrent() {
-
- return (StreamsResultSet) providerQueue;
-
+ Queue<StreamsDatum> batch = Queues.newConcurrentLinkedQueue();
+ int batchSize = 0;
+ while(!this.dataQueue.isEmpty() && batchSize < MAX_SIZE) {
+ StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.dataQueue);
+ if(datum != null) {
+ ++batchSize;
+ batch.add(datum);
+ }
+ }
+ this.isComplete.set(this.scheduler.isComplete() && batch.isEmpty() && this.dataQueue.isEmpty());
+ return new StreamsResultSet(batch);
}
@Override
@@ -153,50 +143,31 @@ public class RssStreamProvider implements StreamsProvider {
@Override
public boolean isRunning() {
- return !executor.isTerminated() && !executor.isShutdown();
+ return !this.isComplete.get();
}
@Override
public void prepare(Object configurationObject) {
-
+ this.executor = new ThreadPoolExecutor(1, 4, 15L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
+ this.dataQueue = Queues.newLinkedBlockingQueue();
+ this.scheduler = getScheduler(this.dataQueue);
+ this.isComplete = new AtomicBoolean(false);
+ this.consecutiveEmptyReads = 0;
+ }
+
+ @VisibleForTesting
+ protected RssFeedScheduler getScheduler(BlockingQueue<StreamsDatum> queue) {
+ if(this.perpetual)
+ return new RssFeedScheduler(this.executor, this.config.getFeeds(), queue);
+ else
+ return new RssFeedScheduler(this.executor, this.config.getFeeds(), queue, 0);
}
@Override
public void cleanUp() {
- stop();
+ this.scheduler.stop();
+ ComponentUtils.shutdownExecutor(this.executor, 10, 10);
}
- private class RssFeedSetupTask implements Runnable {
-
- private RssStreamProvider provider;
- private FeedDetails feedDetails;
- private RssFeedSetupTask(RssStreamProvider provider, FeedDetails feedDetails) {
- this.provider = provider;
- this.feedDetails = feedDetails;
- }
-
- @Override
- public void run() {
-
- URL feedUrl;
- SyndFeed feed;
- try {
- feedUrl = new URL(feedDetails.getUrl());
- SyndFeedInput input = new SyndFeedInput();
- try {
- feed = input.build(new XmlReader(feedUrl));
- executor.submit(new RssStreamProviderTask(provider, feed, feedDetails));
- LOGGER.info("Connected: " + feedDetails.getUrl());
- } catch (FeedException e) {
- LOGGER.warn("FeedException: " + feedDetails.getUrl());
- } catch (IOException e) {
- LOGGER.warn("IOException: " + feedDetails.getUrl());
- }
- } catch (MalformedURLException e) {
- LOGGER.warn("MalformedURLException: " + feedDetails.getUrl());
- }
-
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
index c37eb1e..3d035f8 100644
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
@@ -18,53 +18,227 @@
package org.apache.streams.rss.provider;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndFeed;
+import com.sun.syndication.io.FeedException;
+import com.sun.syndication.io.SyndFeedInput;
+import com.sun.syndication.io.XmlReader;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.data.util.RFC3339Utils;
import org.apache.streams.rss.FeedDetails;
+import org.apache.streams.rss.serializer.SyndEntrySerializer;
+import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Map;
import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
/**
- * Created by sblackmon on 12/10/13.
+ * A {@link java.lang.Runnable} task that queues rss feed data.
+ *
+ * <code>RssStreamProviderTask</code> reads the content of an rss feed and queues the articles from
+ * the feed inform of a {@link com.fasterxml.jackson.databind.node.ObjectNode} wrapped in a {@link org.apache.streams.core.StreamsDatum}.
+ * The task can filter articles by a published date. If the task cannot parse the date of the article or the article does not contain a
+ * published date, by default the task will attempt to queue article.
+ *
+ * A task can be run in perpetual mode which will store the article urls in a static variable. The next time a <code>RssStreamProviderTask</code>
+ * is run, it will not queue data that was seen the previous time the rss feed was read. This is an attempt to reduce
+ * multiple copies of an article from being out put by a {@link org.apache.streams.rss.provider.RssStreamProvider}.
+ *
+ * ** Warning! **
+ * It still is possible to output multiples of the same article. If multiple tasks executions for the same rss feed overlap
+ * in execution time, it possible that the previously seen articles static variable will not have been updated in time.
+ *
*/
public class RssStreamProviderTask implements Runnable {
private final static Logger LOGGER = LoggerFactory.getLogger(RssStreamProviderTask.class);
+ private static final int DEFAULT_TIME_OUT = 10000; // 10 seconds
+ private static final String RSS_KEY = "rssFeed";
+ private static final String URI_KEY = "uri";
+ private static final String LINK_KEY = "link";
+ private static final String DATE_KEY = "publishedDate";
+
+ /**
+ * Map that contains the Set of previously seen articles by an rss feed.
+ */
+ @VisibleForTesting
+ protected static final Map<String, Set<String>> PREVIOUSLY_SEEN = new ConcurrentHashMap<>();
+
+
+ private BlockingQueue<StreamsDatum> dataQueue;
+ private String rssFeed;
+ private int timeOut;
+ private SyndEntrySerializer serializer;
+ private DateTime publishedSince;
+ private boolean perpetual;
+
+
+ /**
+ * Non-perpetual mode, no date filter, time out of 10 sec
+ * @see {@link org.apache.streams.rss.provider.RssStreamProviderTask#RssStreamProviderTask(java.util.concurrent.BlockingQueue, String, org.joda.time.DateTime, int, boolean)}
+ * @param queue
+ * @param rssFeed
+ */
+ public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed) {
+ this(queue, rssFeed, new DateTime().minusYears(30), DEFAULT_TIME_OUT, false);
+ }
+
+ /**
+ * Non-perpetual mode, no date filter
+ * @see {@link org.apache.streams.rss.provider.RssStreamProviderTask#RssStreamProviderTask(java.util.concurrent.BlockingQueue, String, org.joda.time.DateTime, int, boolean)}
+ * @param queue
+ * @param rssFeed
+ * @param timeOut
+ */
+ public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed, int timeOut) {
+ this(queue, rssFeed, new DateTime().minusYears(30), timeOut, false);
+ }
- private RssStreamProvider provider;
- private FeedDetails feedDetails;
- private SyndFeed feed;
+ /**
+ * Non-perpetual mode, time out of 10 sec
+ * @see {@link org.apache.streams.rss.provider.RssStreamProviderTask#RssStreamProviderTask(java.util.concurrent.BlockingQueue, String, org.joda.time.DateTime, int, boolean)}
+ * @param queue
+ * @param rssFeed
+ * @param publishedSince
+ */
+ public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed, DateTime publishedSince) {
+ this(queue, rssFeed, publishedSince, DEFAULT_TIME_OUT, false);
+ }
- private Set<SyndEntry> priorPollResult = Sets.newHashSet();
+ /**
+ * RssStreamProviderTask that reads an rss feed url and queues the resulting articles as StreamsDatums with the documents
+ * being object nodes.
+ * @param queue Queue to push data to
+ * @param rssFeed url of rss feed to read
+ * @param publishedSince DateTime to filter articles by, will queue articles with published times after this
+ * @param timeOut url connection timeout in milliseconds
+ * @param perpetual true, if you want to run in perpetual mode. NOT RECOMMENDED
+ */
+ public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed, DateTime publishedSince, int timeOut, boolean perpetual) {
+ this.dataQueue = queue;
+ this.rssFeed = rssFeed;
+ this.timeOut = timeOut;
+ this.publishedSince = publishedSince;
+ this.serializer = new SyndEntrySerializer();
+ this.perpetual = perpetual;
+ }
- public RssStreamProviderTask(RssStreamProvider provider, SyndFeed feed, FeedDetails feedDetails) {
- this.provider = provider;
- this.feed = feed;
- this.feedDetails = feedDetails;
+ /**
+ * The rss feed url that this task is responsible for reading
+ * @return rss feed url
+ */
+ public String getRssFeed() {
+ return this.rssFeed;
}
@Override
public void run() {
+ try {
+ Set<String> batch = queueFeedEntries(new URL(this.rssFeed));
+ if(this.perpetual)
+ PREVIOUSLY_SEEN.put(this.getRssFeed(), batch);
+ } catch (IOException | FeedException e) {
+ LOGGER.warn("Exception while reading rss stream, {} : {}", this.rssFeed, e);
+ }
+ }
- while(true) {
+ /**
+ * Reads the url and queues the data
+ * @param feedUrl rss feed url
+ * @return set of all article urls that were read from the feed
+ * @throws IOException when it cannot connect to the url or the url is malformed
+ * @throws FeedException when it cannot reed the feed.
+ */
+ @VisibleForTesting
+ protected Set<String> queueFeedEntries(URL feedUrl) throws IOException, FeedException {
+ Set<String> batch = Sets.newConcurrentHashSet();
+ URLConnection connection = feedUrl.openConnection();
+ connection.setConnectTimeout(this.timeOut);
+ connection.setConnectTimeout(this.timeOut);
+ SyndFeedInput input = new SyndFeedInput();
+ SyndFeed feed = input.build(new InputStreamReader(connection.getInputStream()));
+ for (Object entryObj : feed.getEntries()) {
+ SyndEntry entry = (SyndEntry) entryObj;
+ ObjectNode nodeEntry = this.serializer.deserialize(entry);
+ nodeEntry.put(RSS_KEY, this.rssFeed);
+ String entryId = determineId(nodeEntry);
+ batch.add(entryId);
+ StreamsDatum datum = new StreamsDatum(nodeEntry);
try {
- Set<SyndEntry> update = Sets.newHashSet(feed.getEntries());
- Set<SyndEntry> repeats = Sets.intersection(priorPollResult, Sets.newHashSet(update));
- Set<SyndEntry> entrySet = Sets.difference(update, repeats);
- for( SyndEntry item : entrySet) {
- item.setSource(feed);
- provider.inQueue.offer(item);
+ JsonNode published = nodeEntry.get(DATE_KEY);
+ if (published != null) {
+ try {
+ DateTime date = RFC3339Utils.parseToUTC(published.asText());
+ if (date.isAfter(this.publishedSince) && (!this.perpetual || !seenBefore(entryId, this.rssFeed))) {
+ this.dataQueue.put(datum);
+ LOGGER.debug("Added entry, {}, to provider queue.", entryId);
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ } catch (Exception e) {
+ LOGGER.trace("Failed to parse date from object node, attempting to add node to queue by default.");
+ if(!this.perpetual || !seenBefore(entryId, this.rssFeed)) {
+ this.dataQueue.put(datum);
+ LOGGER.debug("Added entry, {}, to provider queue.", entryId);
+ }
+ }
+ } else {
+ LOGGER.debug("No published date present, attempting to add node to queue by default.");
+ if(!this.perpetual || !seenBefore(entryId, this.rssFeed)) {
+ this.dataQueue.put(datum);
+ LOGGER.debug("Added entry, {}, to provider queue.", entryId);
+ }
}
- priorPollResult = update;
- Thread.sleep(feedDetails.getPollIntervalMillis());
- } catch (Exception e) {
- e.printStackTrace();
+ } catch (InterruptedException ie) {
+ LOGGER.error("Interupted Exception.");
+ Thread.currentThread().interrupt();
}
}
+ return batch;
+ }
+ /**
+ * Returns a link to the article to use as the id
+ * @param node
+ * @return
+ */
+ private String determineId(ObjectNode node) {
+ String id = null;
+ if(node.get(URI_KEY) != null && !node.get(URI_KEY).textValue().equals("")) {
+ id = node.get(URI_KEY).textValue();
+ } else if(node.get(LINK_KEY) != null && !node.get(LINK_KEY).textValue().equals("")) {
+ id = node.get(LINK_KEY).textValue();
+ }
+ return id;
}
+ /**
+ * Returns false if the artile was previously seen in another task for this feed
+ * @param id
+ * @param rssFeed
+ * @return
+ */
+ private boolean seenBefore(String id, String rssFeed) {
+ Set<String> previousBatch = PREVIOUSLY_SEEN.get(rssFeed);
+ if(previousBatch == null) {
+ return false;
+ }
+ return previousBatch.contains(id);
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java
new file mode 100644
index 0000000..99ccbf3
--- /dev/null
+++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.streams.rss.provider.perpetual;
+
+import com.google.common.collect.Maps;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.rss.FeedDetails;
+import org.apache.streams.rss.provider.RssStreamProviderTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public class RssFeedScheduler implements Runnable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RssFeedScheduler.class);
+ private static final int DEFAULT_PEROID = 10; // 1 minute
+
+ private ExecutorService service;
+ private List<FeedDetails> feedDetailsList;
+ private int peroid;
+ private AtomicBoolean keepRunning;
+ private AtomicBoolean complete;
+ private Map<String, Long> lastScheduled;
+ private BlockingQueue<StreamsDatum> dataQueue;
+
+ public RssFeedScheduler(ExecutorService service, List<FeedDetails> feedDetailsList, BlockingQueue<StreamsDatum> dataQueue) {
+ this(service, feedDetailsList, dataQueue, DEFAULT_PEROID);
+ }
+
+ public RssFeedScheduler(ExecutorService service, List<FeedDetails> feedDetailsList, BlockingQueue<StreamsDatum> dataQueue, int peroid) {
+ this.service = service;
+ this.feedDetailsList = feedDetailsList;
+ this.peroid = peroid;
+ this.keepRunning = new AtomicBoolean(true);
+ this.lastScheduled = Maps.newHashMap();
+ this.dataQueue = dataQueue;
+ this.complete = new AtomicBoolean(false);
+ }
+
+ public void stop() {
+ this.keepRunning.set(false);
+ }
+
+ public boolean isComplete() {
+ return this.complete.get();
+ }
+
+ @Override
+ public void run() {
+ this.complete.set(false);
+ try {
+ if(this.peroid <= 0) {
+ scheduleFeeds();
+ } else {
+ while (this.keepRunning.get()) {
+ scheduleFeeds();
+ Thread.sleep(this.peroid * 60000);
+ }
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ } finally {
+ this.service = null;
+ LOGGER.info("{} completed scheduling of feeds.", this.getClass().getName());
+ this.complete.set(true);
+ }
+ }
+
+ public void scheduleFeeds() {
+ for(FeedDetails detail : this.feedDetailsList) {
+ Long lastTime = null;
+ if((lastTime = this.lastScheduled.get(detail.getUrl())) == null) {
+ lastTime = 0L;
+ }
+ long currentTime = System.currentTimeMillis();
+ long pollInterval;
+ if(detail.getPollIntervalMillis() == null) {
+ pollInterval = 0;
+ } else {
+ pollInterval = detail.getPollIntervalMillis();
+ }
+ if(currentTime - lastTime > pollInterval) {
+ this.service.execute(new RssStreamProviderTask(this.dataQueue, detail.getUrl()));
+ this.LOGGER.trace("Scheduled data collection on rss feed, {}", detail.getUrl());
+ this.lastScheduled.put(detail.getUrl(), currentTime);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskTest.java
new file mode 100644
index 0000000..48984d2
--- /dev/null
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.streams.rss.provider;
+
+import org.apache.streams.core.StreamsDatum;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URL;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for {@link org.apache.streams.rss.provider.RssStreamProviderTask}
+ */
+public class RssStreamProviderTaskTest {
+
+
+ @Before
+ public void clearPreviouslySeen() {
+ //some test runners run in parallel so needs to be synchronized
+ //if tests are run in parallel test will have undetermined results.
+ synchronized (RssStreamProviderTask.PREVIOUSLY_SEEN) {
+ RssStreamProviderTask.PREVIOUSLY_SEEN.clear();
+ }
+ }
+
+ /**
+ * Test that a task can read a valid rss from a url and queue the data
+ * @throws Exception
+ */
+ @Test
+ public void testNonPerpetualNoTimeFramePull() throws Exception {
+ com.healthmarketscience.common.util.resource.Handler.init();
+ BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
+ RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url");
+ Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals("Expected batch size to be the same as amount of queued datums", batch.size(), queue.size());
+ RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ //Test that it will out previously seen articles
+ queue.clear();
+ batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals("Expected batch size to be the same as amount of queued datums", batch.size(), queue.size());
+ }
+
+ /**
+ * Test that perpetual streams will not output previously seen articles
+ * @throws Exception
+ */
+ @Test
+ public void testPerpetualNoTimeFramePull() throws Exception {
+ com.healthmarketscience.common.util.resource.Handler.init();
+ BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
+ RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", new DateTime().minusYears(1), 10000, true);
+ Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals("Expected batch size to be the same as amount of queued datums", batch.size(), queue.size());
+ RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ //Test that it will not out previously seen articles
+ queue.clear();
+ batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals("Expected queue size to be 0", 0, queue.size());
+ assertEquals("Expected batch size to be 20", 20, batch.size());
+ RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ //Test that not seen urls aren't blocked.
+ queue.clear();
+ batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml"));
+ assertEquals(batch.size(), queue.size());
+ assertEquals("Expected queue size to be 25", 25, queue.size());
+ assertEquals("Expected batch size to be 25", 25, batch.size());
+ }
+
+ /**
+ * Test that you can task will only output aritcles after a certain published time
+ * @throws Exception
+ */
+ @Test
+ public void testNonPerpetualTimeFramedPull() throws Exception{
+ com.healthmarketscience.common.util.resource.Handler.init();
+ BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
+ DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC);
+ RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, false);
+ Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals( 15, queue.size());
+ assertEquals( 20 , batch.size());
+ assertTrue( queue.size() < batch.size());
+ RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ //Test that it will out previously seen articles
+ queue.clear();
+ batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals( 15, queue.size());
+ assertEquals( 20 , batch.size());
+ assertTrue( queue.size() < batch.size());
+ }
+
+ /**
+ * Test that task will only output articles after a certain published time that it has not seen before.
+ * @throws Exception
+ */
+ @Test
+ public void testPerpetualTimeFramedPull() throws Exception {
+ com.healthmarketscience.common.util.resource.Handler.init();
+ BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
+ DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC);
+ RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, true);
+ Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals( 15, queue.size());
+ assertEquals( 20 , batch.size());
+ assertTrue( queue.size() < batch.size());
+ RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+ //Test that it will not out put previously seen articles
+ queue.clear();
+ batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
+ assertEquals( 0, queue.size());
+ assertEquals( 20 , batch.size());
+ assertTrue( queue.size() < batch.size());
+ RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch);
+
+ batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml"));
+ assertTrue( queue.size() < batch.size());
+ assertEquals("Expected queue size to be 0", 3, queue.size());
+ assertEquals("Expected batch size to be 0", 25, batch.size());
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java
new file mode 100644
index 0000000..bf5e105
--- /dev/null
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java
@@ -0,0 +1,103 @@
+package org.apache.streams.rss.provider;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Queues;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.rss.FeedDetails;
+import org.apache.streams.rss.RssStreamConfiguration;
+import org.apache.streams.rss.provider.perpetual.RssFeedScheduler;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Created by rebanks on 9/25/14.
+ */
+public class RssStreamProviderTest extends RandomizedTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RssStreamProviderTest.class);
+
+ @Test
+ public void testRssFeedShutdownsNonPerpetual() throws Exception {
+ RssStreamProvider provider = null;
+ try {
+ final CountDownLatch latch = new CountDownLatch(1);
+ BlockingQueue<StreamsDatum> datums = Queues.newLinkedBlockingQueue();
+ provider = new RssStreamProvider(new RssStreamConfiguration()) {
+ @Override
+ protected RssFeedScheduler getScheduler(BlockingQueue<StreamsDatum> queue) {
+ return new MockScheduler(latch, queue);
+ }
+ };
+ provider.prepare(null);
+ int datumCount = 0;
+ provider.startStream();
+ while (!provider.scheduler.isComplete()) {
+ StreamsResultSet batch = provider.readCurrent();
+ LOGGER.debug("Batch size : {}", batch.size());
+ datumCount += batch.size();
+ Thread.sleep(randomIntBetween(0, 3000));
+ }
+ latch.await();
+
+ //one last pull incase of race condition
+ StreamsResultSet batch = provider.readCurrent();
+ LOGGER.debug("Batch size : {}", batch.size());
+ datumCount += batch.size();
+
+ assertTrue(provider.scheduler.isComplete());
+ assertFalse(provider.isRunning());
+ assertEquals(0, datums.size());
+ assertEquals(20, datumCount);
+ provider.cleanUp();
+ } finally {
+ if(provider != null)
+ provider.cleanUp();
+ }
+ }
+
+
+ private class MockScheduler extends RssFeedScheduler {
+
+ private BlockingQueue<StreamsDatum> queue;
+ private CountDownLatch latch;
+ private volatile boolean complete = false;
+
+ public MockScheduler(CountDownLatch latch, BlockingQueue<StreamsDatum> dataQueue) {
+ super(null, null, dataQueue);
+ this.latch = latch;
+ this.queue = dataQueue;
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < 20; ++i) {
+ this.queue.put(new StreamsDatum(null));
+ Thread.sleep(randomIntBetween(0, 5000));
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ } finally {
+ this.complete = true;
+ this.latch.countDown();
+ }
+ }
+
+
+ @Override
+ public boolean isComplete() {
+ return this.complete;
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
new file mode 100644
index 0000000..2bd0b69
--- /dev/null
+++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.streams.rss.provider.perpetual;
+
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.rss.FeedDetails;
+import org.apache.streams.rss.provider.RssStreamProviderTask;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link org.apache.streams.rss.provider.perpetual.RssFeedScheduler}
+ */
+public class RssFeedSchedulerTest {
+
+
+ /**
+ * Test that feeds are scheduled based on elapsed time correctly.
+ * Takes 1 minute to run.
+ */
+ @Test
+ public void testScheduleFeeds() {
+ ExecutorService mockService = mock(ExecutorService.class);
+ final List<String> queuedTasks = new ArrayList<>(5);
+ doAnswer(new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+ queuedTasks.add(((RssStreamProviderTask) invocationOnMock.getArguments()[0]).getRssFeed());
+ return null;
+ }
+ }).when(mockService).execute(any(Runnable.class));
+
+ RssFeedScheduler scheduler = new RssFeedScheduler(mockService, createFeedList(), new LinkedBlockingQueue<StreamsDatum>(), 1);
+ scheduler.scheduleFeeds();
+ assertEquals("Expected 2 Feeds to be scheduled", 2, queuedTasks.size());
+ assertEquals("Expected Feed 1 to be queued first", "1", queuedTasks.get(0));
+ assertEquals("Expected Feed 2 to be queued second", "2", queuedTasks.get(1));
+
+ safeSleep(1);
+ scheduler.scheduleFeeds();
+ assertEquals("Only feed 1 should have been re-queued", 3, queuedTasks.size());
+ assertEquals("Only feed 1 should have been re-queued", "1", queuedTasks.get(2));
+
+ safeSleep(60 * 1000);
+ scheduler.scheduleFeeds();
+ assertEquals("Both feeds should have been re-queued", 5, queuedTasks.size());
+ assertEquals("1", queuedTasks.get(3));
+ assertEquals("2", queuedTasks.get(4));
+ }
+
+ private List<FeedDetails> createFeedList() {
+ List<FeedDetails> list = Lists.newLinkedList();
+ FeedDetails fd = new FeedDetails();
+ fd.setPollIntervalMillis(1L);
+ fd.setUrl("1");
+ list.add(fd);
+
+ fd = new FeedDetails();
+ fd.setPollIntervalMillis( 60L * 1000);
+ fd.setUrl("2");
+ list.add(fd);
+ return list;
+ }
+
+ private void safeSleep(long milliseconds) {
+ try {
+ Thread.sleep(milliseconds);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ac5ac99d/streams-contrib/streams-provider-rss/src/test/resources/test_rss_xml/economist1.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-rss/src/test/resources/test_rss_xml/economist1.xml b/streams-contrib/streams-provider-rss/src/test/resources/test_rss_xml/economist1.xml
new file mode 100644
index 0000000..cf144b5
--- /dev/null
+++ b/streams-contrib/streams-provider-rss/src/test/resources/test_rss_xml/economist1.xml
@@ -0,0 +1,233 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
+ <channel>
+ <title>The Economist: Finance and economics</title>
+ <description>Finance and economics</description>
+ <link>http://www.economist.com</link>
+ <image>
+ <link>http://www.economist.com</link>
+ <width>125</width>
+ <title>Economist.com</title>
+ <url>//cdn.static-economist.com/sites/all/themes/econfinal/images/the-economist-logo.gif</url>
+ <height>34</height>
+ </image>
+ <language>en-gb</language>
+ <lastBuildDate>Thu, 18 September 2014 15:01:07 GMT</lastBuildDate>
+ <webMaster>webmaster@economist.com</webMaster>
+ <managingEditor>rondiorio@economist.com</managingEditor>
+ <ttl>120</ttl>
+ <docs>http://www.economist.com/rss/</docs>
+ <item>
+ <title>Free exchange: Leaving dead presidents in peace</title>
+ <description><p><object id="myExperience_1425961410001_3801830880001" class="BrightcoveExperience"><param name="bgcolor" value="#FFFFFF" /><param name="isUI" value="true" /><param name="isVid" value="true" /><param name="dynamicStreaming" value="true" /><param name="autoStart" value="false" /><param name="wmode" value="opaque" /><param name="templateLoadHandler" value="ECOmnitureBrightCoveHandlers62fe076522073444d4aae0e909510e4c.omnitureBCTemplateLoaded" /><param name="handlerID" value="ECOmnitureBrightCoveHandlers62fe076522073444d4aae0e909510e4c" /><param name="includeAPI" value="true" /><param name="linkBaseURL" value="http://www.economist.com/feeds/print-sections/79/finance-and-economics.x
ml" /><param name="iid" value="71241" /><param name="vid" value="3801830880001" /><param name="pid" value="1425961410001" /><param name="key" value="AQ~~,AAABDH-R__E~,dB4S9tmhdOo20g03jDsDgNBGDcclfHEU" /><param name="cssclass" value="" /><param name="width" value="595" /><param name="height" value="400" /><param name="labels" value="http://cdn.static-economist.com/sites/all/modules/custom/ec_brightcove/EcBcLables.xml" /><param name="playerID" value="1425961410001" /><param name="playerKey" value="AQ~~,AAABDH-R__E~,dB4S9tmhdOo20g03jDsDgNBGDcclfHEU" /><param name="@videoPlayer" value="3801830880001" /></object>SINCE cash was invented in the seventh century BC, it has generally been the most
convenient way...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618886-abolishing-notes-and-coins-would-bring-huge-economic-benefits-leaving-dead?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618886-abolishing-notes-and-coins-would-bring-huge-economic-benefits-leaving-dead</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>Saving in poor countries: Beyond cows</title>
+ <description><p><div class="content-image-float-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/290-width/images/print-edition/20140920_FNP002_0.jpg" alt="" title="" width="290" height="395" />
+ <span class="caption">A cumbersome savings plan</span>
+ </div>ADULTS in developing countries are half as likely to have an account at a formal financial institution as those in the rich world. Only 18% of people in the Middle East and north Africa do, compared with 89% in high-income countries. Economists would like the world’s poorest to save more. That would help them to pay for big or unexpected expenses, such as school fees or medical treatment. It would also boost investment and thus accelerate economic growth.But getting people to save is hard. One reason is the economic version of myopia: the failure to give adequate weight to future benefits over immediate pleasures. Most people are myopic, but for those in grinding poverty, the self-discipline required to save is greater and the consequences of failure worse.For many, the answer is to tie up money in livestock, which can be sold if necessary, or to join a rotating savings and credit association (ROSCA), which pools members’ savings and disburses them to
those in need. But these mechanisms are far from perfect. A survey in Uganda found that 99% of people using informal savings schemes had at some point lost...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618900-coaxing-does-more-boost-saving-compelling-beyond-cows?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618900-coaxing-does-more-boost-saving-compelling-beyond-cows</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>Corporate tax dodging: Transfer policing</title>
+ <description><p><div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140920_FNC727.png" alt="" title="" width="580" height="562" />
+
+ </div>POLITICIANS in the rich world like to splutter about the ever more elaborate dodges that big multinational firms undertake to minimise their tax bills. But doing something about them is trickier. America’s Congress is struggling to agree on ways to stop companies “inverting”—switching domicile to reduce tax bills (see&nbsp;<a href="http://www.economist.com/news/finance-and-economics/21618912-america-weighs-action-discourage-corporate-exodus-inverse-logic" target="_blank" rel="nofollow">article</a>). The European Union is locked in a protracted debate about whether the favourable treatment that some of its members give to particular forms of corporate revenue are tantamount to illegal subsidies. So the news that the world’s biggest economies have agreed on a plan to limit “base erosion and profit shifting” in corporate tax is something of a watershed.It has become the norm for multinationals to pa
rk themselves or large chunks of their assets—especially intangible ones, such as rights to royalties—in low- or no-tax places such as Bermuda, Luxembourg and Ireland. The wiliest, including Apple, have even discovered ways to re-route funds so as to...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618911-big-economies-take-aim-firms-running-circles-around-their-taxmen-transfer?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618911-big-economies-take-aim-firms-running-circles-around-their-taxmen-transfer</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>Tax inversions: Inverse logic</title>
+ <description><p>MANY countries fret about losing corporate taxes to accounting gymnastics; America worries about losing the companies themselves. This year several big firms have announced plans to merge with foreign partners and in the process shift their headquarters abroad, prompting much hand-wringing.The incentive is simple. America taxes profits no matter where they are earned, at a rate of 39%—higher than in any other rich country. When a company becomes foreign through a merger, or “inverts”, it no longer owes American tax on its foreign profit. It still owes American tax on its American profit. But that, too, can be minimised. Often, the group can shift debt to the American unit, or have it borrow from the foreign parent. It can then pay interest to the parent while deducting the sums involved from its American taxes. Several studies have found such “earnings stripping” common when companies invert. When Walgreens, an American chemist, announced
plans to merge with Swiss-based Alliance Boots, Barclays, a bank, reckoned the move could save $783m a year in taxes in this way.Walgreens is staying put, but fear of further defections is goading politicians. At present, tax-deductible interest paid to a related party may not exceed 50% of a firm’s cashflow. Two bills before Congress would lower that to 25%. Ron Wyden, the Democratic chairman of the Senate’s tax-writing committee, wants...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618912-america-weighs-action-discourage-corporate-exodus-inverse-logic?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618912-america-weighs-action-discourage-corporate-exodus-inverse-logic</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>CaixaBank: Now, make money</title>
+ <description><p>CAIXABANK is a rare breed in Spanish banking. The financial crisis brought most of the country’s 45 <em class="Italic">cajas—</em>regional savings banks with a commitment to social welfare—to their knees. But CaixaBank, based in Barcelona, has grown. On September 1st it became the largest domestic bank by loans, after buying the Spanish retail business of Barclays, a British bank.Many southern European banks are still defined by how they weathered the euro-zone crisis. A handful are expected to fail upcoming stress tests conducted by the European Central Bank. CaixaBank not only survived without state aid, but used the crisis to expand. Its new chief executive, Gonzalo Gortázar, is unperturbed by the ECB’s probe. But even though the economy is growing again, banking in Spain is still a tough business.The bank avoided the worst of the euphoric lending during Spain’s decade-long construction boom that felled many of it
s peers. Its current exposure to property is €31 billion ($40 billion), but it has set aside 42% of that sum in provisions. Some of its doubtful debts came from hoovering up troubled savings banks like Banca Cívica in 2012; that deal increased its loans by a fifth but the dud ones by half. The share of its loans that have soured, at 10.8%, is lower than Spanish average of 13.1%, and falling. The bank also has a bigger cushion of capital than most of its peers....</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618914-rare-survivor-among-spanish-savings-banks-tries-boost-profits-now-make-money?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618914-rare-survivor-among-spanish-savings-banks-tries-boost-profits-now-make-money</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>Shareholder rights: Out of control</title>
+ <description><p><div class="content-image-full">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/full-width/20140920_FNP001.jpg" alt="" title="" width="595" height="335" />
+
+ </div>FOR New York Stock Exchange (NYSE), the listing of Alibaba, a giant Chinese e-commerce website, seems like a triumph. As <em class="Italic">The Economist</em> went to press, the firm was pricing the offering; its shares were due to begin trading on September 19th. Amid all the excitement about whether the IPO would prove the world’s biggest, another of its striking features has been largely forgotten: shareholders will have little control over how the firm is run.Alibaba only listed in New York because Hong Kong Stock Exchange, a more natural home, insists that shareholders have a say over management in keeping with their stake. The firm’s owners, who balked at this notion, took their business to a more pliable venue. Technically, every Alibaba share has equal rights, but they are circumscribed ones. A pre-defined cabal of 30 managers of Alibaba or related companies, including the firm’s chairman, Jack Ma (pictured above), will
control nominations to a majority of seats on the board. According to Alibaba’s prospectus, this group “may make decisions with which you [the shareholder] disagree, including decisions on important topics such as compensation, management succession, acquisition...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618889-more-worlds-big-stockmarkets-are-allowing-firms-alibaba-sideline?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618889-more-worlds-big-stockmarkets-are-allowing-firms-alibaba-sideline</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>Buttonwood: Can’t pay, won’t pay</title>
+ <description><p><div class="content-image-full">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/full-width/images/print-edition/20140920_FND001_0.jpg" alt="" title="" width="595" height="335" />
+
+ </div>HAVE the whizz kids of finance lost their va-va-voom? For decades, hedge funds have been portrayed as the smart money, with the power to frighten chief executives and destabilise governments; rich individuals and powerful institutions competed to give them money. But the announcement on September 15th, by CalPERS, California’s main public pension fund, that it was unwinding its $4 billion hedge-fund portfolio, is a significant blow to the sector’s appeal.The CalPERS press release specifically says that the decision is not based on the performance of the programme. But Ted Eliopoulos, the chief investment officer, said that “when judged against their complexity, cost and the lack of ability to scale at CalPERS’ size”, the allocation to hedge funds was no longer warranted.It is hard to believe that, if the performance of the programme had been stellar, the pension fund would have axed it. But the reference to scale is also striking. Very small pen
sion funds tend not to have the money or the expertise to invest in hedge funds. Now CalPERS (the sixth-biggest pension fund in the world, according to TowersWatson, an actuarial consultant) is saying that it is too big to...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618899-hiring-hedge-funds-was-never-going-make-pension-deficits-disappear-cant-pay?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618899-hiring-hedge-funds-was-never-going-make-pension-deficits-disappear-cant-pay</guid>
+ <pubDate>Thu, 18 September 2014 15:01:07 GMT</pubDate>
+ </item>
+ <item>
+ <title>China’s economy: A test of will</title>
+ <description><p><div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140920_FNC733.png" alt="" title="" width="580" height="562" />
+
+ </div>WHEN Li Keqiang, China’s prime minister, spoke at a big business meeting earlier this month, he trumpeted two achievements. Not only had the government overseen steady economic growth, he said, but it had done so without resorting to a big stimulus. Both assertions are now looking rather doubtful.A barrage of data for August pointed to a sudden weakening in growth, catching many analysts and investors by surprise. Although it is unwise to read too much into one month’s numbers, the figures had a distressingly uniform downward tilt. Investment, retail sales and credit issuance all slowed. Industrial output, which is closely correlated with GDP given the size of China’s manufacturing sector, grew at its weakest pace since late 2008, when the global financial crisis was battering the economy. Housing sales, already struggling, contracted further; they have fallen 8% so far this year. That has started to eat into the revenues of local governments, since
property developers are holding back on land purchases. Yao Wei of Société Générale, a French bank, called it a “shockingly sharp” deceleration.Until the gloomy data started to pile up, China’s economy...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21618913-after-sharp-slowdown-stimulus-back-agenda-test-will?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21618913-after-sharp-slowdown-stimulus-back-agenda-test-will</guid>
+ <pubDate>Thu, 18 September 2014 10:33:16 GMT</pubDate>
+ </item>
+ <item>
+ <title>Buttonwood: Double agents</title>
+ <description><p><div class="content-image-full">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/full-width/images/print-edition/20140913_FND001_0.jpg" alt="" title="" width="595" height="335" />
+
+ </div>MOST financial assets these days are not held directly by investors but by professional fund managers on their behalf. And that separation may explain a couple of big anomalies that undermine belief in the efficiency of markets.The most striking anomaly is momentum—the fact that an asset that has recently been rising in price typically continues to do so. If markets were completely efficient, then past price movements should tell you nothing about the future. But lots of commodity-trading advisers, a type of hedge fund, make their living using sophisticated formulae to exploit this mysterious but abiding tendency.A new <a href="http://www.risk.net/journal-of-investment-strategies/technical-paper/2349968/two-centuries-of-trend-following" target="_blank" rel="nofollow">paper</a> from fund managers at one such group, Capital Fund Management, says the approach has been remarkably and persistently successful. Using d
ata for stockmarkets and commodities, the authors were able to go all the way back to 1693 (for British shares) and 1784 (for sugar).Their back-tested strategy was to buy those assets whose price was higher than its average over the previous five months,...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617013-way-investors-choose-fund-managers-may-cause-anomalies-markets-double?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617013-way-investors-choose-fund-managers-may-cause-anomalies-markets-double</guid>
+ <pubDate>Thu, 11 September 2014 14:54:23 GMT</pubDate>
+ </item>
+ <item>
+ <title>Islamic finance: Big interest, no interest</title>
+ <description><p><div class="content-image-full">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/full-width/images/print-edition/20140913_FNP001_0.jpg" alt="" title="" width="595" height="335" />
+
+ </div>AFTER morning coffee but before the keynote speaker came the muezzin’s recitation from the Koran: “Those who consume interest cannot stand except as one stands who is being beaten by Satan into insanity.” But those attending the Global Islamic Financial Forum needed no reminders that Muslims are supposed to eschew interest: the industry based on that premise is booming. Ernst & Young, a consultancy and accounting firm, estimates that Islamic banking assets grew at an annual rate of 17.6% between 2009 and 2013, and will grow by an average of 19.7% a year to 2018. Khalid Howladar of Moody’s, a rating agency, calls this “a landmark year” for Islamic finance, in that it is moving from “a very esoteric asset class to one that’s more… global.”Most of the world’s Muslims are not so devout that they completely abjure conventional finance: even in Saudi Arabia, the assets of Islamic banks account for barely half of all banking assets. Mus
lim account-holders, Mr Howladar explains, tend to be more concerned with the products and service on offer than with the strictures of <em class="Italic">sharia</em> (rules based on Muslim scripture). But Islamic finance, he says, has...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617014-market-islamic-financial-products-growing-fast-big-interest-no-interest?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617014-market-islamic-financial-products-growing-fast-big-interest-no-interest</guid>
+ <pubDate>Thu, 11 September 2014 14:54:23 GMT</pubDate>
+ </item>
+ <item>
+ <title>“Coco” bonds: Mass conversion</title>
+ <description><p><div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140913_FNC673.png" alt="" title="" width="580" height="562" />
+
+ </div>IT TOOK three separate bail-outs to get KBC, a Belgian bank, through the financial crisis. So one might expect bonds that automatically get wiped out if the bank runs into trouble again to reward investors handsomely. Not so: KBC’s “contingent convertible” bonds, designed to lose all their value in a crisis, yield a meagre 4% a year. Egged on by investors, banks are issuing such securities in growing amounts. Regulators are watching anxiously.Cocos, as these instruments are known, are a newish hybrid of bank equity (the money invested by shareholders, which absorbs any losses in the first instance) and debt (which must be repaid unless a bank runs out of equity). Regulators globally are keen on banks having more equity. This makes bail-outs less likely and, if they prove inevitable, less painful for taxpayers. Bankers prefer debt because it lowers their tax bill, and juices both profits and bonuses. Cocos are the compromise.Cocos take multiple forms
, but all are intended to behave like bonds when times are good, yet absorb losses, equity-like, in a crisis. At a given trigger point, when equity levels are so low that bankruptcy threatens, cocos...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617028-investors-are-panting-risky-new-form-bonds-issued-banks-mass-conversion?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617028-investors-are-panting-risky-new-form-bonds-issued-banks-mass-conversion</guid>
+ <pubDate>Thu, 11 September 2014 14:54:23 GMT</pubDate>
+ </item>
+ <item>
+ <title>The bank that Botín built: A banking giant and a giant bank</title>
+ <description><p><div class="content-image-full retina-595">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140913_FNC680.png" alt="" title="" width="1190" height="596" />
+
+ </div>Over the past 30 years Santander has grown from a minnow to the euro zone’s biggest bank, thanks to the dealmaking of its chairman, Emilio Botín, who died on September 9th. It even did well out of the carve-up of ABN Amro—a deal that ruined two bigger rivals, RBS and Fortis—snapping up Banco Real, its resilient Brazilian unit.</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617029-banking-giant-and-giant-bank?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617029-banking-giant-and-giant-bank</guid>
+ <pubDate>Thu, 11 September 2014 14:54:23 GMT</pubDate>
+ </item>
+ <item>
+ <title>The euro-zone economy: Asset-backed indolence</title>
+ <description><p><div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140913_FNC666.png" alt="" title="" width="580" height="562" />
+
+ </div>THE euro-zone recovery has stalled and inflation is close to zero. Belatedly, the European Central Bank (ECB) is trying to galvanise a lifeless economy, in two ways beyond ultra-low interest rates: a funding scheme to boost banks’ lending to firms and the purchase of private-sector assets.On September 18th it will conduct the first of two special lending operations this year that will make up to €400 billion ($520 billion, or 4% of euro-zone GDP) available to banks on extremely generous terms: for up to four years at a fixed annual interest rate of just 0.15%. The only condition is that banks raise the trajectory of their lending to business, which for many may mean that their stock of corporate loans keeps on shrinking but more slowly than before. In 2015 and 2016 more funding will be available provided that banks actually raise their net lending.These “targeted” long-term loans tackle a flaw in the ECB’s previous such scheme, in the winter of
2011-12, when banks in southern Europe used cheap three-year funding to buy sovereign debt rather than to lend more to companies. But not all the funding on offer will be taken up. Banks in southern Europe...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617030-european-central-banks-plan-economic-revival-underwhelming-asset-backed?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617030-european-central-banks-plan-economic-revival-underwhelming-asset-backed</guid>
+ <pubDate>Thu, 11 September 2014 14:54:23 GMT</pubDate>
+ </item>
+ <item>
+ <title>Japan’s economy: Slings and arrows</title>
+ <description><p><div class="content-image-float-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/290-width/images/print-edition/20140913_FNP002_0.jpg" alt="" title="" width="290" height="502" />
+ <span class="caption">Looking for the third arrow</span>
+ </div>IT IS crisis mode in the <em class="Italic">Kantei</em>, the office of Shinzo Abe, Japan’s prime minister. A succession of awful data has pummelled his economic programme, which consists of three “arrows”: a radical monetary easing, a big fiscal stimulus and a series of structural reforms. On September 8th revised figures showed that GDP shrank by 1.8% in the second quarter, or by 7.1% on an annualised basis, even worse than the initial estimate of 1.7%.Architects of Abenomics had issued dire warnings against raising the consumption tax from 5% to 8% in April. The increase, aimed at improving Japan’s parlous finances, was decided in 2012 under a previous government. As in 1997, the last time politicians dared to raise the tax, consumer spending has wilted.The government also misjudged the effects of the Bank of Japan’s huge asset purchases (quantitative easing, or QE, in the jargon). It is buying ¥7 trillion ($65 billion)
of Japanese government bonds a month, and now owns a fifth of the government’s outstanding debt. A weaker yen, one of QE’s chief consequences, has failed to produce the boom in exports...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617031-harmful-tax-hike-and-reticent-employers-take-their-toll-slings-and-arrows?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617031-harmful-tax-hike-and-reticent-employers-take-their-toll-slings-and-arrows</guid>
+ <pubDate>Thu, 11 September 2014 14:54:23 GMT</pubDate>
+ </item>
+ <item>
+ <title>Free exchange: No country for old money</title>
+ <description><p><div class="content-image-full retina-595">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140913_FNC679.png" alt="" title="" width="1190" height="696" />
+
+ </div>WHEN it comes to money and banking, no country has a richer history than Scotland. It was a Scot, David Hume, who in a 1748 essay set out the first coherent theory of the links between money, inflation and growth. And it is a Scottish idea, the joint-stock bank that, starting in the mid-1800s, became the backbone of global finance. Yet Scotland’s monetary future is as uncertain as its past is proud. Nationalist leaders say that if voters opt for independence in a poll scheduled for September 18th, the country will retain the pound in a currency union with the residual United Kingdom (rUK). That is a terrible idea.The currency union that Scotland’s nationalist party, the SNP, envisages would be similar to the euro area. Sterling notes and coins would continue to circulate in Scotland, with the Bank of England setting a single interest rate for both countries and standing behind Scottish lenders in times of crisis. Such a union would eliminate exchange-
rate risk and the cost of currency conversions, bolstering trade. Since the trade in goods and services between Scotland and rUK ran to £110 billion ($178 billion) in 2013—around two-thirds of Scotland’s...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21617015-if-scotland-gains-independence-it-will-need-new-currency-and-new-central?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21617015-if-scotland-gains-independence-it-will-need-new-currency-and-new-central</guid>
+ <pubDate>Thu, 11 September 2014 09:33:12 GMT</pubDate>
+ </item>
+ <item>
+ <title>Commodity trading: End-to-end game</title>
+ <description><p><div class="content-image-float-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/290-width/images/print-edition/20140906_FNP004_0.jpg" alt="" title="" width="290" height="435" />
+
+ </div>BANKS, harried by regulators and short of capital, are fleeing the commodities business. Deutsche Bank, Morgan Stanley and UBS either shuttered or shrank their commodities operations last year; this year Barclays, Credit Suisse and JPMorgan Chase have scaled back. But even as they retreat, commodity-trading houses, most of which began life as simple middlemen, are getting ever more deeply involved in the extraction, shipping and refining of raw materials.The buyer of JPMorgan Chase’s physical commodities unit, for instance, was Mercuria, a ten-year-old firm based in Switzerland that started out trading oil but now owns (or has joint ventures with) oil-exploration companies, oil-terminal and pipeline operators, coal and iron-ore mines and biofuel refineries. Vertical integration of this sort gives trading operations more flexibility and brings valuable commercial intelligence, but it also pushes the firms into capital-intensive businesses and compounds t
heir exposure to the commodities cycle.America has some big commodity firms, including Archer Daniels Midland, Cargill and Koch Industries. But the real behemoths are based in Switzerland. Vitol, which started...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21615620-commodity-trading-houses-are-growingand-running-more-risks-end-end-game?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21615620-commodity-trading-houses-are-growingand-running-more-risks-end-end-game</guid>
+ <pubDate>Thu, 04 September 2014 15:08:06 GMT</pubDate>
+ </item>
+ <item>
+ <title>Brazil’s economy: Better than Ukraine</title>
+ <description><p><div class="content-image-full">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/full-width/images/print-edition/20140906_FNP001_0.jpg" alt="" title="" width="595" height="335" />
+
+ </div>“WE ARE not in a recession,” insisted Guido Mantega, Brazil’s finance minister, on August 29th. According to the most common definition—two consecutive quarters of falling output—he is wrong. Official figures released earlier that day showed that GDP fell by 0.6% between the first and second quarters (an annualised contraction of 2.4%). Output also fell, by 0.2%, in the first three months of the year.Brazil’s economy has now shrunk in three of the last four quarters. Most analysts think it will not grow at all this year; a year ago they were expecting growth of 3%. In 2015 the economy is likely to expand by only 1%. Not even Mr Mantega can deny that Brazil is going through a rough patch.The government blames a weak global recovery from the financial crisis of 2008-09 and a surfeit of public holidays during the month-long football World Cup, which concluded in Brazil on July 13th. These were decreed by federal, state and municipal authorities t
o ease pressure on public transport as hordes of fans descended on host cities. Itaú, a big Brazilian bank, reckons fewer working days account for half the latest fall in GDP. (Critics note that the <em class="Italic">Copa...</em></p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21615624-government-shrugs-brazils-descent-recession-better-ukraine?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21615624-government-shrugs-brazils-descent-recession-better-ukraine</guid>
+ <pubDate>Thu, 04 September 2014 15:08:06 GMT</pubDate>
+ </item>
+ <item>
+ <title>China’s shadow banks: A moving target</title>
+ <description><p><div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140906_FNC623.png" alt="" title="" width="580" height="526" />
+
+ </div>WILL rising defaults and stricter rules halt the breakneck growth of China’s shadow banks? When one of the country’s many trust companies, which sell high-yield investments, warned earlier this year of a looming default on one of its products, its clients reacted with anger and the wider market with alarm. As panic spread, regulators orchestrated a bail-out of the product, reassuringly named “Credit Equals Gold #1”. But in recent weeks investors in its sibling, “Credit Equals Gold #2”, have met a crueler fate. It is backed by loans to a bankrupt coal-mining firm which came due in July and have since gone unpaid. Investors will not get their money back until collateral can be seized and sold. That process may take more than a year. The episode, naturally, calls into question the widespread belief that such investments are safe because they are marketed by big, state-owned financial institutions.Shadow banks, which barely existed before China’
s credit surge in 2009, now have assets of at least 30 trillion yuan ($4.9 trillion), or more than 50% of GDP, according to estimates by ANZ, a bank. The government’s attempts to slow the pell-mell growth in...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21615625-chinas-shape-shifting-shadow-banks-evolve-once-more-moving-target?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21615625-chinas-shape-shifting-shadow-banks-evolve-once-more-moving-target</guid>
+ <pubDate>Thu, 04 September 2014 15:08:06 GMT</pubDate>
+ </item>
+ <item>
+ <title>Wage stagnation: The big freeze</title>
+ <description><p><div class="content-image-full">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/full-width/20140906_FND001.jpg" alt="" title="" width="595" height="335" />
+
+ </div>CENTRAL bankers once used to inveigh against wage inflation. Guarding against a return to the ruinous price-wage spirals of the 1970s was a constant preoccupation. Since the financial crisis, however, they have started to fret about the opposite concern: stagnant wages and the growing risk of deflation.There has been a squeeze on pay in the rich world for several years now. Between 2010 and 2013 real (inflation-adjusted) wages were flat across the OECD, according to its annual “Employment Outlook”, published on September 3rd. Real wages have barely grown at all in America over that period and have fallen in the euro area and Japan (see chart). Declines have been particularly sharp in the troubled peripheral economies of the euro zone, such as Portugal and Spain, but real wages have also tumbled in Britain.<div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/20140906_FNC637.png" alt="" title="" width="580" height="598" />
+
+ </div>These sharp adjustments have hurt but were in large part unavoidable. Real wages can grow in the long run only at the pace of productivity. If productivity has deteriorated, as...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21615589-throughout-rich-world-wages-are-stuck-big-freeze?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21615589-throughout-rich-world-wages-are-stuck-big-freeze</guid>
+ <pubDate>Thu, 04 September 2014 15:08:06 GMT</pubDate>
+ </item>
+ <item>
+ <title>Buttonwood: American exceptionalism</title>
+ <description><p><div class="content-image-float-290 retina-290">
+
+ <img src="http://cdn.static-economist.com/sites/default/files/imagecache/original-size/images/print-edition/20140906_FNC622.png" alt="" title="" width="580" height="562" />
+
+ </div>FOR more than a decade, Japan has been perceived as the rich world’s exception; it has been mired in deflation, with slow growth and ultra-low bond yields. Economists have debated whether its problems are down to its ageing population, conservative monetary policy or outdated business models.But these days western Europe looks quite Japan-like. Inflation in the euro zone has been heading remorselessly downwards and was just 0.3% in the year to August. The trend rate of economic growth is perceived to be 1% or below. Ten-year bond yields in Germany and Switzerland are well under 1%. Albert Edwards, a strategist at Société Générale, has dubbed these conditions an “ice age” which he predicts will extend across the rich world.If there is an exception in the rich world, it looks like America. The gap between yields on its bonds and on Germany’s is now almost as wide as it has been at any point since the euro was created (see chart). Despite a rally
since the depths of the euro crisis, European equities trade at much lower valuations than those on Wall Street: the cyclically-adjusted price-earnings ratio (or CAPE) for the euro zone is 14.7, according to...</p></description>
+ <link>http://www.economist.com/news/finance-and-economics/21615598-japan-no-longer-odd-one-out-rich-world-american-exceptionalism?fsrc=rss|fec</link>
+ <guid isPermaLink = "false">http://www.economist.com/news/finance-and-economics/21615598-japan-no-longer-odd-one-out-rich-world-american-exceptionalism</guid>
+ <pubDate>Thu, 04 September 2014 15:08:06 GMT</pubDate>
+ </item>
+ </channel>
+</rss>
\ No newline at end of file