You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-dev@jackrabbit.apache.org by Ryan Chesnut <RC...@manh.com> on 2015/03/23 16:33:38 UTC
Oak, Mongo Issues
To whom it may concern,
I'm currently researching using Oak in conjunction with Mongo for a scalable content repository. I've done a proof of concept without any issues but now that I'm running some tests I keep coming upon a problem. Below is exception I keep getting. This happens when the number of files is over 100k.
Exception in thread "main" org.apache.jackrabbit.oak.plugins.document.DocumentStoreException
at org.apache.jackrabbit.oak.plugins.document.DocumentStoreException.convert(DocumentStoreException.java:48)
at org.apache.jackrabbit.oak.plugins.document.DocumentStoreException.convert(DocumentStoreException.java:41)
at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.findAndModify(MongoDocumentStore.java:633)
at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.createOrUpdate(MongoDocumentStore.java:645)
at org.apache.jackrabbit.oak.plugins.document.Commit.createOrUpdateNode(Commit.java:419)
at org.apache.jackrabbit.oak.plugins.document.Commit.applyToDocumentStore(Commit.java:323)
at org.apache.jackrabbit.oak.plugins.document.Commit.prepare(Commit.java:205)
at org.apache.jackrabbit.oak.plugins.document.Commit.apply(Commit.java:171)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:308)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:278)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.access$200(DocumentNodeStoreBranch.java:51)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$Persisted.persistTransientHead(DocumentNodeStoreBranch.java:653)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$Persisted.access$000(DocumentNodeStoreBranch.java:543)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$BranchState.persist(DocumentNodeStoreBranch.java:370)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.setRoot(DocumentNodeStoreBranch.java:494)
at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.setRoot(DocumentNodeStoreBranch.java:161)
at org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.purge(DocumentRootBuilder.java:200)
at org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.updated(DocumentRootBuilder.java:100)
at org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder.updated(MemoryNodeBuilder.java:214)
at org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder.setProperty(MemoryNodeBuilder.java:501)
at org.apache.jackrabbit.oak.core.SecureNodeBuilder.setProperty(SecureNodeBuilder.java:258)
at org.apache.jackrabbit.oak.plugins.tree.impl.AbstractMutableTree.updateChildOrder(AbstractMutableTree.java:110)
at org.apache.jackrabbit.oak.plugins.tree.impl.AbstractMutableTree.setOrderableChildren(AbstractMutableTree.java:94)
at org.apache.jackrabbit.oak.core.MutableTree.setOrderableChildren(MutableTree.java:210)
at org.apache.jackrabbit.oak.util.TreeUtil.addChild(TreeUtil.java:207)
at org.apache.jackrabbit.oak.jcr.delegate.NodeDelegate.addChild(NodeDelegate.java:692)
at org.apache.jackrabbit.oak.jcr.session.NodeImpl$5.perform(NodeImpl.java:287)
at org.apache.jackrabbit.oak.jcr.session.NodeImpl$5.perform(NodeImpl.java:254)
at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.perform(SessionDelegate.java:237)
at org.apache.jackrabbit.oak.jcr.session.ItemImpl.perform(ItemImpl.java:113)
at org.apache.jackrabbit.oak.jcr.session.NodeImpl.addNode(NodeImpl.java:254)
at org.apache.jackrabbit.oak.jcr.session.NodeImpl.addNode(NodeImpl.java:239)
at OakTest.addNode(OakTest.java:252)
at OakTest.createTree(OakTest.java:299)
at OakTest.main(OakTest.java:454)
Caused by: com.mongodb.CommandFailureException: { "serverUsed" : "127.0.0.1:27017" , "value" : { "_id" : "1:/SHIP" , ":childOrder" : { "r14c382dbec5-0-1" : "[\"nam:SHIP20180\",\"nam:SHIP20181\",\"nam:SHIP20182\",\"nam:SHIP20183\",\"nam:SHIP20184\",\"nam:SHIP20185\",\"nam:SHIP20186\",\"nam:SHIP20187\",\"nam:SHIP20188\",\"nam:SHIP20189\",\"nam:SHIP201810\",\"nam:SHIP201811\",\"nam:SHIP201812\",\"nam:SHIP201813\",\"nam:SHIP201814\",\"nam:SHIP201815\",\"nam:SHIP201816\",\"nam:SHIP201817\",\"nam:SHIP201818\",\"nam:SHIP201819\",\"nam:SHIP201820\",\"nam:SHIP201821\",\"nam:SHIP201822\",\"nam:SHIP201823\",\"nam:SHIP201824\",\"nam:SHIP201825\",\"nam:SHIP201826\",\"nam:SHIP201827\",\"nam:SHIP201828\",\"nam:SHIP201829\",\"nam:SHIP201830\",\"nam:SHIP201831\",\"nam:SHIP201832\",\"nam:SHIP201833\",\"nam:SHIP201834\",\"nam:SHIP201835\",\"nam:SHIP201836\",\"nam:SHIP201837\",\"nam:SHIP201838\",\"nam:SHIP201839\",\"nam:SHIP201840\",\"nam:SHIP201841\",\"nam:SHIP201842\",\"nam:SHIP201843\",\"nam:SHIP201844\",\"nam:SHIP201845\",\"nam:SHIP201846\",\"nam:SHIP201847\",\"nam:SHIP201848\",\"nam:SHIP201849\",\"nam:SHIP201850\",\"nam:SHIP201851\",\"nam:SHIP201852\",\"nam:SHIP201853\",\"nam:SHIP201854\",\"nam:SHIP201855\",\. ...........................................................................................................
.....................................................................................................................................
, "_modified" : 1427123335 , "_commitRoot" : { "r14c382dbec5-0-1" : "0" , "r14c382dc37c-0-1" : "0" , "r14c382dee74-0-2" : "0" , "r14c382df715-0-2" : "0" , "r14c3830655e-0-3" : "0" , "r14c3830a0cb-0-3" : "0" , "r14c3831e429-0-4" : "0" , "r14c383236f0-0-4" : "0" , "r14c383756f7-0-2" : "0" , "r14c38379f38-0-2" : "0" , "r14c3837aee3-0-5" : "0" , "r14c38380de4-0-5" : "0" , "r14c383a3c83-0-1" : "0" , "r14c383a97b1-0-1" : "0" , "r14c383c878d-0-3" : "0" , "r14c383d1951-0-3" : "0" , "r14c3846ae8b-0-2" : "0" , "r14c384745e9-0-2" : "0" , "r14c38479c39-0-4" : "0" , "r14c38489f3e-0-4" : "0" , "r14c3848b460-0-5" : "0" , "r14c38497f5f-0-5" : "0" , "r14c384f7f88-0-6" : "0" , "r14c3850ce10-0-6" : "0" , "r14c38544a70-0-4" : "0" , "r14c3854f4d1-0-4" : "0" , "r14c385959e0-0-1" : "0" , "r14c385a428a-0-1" : "0" , "r14c3869970f-0-3" : "0" , "r14c386a4a11-0-3" : "0" , "r14c466d3069-0-1" : "0" , "r14c466d46f5-0-1" : "0" , "r14c46702c86-0-2" : "0" , "r14c46704e48-0-2" : "0" , "r14c467137d4-0-3" : "0" , "r14c46716768-0-3" : "0" , "r14c4672e42c-0-4" : "0" , "r14c467332e3-0-4" : "0" , "r14c4677550d-0-5" : "0" , "r14c4677cea4-0-5" : "0" , "r14c468787a1-0-6" : "0" , "r14c46884aaf-0-6" : "0" , "r14c46ee618a-0-1" : "0" , "r14c46ef52e2-0-1" : "0"} , "_children" : true , "_deleted" : { "r14c382dbec5-0-1" : "false"} , "jcr:primaryType" : { "r14c382dbec5-0-1" : "\"nam:nt:unstructured\""} , "_modCount" : 11938 , "_lastRev" : { "r0-0-1" : "r14c472dfe47-0-1" , "r0-0-2" : "r14c46f7b050-0-2" , "r0-0-3" : "r14c46f8f74d-0-3" , "r0-0-4" : "r14c470245d9-0-4" , "r0-0-5" : "r14c470ab777-0-5" , "r0-0-6" : "r14c4713f3a6-2-6"}} , "errmsg" : "exception: Resulting document after update is larger than 16777216" , "code" : 17419 , "ok" : 0.0}
at com.mongodb.CommandResult.getException(CommandResult.java:76)
at com.mongodb.CommandResult.throwOnError(CommandResult.java:131)
at com.mongodb.DBCollection.findAndModify(DBCollection.java:462)
at com.mongodb.DBCollection.findAndModify(DBCollection.java:402)
at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.findAndModify(MongoDocumentStore.java:620)
... 32 more
Any help in this matter would be appreciated.
Thank you,
Ryan Chesnut
Re: Oak, Mongo Issues
Posted by Julian Sedding <js...@gmail.com>.
Hi Ryan
My guess is that you add all 100k documents under a single parent
node. Additionally your parent node has orderable children. Oak
accommodates large flat file lists in unorderable parent nodes, but
has limits when it comes to orderable parent nodes.
You could try running the same test, but change the node-type of the
parent folder to e.g. oak:Unstructured (or another node-type that
doesn't have orderable children).
Alternatively, you can introduce more hierarchy and thus reduce the
amount of documents within one parent node.
Regards
Julian
On Mon, Mar 23, 2015 at 4:33 PM, Ryan Chesnut <RC...@manh.com> wrote:
> To whom it may concern,
>
> I'm currently researching using Oak in conjunction with Mongo for a scalable content repository. I've done a proof of concept without any issues but now that I'm running some tests I keep coming upon a problem. Below is exception I keep getting. This happens when the number of files is over 100k.
>
>
> Exception in thread "main" org.apache.jackrabbit.oak.plugins.document.DocumentStoreException
> at org.apache.jackrabbit.oak.plugins.document.DocumentStoreException.convert(DocumentStoreException.java:48)
> at org.apache.jackrabbit.oak.plugins.document.DocumentStoreException.convert(DocumentStoreException.java:41)
> at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.findAndModify(MongoDocumentStore.java:633)
> at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.createOrUpdate(MongoDocumentStore.java:645)
> at org.apache.jackrabbit.oak.plugins.document.Commit.createOrUpdateNode(Commit.java:419)
> at org.apache.jackrabbit.oak.plugins.document.Commit.applyToDocumentStore(Commit.java:323)
> at org.apache.jackrabbit.oak.plugins.document.Commit.prepare(Commit.java:205)
> at org.apache.jackrabbit.oak.plugins.document.Commit.apply(Commit.java:171)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:308)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:278)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.access$200(DocumentNodeStoreBranch.java:51)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$Persisted.persistTransientHead(DocumentNodeStoreBranch.java:653)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$Persisted.access$000(DocumentNodeStoreBranch.java:543)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$BranchState.persist(DocumentNodeStoreBranch.java:370)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.setRoot(DocumentNodeStoreBranch.java:494)
> at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.setRoot(DocumentNodeStoreBranch.java:161)
> at org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.purge(DocumentRootBuilder.java:200)
> at org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.updated(DocumentRootBuilder.java:100)
> at org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder.updated(MemoryNodeBuilder.java:214)
> at org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder.setProperty(MemoryNodeBuilder.java:501)
> at org.apache.jackrabbit.oak.core.SecureNodeBuilder.setProperty(SecureNodeBuilder.java:258)
> at org.apache.jackrabbit.oak.plugins.tree.impl.AbstractMutableTree.updateChildOrder(AbstractMutableTree.java:110)
> at org.apache.jackrabbit.oak.plugins.tree.impl.AbstractMutableTree.setOrderableChildren(AbstractMutableTree.java:94)
> at org.apache.jackrabbit.oak.core.MutableTree.setOrderableChildren(MutableTree.java:210)
> at org.apache.jackrabbit.oak.util.TreeUtil.addChild(TreeUtil.java:207)
> at org.apache.jackrabbit.oak.jcr.delegate.NodeDelegate.addChild(NodeDelegate.java:692)
> at org.apache.jackrabbit.oak.jcr.session.NodeImpl$5.perform(NodeImpl.java:287)
> at org.apache.jackrabbit.oak.jcr.session.NodeImpl$5.perform(NodeImpl.java:254)
> at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.perform(SessionDelegate.java:237)
> at org.apache.jackrabbit.oak.jcr.session.ItemImpl.perform(ItemImpl.java:113)
> at org.apache.jackrabbit.oak.jcr.session.NodeImpl.addNode(NodeImpl.java:254)
> at org.apache.jackrabbit.oak.jcr.session.NodeImpl.addNode(NodeImpl.java:239)
> at OakTest.addNode(OakTest.java:252)
> at OakTest.createTree(OakTest.java:299)
> at OakTest.main(OakTest.java:454)
> Caused by: com.mongodb.CommandFailureException: { "serverUsed" : "127.0.0.1:27017" , "value" : { "_id" : "1:/SHIP" , ":childOrder" : { "r14c382dbec5-0-1" : "[\"nam:SHIP20180\",\"nam:SHIP20181\",\"nam:SHIP20182\",\"nam:SHIP20183\",\"nam:SHIP20184\",\"nam:SHIP20185\",\"nam:SHIP20186\",\"nam:SHIP20187\",\"nam:SHIP20188\",\"nam:SHIP20189\",\"nam:SHIP201810\",\"nam:SHIP201811\",\"nam:SHIP201812\",\"nam:SHIP201813\",\"nam:SHIP201814\",\"nam:SHIP201815\",\"nam:SHIP201816\",\"nam:SHIP201817\",\"nam:SHIP201818\",\"nam:SHIP201819\",\"nam:SHIP201820\",\"nam:SHIP201821\",\"nam:SHIP201822\",\"nam:SHIP201823\",\"nam:SHIP201824\",\"nam:SHIP201825\",\"nam:SHIP201826\",\"nam:SHIP201827\",\"nam:SHIP201828\",\"nam:SHIP201829\",\"nam:SHIP201830\",\"nam:SHIP201831\",\"nam:SHIP201832\",\"nam:SHIP201833\",\"nam:SHIP201834\",\"nam:SHIP201835\",\"nam:SHIP201836\",\"nam:SHIP201837\",\"nam:SHIP201838\",\"nam:SHIP201839\",\"nam:SHIP201840\",\"nam:SHIP201841\",\"nam:SHIP201842\",\"nam:SHIP201843\",\"nam:SHIP201844\",\"nam:SHIP201845\",\"nam:SHIP201846\",\"nam:SHIP201847\",\"nam:SHIP201848\",\"nam:SHIP201849\",\"nam:SHIP201850\",\"nam:SHIP201851\",\"nam:SHIP201852\",\"nam:SHIP201853\",\"nam:SHIP201854\",\"nam:SHIP201855\",\. ...........................................................................................................
> .....................................................................................................................................
>
> , "_modified" : 1427123335 , "_commitRoot" : { "r14c382dbec5-0-1" : "0" , "r14c382dc37c-0-1" : "0" , "r14c382dee74-0-2" : "0" , "r14c382df715-0-2" : "0" , "r14c3830655e-0-3" : "0" , "r14c3830a0cb-0-3" : "0" , "r14c3831e429-0-4" : "0" , "r14c383236f0-0-4" : "0" , "r14c383756f7-0-2" : "0" , "r14c38379f38-0-2" : "0" , "r14c3837aee3-0-5" : "0" , "r14c38380de4-0-5" : "0" , "r14c383a3c83-0-1" : "0" , "r14c383a97b1-0-1" : "0" , "r14c383c878d-0-3" : "0" , "r14c383d1951-0-3" : "0" , "r14c3846ae8b-0-2" : "0" , "r14c384745e9-0-2" : "0" , "r14c38479c39-0-4" : "0" , "r14c38489f3e-0-4" : "0" , "r14c3848b460-0-5" : "0" , "r14c38497f5f-0-5" : "0" , "r14c384f7f88-0-6" : "0" , "r14c3850ce10-0-6" : "0" , "r14c38544a70-0-4" : "0" , "r14c3854f4d1-0-4" : "0" , "r14c385959e0-0-1" : "0" , "r14c385a428a-0-1" : "0" , "r14c3869970f-0-3" : "0" , "r14c386a4a11-0-3" : "0" , "r14c466d3069-0-1" : "0" , "r14c466d46f5-0-1" : "0" , "r14c46702c86-0-2" : "0" , "r14c46704e48-0-2" : "0" , "r14c467137d4-0-3" : "0" , "r14c46716768-0-3" : "0" , "r14c4672e42c-0-4" : "0" , "r14c467332e3-0-4" : "0" , "r14c4677550d-0-5" : "0" , "r14c4677cea4-0-5" : "0" , "r14c468787a1-0-6" : "0" , "r14c46884aaf-0-6" : "0" , "r14c46ee618a-0-1" : "0" , "r14c46ef52e2-0-1" : "0"} , "_children" : true , "_deleted" : { "r14c382dbec5-0-1" : "false"} , "jcr:primaryType" : { "r14c382dbec5-0-1" : "\"nam:nt:unstructured\""} , "_modCount" : 11938 , "_lastRev" : { "r0-0-1" : "r14c472dfe47-0-1" , "r0-0-2" : "r14c46f7b050-0-2" , "r0-0-3" : "r14c46f8f74d-0-3" , "r0-0-4" : "r14c470245d9-0-4" , "r0-0-5" : "r14c470ab777-0-5" , "r0-0-6" : "r14c4713f3a6-2-6"}} , "errmsg" : "exception: Resulting document after update is larger than 16777216" , "code" : 17419 , "ok" : 0.0}
> at com.mongodb.CommandResult.getException(CommandResult.java:76)
> at com.mongodb.CommandResult.throwOnError(CommandResult.java:131)
> at com.mongodb.DBCollection.findAndModify(DBCollection.java:462)
> at com.mongodb.DBCollection.findAndModify(DBCollection.java:402)
> at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.findAndModify(MongoDocumentStore.java:620)
> ... 32 more
>
>
>
>
> Any help in this matter would be appreciated.
>
> Thank you,
> Ryan Chesnut
>
>
>
>
>