You are viewing a plain text version of this content. The canonical link for it is here.
Posted to triplesoup-commits@incubator.apache.org by le...@apache.org on 2007/04/13 08:56:16 UTC
svn commit: r528394 [21/35] - in
/incubator/triplesoup/donations/TRIPLES-3-RDFStore: ./ dbms/ dbms/client/
dbms/client/t/ dbms/dbmsproxy/ dbms/deamon/ dbms/doc/ dbms/include/
dbms/libdbms/ dbms/utils/ doc/ include/ lib/ lib/DBD/ lib/RDFStore/
lib/RDFSt...
Added: incubator/triplesoup/donations/TRIPLES-3-RDFStore/rdfstore_kernel.c
URL: http://svn.apache.org/viewvc/incubator/triplesoup/donations/TRIPLES-3-RDFStore/rdfstore_kernel.c?view=auto&rev=528394
==============================================================================
--- incubator/triplesoup/donations/TRIPLES-3-RDFStore/rdfstore_kernel.c (added)
+++ incubator/triplesoup/donations/TRIPLES-3-RDFStore/rdfstore_kernel.c Fri Apr 13 01:56:01 2007
@@ -0,0 +1,11508 @@
+/*
+ * Copyright (c) 2000-2006 All rights reserved,
+ * Alberto Reggiori <ar...@webweaving.org>,
+ * Dirk-Willem van Gulik <di...@webweaving.org>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if any, must
+ * include the following acknowledgment: "This product includes software
+ * developed by Alberto Reggiori <ar...@webweaving.org> and Dirk-Willem
+ * van Gulik <di...@webweaving.org>." Alternately, this acknowledgment may
+ * appear in the software itself, if and wherever such third-party
+ * acknowledgments normally appear.
+ *
+ * 4. All advertising materials mentioning features or use of this software must
+ * display the following acknowledgement: This product includes software
+ * developed by the University of California, Berkeley and its contributors.
+ *
+ * 5. Neither the name of the University nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * 6. Products derived from this software may not be called "RDFStore" nor may
+ * "RDFStore" appear in their names without prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * ====================================================================
+ *
+ * This software consists of work developed by Alberto Reggiori and Dirk-Willem
+ * van Gulik. The RDF specific part is based based on public domain software
+ * written at the Stanford University Database Group by Sergey Melnik. For
+ * more information on the RDF API Draft work, please see
+ * <http://www-db.stanford.edu/~melnik/rdf/api.html> The DBMS TCP/IP server
+ * part is based on software originally written by Dirk-Willem van Gulik for
+ * Web Weaving Internet Engineering m/v Enschede, The Netherlands.
+ *
+ * $Id: rdfstore_kernel.c,v 1.113 2006/06/19 10:10:21 areggiori Exp $
+ *
+ */
+
+#if !defined(WIN32)
+#include <sys/param.h>
+#endif
+
+#include <sys/types.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <strings.h>
+#include <fcntl.h>
+
+#include <netinet/in.h>
+
+#include <time.h>
+#include <sys/stat.h>
+
+#include "rdfstore_log.h"
+#include "rdfstore_ms.h"
+#include "rdfstore.h"
+#include "rdfstore_iterator.h"
+#include "rdfstore_serializer.h"
+#include "rdfstore_digest.h"
+#include "rdfstore_bits.h"
+#include "rdfstore_utf8.h"
+#include "rdfstore_xsd.h"
+
+/*
+#define MX { printf(" MX %s:%d - %p\n",__FILE__,__LINE__,me->nindex->free); }
+*/
+#define MX
+
+/*
+ * #define RDFSTORE_DEBUG
+ */
+
+/*
+ * #define RDFSTORE_CONNECTIONS
+ */
+/*
+ * #define RDFSTORE_DEBUG_CONNECTIONS
+ */
+/*
+ * #define RDFSTORE_CONNECTIONS_REINDEXING
+ */
+
+char * rdfstore_get_version (
+ rdfstore * me
+ ) {
+ return VERSION;
+ };
+
+int
+rdfstore_connect(
+ rdfstore * *mme,
+ char *name,
+ int flags,
+ int freetext,
+ int sync,
+ int remote,
+ char *host,
+ int port,
+/* Callbacks for memory management and error handling. */
+ void *(*_mmalloc) (size_t s),
+ void (*_mfree) (void *adr),
+ void (*_mcallback) (dbms_cause_t cause, int cnt),
+ void (*_merror) (char *err, int erx)
+)
+{
+ rdfstore *me = NULL;
+ DBT key, data;
+ int err = 0,comp_alg=RDFSTORE_COMPRESSION_TYPE_DEFAULT;
+#ifdef RDFSTORE_CONNECTIONS
+ int comp_alg_connections=RDFSTORE_COMPRESSION_TYPE_BLOCK;/* comparing on some thesaurus like 10k triples this is the best */
+#endif
+
+ memset(&key, 0, sizeof(key));
+ memset(&data, 0, sizeof(data));
+
+ *mme = NULL;
+
+ me = (rdfstore *) RDFSTORE_MALLOC(sizeof(rdfstore));
+
+ if (me == NULL)
+ return -1;
+
+ me->model = NULL;
+ me->nodes = NULL;
+ me->subjects = NULL;
+ me->predicates = NULL;
+ me->objects = NULL;
+#ifdef RDFSTORE_CONNECTIONS
+ me->s_connections = NULL;
+ me->p_connections = NULL;
+ me->o_connections = NULL;
+#endif
+ me->languages = NULL;
+ me->datatypes = NULL;
+ me->xsd_integer = NULL;
+ me->xsd_double = NULL;
+ me->xsd_date = NULL;
+ me->contexts = NULL;
+ me->freetext = 0;
+ me->statements = NULL;
+ me->cursor = NULL;
+
+ /* prefixes mapping stuff */
+ me->prefixes = NULL;
+
+ /* set/get options */
+ me->flag = flags;
+ me->sync = sync;
+ me->remote = remote;
+
+ if (me->remote) {
+ if ((host != NULL) &&
+ (strlen(host) > 0))
+ strcpy(me->host, host);
+ me->port = port;
+ } else {
+ strcpy(me->host, "");
+ me->port = 0;
+ };
+ me->context = NULL;
+
+ /* name can also be file://.... for local or rdfstore://demo.asemantics.com:1234/nb for remote */
+ if (!strncmp(name,"file://",(size_t)7)) {
+ fprintf(stderr,"Aborted: RDF/XML or N-Triples file will be supported with in-memory model and C level parsing done.\n");
+ goto exitandclean;
+ } else if (!strncmp(name,"rdfstore://",(size_t)11)) {
+ char url_port[255];
+ char * p;
+ char * p1;
+ name+=11;
+ p = strstr(name,":");
+ p1 = strstr(name,"/");
+ if(p!=NULL) {
+ /* get the host out */
+ strncpy(me->host,name,p-name);
+ me->host[p-name] = '\0';
+ if (strlen(me->host)<=0) {
+ fprintf(stderr,"Aborted: You really want an Internet hostname.\n");
+ goto exitandclean;
+ };
+ host = me->host;
+ /* get the port out */
+ strncpy(url_port,p+1,p1-(p+1));
+ port = atoi(url_port);
+ if (port<=1) {
+ fprintf(stderr,"Aborted: You really want a port number >1.\n");
+ goto exitandclean;
+ };
+ name=p1+1;
+ me->port = port;
+
+ remote = 1;
+ me->remote = 1;
+ } else if(p1!=NULL) {
+ /* get the host out */
+ strncpy(me->host,name,p1-name);
+ me->host[p1-name] = '\0';
+ if (strlen(me->host)<=0) {
+ remote = 0;
+ me->remote = 0;
+ } else {
+ host = me->host;
+ remote = 1;
+ me->remote = 1;
+ name=p1+1;
+ };
+ };
+ } else if (!strncmp(name,"http://",(size_t)7)) {
+ fprintf(stderr,"Aborted: What are you trying to do? That's DAV like isn't it? ;-)\n");
+ goto exitandclean;
+ };
+
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->model,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/model"),
+ (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0)
+ goto exitandclean;
+
+ /* check indexing version first or croak */
+ key.data = RDFSTORE_INDEXING_VERSION_KEY;
+ key.size = sizeof(RDFSTORE_INDEXING_VERSION_KEY);
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ if (!(me->flag)) {
+ data.data = RDFSTORE_INDEXING_VERSION;
+ data.size = strlen(RDFSTORE_INDEXING_VERSION) + 1;
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in table model for store '%s': %s\n", (int)data.size, (char *)key.data, (char *)data.data, rdfstore_flat_store_get_error(me->model));
+
+ goto exitandclean;
+ };
+ } else {
+ if( (name != NULL) &&
+ (strlen(name) > 0) ) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Incompatible RDF database indexing version. This is version %s. You need to upgrade your database; dump your data, remove old database and re-ingest data.\n", RDFSTORE_INDEXING_VERSION );
+ goto exitandclean;
+ };
+ };
+ strcpy(me->version, RDFSTORE_INDEXING_VERSION);
+ } else {
+ /* just croak if different version for the moment */
+ if (strncmp( RDFSTORE_INDEXING_VERSION,
+ data.data, data.size )) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Incompatible RDF database indexing version %s. This is version %s. You need to upgrade your database; dump your data, remove old database and re-ingest data.\n", (char *)data.data, RDFSTORE_INDEXING_VERSION );
+ goto exitandclean;
+ };
+ strcpy(me->version, data.data);
+ RDFSTORE_FREE(data.data);
+ };
+
+ if ((name != NULL) &&
+ (strlen(name) > 0)) {
+ key.data = RDFSTORE_NAME_KEY;
+ key.size = sizeof(RDFSTORE_NAME_KEY);
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ if (!(me->flag)) {
+ data.data = name;
+ data.size = strlen(name) + 1;
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in table model for store '%s': %s\n", (int)data.size, (char *)key.data, (char *)data.data, rdfstore_flat_store_get_error(me->model));
+
+ goto exitandclean;
+ };
+ } else {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Store '%s' does not exist or is corrupted\n", name);
+ goto exitandclean;
+ };
+ strcpy(me->name, name);
+ } else {
+ if (strncmp(name, data.data, strlen(name))) { /* which is obvioulsy
+ * wrong but we avoid
+ * bother of ending
+ * slashes ;-) */
+ RDFSTORE_FREE(data.data);
+ perror("rdfstore_connect");
+ fprintf(stderr,"It seems you have got the wrong store name '%s' instead of '%s'\n", name, (char *)data.data);
+ goto exitandclean;
+ };
+ strcpy(me->name, data.data);
+ RDFSTORE_FREE(data.data);
+ };
+ };
+
+ key.data = RDFSTORE_FREETEXT_KEY;
+ key.size = sizeof(RDFSTORE_FREETEXT_KEY);
+
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ if (!(me->flag)) {
+ data.data = (freetext) ? "1" : "0";
+ data.size = sizeof((freetext) ? "1" : "0") + 1;
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in model for store '%s': %s\n", (int)data.size, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ goto exitandclean;
+ };
+ } else if ((me->name != NULL) &&
+ (strlen(me->name) > 0)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Store '%s' seems corrupted\n", me->name);
+ goto exitandclean;
+ };
+ me->freetext = (freetext) ? 1 : 0;
+ } else {
+ me->freetext = (strcmp(data.data, "0")) ? 1 : 0;
+ RDFSTORE_FREE(data.data);
+ };
+
+ key.data = RDFSTORE_COMPRESSION_KEY;
+ key.size = sizeof(RDFSTORE_COMPRESSION_KEY);
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ if (!(me->flag)) {
+ unsigned char outbuf[256];
+ packInt(comp_alg, outbuf);
+ data.data = outbuf;
+ data.size = sizeof(int);
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in model for store '%s': %s\n", (int)data.size, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ goto exitandclean;
+ };
+ };
+ } else {
+ unpackInt(data.data, &comp_alg);
+ RDFSTORE_FREE(data.data);
+ };
+
+ if (rdfstore_compress_init(comp_alg,&(me->func_decode),&(me->func_encode))) {
+#ifdef RDFSTORE_DEBUG
+ fprintf(stderr,"Could not init default compression function for algorithm '%d'\n",comp_alg);
+#endif
+ goto exitandclean;
+ };
+
+#ifdef RDFSTORE_CONNECTIONS
+ key.data = RDFSTORE_COMPRESSION_CONNECTIONS_KEY;
+ key.size = sizeof(RDFSTORE_COMPRESSION_CONNECTIONS_KEY);
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ if (!(me->flag)) {
+ unsigned char outbuf[256];
+ packInt(comp_alg_connections, outbuf);
+ data.data = outbuf;
+ data.size = sizeof(int);
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in model for store '%s': %s\n", (int)data.size, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ goto exitandclean;
+ };
+ };
+ } else {
+ unpackInt(data.data, &comp_alg_connections);
+ RDFSTORE_FREE(data.data);
+ };
+
+ /* now bear in mind we using this for sake of experiment on connections tables to see if it compresses better.... */
+ if (rdfstore_compress_init(comp_alg_connections,&(me->func_decode_connections),&(me->func_encode_connections))) {
+#ifdef RDFSTORE_DEBUG
+ fprintf(stderr,"Could not init connections compression function for algorithm '%d'\n",comp_alg_connections);
+#endif
+ goto exitandclean;
+ };
+#endif
+
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->nodes,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/nodes"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ goto exitandclean;
+ };
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->subjects,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/subjects"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ goto exitandclean;
+ };
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->predicates,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/predicates"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ goto exitandclean;
+ };
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->objects,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/objects"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ goto exitandclean;
+ };
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->contexts,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/contexts"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ goto exitandclean;
+ };
+#ifdef RDFSTORE_CONNECTIONS
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->s_connections,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/s_connections"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0 );
+ if (err != 0) {
+ goto exitandclean;
+ };
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->p_connections,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/p_connections"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0 );
+ if (err != 0) {
+ goto exitandclean;
+ };
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->o_connections,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/o_connections"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0 );
+ if (err != 0) {
+ goto exitandclean;
+ };
+#endif
+
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->languages,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/languages"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0 );
+ if (err != 0) {
+ goto exitandclean;
+ };
+
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->datatypes,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/datatypes"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0 );
+ if (err != 0) {
+ goto exitandclean;
+ };
+
+ /* special table for integers */
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->xsd_integer,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/xsd_integer"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ FLAT_STORE_BT_COMP_INT );
+ if (err != 0) {
+ goto exitandclean;
+ };
+
+ /* special table for doubles */
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->xsd_double,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/xsd_double"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ FLAT_STORE_BT_COMP_DOUBLE );
+ if (err != 0) {
+ goto exitandclean;
+ };
+
+ /* special table for dates */
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->xsd_date,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/xsd_date"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0); /* I guess lexicographical order for xsd:date and xsd:dateTime should work */
+ if (err != 0) {
+ goto exitandclean;
+ };
+
+ if (me->freetext) { /* just if we need free-text indexing */
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->windex,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/windex"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0); /* might be default lexicographical order for words is not UTF-8 safe? yeah..... */
+ if (err != 0) {
+ goto exitandclean;
+ };
+ };
+
+ err = rdfstore_flat_store_open(remote,
+ flags,
+ &me->statements,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/statements"), (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ goto exitandclean;
+ };
+
+ /* this is just used internally and never gets returned to the user */
+ me->cursor = NULL;
+ me->cursor = (rdfstore_iterator *) RDFSTORE_MALLOC(sizeof(rdfstore_iterator));
+ if (me->cursor == NULL) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Cannot create internal results cursor/iterator for store '%s'\n", (name != NULL) ? name : "(in-memory)");
+ goto exitandclean;
+ };
+
+ /* initialize statements counters if necessary */
+
+ /* keep the number of zapped statements */
+ key.data = RDFSTORE_COUNTER_REMOVED_KEY;
+ key.size = sizeof(RDFSTORE_COUNTER_REMOVED_KEY);
+ if ((rdfstore_flat_store_exists(me->model, key)) != 0) {
+ if (!(me->flag)) {
+ unsigned char outbuf[256];
+ packInt(0, outbuf);
+ data.data = outbuf;
+ data.size = sizeof(int);
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in model for store '%s': %s\n", (int)data.size, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ goto exitandclean;
+ };
+ } else if ((me->name != NULL) &&
+ (strlen(me->name) > 0)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Store '%s' seems corrupted\n", (me->name != NULL) ? me->name : "(in-memory)");
+ goto exitandclean;
+ };
+ };
+
+ /* keep the total number of statements */
+ key.data = RDFSTORE_COUNTER_KEY;
+ key.size = sizeof(RDFSTORE_COUNTER_KEY);
+ if ((rdfstore_flat_store_exists(me->model, key)) != 0) {
+ if (!(me->flag)) {
+ unsigned char outbuf[256];
+ packInt(0, outbuf);
+ data.data = outbuf;
+ data.size = sizeof(int);
+ err = rdfstore_flat_store_store(me->model, key, data);
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Could not store '%d' bytes for key '%s' in model for store '%s': %s\n", (int)data.size, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ goto exitandclean;
+ };
+ } else if ((me->name != NULL) &&
+ (strlen(me->name) > 0)) {
+ perror("rdfstore_connect");
+ fprintf(stderr,"Store '%s' seems corrupted\n", (me->name != NULL) ? me->name : "(in-memory)");
+ goto exitandclean;
+ };
+ };
+
+ me->cursor->store = me;
+ /* bzero(me->cursor->ids,sizeof(unsigned char)*(RDFSTORE_MAXRECORDS_BYTES_SIZE)); */
+ me->cursor->remove_holes = 0; /* reset the total of holes */
+ me->cursor->st_counter = 0;
+ if ((me->name != NULL) &&
+ (strlen(me->name) > 0)) {
+ rdfstore_size(me, &me->cursor->size);
+ } else {
+ me->cursor->size = 0;
+ };
+ me->cursor->ids_size = (me->cursor->size / 8);
+ if (me->cursor->size % 8)
+ me->cursor->ids_size++;
+ me->cursor->pos = 0;
+
+ me->attached = 0; /* reset the number of items (cursors)
+ * currenlty attached */
+ me->tobeclosed = 0;
+
+ /* it seems BDB 1.8x needs this to start properly */
+ if (!(me->flag)) {
+ /*
+ * fprintf(stderr,"Initial sync for BDB 1.8x and flag = '%d' (must be
+ * 0)",me->flag);
+ */
+ rdfstore_flat_store_sync(me->model);
+ rdfstore_flat_store_sync(me->nodes);
+ rdfstore_flat_store_sync(me->subjects);
+ rdfstore_flat_store_sync(me->predicates);
+ rdfstore_flat_store_sync(me->objects);
+ if (me->contexts)
+ rdfstore_flat_store_sync(me->contexts);
+#ifdef RDFSTORE_CONNECTIONS
+ if (me->s_connections)
+ rdfstore_flat_store_sync(me->s_connections);
+ if (me->p_connections)
+ rdfstore_flat_store_sync(me->p_connections);
+ if (me->o_connections)
+ rdfstore_flat_store_sync(me->o_connections);
+#endif
+ if (me->languages)
+ rdfstore_flat_store_sync(me->languages);
+ if (me->datatypes)
+ rdfstore_flat_store_sync(me->datatypes);
+ if (me->xsd_integer)
+ rdfstore_flat_store_sync(me->xsd_integer);
+ if (me->xsd_double)
+ rdfstore_flat_store_sync(me->xsd_double);
+ if (me->xsd_date)
+ rdfstore_flat_store_sync(me->xsd_date);
+ if (me->freetext)
+ rdfstore_flat_store_sync(me->windex);
+ rdfstore_flat_store_sync(me->statements);
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ unsigned int size = 0;
+ if ((name != NULL) && (strlen(name) > 0)) {
+ if (rdfstore_size(me, &size)) {
+ perror("rdfstore_is_empty");
+ fprintf(stderr,"Could not carry out model size for store '%s'\n", (me->name != NULL) ? me->name : "(in-memory)");
+ goto exitandclean;
+ };
+ };
+ fprintf(stderr,"Connected to database \n\tname='%s'\n\tremote='%d'\n\thost='%s'\n\tport='%d'\n\tfreetext='%d'\n\tsync='%d'\n\tsize='%d'", me->name, me->remote, me->host, me->port, me->freetext, me->sync, size);
+ };
+#endif
+
+ *mme = me;
+
+MX;
+ return 0;
+
+exitandclean:
+ /* XX should we not also free the me->model, me->nodes and so on ? or at least close them ? */
+ if (me->model) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->model);
+ rdfstore_flat_store_close(me->model);
+ };
+ if (me->nodes) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->nodes);
+ rdfstore_flat_store_close(me->nodes);
+ };
+ if (me->subjects) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->subjects);
+ rdfstore_flat_store_close(me->subjects);
+ };
+ if (me->predicates) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->predicates);
+ rdfstore_flat_store_close(me->predicates);
+ };
+ if (me->objects) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->objects);
+ rdfstore_flat_store_close(me->objects);
+ };
+ if (me->contexts) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->contexts);
+ rdfstore_flat_store_close(me->contexts);
+ };
+#ifdef RDFSTORE_CONNECTIONS
+ if (me->s_connections) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->s_connections);
+ rdfstore_flat_store_close(me->s_connections);
+ };
+ if (me->p_connections) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->p_connections);
+ rdfstore_flat_store_close(me->p_connections);
+ };
+ if (me->o_connections) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->o_connections);
+ rdfstore_flat_store_close(me->o_connections);
+ };
+#endif
+ if (me->languages) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->languages);
+ rdfstore_flat_store_close(me->languages);
+ };
+ if (me->datatypes) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->datatypes);
+ rdfstore_flat_store_close(me->datatypes);
+ };
+ if (me->xsd_integer) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->xsd_integer);
+ rdfstore_flat_store_close(me->xsd_integer);
+ };
+ if (me->xsd_double) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->xsd_double);
+ rdfstore_flat_store_close(me->xsd_double);
+ };
+ if (me->xsd_date) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->xsd_date);
+ rdfstore_flat_store_close(me->xsd_date);
+ };
+ if (me->windex) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->windex);
+ rdfstore_flat_store_close(me->windex);
+ };
+ if (me->statements) {
+ if (!(me->flag)) rdfstore_flat_store_sync(me->statements);
+ rdfstore_flat_store_close(me->statements);
+ };
+ if (me->cursor) RDFSTORE_FREE(me->cursor);
+ RDFSTORE_FREE(me);
+ return -1;
+};
+
+int
+rdfstore_disconnect(rdfstore * me)
+{
+
+ if (me == NULL) {
+#ifdef RDFSTORE_DEBUG
+ printf(">>>>>>>>>>>>>>>>>>>>%p IMPOSSIBLE TO CLOSE\n", me);
+#endif
+ return -1;
+ };
+
+ if (me->attached > 0) {
+#ifdef RDFSTORE_DEBUG
+ printf(">>>>>>>>>>>>>>>>>>>>%p TO BE CLOSED\n", me);
+#endif
+ me->tobeclosed = 1;
+ return 1; /* wait the cursors to call me back :-) */
+ } else {
+#ifdef RDFSTORE_DEBUG
+ printf("<<<<<<<<<<<<<<<<<<<<%p CLOSING\n", me);
+#endif
+ me->tobeclosed = 0;
+ };
+MX;
+
+ if ((me->sync) &&
+ (!(me->flag))) {
+ rdfstore_flat_store_sync(me->model);
+ rdfstore_flat_store_sync(me->nodes);
+ rdfstore_flat_store_sync(me->subjects);
+ rdfstore_flat_store_sync(me->predicates);
+ rdfstore_flat_store_sync(me->objects);
+#ifdef RDFSTORE_CONNECTIONS
+ if (me->s_connections)
+ rdfstore_flat_store_sync(me->s_connections);
+ if (me->p_connections)
+ rdfstore_flat_store_sync(me->p_connections);
+ if (me->o_connections)
+ rdfstore_flat_store_sync(me->o_connections);
+#endif
+ if (me->languages)
+ rdfstore_flat_store_sync(me->languages);
+ if (me->datatypes)
+ rdfstore_flat_store_sync(me->datatypes);
+ if (me->xsd_integer)
+ rdfstore_flat_store_sync(me->xsd_integer);
+ if (me->xsd_double)
+ rdfstore_flat_store_sync(me->xsd_double);
+ if (me->xsd_date)
+ rdfstore_flat_store_sync(me->xsd_date);
+ if (me->freetext)
+ rdfstore_flat_store_sync(me->windex);
+ if (me->contexts)
+ rdfstore_flat_store_sync(me->contexts);
+ rdfstore_flat_store_sync(me->statements);
+ };
+
+ if (me->cursor != NULL)
+ RDFSTORE_FREE(me->cursor);
+
+ if (me->context != NULL) {
+ RDFSTORE_FREE(me->context->value.resource.identifier);
+ RDFSTORE_FREE(me->context);
+ };
+
+ rdfstore_flat_store_close(me->model);
+ rdfstore_flat_store_close(me->nodes);
+ rdfstore_flat_store_close(me->subjects);
+ rdfstore_flat_store_close(me->predicates);
+ rdfstore_flat_store_close(me->objects);
+ if (me->contexts)
+ rdfstore_flat_store_close(me->contexts);
+#ifdef RDFSTORE_CONNECTIONS
+ if (me->s_connections)
+ rdfstore_flat_store_close(me->s_connections);
+ if (me->p_connections)
+ rdfstore_flat_store_close(me->p_connections);
+ if (me->o_connections)
+ rdfstore_flat_store_close(me->o_connections);
+#endif
+ if (me->languages)
+ rdfstore_flat_store_close(me->languages);
+ if (me->datatypes)
+ rdfstore_flat_store_close(me->datatypes);
+ if (me->xsd_integer)
+ rdfstore_flat_store_close(me->xsd_integer);
+ if (me->xsd_double)
+ rdfstore_flat_store_close(me->xsd_double);
+ if (me->xsd_date)
+ rdfstore_flat_store_close(me->xsd_date);
+ if (me->freetext)
+ rdfstore_flat_store_close(me->windex);
+ rdfstore_flat_store_close(me->statements);
+
+ RDFSTORE_FREE(me);
+ me = NULL;
+
+ return 0;
+};
+
+int
+rdfstore_isconnected(
+ rdfstore * me
+) {
+ return (me->model != NULL) ? 0 : 1;
+ };
+
+/*
+ * perhaps in the future we might have different tables on different servers
+ * or some local/in-memory
+ */
+int
+rdfstore_isremote(
+ rdfstore * me
+) {
+ return ( (rdfstore_isconnected(me)==0) && ( rdfstore_flat_store_isremote(me->model) == 1 ) ) ? 0 : 1;
+};
+
+int
+rdfstore_size(
+ rdfstore * me,
+ unsigned int *size)
+{
+
+ DBT key, data;
+ unsigned int removed = 0;
+
+ memset(&key, 0, sizeof(key));
+ memset(&data, 0, sizeof(data));
+
+ key.data = RDFSTORE_COUNTER_REMOVED_KEY;
+ key.size = sizeof(RDFSTORE_COUNTER_REMOVED_KEY);
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ perror("rdfstore_size");
+ fprintf(stderr,"Could not find counter_removed_key for store '%s': %s\n", (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ return -1;
+ };
+
+ /* cast values to int */
+ unpackInt(data.data, &removed);
+ RDFSTORE_FREE(data.data);
+
+ memset(&data, 0, sizeof(data));
+ key.data = RDFSTORE_COUNTER_KEY;
+ key.size = sizeof(RDFSTORE_COUNTER_KEY);
+ if ((rdfstore_flat_store_fetch(me->model, key, &data)) != 0) {
+ perror("rdfstore_size");
+ fprintf(stderr,"Could not find counter_key for store '%s': %s\n", (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+ return -1;
+ };
+ unpackInt(data.data, size);
+ RDFSTORE_FREE(data.data);
+
+#ifdef RDFSTORE_DEBUG
+ fprintf(stderr,"size = %d - %d\n", (*size), removed);
+#endif
+
+ /* sum them */
+ (*size) -= removed;
+
+ return 0;
+};
+
+/* try to be as lightweight as possible here without requiring full-blown rdfstore connections, but just one to model */
+int rdfstore_if_modified_since (
+ char * name,
+ char * since,
+ /* Callbacks for memory management and error handling. */
+ void *(*_mmalloc) (size_t s),
+ void (*_mfree) (void *adr),
+ void (*_mcallback) (dbms_cause_t cause, int cnt),
+ void (*_merror) (char *err, int erx)
+ ) {
+ struct tm thedateval_tm;
+ char thedateval[RDFSTORE_XSD_DATETIME_FORMAT_SIZE];
+ DBT key, data;
+ int err=0;
+ FLATDB * model;
+ int port=0;
+ int remote=0;
+ char host[ MAXPATHLEN ];
+
+ strcpy(host, "");
+
+ if( name == NULL ) {
+ return 0;
+ };
+
+ memset(&key, 0, sizeof(key));
+ memset(&data, 0, sizeof(data));
+
+ if( ! rdfstore_xsd_deserialize_dateTime( since, &thedateval_tm ) ) { /* get/normalize passed xsd:dateTime */
+ return -1;
+ };
+
+ rdfstore_xsd_serialize_dateTime( thedateval_tm, thedateval );
+
+ /* get the DB name */
+ if (!strncmp(name,"rdfstore://",(size_t)11)) {
+ char url_port[255];
+ char * p;
+ char * p1;
+ name+=11;
+ p = strstr(name,":");
+ p1 = strstr(name,"/");
+ if(p!=NULL) {
+ /* get the host out */
+ strncpy(host,name,p-name);
+ host[p-name] = '\0';
+ if (strlen(host)<=0) {
+ return -1;
+ };
+ /* get the port out */
+ strncpy(url_port,p+1,p1-(p+1));
+ port = atoi(url_port);
+ if (port<=1) {
+ return -1;
+ };
+ name=p1+1;
+ remote = 1;
+ } else if(p1!=NULL) {
+ /* get the host out */
+ strncpy(host,name,p1-name);
+ host[p1-name] = '\0';
+ if (strlen(host)<=0) {
+ remote = 0;
+ } else {
+ name=p1+1;
+ remote = 1;
+ };
+ };
+ } else if ( (!strncmp(name,"file://",(size_t)7)) ||
+ (!strncmp(name,"http://",(size_t)7)) ) {
+ return -1;
+ };
+
+ /* just one quick / simple connection to model */
+ err = rdfstore_flat_store_open(remote,
+ 1, /* read only */
+ &model,
+ name, (((name == NULL) || (strlen(name) == 0)) ? NULL : "/model"),
+ (unsigned int)(32 * 1024), host, port,
+ _mmalloc, _mfree, _mcallback, _merror,
+ 0);
+ if (err != 0) {
+ return -1;
+ };
+
+ key.data = RDFSTORE_LASTMODIFIED_KEY;
+ key.size = sizeof(RDFSTORE_LASTMODIFIED_KEY);
+
+ err = rdfstore_flat_store_fetch(model, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_if_modified_since");
+ fprintf(stderr,"Could not find %s key for store '%s': %s\n", RDFSTORE_LASTMODIFIED_KEY, (name != NULL) ? name : "(in-memory)", rdfstore_flat_store_get_error(model));
+ rdfstore_flat_store_close(model);
+
+ return -1;
+ } else {
+ rdfstore_flat_store_close(model);
+
+ return 1;
+ };
+ } else {
+ if( strcmp( thedateval, data.data ) < 0 ) {
+#ifdef RDFSTORE_DEBUG
+ printf(" %s < %s \n", thedateval, (char*)(data.data) );
+#endif
+ rdfstore_flat_store_close(model);
+
+ return 0;
+ } else {
+#ifdef RDFSTORE_DEBUG
+ printf(" %s >= %s \n", thedateval, (char*)(data.data) );
+#endif
+ rdfstore_flat_store_close(model);
+
+ return 1;
+ };
+ };
+ };
+
+int
+rdfstore_insert(
+ rdfstore * me,
+ RDF_Statement * statement,
+ RDF_Node * given_context
+)
+{
+ RDF_Node *context = NULL;
+ char *buff = NULL;
+ char *_buff = NULL;
+ unsigned int outsize = 0;
+ unsigned int st_id = 0;
+ DBT key, data;
+ unsigned char outbuf[256];
+ unsigned char outbuf1[256];
+ unsigned char nodebuf[ 32 * 1024 ];
+ unsigned char *word;
+ unsigned char mask = 0;
+ unsigned char *utf8_casefolded_buff;
+ unsigned int utf8_size = 0;
+ char *sep = RDFSTORE_WORD_SPLITS;
+ int err, l, i = 0;
+ rdf_store_digest_t hc = 0;
+
+ int islval=0;
+ int isdval=0;
+ long thelval;
+ double thedval;
+ struct tm thedateval_tm;
+ struct tm* ptm;
+ time_t now;
+ char thedateval[RDFSTORE_XSD_DATETIME_FORMAT_SIZE];
+
+ assert(sizeof(unsigned char)==1);
+
+#ifdef RDFSTORE_CONNECTIONS
+ /* buffers for connections matrixes */
+ static unsigned char s_connections[RDFSTORE_MAXRECORDS_BYTES_SIZE];
+ static unsigned char p_connections[RDFSTORE_MAXRECORDS_BYTES_SIZE];
+ static unsigned char o_connections[RDFSTORE_MAXRECORDS_BYTES_SIZE];
+ unsigned int s_outsize = 0;
+ unsigned int p_outsize = 0;
+ unsigned int o_outsize = 0;
+
+ bzero(s_connections,sizeof(RDFSTORE_MAXRECORDS_BYTES_SIZE));
+ bzero(p_connections,sizeof(RDFSTORE_MAXRECORDS_BYTES_SIZE));
+ bzero(o_connections,sizeof(RDFSTORE_MAXRECORDS_BYTES_SIZE));
+
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator *reindex;
+ unsigned int pos = 0;
+ RDF_Statement * neighbour;
+ unsigned int outsize_reindex = 0;
+ static unsigned char reindex_encode[RDFSTORE_MAXRECORDS_BYTES_SIZE];
+ static unsigned char reindex_decode[RDFSTORE_MAXRECORDS_BYTES_SIZE];
+
+ bzero(reindex_encode,sizeof(RDFSTORE_MAXRECORDS_BYTES_SIZE));
+ bzero(reindex_decode,sizeof(RDFSTORE_MAXRECORDS_BYTES_SIZE));
+#endif
+
+#endif
+
+ /*
+ int ttime=0; struct timeval tstart,tnow;
+ */
+
+ /*
+ gettimeofday(&tstart,NULL);
+ */
+
+#ifdef RDFSTORE_FLAT_STORE_DEBUG /*&& RDFSTORE_COUNT_OPERATIONS_PER_STATEMENT*/
+ rdfstore_flat_store_reset_debuginfo(me->model);
+ rdfstore_flat_store_reset_debuginfo(me->statements);
+ rdfstore_flat_store_reset_debuginfo(me->nodes);
+ rdfstore_flat_store_reset_debuginfo(me->subjects);
+ rdfstore_flat_store_reset_debuginfo(me->predicates);
+ rdfstore_flat_store_reset_debuginfo(me->objects);
+#ifdef RDFSTORE_CONNECTIONS
+ if(me->s_connections)
+ rdfstore_flat_store_reset_debuginfo(me->s_connections);
+ if(me->p_connections)
+ rdfstore_flat_store_reset_debuginfo(me->p_connections);
+ if(me->o_connections)
+ rdfstore_flat_store_reset_debuginfo(me->o_connections);
+#endif
+ if(me->languages)
+ rdfstore_flat_store_reset_debuginfo(me->languages);
+ if(me->datatypes)
+ rdfstore_flat_store_reset_debuginfo(me->datatypes);
+ if(me->xsd_integer)
+ rdfstore_flat_store_reset_debuginfo(me->xsd_integer);
+ if(me->xsd_double)
+ rdfstore_flat_store_reset_debuginfo(me->xsd_double);
+ if(me->xsd_date)
+ rdfstore_flat_store_reset_debuginfo(me->xsd_date);
+ if (context != NULL)
+ rdfstore_flat_store_reset_debuginfo(me->contexts);
+ if (me->freetext)
+ rdfstore_flat_store_reset_debuginfo(me->windex);
+ fprintf(stderr,"rdfstore_insert BEGIN (reset number of DB operations)\n");
+#endif
+
+ if ((statement == NULL) ||
+ (statement->subject == NULL) ||
+ (statement->predicate == NULL) ||
+ (statement->subject->value.resource.identifier == NULL) ||
+ (statement->predicate->value.resource.identifier == NULL) ||
+ (statement->object == NULL) ||
+ ((statement->object->type != 1) &&
+ (statement->object->value.resource.identifier == NULL)) ||
+ ((given_context != NULL) &&
+ (given_context->value.resource.identifier == NULL)) ||
+ ((statement->node != NULL) &&
+ (statement->node->value.resource.identifier == NULL))) {
+#ifdef RDFSTORE_DEBUG
+ fprintf(stderr,"Wrong params\n");
+#endif
+ return -1;
+ }
+
+ if (given_context == NULL) {
+ if (statement->context != NULL)
+ context = statement->context;
+ else {
+ /* use default context */
+ if (me->context != NULL)
+ context = me->context;
+ };
+ } else {
+ /* use given context instead */
+ context = given_context;
+ };
+
+#ifdef RDFSTORE_DEBUG
+ fprintf(stderr,"TO ADD:\n");
+ fprintf(stderr,"\tS='%s'\n", statement->subject->value.resource.identifier);
+ fprintf(stderr,"\tP='%s'\n", statement->predicate->value.resource.identifier);
+ if (statement->object->type != 1) {
+ fprintf(stderr,"\tO='%s'\n", statement->object->value.resource.identifier);
+ } else {
+ fprintf(stderr,"\tOLIT='%s'", statement->object->value.literal.string);
+ fprintf(stderr," LANG='%s'", statement->object->value.literal.lang);
+ fprintf(stderr," TYPE='%s'", statement->object->value.literal.dataType);
+ fprintf(stderr," PARSETYPE='%d'", statement->object->value.literal.parseType);
+ fprintf(stderr,"\n");
+ };
+ if (context != NULL) {
+ fprintf(stderr,"\tC='%s'\n", context->value.resource.identifier);
+ };
+ if (statement->node != NULL)
+ fprintf(stderr,"\tSRES='%s'\n", statement->node->value.resource.identifier);
+ fprintf(stderr," with options freetext='%d'\n", me->freetext);
+ if ((buff = rdfstore_ntriples_statement(statement, context)) != NULL) {
+ fprintf(stderr," N-triples: %s\n", buff);
+ RDFSTORE_FREE(buff);
+ };
+#endif
+
+ memset(&key, 0, sizeof(key));
+ memset(&data, 0, sizeof(data));
+
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ /* init re-index iterator (needed below when filling up adjacency matrixes) */
+ reindex = (rdfstore_iterator *) RDFSTORE_MALLOC(sizeof(rdfstore_iterator));
+ if (reindex == NULL) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Cannot create reindex cursor/iterator for store '%s'\n", (me->name != NULL) ? me->name : "(in-memory)");
+ return -1;
+ };
+ reindex->store = me;
+ reindex->store->attached++;
+ reindex->remove_holes = 0; /* reset the total number of holes */
+ reindex->st_counter = 0;
+ reindex->pos = 0;
+ reindex->ids_size = 0;
+ reindex->size = 0;
+#endif
+
+ /* compute statement hashcode */
+ hc = rdfstore_digest_get_statement_hashCode(statement, context);
+
+ /* cache the hashcode if the statement has a "proper" identity */
+ if ((given_context == NULL) &&
+ (me->context == NULL))
+ statement->hashcode = hc;
+
+ /* we do not want duplicates (in the same context) */
+ packInt(hc, outbuf);
+
+#ifdef RDFSTORE_DEBUG
+ printf("Statement hashcode is '%d' while packed is '", hc);
+ for (i = 0; i < sizeof(int); i++) {
+ printf("%02X", outbuf[i]);
+ };
+ printf("'\n");
+#endif
+
+ key.data = outbuf;
+ key.size = sizeof(int);
+ if ((rdfstore_flat_store_exists(me->statements, key)) == 0) {
+#ifdef RDFSTORE_DEBUG
+ if ((buff = rdfstore_ntriples_statement(statement, context)) != NULL) {
+ fprintf(stderr,"Cannot insert multiple copies of the statement '%s' for store '%s' on key=%x\n",
+ buff, (me->name != NULL) ? me->name : "(in-memory)",(int)((int *)(outbuf)));
+ RDFSTORE_FREE(buff);
+ };
+#endif
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return 1;
+ };
+
+ /*
+ * NOTE about interpretation of contexts:
+ *
+ * E.g. If the context is the timestamp of yesterday we can insert a
+ * statement in that context (temporal context); then we can retrieve
+ * it in the given context 'has somebody said something yesterday
+ * about the statement?'. The answer would be affermative. In case
+ * the context is not set it make a lot of sense ask to the database
+ * 'has never been said something about the statement (ever!)?'; the
+ * answer would be the same and affermative. If the context would be
+ * another one for example the timestamp of today, than the answer to
+ * the question 'has somebody said something today about the
+ * statement?' would be definitively *false*. This example can be
+ * applied to RSS1.0 feeds of news if you wish....
+ *
+ * This to me is really much the same of 'views' (or grouping??!?)
+ * concept in traditional relational databases
+ */
+
+ /*
+ * About the indexing algorithm see doc/SWADe-rdfstore.html
+ */
+
+ /* store the STATEMENT */
+ key.data = RDFSTORE_COUNTER_KEY;
+ key.size = sizeof(RDFSTORE_COUNTER_KEY);
+
+ /* increment statement counter */
+ if ((rdfstore_flat_store_inc(me->model, key, &data)) != 0) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not increment statement counter for store '%s': %s\n", (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ unpackInt(data.data,&st_id);
+
+ RDFSTORE_FREE(data.data);
+
+#ifdef RDFSTORE_DEBUG
+ {
+ fprintf(stderr, "New statement identifier: %d > %d\n", (int)st_id, (int)RDFSTORE_MAXRECORDS);
+ };
+#endif
+
+ if (st_id > RDFSTORE_MAXRECORDS) {
+ if ((rdfstore_flat_store_dec(me->model, key, &data)) == 0)
+ RDFSTORE_FREE(data.data);
+ perror("rdfstore_insert");
+ fprintf(stderr,"RDFSTORE_MAXRECORDS(%d) reached (st_id=%d) - can not insert more statements in store '%s': %s\n", RDFSTORE_MAXRECORDS, st_id, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->model));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+
+ /* the counter starts from zero anyway! */
+ st_id--;
+
+ /* force this (or warning/error returned ?? ) */
+ /*
+ * rdf:parseType="Literal" is like
+ * rdf:datatype="http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral" - is it still after new RDF specs?
+ */
+ if (statement->object->type == 1) {
+ if ((statement->object->value.literal.parseType == 1) &&
+ (statement->object->value.literal.dataType != NULL) &&
+ (strcmp(statement->object->value.literal.dataType, RDFSTORE_RDF_PARSETYPE_LITERAL))) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Statement object '%s' has rdf:parseType='Literal' but rdf:dataType='%s'\n", statement->object->value.literal.string, statement->object->value.literal.dataType);
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else if ((statement->object->value.literal.dataType != NULL) &&
+ (strcmp(statement->object->value.literal.dataType, RDFSTORE_RDF_PARSETYPE_LITERAL) == 0) &&
+ (statement->object->value.literal.parseType != 1)) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Statement object '%s' has rdf:dataType='%s' but rdf:parseType='Resource'\n", statement->object->value.literal.string, RDFSTORE_RDF_PARSETYPE_LITERAL);
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+ /* nodes table */
+
+ /* try to allocate just the necessary - this means that
+ * we must use memcpy (and not strcpy) to fill it as other
+ * wise the extra \0 throws us one off.
+ */
+ l = (sizeof(int) * 7) + sizeof(char) +
+ (sizeof(char) * statement->subject->value.resource.identifier_len)+
+ (sizeof(char) * statement->predicate->value.resource.identifier_len) +
+ ((statement->object->type != 1) ?
+ (sizeof(char) * statement->object->value.resource.identifier_len) :
+ ((sizeof(char) * ((statement->object->value.literal.string != NULL) ? statement->object->value.literal.string_len : 0)) +
+ (((statement->object->value.literal.lang != NULL) && (strlen(statement->object->value.literal.lang) > 0)) ?
+ (sizeof(char) * strlen(statement->object->value.literal.lang)) : 0) +
+ ((statement->object->value.literal.dataType != NULL) ? (sizeof(char) * strlen(statement->object->value.literal.dataType)) : 0))) +
+ ((context != NULL) ?
+ (sizeof(char) * context->value.resource.identifier_len) : 0) +
+ ((statement->node != NULL) ?
+ (sizeof(char) * statement->node->value.resource.identifier_len) : 0);
+
+ if (l < sizeof(nodebuf))
+ buff = nodebuf;
+ else
+ buff = _buff = (char *)RDFSTORE_MALLOC(l);
+
+ if (buff == NULL) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not allocate memory for statement in store '%s'\n", (me->name != NULL) ? me->name : "(in-memory)");
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+
+ assert(sizeof(int) == 4);
+
+ /* offsets */
+ i = 0;
+
+ /* subject */
+ packInt(statement->subject->value.resource.identifier_len, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* predicate */
+ packInt(statement->predicate->value.resource.identifier_len, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* object */
+ packInt((statement->object->type != 1) ?
+ statement->object->value.resource.identifier_len :
+ (statement->object->value.literal.string != NULL) ? statement->object->value.literal.string_len : 0, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* object literal language */
+ packInt(((statement->object->type == 1) && (statement->object->value.literal.lang != NULL) && (strlen(statement->object->value.literal.lang) > 0)) ? strlen(statement->object->value.literal.lang) : 0, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* object literal data type */
+ packInt(((statement->object->type == 1) && (statement->object->value.literal.dataType != NULL)) ? strlen(statement->object->value.literal.dataType) : 0, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* context */
+ packInt((context != NULL) ? context->value.resource.identifier_len : 0, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* statement resource hashcode (if statement is a resource) */
+ packInt((statement->node != NULL) ? statement->node->value.resource.identifier_len : 0, outbuf);
+ memcpy(buff + i, outbuf, sizeof(int));
+ i += sizeof(int);
+
+ /* mask for special byte */
+ if (statement->object->type == 1)
+ mask |= 1;
+
+ if (statement->subject->type == 2)
+ mask |= 2;
+
+ if (statement->predicate->type == 2)
+ mask |= 4;
+
+ if (statement->object->type == 2)
+ mask |= 8;
+
+ if ((context != NULL) &&
+ (context->type == 2))
+ mask |= 16;
+
+ if (statement->isreified == 1)
+ mask |= 32;
+
+ memcpy(buff + i, &mask, 1);
+ i++;
+
+ /* subject */
+ memcpy(buff + i, statement->subject->value.resource.identifier, statement->subject->value.resource.identifier_len);
+ i += statement->subject->value.resource.identifier_len;
+
+ /* predicate */
+ memcpy(buff + i, statement->predicate->value.resource.identifier, statement->predicate->value.resource.identifier_len);
+ i += statement->predicate->value.resource.identifier_len;
+
+ /* object */
+ if (statement->object->type == 1) {
+ /* object literal string itself */
+ if (statement->object->value.literal.string != NULL) {
+ memcpy(buff + i, statement->object->value.literal.string, statement->object->value.literal.string_len);
+ i += statement->object->value.literal.string_len;
+ };
+ /* object literal language */
+ if (statement->object->value.literal.lang != NULL) {
+ memcpy(buff + i, statement->object->value.literal.lang, strlen(statement->object->value.literal.lang));
+ i += strlen(statement->object->value.literal.lang);
+ };
+ /* object literal data type */
+ if (statement->object->value.literal.dataType != NULL) {
+ memcpy(buff + i, statement->object->value.literal.dataType, strlen(statement->object->value.literal.dataType));
+ i += strlen(statement->object->value.literal.dataType);
+ };
+ } else {
+ memcpy(buff + i, statement->object->value.resource.identifier, statement->object->value.resource.identifier_len);
+ i += statement->object->value.resource.identifier_len;
+ };
+
+ /* context */
+ if (context != NULL) {
+ memcpy(buff + i, context->value.resource.identifier, context->value.resource.identifier_len);
+ i += context->value.resource.identifier_len;
+ };
+
+ /* statement as resource stuff */
+ if (statement->node != NULL) {
+ memcpy(buff + i, statement->node->value.resource.identifier, statement->node->value.resource.identifier_len);
+ i += statement->node->value.resource.identifier_len;
+ };
+
+ /* Check out lenght calcuation.. */
+ assert(l == i);
+
+ /* store the whole content */
+ packInt(st_id, outbuf);
+
+ key.data = outbuf;
+ key.size = sizeof(int);
+
+ buff[i++] = '\0'; /* Terminate the string and increase the length */
+ data.data = buff;
+ data.size = i;
+
+ err = rdfstore_flat_store_store(me->nodes, key, data);
+
+ if (_buff)
+ RDFSTORE_FREE(_buff);
+
+ if ((err != 0) &&
+ (err != FLAT_STORE_E_KEYEXIST)) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for statememt in nodes for store '%s': %s\n", (int)data.size, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->nodes));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+
+ /* index special literal stuff */
+ if (statement->object->type == 1) {
+ if ( (me->freetext) &&
+ (statement->object->value.literal.string != NULL) &&
+ (statement->object->value.literal.string_len > 0) ) {
+ utf8_casefolded_buff = (unsigned char *)RDFSTORE_MALLOC(statement->object->value.literal.string_len * sizeof(unsigned char) * (RDFSTORE_UTF8_MAXLEN_FOLD + 1)); /* what about the ending '\0' here ?? */
+ if (utf8_casefolded_buff == NULL) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Cannot compute case-folded string out of input literal for store '%s'\n", (me->name != NULL) ? me->name : "(in-memory)");
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ if (rdfstore_utf8_string_to_utf8_foldedcase(statement->object->value.literal.string_len, statement->object->value.literal.string, &utf8_size, utf8_casefolded_buff)) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Cannot compute case-folded string out of input literal for store '%s'\n", (me->name != NULL) ? me->name : "(in-memory)");
+ RDFSTORE_FREE(utf8_casefolded_buff);
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+
+ /* we do not even try to avoid duplicates for the moment */
+ for ( word = strtok(utf8_casefolded_buff, sep);
+ word;
+ word = strtok(NULL, sep) ) {
+ int jj=0;
+ int kk=0;
+
+ key.data = word;
+ key.size = strlen(word);
+
+ /*
+ *
+ * bzero(me->bits_encode,sizeof(me->bits_encode
+ * ));
+ * bzero(me->bits_decode,sizeof(me->bits_decod
+ * e));
+ */
+ err = rdfstore_flat_store_fetch_compressed(me->windex, me->func_decode, key, &outsize, me->bits_decode);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch windex of word '%s' for store '%s': %s\n", word, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->windex));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ };
+
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ if (outsize) {
+ err = rdfstore_flat_store_store_compressed(me->windex, me->func_encode, key, outsize, me->bits_decode,me->bits_encode );
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for '%s' in windex for store '%s'\n", outsize, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for word '%s' in windex for store '%s': %s\n", (int)data.size, word, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->windex));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->windex, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) windex for case-folded word '%s' -->'", st_id, word);
+ for(i=0;i<8*outsize;i++) {
+ printf("Rec %d %c\n", i, (me->bits_decode[i>>3] & (1<<(i&7))) ? '1':'0');
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+
+ /*
+ stemming code for ISO-Latin languiages (left-to-right) and shouild be UTF-8 aware
+ ie. word = "stemming" -- do index --> "s", "st", "ste", ....
+ */
+#if RDFSTORE_WORD_STEMMING > 0
+
+ if( ( rdfstore_xsd_deserialize_integer( word, &thelval ) ) ||
+ ( rdfstore_xsd_deserialize_double( word, &thedval ) ) ||
+ ( rdfstore_xsd_deserialize_dateTime( word,
+ &thedateval_tm ) ) ||
+ ( rdfstore_xsd_deserialize_date( word,
+ &thedateval_tm ) ) || /* dates are skipped, even if rdf:datatype is not set */
+ (strlen(word)<=1) )
+ continue;
+
+ /* for efficency we should check if the given partial stem has been already indexed for the same word!!! */
+ jj=1;
+ while ( ( jj < strlen(word) ) &&
+ ( kk < RDFSTORE_WORD_STEMMING ) ) {
+ char stem[MIN((RDFSTORE_WORD_STEMMING*RDFSTORE_UTF8_MAXLEN_FOLD),strlen(word))+1];
+
+ bzero(stem,MIN((RDFSTORE_WORD_STEMMING*RDFSTORE_UTF8_MAXLEN_FOLD),strlen(word))+1);
+
+ /* look for next utf8 char to add to stemming string */
+ utf8_size=0;
+ while ( ( jj < strlen(word) ) &&
+ (!( rdfstore_utf8_is_utf8( word+jj, &utf8_size ) )) ) {
+ jj++;
+ };
+
+ if (jj>strlen(word)) {
+ strncpy(stem, word, jj-1);
+ } else {
+ strncpy(stem, word, jj);
+ };
+
+ key.data = stem;
+ key.size = strlen(stem);
+
+ err = rdfstore_flat_store_fetch(me->windex, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch windex of stemming '%s' of word '%s' for store '%s': %s\n", stem, word, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->windex));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode); /* perhaps the
+ * compression for
+ * single bits could be
+ * different */
+ RDFSTORE_FREE(data.data);
+ };
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->windex, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for '%s' in windex for store '%s'\n", outsize, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for stemming '%s' in windex for store '%s': %s\n", (int)data.size, stem, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->windex));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->windex, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) windex for case-folded stemming '%s' of word '%s' -->'", st_id, stem, word);
+ for (i = 0; i < outsize; i++) {
+ printf("%02X", me->bits_decode[i]);
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+ jj++;
+ kk++;
+ };
+#endif
+ };
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ };
+
+ /* languages table */
+ if ( (statement->object->value.literal.lang != NULL) &&
+ (strlen(statement->object->value.literal.lang) > 0) ) {
+ utf8_casefolded_buff = (unsigned char *)RDFSTORE_MALLOC(strlen(statement->object->value.literal.lang) * sizeof(unsigned char) * (RDFSTORE_UTF8_MAXLEN_FOLD + 1));
+ if (utf8_casefolded_buff == NULL) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Cannot compute case-folded string for literal language code '%s' for store '%s'\n", statement->object->value.literal.lang, (me->name != NULL) ? me->name : "(in-memory)");
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ if (rdfstore_utf8_string_to_utf8_foldedcase(strlen(statement->object->value.literal.lang), statement->object->value.literal.lang, &utf8_size, utf8_casefolded_buff)) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Cannot compute case-folded string for literal language code '%s' for store '%s'\n", statement->object->value.literal.lang, (me->name != NULL) ? me->name : "(in-memory)");
+ RDFSTORE_FREE(utf8_casefolded_buff);
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+
+ key.data = utf8_casefolded_buff;
+ key.size = utf8_size;
+
+ err = rdfstore_flat_store_fetch(me->languages, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch language '%s' of literal '%s' for store '%s': %s\n", statement->object->value.literal.lang, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->languages));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->languages, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for '%s' in languages for store '%s'\n", outsize, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for language '%s' in languages for store '%s': %s\n", (int)data.size, statement->object->value.literal.lang, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->languages));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->languages, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) language '%s' of literal '%s' -->'", st_id, statement->object->value.literal.lang, statement->object->value.literal.string);
+ for (i = 0; i < outsize; i++) {
+ printf("%02X", me->bits_decode[i]);
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+
+ RDFSTORE_FREE(utf8_casefolded_buff);
+ };
+
+ /* datatypes table */
+ if ( (statement->object->value.literal.dataType != NULL) &&
+ (strlen(statement->object->value.literal.dataType) > 0) ) {
+ key.data = statement->object->value.literal.dataType;
+ key.size = strlen(statement->object->value.literal.dataType);
+
+ err = rdfstore_flat_store_fetch(me->datatypes, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch datatype '%s' of literal '%s' for store '%s': %s\n", statement->object->value.literal.dataType, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->datatypes));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->datatypes, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for '%s' in datatypes for store '%s'\n", outsize, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for datatype '%s' in datatypes for store '%s': %s\n", (int)data.size, statement->object->value.literal.dataType, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->datatypes));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->datatypes, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) datatype '%s' of literal '%s' -->'", st_id, statement->object->value.literal.dataType, statement->object->value.literal.string);
+ for (i = 0; i < outsize; i++) {
+ printf("%02X", me->bits_decode[i]);
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+
+ /* date type indexing only if rdf:datatype is set accordingly to xsd:date or xsd:dateTime */
+ if( (strcmp(statement->object->value.literal.dataType,RDFSTORE_MS_XSD_DATE)==0) ||
+ (strcmp(statement->object->value.literal.dataType,RDFSTORE_MS_XSD_DATETIME)==0) ) {
+ if ( ( rdfstore_xsd_deserialize_dateTime( statement->object->value.literal.string,
+ &thedateval_tm ) ) ||
+ ( rdfstore_xsd_deserialize_date( statement->object->value.literal.string,
+ &thedateval_tm ) ) ) {
+
+ rdfstore_xsd_serialize_dateTime( thedateval_tm, thedateval ); /* we index xsd:dataTime version anyway */
+
+ key.data = thedateval;
+ key.size = strlen(thedateval)+1;
+
+#ifdef RDFSTORE_DEBUG
+fprintf(stderr, "INDEX DATE '%s' for LITERAL '%s' \n",thedateval, statement->object->value.literal.string);
+#endif
+
+ err = rdfstore_flat_store_fetch(me->xsd_date, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch from date table value '%s' for store '%s': %s\n", statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->xsd_date));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->xsd_date, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for date '%s' in literal '%s' in integer table for store '%s'\n", outsize, thedateval, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for date '%s' in literal '%s' in date table for store '%s': %s\n", (int)data.size, thedateval, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->xsd_date));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->xsd_date, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) date '%s' of literal '%s' -->'", st_id, thedateval, statement->object->value.literal.string);
+ for (i = 0; i < outsize; i++) {
+ printf("%02X", me->bits_decode[i]);
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+ };
+ }; /* end of date indexing */
+ };
+
+ /* for xsd:integer alike literals use special b-tree sorted index if strtol() works.... */
+ if( ( islval = rdfstore_xsd_deserialize_integer( statement->object->value.literal.string, &thelval ) ) != 0 ) {
+ key.data = (long*) &thelval; /* should pack int perhaps... */
+ key.size = sizeof(long);
+
+#ifdef RDFSTORE_DEBUG
+fprintf(stderr, "INDEX INTEGER '%ld' for LITERAL '%s' \n",(long)thelval, statement->object->value.literal.string);
+#endif
+
+ err = rdfstore_flat_store_fetch(me->xsd_integer, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch from integer table value '%s' for store '%s': %s\n", statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->xsd_integer));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->xsd_integer, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for integer '%ld' in literal '%s' in integer table for store '%s'\n", outsize, (long)thelval, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for integer '%ld' in literal '%s' in integer table for store '%s': %s\n", (int)data.size, (long)thelval, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->xsd_integer));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->xsd_integer, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) integer '%ld' of literal '%s' -->'", st_id, (long)thelval, statement->object->value.literal.string);
+ for (i = 0; i < outsize; i++) {
+ printf("%02X", me->bits_decode[i]);
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+
+ };
+
+ /* for xsd:double or xsd:float alike literals use special b-tree sorted index if strtod() works.... */
+ if( ( islval == 0 ) && /* do not index xsd:integer(s) twice also as xsd:double */
+ ( ( isdval = rdfstore_xsd_deserialize_double( statement->object->value.literal.string, &thedval ) ) != 0 ) ) {
+ key.data = (double*) &thedval; /* should pack int perhaps... */
+ key.size = sizeof(double);
+
+#ifdef RDFSTORE_DEBUG
+fprintf(stderr, "INDEX DOUBLE '%f' for LITERAL '%s' \n",thedval, statement->object->value.literal.string);
+#endif
+
+ err = rdfstore_flat_store_fetch(me->xsd_double, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch from double table value '%s' for store '%s': %s\n", statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->xsd_double));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->xsd_double, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for double '%f' in literal '%s' in double table for store '%s'\n", outsize, thedval, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for double '%f' in literal '%s' in double table for store '%s': %s\n", (int)data.size, thedval, statement->object->value.literal.string, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->xsd_double));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i;
+ if ((rdfstore_flat_store_fetch(me->xsd_double, key, &data)) == 0) {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode);
+ RDFSTORE_FREE(data.data);
+ };
+ printf("ADDED (%d) double '%f' of literal '%s' -->'", st_id, thedval, statement->object->value.literal.string);
+ for (i = 0; i < outsize; i++) {
+ printf("%02X", me->bits_decode[i]);
+ };
+ printf("'\n");
+ }
+#endif
+
+ };
+
+ };
+ }; /* end index special literal stuff */
+
+ /*
+ * gettimeofday(&tnow,NULL); ttime = ( tnow.tv_sec - tstart.tv_sec ) *
+ * 1000000 + ( tnow.tv_usec - tstart.tv_usec ) * 1;
+ * printf("rdfstore_insert DONE [%d micro sec]\n",ttime);
+ */
+
+ /* adjacency matrixes (tables) i.e. subjects, predicates, objects, contexts and connections */
+
+ /*
+ * compute other hashcodes (the should be cached because the
+ * underlying digest is carried out also for the statement->hashcode
+ * above)
+ */
+ statement->subject->hashcode = rdfstore_digest_get_node_hashCode(statement->subject, 0);
+ statement->predicate->hashcode = rdfstore_digest_get_node_hashCode(statement->predicate, 0);
+ statement->object->hashcode = rdfstore_digest_get_node_hashCode(statement->object, 0);
+ if (context != NULL)
+ context->hashcode = rdfstore_digest_get_node_hashCode(context, 0);
+
+ /*
+ * possible connections (see doc/SWADe-rdfstore.html)
+ *
+ *
+ * subjects table template:
+ * -------------------------------------
+ * st_num 01234567 89..
+ * node
+ * -------------------------------------
+ * subject-node 10000010 00
+ * -------------------------------------
+ * ^ ^
+ * | |
+ * st_num(0)->subject --+ |
+ * |
+ * st_num(6)->subject --------+
+ *
+ *
+ * Similarly the other predicates and objects tables
+ * are being generated - similarly contexts table is
+ * filled in for particulat context-node(s) for a
+ * a given statement.
+ *
+ *
+ * connections tables template: (to be completed)
+ *
+ * --> it maps DUG (normalized DLG) graph nodes to their connections and
+ * corresponding statements
+ *
+ *
+ * Anoatomy of connections insertion/updation
+ *
+ * here are all the possible connections a new statement might get into (i.e. "swiss army knife problem"):
+ *
+ * a4 a6 a8
+ * \ \ \
+ * \ \ \
+ * \---> A =============> B =============> C------>b9----->c9
+ * / ^\ /^\ ^\
+ * / | \ / | \ | \
+ * b5<---/ | \--->c4 / | \--->c6 | \--->c8
+ * / b1 / b2 b3
+ * / ^ b7<---/ ^ ^
+ * c5<---/ | \ | |
+ * | \ | |
+ * a1 \--->c7 a2 a3
+ *
+ *
+ * where A======>B======>C is the new statement to add
+ *
+ * then each A, B or C gets it respective connections (s_connection, p_connections or o_connections) table
+ * set with ALL the possible other connections with the existing statements - the possible connections are
+ * carried out by a permutation on all possible positions of each s,p,o component node on the subject, predicates
+ * or objects table values (who has which node). Due that no intrisic ordering of statement can be assumed, quite a
+ * lot of re-indexing is required when inserting a new statement - such overhaead can be roughly estimated up to 6*3
+ * re-indexing operations per statement. This is the case when the new statement is being connected to mostly any other
+ * neighbour statement already existing - the re-indexing require to set the right bitno into connections tables for each
+ * neighbour its components (see code below now). Of course molteplicity of connections on arcs is assumed not to be a
+ * problem in the resulting Directed Unlabelled Graph (DUG) of nodes.
+ *
+ */
+
+ /* subjects */
+ packInt(statement->subject->hashcode, outbuf);
+ key.data = outbuf;
+ key.size = sizeof(int);
+
+ /*
+ * bzero(me->bits_encode,sizeof(me->bits_encode));
+ * bzero(me->bits_decode,sizeof(me->bits_decode));
+ */
+
+ err = rdfstore_flat_store_fetch(me->subjects, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch key '%s' for subject in subjects table for store '%s': %s\n", (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->subjects));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize = 0;
+ };
+ } else {
+ me->func_decode(data.size, data.data, &outsize, me->bits_decode); /* perhaps the
+ * compression for
+ * single bits could be
+ * different */
+ RDFSTORE_FREE(data.data);
+ };
+
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ /*
+ We need to re-index quite a lot for each new statement component now - hope caching will
+ help here!! i.e. the swiss army nife problem (see SWAD-E paper and preso)
+
+ 1.1) add this new statement (st_id) to p_connections(neighbour->predicate) and o_connections(neighbour->object)
+ tables of each other statement (neighbour) connected to this one via subjects(SUBJECT) node
+ 1.2) add this new statement (st_id) to s_connections(neighbour->subject) and o_connections(neighbour->object)
+ tables of each other statement (neighbour) connected to this one via predicates(SUBJECT) node
+ 1.3) add this new statement (st_id) to s_connections(neighbour->subject) and p_connections(neighbour->predicate)
+ tables of each other statement (neighbour) connected to this one via objects(SUBJECT) node
+ */
+
+ /* 1.1) reindex st_id for connections to subjects(SUBJECT) node */
+
+ /* copy the subjects(SUBJECT) bits through the reindex iterator array */
+ memcpy(reindex->ids, me->bits_decode, outsize);
+ reindex->ids_size = outsize;
+ /* set the size - inefficient!! */
+ pos = 0;
+ reindex->size = 0;
+ /* count the ones (inefficient still) */
+ while ((pos = rdfstore_bits_getfirstsetafter(outsize, me->bits_decode, pos)) < 8 * outsize) {
+ reindex->size++;
+ pos++;
+ };
+
+ /* scan the obtained iterator */
+ while ( ( neighbour = rdfstore_iterator_each ( reindex ) ) != NULL ) {
+
+#ifdef RDFSTORE_DEBUG_CONNECTIONS
+ printf("(insert)REINDEXING predicate '%s' for connections to subjects('%s') node in st_id=%d\n",neighbour->predicate->value.resource.identifier,statement->subject->value.resource.identifier,st_id);
+#endif
+ neighbour->predicate->hashcode = rdfstore_digest_get_node_hashCode(neighbour->predicate, 0);
+
+ /* fetch p_connections(neighbour->predicate) */
+ packInt(neighbour->predicate->hashcode, outbuf); /* wrong */
+ key.data = outbuf;
+ key.size = sizeof(int);
+
+ err = rdfstore_flat_store_fetch(me->p_connections, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch key '%s' for predicate in p_connections table for store '%s': %s\n", (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->p_connections));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize_reindex = 0;
+ };
+ } else {
+ me->func_decode_connections(data.size, data.data, &outsize_reindex, reindex_decode);
+ RDFSTORE_FREE(data.data);
+ };
+
+ /* set the corresponding bit of this statement */
+ rdfstore_bits_setmask(&outsize_reindex, reindex_decode, st_id, 1, 1, sizeof(reindex_decode));
+
+ /* store it back */
+ me->func_encode_connections(outsize_reindex, reindex_decode, &outsize_reindex, reindex_encode);
+
+ data.data = reindex_encode;
+ data.size = outsize_reindex;
+ err = rdfstore_flat_store_store(me->p_connections, key, data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for predicate in p_connections table for store '%s': %s\n", (int)data.size, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->p_connections));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG_CONNECTIONS
+ printf("(insert)REINDEXING object '%s' for connections to subjects('%s') node in st_id=%d\n",(neighbour->object->type==1) ? neighbour->object->value.literal.string : neighbour->object->value.resource.identifier,statement->subject->value.resource.identifier,st_id);
+#endif
+ neighbour->object->hashcode = rdfstore_digest_get_node_hashCode(neighbour->object, 0);
+
+ /* fetch o_connections(neighbour->object) */
+ packInt(neighbour->object->hashcode, outbuf); /* wrong */
+ key.data = outbuf;
+ key.size = sizeof(int);
+
+ err = rdfstore_flat_store_fetch(me->o_connections, key, &data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_NOTFOUND) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not fetch key '%s' for object in o_connections table for store '%s': %s\n", (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->o_connections));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ } else {
+ outsize_reindex = 0;
+ };
+ } else {
+ me->func_decode_connections(data.size, data.data, &outsize_reindex, reindex_decode);
+ RDFSTORE_FREE(data.data);
+ };
+
+ /* set the corresponding bit of this statement */
+ rdfstore_bits_setmask(&outsize_reindex, reindex_decode, st_id, 1, 1, sizeof(reindex_decode));
+
+ /* store it back */
+ me->func_encode_connections(outsize_reindex, reindex_decode, &outsize_reindex, reindex_encode);
+
+ data.data = reindex_encode;
+ data.size = outsize_reindex;
+ err = rdfstore_flat_store_store(me->o_connections, key, data);
+ if (err != 0) {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for object in o_connections table for store '%s': %s\n", (int)data.size, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->o_connections));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+ /* free neighbour */
+ RDFSTORE_FREE( neighbour->subject->value.resource.identifier );
+ RDFSTORE_FREE( neighbour->subject );
+ RDFSTORE_FREE( neighbour->predicate->value.resource.identifier );
+ RDFSTORE_FREE( neighbour->predicate );
+ if ( neighbour->object->type == 1 ) {
+ if ( neighbour->object->value.literal.dataType != NULL )
+ RDFSTORE_FREE( neighbour->object->value.literal.dataType );
+ RDFSTORE_FREE( neighbour->object->value.literal.string );
+ } else {
+ RDFSTORE_FREE( neighbour->object->value.resource.identifier );
+ };
+ RDFSTORE_FREE( neighbour->object );
+ if ( neighbour->context != NULL ) {
+ RDFSTORE_FREE( neighbour->context->value.resource.identifier );
+ RDFSTORE_FREE( neighbour->context );
+ };
+ if ( neighbour->node != NULL ) {
+ RDFSTORE_FREE( neighbour->node->value.resource.identifier );
+ RDFSTORE_FREE( neighbour->node );
+ };
+ RDFSTORE_FREE( neighbour );
+ };
+#endif
+
+ rdfstore_bits_setmask(&outsize, me->bits_decode, st_id, 1, 1, sizeof(me->bits_decode));
+
+#ifdef RDFSTORE_CONNECTIONS
+ /* COPY subjects(SUBJECT) to s_connections(SUBJECT) */
+ bcopy(me->bits_decode, s_connections, outsize); /* slow? */
+ s_outsize = outsize;
+
+ /* COPY subjects(SUBJECT) to p_connections(PREDICATE) */
+ bcopy(me->bits_decode, p_connections, outsize); /* slow? */
+ p_outsize = outsize;
+
+ /* COPY subjects(SUBJECT) to o_connections(OBJECT) */
+ bcopy(me->bits_decode, o_connections, outsize); /* slow? */
+ o_outsize = outsize;
+#endif
+
+ me->func_encode(outsize, me->bits_decode, &outsize, me->bits_encode);
+ if (outsize) {
+
+ /* regenerate it due to the re-indexing above which uses key already */
+ packInt(statement->subject->hashcode, outbuf);
+ key.data = outbuf;
+ key.size = sizeof(int);
+
+ data.data = me->bits_encode;
+ data.size = outsize;
+ err = rdfstore_flat_store_store(me->subjects, key, data);
+ if (err == 0) {
+#ifdef RDFSTORE_DEBUG_COMPRESSION
+ fprintf(stderr,"Stored %d bytes for '%s' in subjects table for store '%s'\n", outsize, (char *)key.data, (me->name != NULL) ? me->name : "(in-memory)");
+#endif
+ } else {
+ if (err != FLAT_STORE_E_KEYEXIST) {
+ perror("rdfstore_insert");
+ fprintf(stderr,"Could not store '%d' bytes for subject in subjects table for store '%s': %s\n", (int)data.size, (me->name != NULL) ? me->name : "(in-memory)", rdfstore_flat_store_get_error(me->subjects));
+#if defined(RDFSTORE_CONNECTIONS) && defined(RDFSTORE_CONNECTIONS_REINDEXING)
+ rdfstore_iterator_close(reindex);
+#endif
+ return -1;
+ };
+ };
+
+#ifdef RDFSTORE_DEBUG
+ {
+ int i=0;
+ if ((rdfstore_flat_store_fetch(me->subjects, key, &data)) == 0) {
[... 9138 lines stripped ...]