You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2012/01/23 16:24:40 UTC
svn commit: r1234853 - in /incubator/jena/Jena2/Fuseki/trunk: src-dev/dev/
src/main/java/org/apache/jena/fuseki/
src/main/java/org/apache/jena/fuseki/mgt/
src/main/java/org/apache/jena/fuseki/server/
Author: andy
Date: Mon Jan 23 15:24:40 2012
New Revision: 1234853
URL: http://svn.apache.org/viewvc?rev=1234853&view=rev
Log:
Framework for management functions (backup, stop server, others to be decided)
Added:
incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ManagementServer.java
Modified:
incubator/jena/Jena2/Fuseki/trunk/src-dev/dev/RunFuseki.java
incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/FusekiCmd.java
incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ServerServlet.java
incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/FusekiConfig.java
incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/SPARQLServer.java
Modified: incubator/jena/Jena2/Fuseki/trunk/src-dev/dev/RunFuseki.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/Fuseki/trunk/src-dev/dev/RunFuseki.java?rev=1234853&r1=1234852&r2=1234853&view=diff
==============================================================================
--- incubator/jena/Jena2/Fuseki/trunk/src-dev/dev/RunFuseki.java (original)
+++ incubator/jena/Jena2/Fuseki/trunk/src-dev/dev/RunFuseki.java Mon Jan 23 15:24:40 2012
@@ -45,6 +45,7 @@ public class RunFuseki
//"--timeout=1000,5000",
//"--set=arq:queryTimeout=1000",
//"--port=3030",
+ //"--mgtPort=3031",
//"--host=localhost",
"--mem",
//"--loc=DB",
Modified: incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/FusekiCmd.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/FusekiCmd.java?rev=1234853&r1=1234852&r2=1234853&view=diff
==============================================================================
--- incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/FusekiCmd.java (original)
+++ incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/FusekiCmd.java Mon Jan 23 15:24:40 2012
@@ -18,15 +18,19 @@
package org.apache.jena.fuseki;
+import static org.apache.jena.fuseki.Fuseki.serverLog ;
+
import java.io.InputStream ;
import java.net.InetAddress ;
import java.net.UnknownHostException ;
import java.util.Arrays ;
import java.util.List ;
+import org.apache.jena.fuseki.mgt.ManagementServer ;
import org.apache.jena.fuseki.server.DatasetRef ;
import org.apache.jena.fuseki.server.FusekiConfig ;
import org.apache.jena.fuseki.server.SPARQLServer ;
+import org.eclipse.jetty.server.Server ;
import org.openjena.atlas.io.IO ;
import org.openjena.atlas.lib.FileOps ;
import org.openjena.atlas.lib.Sink ;
@@ -68,6 +72,7 @@ public class FusekiCmd extends CmdARQ
// pages/control/
// pages/query/ or /pages/sparql/
+ private static ArgDecl argMgtPort = new ArgDecl(ArgDecl.HasValue, "mgtPort", "mgtport") ;
private static ArgDecl argMem = new ArgDecl(ArgDecl.NoValue, "mem") ;
private static ArgDecl argAllowUpdate = new ArgDecl(ArgDecl.NoValue, "update", "allowUpdate") ;
private static ArgDecl argFile = new ArgDecl(ArgDecl.HasValue, "file") ;
@@ -93,8 +98,9 @@ public class FusekiCmd extends CmdARQ
new FusekiCmd(argv).mainRun() ;
}
- private int port = 3030 ;
- private String clientHost = null;
+ private int port = 3030 ;
+ private int mgtPort = -1 ;
+ private String clientHost = null;
private DatasetGraph dsg ;
private String datasetPath ;
@@ -119,6 +125,7 @@ public class FusekiCmd extends CmdARQ
add(argAllowUpdate, "--update", "Allow updates (via SPARQL Update and SPARQL HTTP Update)") ;
add(argFusekiConfig, "--config=", "Use a configuration file to determine the services") ;
add(argJettyConfig, "--jetty-config=", "Set up the server (not services) with a Jetty XML file") ;
+ add(argMgtPort, "--mgt=port", "Enable the management commands on the given port") ;
super.modVersion.addClass(TDB.class) ;
super.modVersion.addClass(Fuseki.class) ;
}
@@ -234,6 +241,17 @@ public class FusekiCmd extends CmdARQ
}
}
+ if ( contains(argMgtPort) )
+ {
+ String mgtPortStr = getValue(argMgtPort) ;
+ try {
+ mgtPort = Integer.parseInt(mgtPortStr) ;
+ } catch (NumberFormatException ex)
+ {
+ throw new CmdException(argMgtPort.getKeyName()+" : bad port number: "+mgtPortStr) ;
+ }
+ }
+
if ( contains(argHost) )
{
clientHost = getValue(argHost);
@@ -295,9 +313,28 @@ public class FusekiCmd extends CmdARQ
}
// Temporary
Fuseki.setServer(server) ;
+
+ Server mgtServer = null ;
+
+ if ( mgtPort > 0 )
+ {
+ mgtServer = ManagementServer.createManagementServer(mgtPort) ;
+ try { mgtServer.start() ; }
+ catch (java.net.BindException ex)
+ { serverLog.error("SPARQLServer: Failed to start management server: " + ex.getMessage()) ; System.exit(1) ; }
+ catch (Exception ex)
+ { serverLog.error("SPARQLServer: Failed to start management server: " + ex.getMessage(), ex) ; System.exit(1) ; }
+ }
server.start() ;
try { server.getServer().join() ; } catch (Exception ex) {}
+
+ if ( mgtServer != null )
+ {
+ try { mgtServer.stop() ; }
+ catch (Exception e) { serverLog.warn("Failed to cleanly stop the management server", e) ; }
+ }
+ System.exit(0) ;
}
Added: incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ManagementServer.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ManagementServer.java?rev=1234853&view=auto
==============================================================================
--- incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ManagementServer.java (added)
+++ incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ManagementServer.java Mon Jan 23 15:24:40 2012
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.fuseki.mgt;
+
+import static org.apache.jena.fuseki.Fuseki.serverLog ;
+
+import java.util.List ;
+
+import javax.servlet.http.HttpServlet ;
+
+import org.apache.jena.fuseki.Fuseki ;
+import org.apache.jena.fuseki.server.FusekiErrorHandler ;
+import org.eclipse.jetty.server.Connector ;
+import org.eclipse.jetty.server.Server ;
+import org.eclipse.jetty.server.nio.BlockingChannelConnector ;
+import org.eclipse.jetty.servlet.ServletContextHandler ;
+import org.eclipse.jetty.servlet.ServletHolder ;
+
+public class ManagementServer
+{
+
+ public static Server createManagementServer(int mgtPort)
+ {
+ Fuseki.serverLog.info("Adding management functions") ;
+ // This is actually a separate Jetty server
+ Server server = new Server() ;
+
+ BlockingChannelConnector bcConnector = new BlockingChannelConnector() ;
+ //bcConnector.setUseDirectBuffers(false) ;
+
+ Connector connector = bcConnector ;
+ // Ignore. If set, then if this goes off, it keeps going off
+ // and you get a lot of log messages.
+ connector.setMaxIdleTime(0) ; // Jetty outputs a lot of messages if this goes off.
+ connector.setPort(mgtPort);
+ server.addConnector(connector) ;
+
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setErrorHandler(new FusekiErrorHandler()) ;
+ server.setHandler(context);
+
+ // Add the server control servlet
+ addServlet(context, new ServerServlet(), "/mgt") ;
+
+ return server ;
+ // Old plan
+// // Development : server control panel.
+// addServlet(context, new ServerServlet(), "/server") ;
+// addServlet(context, new ActionBackup(), "/backup") ;
+
+
+ }
+
+ // SHARE
+ private static void addServlet(ServletContextHandler context, String datasetPath, HttpServlet servlet, List<String> pathSpecs)
+ {
+ for ( String pathSpec : pathSpecs )
+ {
+ if ( pathSpec.endsWith("/") )
+ pathSpec = pathSpec.substring(0, pathSpec.length()-1) ;
+ if ( pathSpec.startsWith("/") )
+ pathSpec = pathSpec.substring(1, pathSpec.length()) ;
+ addServlet(context, servlet, datasetPath+"/"+pathSpec) ;
+ }
+ }
+
+ private static void addServlet(ServletContextHandler context, HttpServlet servlet, String pathSpec)
+ {
+ ServletHolder holder = new ServletHolder(servlet) ;
+ addServlet(context, holder, pathSpec) ;
+ }
+
+ private static void addServlet(ServletContextHandler context, ServletHolder holder, String pathSpec)
+ {
+ serverLog.debug("Add servlet @ "+pathSpec) ;
+ context.addServlet(holder, pathSpec) ;
+ }
+
+}
+
Modified: incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ServerServlet.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ServerServlet.java?rev=1234853&r1=1234852&r2=1234853&view=diff
==============================================================================
--- incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ServerServlet.java (original)
+++ incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/mgt/ServerServlet.java Mon Jan 23 15:24:40 2012
@@ -31,8 +31,10 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse ;
import org.apache.jena.fuseki.Fuseki ;
+import org.apache.jena.fuseki.http.HttpSC ;
import org.apache.jena.fuseki.server.DatasetRef ;
import org.apache.jena.fuseki.server.SPARQLServer ;
+import org.slf4j.Logger ;
import com.hp.hpl.jena.Jena ;
import com.hp.hpl.jena.query.ARQ ;
@@ -42,6 +44,8 @@ import com.hp.hpl.jena.tdb.TDB ;
public class ServerServlet extends HttpServlet
{
+ private static Logger log = Fuseki.serverLog ;
+
public ServerServlet()
{
@@ -52,6 +56,51 @@ public class ServerServlet extends HttpS
{
return ;
}
+
+ public static String paramCmd = "cmd" ;
+ public static String cmdBackup = "backup" ;
+ public static String cmdRestart = "restart" ;
+ public static String cmdShutdown = "shutdown" ;
+
+ ActionBackup actionBackup = new ActionBackup() ;
+
+ @Override
+ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException
+ {
+ //Commands:
+ // ?cmd=backup
+
+ String[] args = req.getParameterValues(paramCmd) ;
+ if ( args == null )
+ {
+ resp.setContentType("text/plain") ;
+ resp.setStatus(HttpSC.BAD_REQUEST_400) ;
+
+ return ;
+ }
+ for ( String cmd : args )
+ {
+ if ( log.isInfoEnabled() )
+ log.info("Management command: "+cmd) ;
+
+ if ( cmd.equalsIgnoreCase(cmdBackup))
+ {
+ actionBackup.doPost(req, resp) ;
+ continue ;
+ }
+ if ( cmd.equalsIgnoreCase(cmdRestart))
+ {
+ continue ;
+ }
+ if ( cmd.equalsIgnoreCase(cmdShutdown))
+ {
+ Fuseki.getServer().stop() ;
+ continue ;
+ }
+ log.warn("Unrecognized command : "+cmd) ;
+
+ }
+ }
@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp)
Modified: incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/FusekiConfig.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/FusekiConfig.java?rev=1234853&r1=1234852&r2=1234853&view=diff
==============================================================================
--- incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/FusekiConfig.java (original)
+++ incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/FusekiConfig.java Mon Jan 23 15:24:40 2012
@@ -27,7 +27,6 @@ import org.apache.jena.fuseki.FusekiConf
import org.apache.jena.fuseki.HttpNames ;
import org.openjena.atlas.iterator.Iter ;
import org.openjena.atlas.lib.StrUtils ;
-import org.openjena.atlas.logging.Log ;
import org.slf4j.Logger ;
import com.hp.hpl.jena.assembler.Assembler ;
@@ -59,8 +58,17 @@ import com.hp.hpl.jena.vocabulary.RDFS ;
public class FusekiConfig
{
- static { Log.setLog4j() ; }
+ static { Fuseki.init(); }
+ // The datastructure that captures a servers configuration.
+
+ // Server port
+ int port ;
+ // Management command port - -1 for none.
+ int mgtPort ;
+ List<DatasetRef> datasets = null ;
+
+
private static Logger log = Fuseki.configLog ;
private static String prefixes = StrUtils.strjoinNL(
Modified: incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/SPARQLServer.java
URL: http://svn.apache.org/viewvc/incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/SPARQLServer.java?rev=1234853&r1=1234852&r2=1234853&view=diff
==============================================================================
--- incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/SPARQLServer.java (original)
+++ incubator/jena/Jena2/Fuseki/trunk/src/main/java/org/apache/jena/fuseki/server/SPARQLServer.java Mon Jan 23 15:24:40 2012
@@ -28,10 +28,14 @@ import javax.servlet.http.HttpServlet ;
import org.apache.jena.fuseki.Fuseki ;
import org.apache.jena.fuseki.FusekiException ;
-import org.apache.jena.fuseki.mgt.ActionBackup ;
import org.apache.jena.fuseki.mgt.ActionDataset ;
-import org.apache.jena.fuseki.mgt.ServerServlet ;
-import org.apache.jena.fuseki.servlets.* ;
+import org.apache.jena.fuseki.servlets.DumpServlet ;
+import org.apache.jena.fuseki.servlets.SPARQL_QueryDataset ;
+import org.apache.jena.fuseki.servlets.SPARQL_QueryGeneral ;
+import org.apache.jena.fuseki.servlets.SPARQL_REST_R ;
+import org.apache.jena.fuseki.servlets.SPARQL_REST_RW ;
+import org.apache.jena.fuseki.servlets.SPARQL_Update ;
+import org.apache.jena.fuseki.servlets.SPARQL_Upload ;
import org.apache.jena.fuseki.validation.DataValidator ;
import org.apache.jena.fuseki.validation.IRIValidator ;
import org.apache.jena.fuseki.validation.QueryValidator ;
@@ -115,7 +119,6 @@ public class SPARQLServer
server = defaultServerConfig(port) ;
// Keep the server to a maximum number of threads.
//server.setThreadPool(new QueuedThreadPool(ThreadPoolSize)) ;
-
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setErrorHandler(new FusekiErrorHandler()) ;
@@ -161,10 +164,6 @@ public class SPARQLServer
// Action when control panel selects a dataset.
HttpServlet datasetChooser = new ActionDataset() ;
addServlet(context, datasetChooser, "/dataset") ;
-
-// // Development : server control panel.
-// addServlet(context, new ServerServlet(), "/server") ;
-// addServlet(context, new ActionBackup(), "/backup") ;
}
if ( installServices )
@@ -240,7 +239,7 @@ public class SPARQLServer
throw new FusekiException("Failed to configure a server using configuration file '"+jettyConfig+"'") ;
}
}
-
+
private static Server defaultServerConfig(int port)
{
// Server, with one NIO-based connector, large input buffer size (for long URLs, POSTed forms (queries, updates)).
@@ -279,6 +278,7 @@ public class SPARQLServer
addServlet(context, staticContent, pathSpec) ;
}
+ // SHARE
private static void addServlet(ServletContextHandler context, String datasetPath, HttpServlet servlet, List<String> pathSpecs)
{
for ( String pathSpec : pathSpecs )
Re: svn commit: r1234853 - in /incubator/jena/Jena2/Fuseki/trunk:
src-dev/dev/ src/main/java/org/apache/jena/fuseki/ src/main/java/org/apache/jena/fuseki/mgt/
src/main/java/org/apache/jena/fuseki/server/
Posted by Andy Seaborne <an...@apache.org>.
On 24/01/12 08:36, Paolo Castagna wrote:
> andy@apache.org wrote:
>> Framework for management functions (backup, stop server, others to be decided)
>
> Thank you Andy, I've just look at it (not tried yet).
> But, it seems to me quite an useful thing (for production systems).
>
> While ago, when I was thinking how a very simple replication scheme (using rsync)
> would work, I was planning to have a management function to set Fuseki in read-only
> mode (i.e. it sync changes on disk and it stops listening to updates) and then set
> it back to read-write mode (if that was the starting configuration). This way people
> are sure that all changes have been flushed on disk, no more changes are possible,
> therefore it is (right?) possible to copy, backup, manipulate the TDB indexes using
> filesystem operations/tools (including rsync).
>
> Does a management function to switch between read-only and read-write mode make
> sense to you?
>
> Paolo
A feature to put the server (or, rather, a dataset) into read-only mode
would of itself be useful.
To copy, or rsync, the dataset you need to
(1) make it go read only
(2) wait for write transactions to end
(3) ensure the journal has been written back to the DB
then the DB on disk is consistent and can be copied.
It's not possible to manipulate the indexes while the system is running
currently so you can't rsync in changes. Cached information would need
to (atomically) be dropped and then restart from disk.
A way to have two copies of the DB, one active, one not, and switch
between them would be a way.
Another way is to put a load balancer in front of two copies and
start/stop the copies.
I have (this week) used both HAProxy and AWS Load Balancer to do exactly
that.
Andy
Re: svn commit: r1234853 - in /incubator/jena/Jena2/Fuseki/trunk:
src-dev/dev/ src/main/java/org/apache/jena/fuseki/ src/main/java/org/apache/jena/fuseki/mgt/
src/main/java/org/apache/jena/fuseki/server/
Posted by Paolo Castagna <ca...@googlemail.com>.
andy@apache.org wrote:
> Framework for management functions (backup, stop server, others to be decided)
Thank you Andy, I've just look at it (not tried yet).
But, it seems to me quite an useful thing (for production systems).
While ago, when I was thinking how a very simple replication scheme (using rsync)
would work, I was planning to have a management function to set Fuseki in read-only
mode (i.e. it sync changes on disk and it stops listening to updates) and then set
it back to read-write mode (if that was the starting configuration). This way people
are sure that all changes have been flushed on disk, no more changes are possible,
therefore it is (right?) possible to copy, backup, manipulate the TDB indexes using
filesystem operations/tools (including rsync).
Does a management function to switch between read-only and read-write mode make
sense to you?
Paolo