From 76236787e00237d30bf6cdd5e0165924cbdc90b6 Mon Sep 17 00:00:00 2001 From: physikerwelt Date: Mon, 10 Feb 2014 15:04:19 +0000 Subject: [PATCH] Separate Mathoid from the Math extension Mathoid is supposed to be run at a server separated from the MediaWiki math extension. It should use the MathJax version customized to the MediaWiki needs as submodule. It has been moved to the new repository mediawiki/services/mathoid Change-Id: I00b42926ba91043570c5058e0352c0e743a400b0 --- mathoid/.gitignore | 3 - mathoid/.jshintignore | 3 - mathoid/.jshintrc | 32 ------- mathoid/LICENSE.txt | 21 ----- mathoid/MOVED | 2 + mathoid/README.md | 38 -------- mathoid/engine.js | 135 --------------------------- mathoid/index.html | 24 ----- mathoid/main.js | 78 ---------------- mathoid/mathoid-worker.js | 176 ------------------------------------ mathoid/mathoid.config.json | 1 - mathoid/mathoid.js | 47 ---------- mathoid/package.json | 10 -- 13 files changed, 2 insertions(+), 568 deletions(-) delete mode 100644 mathoid/.gitignore delete mode 100644 mathoid/.jshintignore delete mode 100644 mathoid/.jshintrc delete mode 100644 mathoid/LICENSE.txt create mode 100644 mathoid/MOVED delete mode 100644 mathoid/README.md delete mode 100644 mathoid/engine.js delete mode 100644 mathoid/index.html delete mode 100644 mathoid/main.js delete mode 100644 mathoid/mathoid-worker.js delete mode 100644 mathoid/mathoid.config.json delete mode 100755 mathoid/mathoid.js delete mode 100644 mathoid/package.json diff --git a/mathoid/.gitignore b/mathoid/.gitignore deleted file mode 100644 index 2c16869..0000000 --- a/mathoid/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# npm package sources # -################### -node_modules diff --git a/mathoid/.jshintignore b/mathoid/.jshintignore deleted file mode 100644 index c276fac..0000000 --- a/mathoid/.jshintignore +++ /dev/null @@ -1,3 +0,0 @@ -# browser code in the middle of node code.. -main.js -engine.js diff --git a/mathoid/.jshintrc b/mathoid/.jshintrc deleted file mode 100644 index 73f4fdd..0000000 --- a/mathoid/.jshintrc +++ /dev/null @@ -1,32 +0,0 @@ -{ - "predef": [ - "ve", - - "setImmediate", - - "QUnit" - ], - - "bitwise": true, - "curly": true, - "eqeqeq": true, - "immed": true, - "latedef": true, - "newcap": true, - "noarg": true, - "noempty": true, - "nonew": true, - "regexp": false, - "undef": true, - "strict": false, - "trailing": true, - - "smarttabs": true, - "multistr": true, - - "node": true, - - "nomen": false, - "loopfunc": true - //"onevar": true -} diff --git a/mathoid/LICENSE.txt b/mathoid/LICENSE.txt deleted file mode 100644 index 1c178e9..0000000 --- a/mathoid/LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Anton Grbin - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/mathoid/MOVED b/mathoid/MOVED new file mode 100644 index 0000000..c110e38 --- /dev/null +++ b/mathoid/MOVED @@ -0,0 +1,2 @@ +Moved to mediawiki/services/mathoid +Clone via git clone https://gerrit.wikimedia.org/r/mediawiki/services/mathoid diff --git a/mathoid/README.md b/mathoid/README.md deleted file mode 100644 index dab5391..0000000 --- a/mathoid/README.md +++ /dev/null @@ -1,38 +0,0 @@ -svgtex -====== - -Using MathJax and PhantomJS to create SVGs and MathML on server side with minimum overhead. - -MathJax is a great tool! Why not use it on a server side too? - -To avoid loading whole phantomjs and MathJax into memory with every call the service is exposed via HTTP. - -``` -$ phantomjs main.js & - -loading bench page -server started on port 16000 -you can hit server with http://localhost:16000/?2^n -.. or by sending latex source in POST (not url encoded) -``` - -And then .. curl it up to get back a json array that contains SVG and MathML - -``` -$ curl localhost:16000/?aa=a^2 - -aa=a^2.. 6B query, OK 2393B result, took 15ms. -{"tex":"aa=a^2","svg":"","mml":"\n a\n a\n =\n \n a\n 2\n \n"} -``` - - -Stability ---------- - -experimental. - -Read https://github.com/agrbin/svgtex/wiki for more details! - -Forked from https://github.com/agrbin/svgtex - - diff --git a/mathoid/engine.js b/mathoid/engine.js deleted file mode 100644 index 2b1bb8c..0000000 --- a/mathoid/engine.js +++ /dev/null @@ -1,135 +0,0 @@ -// perfect singleton -window.engine = (new (function() { - - this.Q = MathJax.Hub.queue; - this.math = null; - this.buffer = []; - - function toMathML(jax,callback) { - var mml; - try { - mml = jax.root.toMathML(''); - } catch(err) { - if (!err.restart) {throw err;} // an actual error - return MathJax.Callback.After([toMathML,jax,callback],err.restart); - } - MathJax.Callback(callback)(mml); - } - - // bind helper. - this.bind = function(method) { - var engine = this; - return function() { - return method.apply(engine, arguments); - }; - }; - - // initialize Engine, after MathJax is loaded, this.math will - // point to our jax. - this.init = function() { - this.Q.Push(this.bind(function () { - this.math = MathJax.Hub.getAllJax('math')[0]; - this.processBuffered(); - })); - }; - - // receives input latex string and invokes cb - // function with svg result. - this.processCB = function(latex, cb) { - this.Q.Push(['Text', this.math, latex]); - this.Q.Push(this.bind(function() { - // then, this toSVG call will invoke cb(result). - cb(document.getElementsByTagName('svg')[1].cloneNode(true)); - })); - }; - - // this is a helper for merge, who will want to decide - // whether something went wrong while rendering latex. - // the constant #C00 could be overriden by config!! - this.TextIsError = function(txt) { - return txt.getAttribute('fill') === '#C00' && - txt.getAttribute('stroke') === 'none'; - }; - - // mathjax keeps parts of SVG symbols in one hidden svg at - // the begining of the DOM, this function should take two - // SVGs and return one stand-alone svg which could be - // displayed like an image on some different page. - this.merge = function(svg) { - var uses, - copied = {}, - k, - id, - texts, - i, - tmpDiv, - defs = document.getElementById('MathJax_SVG_Hidden') - .nextSibling.childNodes[0].cloneNode(false); - - // clone and copy all used paths into local defs. - // xlink:href in uses FIX - uses = svg.getElementsByTagName('use'); - for ( k = 0; k < uses.length; ++k) { - id = uses[k].getAttribute('href'); - if (id && copied[id]) { - uses[k].setAttribute('xlink:href', id); - // Already copied, skip - continue; - } - defs.appendChild( - document.getElementById(id.substr(1)).cloneNode(true) - ); - uses[k].setAttribute('xlink:href', id); - copied[id] = true; - } - - // check for errors in svg. - texts = document.getElementsByTagName('text', svg); - for ( i = 0; i < texts.length; ++i) { - if (this.TextIsError(texts[i])) { - return [texts[i].textContent]; - } - } - - svg.style.position = 'static'; - svg.setAttribute('xmlns', 'http://www.w3.org/2000/svg'); - tmpDiv = document.createElement('div'); - tmpDiv.appendChild(svg); - svg.insertBefore(defs, svg.firstChild); - return tmpDiv.innerHTML; - }; - - // if someone calls process before init is complete, - // that call will be stored into buffer. After the init - // is complete, all buffer stuff will get resolved. - this.processBuffered = function() { - for (var i = 0; i < this.buffer.length; ++i) { - this.process(this.buffer[i][0], this.buffer[i][1]); - } - this.buffer = []; - }; - - // callback will be invoked with array [original latex, SVG output] - // if there is an error during the latex rendering then second - // element (instead of SVG output) will be array again with - // only one string element describing the error message. - this.process = function(latex, cb) { - if (this.math === null) { - this.buffer.push( [latex, cb] ); - } else { - try{ - this.processCB(latex, this.bind(function( ) { - var jax = MathJax.Hub.getAllJax(), - mergedSVG = this.merge(document.getElementsByTagName('svg')[1].cloneNode(true)); - toMathML(jax[0],function (mml) { - cb([latex, mergedSVG, mml]); - }); - })); - } catch (err) { - cb([latex, err, err]); - } - } - }; - - this.init(); -})()); diff --git a/mathoid/index.html b/mathoid/index.html deleted file mode 100644 index a6c6c70..0000000 --- a/mathoid/index.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - -
${}$
- - diff --git a/mathoid/main.js b/mathoid/main.js deleted file mode 100644 index e9b3222..0000000 --- a/mathoid/main.js +++ /dev/null @@ -1,78 +0,0 @@ -// this script will set up a HTTP server on this port (local connections only) -// and will receive POST requests (not urlencoded) -var PORT = 16000; - -var server = require('webserver').create(); -var page = require('webpage').create(); -var args = require('system').args; -var activeRequests = {}; -var service = null; - -if (args.length > 1) { - PORT = args[1]; -} - -// thanks to: -// stackoverflow.com/questions/5515869/string-length-in-bytes-in-javascript -function utf8Strlen(str) { - var m = encodeURIComponent(str).match(/%[89ABab]/g); - return str.length + (m ? m.length : 0); -} - -page.onCallback = function(data) { - var out, - log = '', - record = activeRequests[data[0]], - resp = record[0], - t = ', took ' + (((new Date()).getTime() - record[1])) + 'ms.'; - if ((typeof data[1]) === 'string') { - resp.statusCode = 200; - log = data[0].substr(0, 30) + '.. ' + data[0].length + 'B query, OK ' + data[1].length + '/' + data[2].length + 'B result' + t; - out = JSON.stringify({tex:data[0],svg:data[1],mml:data[2],'log':log, 'sucess':true}); - resp.setHeader('Content-Type', 'application/json'); - resp.setHeader('Content-Length', utf8Strlen(out).toString() ); - resp.write(out); - //console.log(log); - } else { - resp.statusCode = 400; - log = data[0].substr(0, 30) + '.. ' + - data[0].length + 'B query, ERR ' + data[1][0] + t; - out = JSON.stringify({err:data[1][0],svg:data[1],mml:data[2],'log':log,'sucess':false}); - resp.write(out); - console.log(log); - phantom.exit(1); - } - resp.close(); -}; - -console.log('loading bench page'); -page.open('index.html', function ( ) { - service = server.listen('127.0.0.1:' + PORT, function(req, resp) { - var query; - if (req.method === 'GET') { - // URL starts with /? and is urlencoded. - query = decodeURI(req.url.substr(2)); - } else { - query = req.post.tex; - } - if (query === undefined) { - return resp.close(); - } - activeRequests[query] = [resp, (new Date()).getTime()]; - // this is just queueing call, it will return at once. - page.evaluate(function(q) { - window.engine.process(q, window.callPhantom); - }, query); - }); - - if (!service) { - console.log('server failed to start on port ' + PORT); - phantom.exit(1); - }/* else { - console.log("server started on port " + PORT); - console.log("you can hit server with http://localhost:" + PORT + "/?2^n"); - console.log(".. or by sending tex source in POST (not url encoded)"); - }*/ -}); - - diff --git a/mathoid/mathoid-worker.js b/mathoid/mathoid-worker.js deleted file mode 100644 index dc5fcb2..0000000 --- a/mathoid/mathoid-worker.js +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Storoid worker. - * - * Configure in storoid.config.json. - */ - -// global includes -var express = require('express'), - cluster = require('cluster'), - http = require('http'), - fs = require('fs'), - child_process = require('child_process'), - request = require('request'), - querystring = require('querystring'); - -var config; - -// Get the config -try { - config = JSON.parse(fs.readFileSync('./mathoid.config.json', 'utf8')); -} catch ( e ) { - // Build a skeleton localSettings to prevent errors later. - console.error("Please set up your mathoid.config.json from the example " + - "storoid.config.json.example"); - process.exit(1); -} - -/** - * The name of this instance. - * @property {string} - */ -var instanceName = cluster.isWorker ? 'worker(' + process.pid + ')' : 'master'; - -console.log( ' - ' + instanceName + ' loading...' ); - - - -/* - * Backend setup - */ -var restarts = 10; - -var backend, - backendStarting = false, - backendPort, - requestQueue = []; - -// forward declaration -var handleRequests; - -var backendCB = function () { - backendStarting = false; - handleRequests(); -}; - -var startBackend = function (cb) { - if (backend) { - backend.removeAllListeners(); - backend.kill('SIGKILL'); - } - backendPort = Math.floor(9000 + Math.random() * 50000); - console.error(instanceName + ': Starting backend on port ' + backendPort); - backend = child_process.spawn('phantomjs', ['main.js', backendPort]); - backend.stdout.pipe(process.stderr); - backend.stderr.pipe(process.stderr); - backend.on('close', startBackend); - backendStarting = true; - // give backend 1 seconds to start up - setTimeout(backendCB, 1000); -}; -startBackend(); - -/* -------------------- Web service --------------------- */ - - -var app = express.createServer(); - -// Increase the form field size limit from the 2M default. -app.use(express.bodyParser({maxFieldsSize: 25 * 1024 * 1024})); -app.use( express.limit( '25mb' ) ); - -app.get('/', function(req, res){ - res.write('\n'); - res.write('Welcome to Mathoid. POST to / with var tex'); - res.write('
'); - res.end(''); -}); - -// robots.txt: no indexing. -app.get(/^\/robots.txt$/, function ( req, res ) { - res.end( "User-agent: *\nDisallow: /\n" ); -}); - - - -function handleRequest(req, res, tex) { - // do the backend request - var reqbody = new Buffer(querystring.stringify({tex: tex})), - options = { - method: 'POST', - uri: 'http://localhost:' + backendPort.toString() + '/', - body: reqbody, - // Work around https://github.com/ariya/phantomjs/issues/11421 by - // setting explicit upper-case headers (request sends them lowercase - // by default) and manually encoding the body. - headers: { - 'Content-Length': reqbody.length, - 'Content-Type': 'application/x-www-form-urlencoded' - }, - timeout: 2000 - }; - request(options, function (err, response, body) { - body = new Buffer(body); - if (err || response.statusCode !== 200) { - var errBuf; - if (err) { - errBuf = new Buffer(JSON.stringify({ - tex: tex, - log: err.toString(), - success: false - })); - } else { - errBuf = body; - } - res.writeHead(500, - { - 'Content-Type': 'application/json', - 'Content-Length': errBuf.length - }); - res.end(errBuf); - // don't retry the request - requestQueue.shift(); - startBackend(); - return handleRequests(); - } - res.writeHead(200, - { - 'Content-Type': 'application/json', - 'Content-length': body.length - }); - res.end(body); - requestQueue.shift(); - handleRequests(); - }); -} - -handleRequests = function() { - // Call the next request on the queue - if (!backendStarting && requestQueue.length) { - requestQueue[0](); - } -}; - - -app.post(/^\/$/, function ( req, res ) { - // First some rudimentary input validation - if (!req.body.tex) { - res.writeHead(400); - return res.end(JSON.stringify({error: "'tex' post parameter is missing!"})); - } - var tex = req.body.tex; - - requestQueue.push(handleRequest.bind(null, req, res, tex)); - // phantomjs only handles one request at a time. Enforce this. - if (requestQueue.length === 1) { - // Start this process - handleRequests(); - } - -}); - - -console.log( ' - ' + instanceName + ' ready' ); - -module.exports = app; - diff --git a/mathoid/mathoid.config.json b/mathoid/mathoid.config.json deleted file mode 100644 index 0967ef4..0000000 --- a/mathoid/mathoid.config.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/mathoid/mathoid.js b/mathoid/mathoid.js deleted file mode 100755 index a1a4688..0000000 --- a/mathoid/mathoid.js +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env node -/** - * A very basic cluster-based server runner. Restarts failed workers, but does - * not much else right now. - */ - -var cluster = require('cluster'); - -if (cluster.isMaster) { - // Start a few more workers than there are cpus visible to the OS, so that we - // get some degree of parallelism even on single-core systems. A single - // long-running request would otherwise hold up all concurrent short requests. - var numCPUs = require('os').cpus().length + 3; - // Fork workers. - for (var i = 0; i < numCPUs; i++) { - cluster.fork(); - } - - cluster.on('exit', function(worker) { - if (!worker.suicide) { - var exitCode = worker.process.exitCode; - console.log('worker', worker.process.pid, - 'died ('+exitCode+'), restarting.'); - cluster.fork(); - } - }); - - process.on('SIGTERM', function() { - console.log('master shutting down, killing workers'); - var workers = cluster.workers; - Object.keys(workers).forEach(function(id) { - console.log('Killing worker ' + id); - workers[id].destroy(); - }); - console.log('Done killing workers, bye'); - process.exit(1); - } ); -} else { - var mathoidWorker = require('./mathoid-worker.js'); - process.on('SIGTERM', function() { - console.log('Worker shutting down'); - process.exit(1); - }); - // when running on appfog.com the listen port for the app - // is passed in an environment variable. Most users can ignore this! - mathoidWorker.listen(process.env.VCAP_APP_PORT || 8010); -} diff --git a/mathoid/package.json b/mathoid/package.json deleted file mode 100644 index fc697a0..0000000 --- a/mathoid/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "Mathoid", - "description": "Render tex to SVG and MathML using MathJax, Phantom. Based on svgtex.", - "version": "0.0.1", - "dependencies": { - "querystring": "0.x.x", - "express": "2.5.x", - "request": "2.x.x" - } -}