Separate Mathoid from the Math extension

Mathoid is supposed to be run at a server separated from
the MediaWiki math extension. It should use the MathJax
version customized to the MediaWiki needs as submodule.
It has been moved to the new repository
mediawiki/services/mathoid

Change-Id: I00b42926ba91043570c5058e0352c0e743a400b0
This commit is contained in:
physikerwelt 2014-02-10 15:04:19 +00:00 committed by Physikerwelt
parent e0b3883c55
commit 76236787e0
13 changed files with 2 additions and 568 deletions

3
mathoid/.gitignore vendored
View File

@ -1,3 +0,0 @@
# npm package sources #
###################
node_modules

View File

@ -1,3 +0,0 @@
# browser code in the middle of node code..
main.js
engine.js

View File

@ -1,32 +0,0 @@
{
"predef": [
"ve",
"setImmediate",
"QUnit"
],
"bitwise": true,
"curly": true,
"eqeqeq": true,
"immed": true,
"latedef": true,
"newcap": true,
"noarg": true,
"noempty": true,
"nonew": true,
"regexp": false,
"undef": true,
"strict": false,
"trailing": true,
"smarttabs": true,
"multistr": true,
"node": true,
"nomen": false,
"loopfunc": true
//"onevar": true
}

View File

@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2013 Anton Grbin
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

2
mathoid/MOVED Normal file
View File

@ -0,0 +1,2 @@
Moved to mediawiki/services/mathoid
Clone via git clone https://gerrit.wikimedia.org/r/mediawiki/services/mathoid

View File

@ -1,38 +0,0 @@
svgtex
======
Using MathJax and PhantomJS to create SVGs and MathML on server side with minimum overhead.
MathJax is a great tool! Why not use it on a server side too?
To avoid loading whole phantomjs and MathJax into memory with every call the service is exposed via HTTP.
```
$ phantomjs main.js &
loading bench page
server started on port 16000
you can hit server with http://localhost:16000/?2^n
.. or by sending latex source in POST (not url encoded)
```
And then .. curl it up to get back a json array that contains SVG and MathML
```
$ curl localhost:16000/?aa=a^2
aa=a^2.. 6B query, OK 2393B result, took 15ms.
{"tex":"aa=a^2","svg":"<svg xmlns:xlink=\"http://www.w3.org/1999/xlink\" style=\"width: 7.875ex; height: 2.125ex; vertical-align: -0.125ex; margin-top: 1px; margin-right: 0px; margin-bottom: 1px; margin-left: 0px; position: static; \" viewBox=\"0 -859.4768963737118 3397.6444800547624 898.0576086653176\" xmlns=\"http://www.w3.org/2000/svg\"><defs id=\"MathJax_SVG_glyphs\"><path id=\"MJMATHI-61\" stroke-width=\"10\" d=\"M33 157Q33 258 109 349T280 441Q331 441 370 392Q386 422 416 422Q429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487Q506 153 506 144Q506 138 501 117T481 63T449 13Q436 0 417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157ZM351 328Q351 334 346 350T323 385T277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z\"></path><path id=\"MJMAIN-3D\" stroke-width=\"10\" d=\"M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z\"></path><path id=\"MJMAIN-32\" stroke-width=\"10\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"></path></defs><g stroke=\"black\" fill=\"black\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use href=\"#MJMATHI-61\" xlink:href=\"#MJMATHI-61\"></use><use href=\"#MJMATHI-61\" x=\"534\" y=\"0\" xlink:href=\"#MJMATHI-61\"></use><use href=\"#MJMAIN-3D\" x=\"1345\" y=\"0\" xlink:href=\"#MJMAIN-3D\"></use><g transform=\"translate(2406,0)\"><use href=\"#MJMATHI-61\" xlink:href=\"#MJMATHI-61\"></use><use transform=\"scale(0.7071067811865476)\" href=\"#MJMAIN-32\" x=\"755\" y=\"513\" xlink:href=\"#MJMAIN-32\"></use></g></g></svg>","mml":"<math xmlns=\"http://www.w3.org/1998/Math/MathML\">\n <mi>a</mi>\n <mi>a</mi>\n <mo>=</mo>\n <msup>\n <mi>a</mi>\n <mn>2</mn>\n </msup>\n</math>"}
```
Stability
---------
experimental.
Read https://github.com/agrbin/svgtex/wiki for more details!
Forked from https://github.com/agrbin/svgtex

View File

@ -1,135 +0,0 @@
// perfect singleton
window.engine = (new (function() {
this.Q = MathJax.Hub.queue;
this.math = null;
this.buffer = [];
function toMathML(jax,callback) {
var mml;
try {
mml = jax.root.toMathML('');
} catch(err) {
if (!err.restart) {throw err;} // an actual error
return MathJax.Callback.After([toMathML,jax,callback],err.restart);
}
MathJax.Callback(callback)(mml);
}
// bind helper.
this.bind = function(method) {
var engine = this;
return function() {
return method.apply(engine, arguments);
};
};
// initialize Engine, after MathJax is loaded, this.math will
// point to our jax.
this.init = function() {
this.Q.Push(this.bind(function () {
this.math = MathJax.Hub.getAllJax('math')[0];
this.processBuffered();
}));
};
// receives input latex string and invokes cb
// function with svg result.
this.processCB = function(latex, cb) {
this.Q.Push(['Text', this.math, latex]);
this.Q.Push(this.bind(function() {
// then, this toSVG call will invoke cb(result).
cb(document.getElementsByTagName('svg')[1].cloneNode(true));
}));
};
// this is a helper for merge, who will want to decide
// whether something went wrong while rendering latex.
// the constant #C00 could be overriden by config!!
this.TextIsError = function(txt) {
return txt.getAttribute('fill') === '#C00' &&
txt.getAttribute('stroke') === 'none';
};
// mathjax keeps parts of SVG symbols in one hidden svg at
// the begining of the DOM, this function should take two
// SVGs and return one stand-alone svg which could be
// displayed like an image on some different page.
this.merge = function(svg) {
var uses,
copied = {},
k,
id,
texts,
i,
tmpDiv,
defs = document.getElementById('MathJax_SVG_Hidden')
.nextSibling.childNodes[0].cloneNode(false);
// clone and copy all used paths into local defs.
// xlink:href in uses FIX
uses = svg.getElementsByTagName('use');
for ( k = 0; k < uses.length; ++k) {
id = uses[k].getAttribute('href');
if (id && copied[id]) {
uses[k].setAttribute('xlink:href', id);
// Already copied, skip
continue;
}
defs.appendChild(
document.getElementById(id.substr(1)).cloneNode(true)
);
uses[k].setAttribute('xlink:href', id);
copied[id] = true;
}
// check for errors in svg.
texts = document.getElementsByTagName('text', svg);
for ( i = 0; i < texts.length; ++i) {
if (this.TextIsError(texts[i])) {
return [texts[i].textContent];
}
}
svg.style.position = 'static';
svg.setAttribute('xmlns', 'http://www.w3.org/2000/svg');
tmpDiv = document.createElement('div');
tmpDiv.appendChild(svg);
svg.insertBefore(defs, svg.firstChild);
return tmpDiv.innerHTML;
};
// if someone calls process before init is complete,
// that call will be stored into buffer. After the init
// is complete, all buffer stuff will get resolved.
this.processBuffered = function() {
for (var i = 0; i < this.buffer.length; ++i) {
this.process(this.buffer[i][0], this.buffer[i][1]);
}
this.buffer = [];
};
// callback will be invoked with array [original latex, SVG output]
// if there is an error during the latex rendering then second
// element (instead of SVG output) will be array again with
// only one string element describing the error message.
this.process = function(latex, cb) {
if (this.math === null) {
this.buffer.push( [latex, cb] );
} else {
try{
this.processCB(latex, this.bind(function( ) {
var jax = MathJax.Hub.getAllJax(),
mergedSVG = this.merge(document.getElementsByTagName('svg')[1].cloneNode(true));
toMathML(jax[0],function (mml) {
cb([latex, mergedSVG, mml]);
});
}));
} catch (err) {
cb([latex, err, err]);
}
}
};
this.init();
})());

View File

@ -1,24 +0,0 @@
<!--
This file is loaded from main.js (phantomjs).
-->
<html>
<head>
<script type="text/javascript">
MathJax = {
jax: ["input/TeX","output/SVG","output/NativeMML"],
extensions: ["tex2jax.js","toMathML.js","TeX/noErrors.js","TeX/noUndefined.js","TeX/AMSmath.js","TeX/AMSsymbols.js"],
showProcessingMessages: false,
messageStyle: "none",
tex2jax: {
inlineMath: [["$","$"],["\\(","\\)"]]
}
};
</script>
<script type="text/javascript" src="../modules/MathJax/unpacked/MathJax.js">
</script>
<script type="text/javascript" src="engine.js"></script>
</head>
<body>
<div id="math">${}$</div>
</body>
</html>

View File

@ -1,78 +0,0 @@
// this script will set up a HTTP server on this port (local connections only)
// and will receive POST requests (not urlencoded)
var PORT = 16000;
var server = require('webserver').create();
var page = require('webpage').create();
var args = require('system').args;
var activeRequests = {};
var service = null;
if (args.length > 1) {
PORT = args[1];
}
// thanks to:
// stackoverflow.com/questions/5515869/string-length-in-bytes-in-javascript
function utf8Strlen(str) {
var m = encodeURIComponent(str).match(/%[89ABab]/g);
return str.length + (m ? m.length : 0);
}
page.onCallback = function(data) {
var out,
log = '',
record = activeRequests[data[0]],
resp = record[0],
t = ', took ' + (((new Date()).getTime() - record[1])) + 'ms.';
if ((typeof data[1]) === 'string') {
resp.statusCode = 200;
log = data[0].substr(0, 30) + '.. ' + data[0].length + 'B query, OK ' + data[1].length + '/' + data[2].length + 'B result' + t;
out = JSON.stringify({tex:data[0],svg:data[1],mml:data[2],'log':log, 'sucess':true});
resp.setHeader('Content-Type', 'application/json');
resp.setHeader('Content-Length', utf8Strlen(out).toString() );
resp.write(out);
//console.log(log);
} else {
resp.statusCode = 400;
log = data[0].substr(0, 30) + '.. ' +
data[0].length + 'B query, ERR ' + data[1][0] + t;
out = JSON.stringify({err:data[1][0],svg:data[1],mml:data[2],'log':log,'sucess':false});
resp.write(out);
console.log(log);
phantom.exit(1);
}
resp.close();
};
console.log('loading bench page');
page.open('index.html', function ( ) {
service = server.listen('127.0.0.1:' + PORT, function(req, resp) {
var query;
if (req.method === 'GET') {
// URL starts with /? and is urlencoded.
query = decodeURI(req.url.substr(2));
} else {
query = req.post.tex;
}
if (query === undefined) {
return resp.close();
}
activeRequests[query] = [resp, (new Date()).getTime()];
// this is just queueing call, it will return at once.
page.evaluate(function(q) {
window.engine.process(q, window.callPhantom);
}, query);
});
if (!service) {
console.log('server failed to start on port ' + PORT);
phantom.exit(1);
}/* else {
console.log("server started on port " + PORT);
console.log("you can hit server with http://localhost:" + PORT + "/?2^n");
console.log(".. or by sending tex source in POST (not url encoded)");
}*/
});

View File

@ -1,176 +0,0 @@
/*
* Storoid worker.
*
* Configure in storoid.config.json.
*/
// global includes
var express = require('express'),
cluster = require('cluster'),
http = require('http'),
fs = require('fs'),
child_process = require('child_process'),
request = require('request'),
querystring = require('querystring');
var config;
// Get the config
try {
config = JSON.parse(fs.readFileSync('./mathoid.config.json', 'utf8'));
} catch ( e ) {
// Build a skeleton localSettings to prevent errors later.
console.error("Please set up your mathoid.config.json from the example " +
"storoid.config.json.example");
process.exit(1);
}
/**
* The name of this instance.
* @property {string}
*/
var instanceName = cluster.isWorker ? 'worker(' + process.pid + ')' : 'master';
console.log( ' - ' + instanceName + ' loading...' );
/*
* Backend setup
*/
var restarts = 10;
var backend,
backendStarting = false,
backendPort,
requestQueue = [];
// forward declaration
var handleRequests;
var backendCB = function () {
backendStarting = false;
handleRequests();
};
var startBackend = function (cb) {
if (backend) {
backend.removeAllListeners();
backend.kill('SIGKILL');
}
backendPort = Math.floor(9000 + Math.random() * 50000);
console.error(instanceName + ': Starting backend on port ' + backendPort);
backend = child_process.spawn('phantomjs', ['main.js', backendPort]);
backend.stdout.pipe(process.stderr);
backend.stderr.pipe(process.stderr);
backend.on('close', startBackend);
backendStarting = true;
// give backend 1 seconds to start up
setTimeout(backendCB, 1000);
};
startBackend();
/* -------------------- Web service --------------------- */
var app = express.createServer();
// Increase the form field size limit from the 2M default.
app.use(express.bodyParser({maxFieldsSize: 25 * 1024 * 1024}));
app.use( express.limit( '25mb' ) );
app.get('/', function(req, res){
res.write('<html><body>\n');
res.write('Welcome to Mathoid. POST to / with var <code>tex</code>');
res.write('<form action="/" method="POST"><input type="text" name="tex"></form>');
res.end('</body></html>');
});
// robots.txt: no indexing.
app.get(/^\/robots.txt$/, function ( req, res ) {
res.end( "User-agent: *\nDisallow: /\n" );
});
function handleRequest(req, res, tex) {
// do the backend request
var reqbody = new Buffer(querystring.stringify({tex: tex})),
options = {
method: 'POST',
uri: 'http://localhost:' + backendPort.toString() + '/',
body: reqbody,
// Work around https://github.com/ariya/phantomjs/issues/11421 by
// setting explicit upper-case headers (request sends them lowercase
// by default) and manually encoding the body.
headers: {
'Content-Length': reqbody.length,
'Content-Type': 'application/x-www-form-urlencoded'
},
timeout: 2000
};
request(options, function (err, response, body) {
body = new Buffer(body);
if (err || response.statusCode !== 200) {
var errBuf;
if (err) {
errBuf = new Buffer(JSON.stringify({
tex: tex,
log: err.toString(),
success: false
}));
} else {
errBuf = body;
}
res.writeHead(500,
{
'Content-Type': 'application/json',
'Content-Length': errBuf.length
});
res.end(errBuf);
// don't retry the request
requestQueue.shift();
startBackend();
return handleRequests();
}
res.writeHead(200,
{
'Content-Type': 'application/json',
'Content-length': body.length
});
res.end(body);
requestQueue.shift();
handleRequests();
});
}
handleRequests = function() {
// Call the next request on the queue
if (!backendStarting && requestQueue.length) {
requestQueue[0]();
}
};
app.post(/^\/$/, function ( req, res ) {
// First some rudimentary input validation
if (!req.body.tex) {
res.writeHead(400);
return res.end(JSON.stringify({error: "'tex' post parameter is missing!"}));
}
var tex = req.body.tex;
requestQueue.push(handleRequest.bind(null, req, res, tex));
// phantomjs only handles one request at a time. Enforce this.
if (requestQueue.length === 1) {
// Start this process
handleRequests();
}
});
console.log( ' - ' + instanceName + ' ready' );
module.exports = app;

View File

@ -1 +0,0 @@
{}

View File

@ -1,47 +0,0 @@
#!/usr/bin/env node
/**
* A very basic cluster-based server runner. Restarts failed workers, but does
* not much else right now.
*/
var cluster = require('cluster');
if (cluster.isMaster) {
// Start a few more workers than there are cpus visible to the OS, so that we
// get some degree of parallelism even on single-core systems. A single
// long-running request would otherwise hold up all concurrent short requests.
var numCPUs = require('os').cpus().length + 3;
// Fork workers.
for (var i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('exit', function(worker) {
if (!worker.suicide) {
var exitCode = worker.process.exitCode;
console.log('worker', worker.process.pid,
'died ('+exitCode+'), restarting.');
cluster.fork();
}
});
process.on('SIGTERM', function() {
console.log('master shutting down, killing workers');
var workers = cluster.workers;
Object.keys(workers).forEach(function(id) {
console.log('Killing worker ' + id);
workers[id].destroy();
});
console.log('Done killing workers, bye');
process.exit(1);
} );
} else {
var mathoidWorker = require('./mathoid-worker.js');
process.on('SIGTERM', function() {
console.log('Worker shutting down');
process.exit(1);
});
// when running on appfog.com the listen port for the app
// is passed in an environment variable. Most users can ignore this!
mathoidWorker.listen(process.env.VCAP_APP_PORT || 8010);
}

View File

@ -1,10 +0,0 @@
{
"name": "Mathoid",
"description": "Render tex to SVG and MathML using MathJax, Phantom. Based on svgtex.",
"version": "0.0.1",
"dependencies": {
"querystring": "0.x.x",
"express": "2.5.x",
"request": "2.x.x"
}
}