From af74865dc433b205ba77b039b59428da29c03594 Mon Sep 17 00:00:00 2001 From: Marsup Date: Fri, 10 Aug 2012 10:14:18 +0200 Subject: [PATCH 01/87] Upgrade mongodb driver and use native BSON parser --- lib/cube/server.js | 2 +- package.json | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cube/server.js b/lib/cube/server.js index 8a2ceda2..23dac91c 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -31,7 +31,7 @@ var wsOptions = { // MongoDB driver configuration. var server_options = {auto_reconnect: true}, - db_options = {}; + db_options = {native_parser: true}; module.exports = function(options) { var server = {}, diff --git a/package.json b/package.json index d04cb783..c1d1cf94 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,14 @@ "repository": {"type": "git", "url": "http://github.com/square/cube.git"}, "main": "./lib/cube", "dependencies": { - "mongodb": "1.0.1", - "node-static": "0.5.9", + "mongodb": "1.1.2", + "node-static": "0.6.0", "pegjs": "0.6.2", "vows": "0.5.11", "websocket": "1.0.3", "websocket-server": "1.4.04" + }, + "scripts": { + "preinstall": "npm install mongodb --mongodb:native" } } From e0d6061f17563d4d7fd1282437aaca1855ede5e7 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 27 Aug 2012 16:34:27 -0500 Subject: [PATCH 02/87] improved logging and meta-cubifying of progress reports. metalog: * metalog.info('foo', {...}) for progress recording -- sent to log by default * metalog.minor('foo', {...}) for verbose recording -- sent nowhere by default * metalog.event('foo', {...}) cubifies the event and sends it to info - metalog.event('foo', {...}, 'silent') to cubify but not log * retargetable: - metalog.loggers.minor = metalog.log to log minor events - metalog.loggers.info = metalog.silent to quash logging also separated db test helpers into own file and added fixture ability. --- lib/cube/index.js | 1 + lib/cube/metalog.js | 46 ++++++++++++++++ lib/cube/server.js | 53 +++++++++---------- package.json | 11 ++-- test/metalog-test.js | 122 +++++++++++++++++++++++++++++++++++++++++++ test/test.js | 39 ++++---------- test/test_db.js | 75 ++++++++++++++++++++++++++ 7 files changed, 283 insertions(+), 64 deletions(-) create mode 100644 lib/cube/metalog.js create mode 100644 test/metalog-test.js create mode 100644 test/test_db.js diff --git a/lib/cube/index.js b/lib/cube/index.js index af67319e..e294d354 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -1,3 +1,4 @@ +exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); exports.server = require("./server"); exports.collector = require("./collector"); diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js new file mode 100644 index 00000000..45b6b993 --- /dev/null +++ b/lib/cube/metalog.js @@ -0,0 +1,46 @@ +var util = require("util"); + +var metalog = { + putter: null, + log: util.log, + silent: function(){ } +} + +// adjust verboseness by reassigning `metalog.loggers.{level}` +// @example: quiet all logs +// metalog.loggers.info = metalog.silent(); +metalog.loggers = { + info: metalog.log, + minor: metalog.silent +} + +// if true, cubify `metalog.event`s +metalog.send_events = true; + +// Cubify an event and (optionally) log it. The last parameter specifies the +// logger -- 'info' (the default), 'minor' or 'silent'. +metalog.event = function(name, hsh, logger){ + metalog[logger||"info"](name, hsh); + if ((! metalog.send_events) || (! metalog.putter)) return; + metalog.putter({ type: name, time: Date.now(), data: hsh }); +}; + +// Events important enough for the production log file. Does not cubify. +metalog.info = function(name, hsh){ + metalog.loggers.info(name + "\t" + JSON.stringify(hsh)); +}; + +// Debug-level statements; loggers.minor is typically mapped to 'silent'. +metalog.minor = function(name, hsh){ + metalog.loggers.minor(name + "\t" + JSON.stringify(hsh)); +}; + +// Dump the 'util.inspect' view of each argument to the console. +metalog.inspectify = function(args){ + args = Array.prototype.slice.call(arguments); + args.map(function (arg){ + util.debug(util.inspect(arg)); + }); +}; + +module.exports = metalog; diff --git a/lib/cube/server.js b/lib/cube/server.js index 8a2ceda2..879a07f0 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -1,3 +1,16 @@ +// Server -- generic HTTP, UDP and websockets server +// +// Used by the collector to accept new events via HTTP or websockets +// Used by the evaluator to serve pages over HTTP, and the continuously-updating +// metrics stream over websockets +// +// holds +// * the primary and secondary websockets connections +// * the HTTP listener connection +// * the MongoDB connection +// * the UDP listener connection +// + var util = require("util"), url = require("url"), http = require("http"), @@ -5,7 +18,8 @@ var util = require("util"), websocket = require("websocket"), websprocket = require("websocket-server"), static = require("node-static"), - mongodb = require("mongodb"); + mongodb = require("mongodb"), + metalog = require("./metalog"); // Don't crash on errors. process.on("uncaughtException", function(error) { @@ -38,7 +52,6 @@ module.exports = function(options) { primary = http.createServer(), secondary = websprocket.createServer(), file = new static.Server("static"), - meta, endpoints = {ws: [], http: []}, mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), db = new mongodb.Db(options["mongo-database"], mongo, db_options), @@ -88,20 +101,10 @@ module.exports = function(options) { e.dispatch(JSON.parse(message.utf8Data || message), callback); }); - meta({ - type: "cube_request", - time: Date.now(), - data: { - ip: connection.remoteAddress, - path: request.url, - method: "WebSocket" - } - }); - + metalog.event('cube_request', { is: 'ws', method: "WebSocket", ip: connection.remoteAddress, path: request.url}, 'minor'); return; } } - connection.close(); } @@ -113,17 +116,7 @@ module.exports = function(options) { for (var i = -1, n = endpoints.http.length, e; ++i < n;) { if ((e = endpoints.http[i]).match(u.pathname, request.method)) { e.dispatch(request, response); - - meta({ - type: "cube_request", - time: Date.now(), - data: { - ip: request.connection.remoteAddress, - path: u.pathname, - method: request.method - } - }); - + metalog.event('cube_request', { method: request.method, ip: request.connection.remoteAddress, path: u.pathname }); return; } } @@ -132,6 +125,7 @@ module.exports = function(options) { request.on("end", function() { file.serve(request, response, function(error) { if (error) { + metalog.event('cube_request', { is: 'failed', msg: error, code: error.status, ip: request.connection.remoteAddress, path: u.pathname }); response.writeHead(error.status, {"Content-Type": "text/plain"}); response.end(error.status + ""); } @@ -141,11 +135,12 @@ module.exports = function(options) { server.start = function() { // Connect to mongodb. - util.log("starting mongodb client"); + mongo_password = options["mongo-password"]; delete options["mongo-password"]; + metalog.info('cube_life', {is: 'mongo_connect', options: options }); db.open(function(error) { if (error) throw error; if (options["mongo-username"] == null) return ready(); - db.authenticate(options["mongo-username"], options["mongo-password"], function(error, success) { + db.authenticate(options["mongo-username"], mongo_password, function(error, success) { if (error) throw error; if (!success) throw new Error("authentication failed"); ready(); @@ -155,11 +150,11 @@ module.exports = function(options) { // Start the server! function ready() { server.register(db, endpoints); - meta = require("./event").putter(db); - util.log("starting http server on port " + options["http-port"]); + metalog.putter = require("./event").putter(db); + metalog.event("cube_life", { is: 'start_http', port: options["http-port"] }); primary.listen(options["http-port"]); if (endpoints.udp) { - util.log("starting udp server on port " + options["udp-port"]); + metalog.event("cube_life", { is: 'start_udp', port: options["udp-port"] }); var udp = dgram.createSocket("udp4"); udp.on("message", function(message) { endpoints.udp(JSON.parse(message.toString("utf8")), ignore); diff --git a/package.json b/package.json index d04cb783..0b0e1890 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,12 @@ "repository": {"type": "git", "url": "http://github.com/square/cube.git"}, "main": "./lib/cube", "dependencies": { - "mongodb": "1.0.1", - "node-static": "0.5.9", - "pegjs": "0.6.2", - "vows": "0.5.11", - "websocket": "1.0.3", + "mongodb": "1.0.1", + "node-static": "0.5.9", + "pegjs": "0.6.2", + "vows": "0.5.11", + "node-gyp": "0.6.8", + "websocket": "1.0.3", "websocket-server": "1.4.04" } } diff --git a/test/metalog-test.js b/test/metalog-test.js new file mode 100644 index 00000000..fee57c54 --- /dev/null +++ b/test/metalog-test.js @@ -0,0 +1,122 @@ +var vows = require("vows"), + assert = require("assert"), + test = require('./test'); + +var suite = vows.describe("metalog"); + +suite.with_log = function(batch){ + suite.addBatch({ + '':{ + topic: function(){ + var metalog = require("../lib/cube/metalog"); + var logged = this.logged = { infoed: [], minored: [], putted: [] }; + this.original = { send_events: metalog.send_events, putter: metalog.putter, info: metalog.loggers.info, minor: metalog.loggers.minor }; + metalog.send_events = true; + metalog.loggers.info = function(line){ logged.infoed.push(line); }; + metalog.loggers.minor = function(line){ logged.minored.push(line); }; + metalog.putter = function(line){ logged.putted.push(line); }; + return metalog; + }, + metalog: batch, + teardown: function(metalog){ + metalog.loggers.info = this.original.info; + metalog.loggers.minor = this.original.minor; + metalog.putter = this.original.putter; + } + } + }); + return suite; +} + +suite.with_log({ + '.info': { + 'logs record to metalog.logers.info': function(metalog){ + metalog.info('reactor_level', { criticality: 7, cores: 'leaking' }); + assert.equal(this.logged.infoed.pop(), 'reactor_level\t{"criticality":7,"cores":"leaking"}'); + assert.deepEqual(this.logged, { infoed: [], minored: [], putted: [] }); + } + } +}).with_log({ + '.minor': { + 'logs record to metalog.loggers.minor': function(metalog){ + metalog.minor('reactor_level', { modacity: 3 }); + assert.equal(this.logged.minored.pop(), 'reactor_level\t{"modacity":3}'); + assert.deepEqual(this.logged, { infoed: [], minored: [], putted: [] }); + } + } +}).with_log({ + '.event': { + 'with send_events=true': { + topic: function(metalog){ + metalog.send_events = true; + metalog.event('reactor_level', { criticality: 9, hemiconducers: 'relucting' }); + return metalog; + }, + 'logs record to metalog.info by default': function(metalog){ + assert.equal(this.logged.infoed.pop(), 'reactor_level\t{"criticality":9,"hemiconducers":"relucting"}'); + }, + 'writes an event to cube itself': function(metalog){ + event = this.logged.putted.pop(); + event.time = 'whatever'; + assert.deepEqual(event, { + data: { hemiconducers: 'relucting', criticality: 9 }, + type: 'reactor_level', + time: 'whatever' + }); + } + } + } +}).with_log({ + '.event': { + 'with send_events=false': { + topic: function(metalog){ + metalog.send_events = false; + metalog.event('reactor_level', { criticality: 10, hemiconducers: 'fremulating' }); + return metalog; + }, + 'logs record to metalog.loggers.info': function(metalog){ + assert.equal(this.logged.infoed.pop(), 'reactor_level\t{"criticality":10,"hemiconducers":"fremulating"}'); + }, + 'does not write an event to cube': function(metalog){ + assert.deepEqual(this.logged, { infoed: [], minored: [], putted: [] }); + } + } + } +}).with_log({ + '.event': { + 'last parameter overrides log target': { + topic: function(metalog) { + metalog.send_events = true; + metalog.event('reactor_level', { criticality: 3, hemiconducers: 'cromulent' }, 'minor'); + metalog.event('reactor_level', { criticality: 2, hemiconducers: 'whispery' }, 'silent'); + return metalog; + }, + '': function(metalog){ + assert.equal(this.logged.minored.pop(), 'reactor_level\t{"criticality":3,"hemiconducers":"cromulent"}'); + assert.equal(this.logged.putted.pop().data.criticality, 2); + assert.equal(this.logged.putted.pop().data.criticality, 3); + assert.deepEqual(this.logged, { infoed: [], minored: [], putted: [] }); + } + } + } +}); + +function dummy_logger(arg){}; + +suite.addBatch({ + 'metalog':{ + topic: function(){ return require("../lib/cube/metalog"); }, + '': { + 'loggers persist across factory invocation': function(metalog){ + metalog.orig_minor = metalog.loggers.minor; + metalog.loggers.minor = dummy_logger; + var ml2 = require("../lib/cube/metalog"); + assert.deepEqual(metalog, ml2); + assert.deepEqual(metalog.loggers.minor, dummy_logger); + assert.notDeepEqual(metalog.loggers.minor, metalog.orig_minor); + }, + teardown: function(metalog){ metalog.loggers.minor = metalog.orig_minor; } + } + }}); + +suite.export(module); diff --git a/test/test.js b/test/test.js index 2498d2de..ccfdd5fb 100644 --- a/test/test.js +++ b/test/test.js @@ -1,37 +1,13 @@ var mongodb = require("mongodb"), - assert = require("assert"), - util = require("util"), - http = require("http"); + assert = require("assert"), + util = require("util"), + metalog = require("../lib/cube/metalog"), + test_db = require("./test_db"), + http = require("http"); exports.port = 1083; -exports.batch = function(batch) { - return { - "": { - topic: function() { - var client = new mongodb.Server("localhost", 27017); - db = new mongodb.Db("cube_test", client), - cb = this.callback; - db.open(function(error) { - var collectionsRemaining = 2; - db.dropCollection("test_events", collectionReady); - db.dropCollection("test_metrics", collectionReady); - function collectionReady() { - if (!--collectionsRemaining) { - cb(null, {client: client, db: db}); - } - } - }); - }, - "": batch, - teardown: function(test) { - if (test.client.isConnected()) { - test.client.close(); - } - } - } - }; -}; +exports.batch = test_db.batch; exports.request = function(options, data) { return function() { @@ -54,4 +30,7 @@ exports.request = function(options, data) { }; // Disable logging for tests. +metalog.loggers.info = metalog.silent; +metalog.loggers.minor = metalog.silent; util.log = function() {}; +metalog.send_events = false; diff --git a/test/test_db.js b/test/test_db.js new file mode 100644 index 00000000..e3b0a90e --- /dev/null +++ b/test/test_db.js @@ -0,0 +1,75 @@ +var mongodb = require("mongodb"), + util = require("util"), + metalog = require("../lib/cube/metalog"); + +var test_db = { }; + +var test_collections = ["test_users", "test_events", "test_metrics"]; + +var options = exports.options = { + "mongo-host": "localhost", + "mongo-port": 27017, + "mongo-database": "cube_test" +}; + +exports.using_objects = function (clxn_name, test_objects){ + metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); + return function(tdb){ + var that = this; + tdb.db.collection(clxn_name, function(err, clxn){ + if (err) throw(err); + that[clxn_name] = clxn; + clxn.remove({ dummy: true }, {safe: true}, function(){ + clxn.insert(test_objects, { safe: true }, function(){ + that.callback(null); + }); }); + }); + }; +}; + +exports.batch = function(batch) { + return { + "": { + topic: function() { + connect(); + setup_db(this.callback); + }, + "": batch, + teardown: function(test) { + if (test.client.isConnected()) { + process.nextTick(function(){ test.client.close(); }); + }; + } + } + }; +}; + +// +// db methods +// + +function setup_db(cb){ + drop_collections(cb); +} + +function connect(){ + metalog.minor('cube_testdb', { state: 'connecting to db', options: options }); + test_db.options = options; + test_db.client = new mongodb.Server(options["mongo-host"], options["mongo-port"], {auto_reconnect: true}); + test_db.db = new mongodb.Db(options["mongo-database"], test_db.client, {}); +} + +function drop_collections(cb){ + metalog.minor('cube_testdb', { state: 'dropping test collections', collections: test_collections }); + test_db.db.open(function(error) { + var collectionsRemaining = test_collections.length; + test_collections.forEach(function(collection_name){ + test_db.db.dropCollection(collection_name, collectionReady); + }); + function collectionReady() { + if (!--collectionsRemaining) { + cb(null, test_db); + } + } + }); +} From b2e9ece57c5ab7d4f81951ebbde0e86a7c7badcc Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 27 Aug 2012 20:03:35 -0500 Subject: [PATCH 03/87] Added authentication: allow_all, read_only, or mongo_cookie * authentication.authenticator(name) gives you the requested authenticator - server.js uses options['authenticator'] - 'allow_all' is default in bin/collector-config.js etc. * authenticator.check(request, auth_ok, auth_no) - calls auth_ok if authenticated, auth_no if rejected - staples an 'authorized' member to the request object: - eg we use 'request.authorized.admin' to govern board editing - in the mongo_cookie authenticator, it's the user record * mongo_cookie authenticator compares bcrypted cookie to a stored hashed secret - you must set the cookie and store that db record in your host client; see 'test/authenticator-test.js' for format. Rails+devise snippet available on request. --- bin/collector-config.js | 3 +- bin/evaluator-config.js | 3 +- lib/cube/authentication.js | 114 ++++++++++++++++++++++++ lib/cube/index.js | 1 + lib/cube/server.js | 63 +++++++++++--- package.json | 4 +- test/authentication-test.js | 169 ++++++++++++++++++++++++++++++++++++ test/collector-test.js | 3 +- 8 files changed, 343 insertions(+), 17 deletions(-) create mode 100644 lib/cube/authentication.js create mode 100644 test/authentication-test.js diff --git a/bin/collector-config.js b/bin/collector-config.js index 01acae60..fcbd19ef 100644 --- a/bin/collector-config.js +++ b/bin/collector-config.js @@ -6,5 +6,6 @@ module.exports = { "mongo-username": null, "mongo-password": null, "http-port": 1080, - "udp-port": 1180 + "udp-port": 1180, + "authenticator": "allow_all" }; diff --git a/bin/evaluator-config.js b/bin/evaluator-config.js index 450589da..84e544f9 100644 --- a/bin/evaluator-config.js +++ b/bin/evaluator-config.js @@ -5,5 +5,6 @@ module.exports = { "mongo-database": "cube_development", "mongo-username": null, "mongo-password": null, - "http-port": 1081 + "http-port": 1081, + "authenticator": "allow_all" }; diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js new file mode 100644 index 00000000..2a34ae91 --- /dev/null +++ b/lib/cube/authentication.js @@ -0,0 +1,114 @@ +// +// authentication -- authenticate user identities and authorize their action +// +// Call an authenticator with the request and ok/no callbacks: +// * if authentication succeeds, a permissions record is attached to the request +// object as `request.authorized`, and sent as the callback parameter. +// * if authentication fails, the callback is called with a short string +// describing the reason. +// +// The authenticators include: +// * allow_all -- uniformly authenticates all requests, authorizing all to write +// * read_only -- uniformly authenticates all requests, authorizing none to write +// * mongo_cookie -- validates bcrypt'ed token in cookie with user record in db. +// +// This module does not provide any means for creating user tokens; do this in +// your front-end app. +// + +var mongodb = require("mongodb"), + cookies = require("cookies"), + bcrypt = require("bcrypt"), + metalog = require('./metalog') + ; + +var authentication = {}; + +authentication.authenticator = function(strategy, db, options){ + metalog.minor('cube_auth', { strategy: strategy }); + return authentication[strategy](db, options); +} + +authentication.allow_all = function(){ + function check(request, auth_ok, auth_no) { + metalog.event('cube_auth', { authenticator: 'allow_all', path: request.url }, 'minor'); + request.authorized = { admin: true }; + return auth_ok(request.authorized); + }; + return { check: check }; +}; + +authentication.read_only = function(){ + function check(request, auth_ok, auth_no) { + metalog.event('cube_auth', { authenticator: 'read_only', path: request.url }, 'minor'); + request.authorized = { admin: false }; + return auth_ok(request.authorized); + }; + return { check: check }; +}; + +authentication.mongo_cookie = function(db, options){ + var users; + + db.collection(options["collection"]||"users", function(error, clxn){ + if (error) throw(error); + // TODO: check if not collection? + users = clxn; + }); + + function check(request, auth_ok, auth_no) { + var cookie = (new cookies(request)).get('_cube_session'), + token = decodeURIComponent(cookie).split('--'), // token[0] = token uid, token[1] = token secret + token_uid = new Buffer(token[0] + '', 'base64').toString('utf8'), + token_secret = new Buffer(token[1] + '', 'base64').toString('utf8'); + + metalog.event('cube_auth', { is: 'hi', tu: token_uid }, 'minor'); + + if (! (cookie && token && token_uid && token_secret)){ + metalog.event('cube_auth', { is: 'no', r: 'no_token_in_request' }); + auth_no('no_token_in_request'); + return; + }; + + validate_token({"tokens.uid": token_uid}, auth_ok, auth_no); + + function validate_token(query, auth_ok, auth_no){ + // Asynchronously load the requested user. + users.findOne(query, function(error, user) { + if((!error) && user) { + metalog.event('cube_auth', { is: 'ok', tu: token_uid, u: user._id }, 'minor'); + var auth_info = user.tokens.filter(function(t){ return t.uid === token_uid; }); + + if(auth_info.length === 1){ + bcrypt.compare(token_secret, auth_info[0].hashed_secret, function(err, res) { + if (!err && res === true ){ + delete auth_info[0].hashed_secret; + request.authorized = auth_info[0]; + auth_ok(request.authorized); + } else { + metalog.event('cube_auth', { is: 'no', r: 'bad_token', tu: token_uid, u: user }); + auth_no('bad_token'); + } + }); + } else { + metalog.event('cube_auth', { is: 'no', r: 'missing_token', tu: token_uid, u: user }); + auth_no('missing_token'); + } + } else { + metalog.event('cube_auth', { is: 'no', r: 'missing_user', tu: token_uid }); + auth_no('missing_user'); + }; + }); + }; + }; + return { check: check }; +}; + +// base-64 encode a uid and bcrypted secret +authentication.gen_cookie = function(session_name, uid, secret){ + encoded_uid = new Buffer(uid, 'utf8').toString('base64'); + encoded_sec = new Buffer(secret, 'utf8').toString('base64'); + return (session_name+"="+encoded_uid+"--"+encoded_sec+";"); +}; + +module.exports = authentication; diff --git a/lib/cube/index.js b/lib/cube/index.js index e294d354..76ac57ba 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -1,3 +1,4 @@ +exports.authentication = require("./authentication"); exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); exports.server = require("./server"); diff --git a/lib/cube/server.js b/lib/cube/server.js index 879a07f0..aec57a86 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -19,6 +19,7 @@ var util = require("util"), websprocket = require("websocket-server"), static = require("node-static"), mongodb = require("mongodb"), + authentication = require("./authentication"), metalog = require("./metalog"); // Don't crash on errors. @@ -55,21 +56,42 @@ module.exports = function(options) { endpoints = {ws: [], http: []}, mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), db = new mongodb.Db(options["mongo-database"], mongo, db_options), - id = 0; + id = 0, + authenticator; secondary.server = primary; + function is_sec_ws_initiation(request){ + return ("sec-websocket-version" in request.headers); + } + function is_ws_initiation(request){ + return (request.method === "GET" + && /^websocket$/i.test(request.headers.upgrade) + && /^upgrade$/i.test(request.headers.connection)); + } + // Register primary WebSocket listener with fallback. primary.on("upgrade", function(request, socket, head) { - if ("sec-websocket-version" in request.headers) { - request = new websocket.request(socket, request, wsOptions); - request.readHandshake(); - connect(request.accept(request.requestedProtocols[0], request.origin), request.httpRequest); - } else if (request.method === "GET" - && /^websocket$/i.test(request.headers.upgrade) - && /^upgrade$/i.test(request.headers.connection)) { - new websprocket.Connection(secondary.manager, secondary.options, request, socket, head); + function auth_ok(perms) { + if (is_sec_ws_initiation(request)) { + request = new websocket.request(socket, request, wsOptions); + request.readHandshake(); + connect(request.accept(request.requestedProtocols[0], request.origin), request.httpRequest); + } else if (is_ws_initiation(request)) { + new websprocket.Connection(secondary.manager, secondary.options, request, socket, head); + } + } + function auth_no(perms) { + if (is_sec_ws_initiation(request)) { + request = new websocket.request(socket, request, wsOptions); + request.readHandshake(); + request.reject(); + } else if (is_ws_initiation(request)) { + res = 'HTTP/1.1 403 Forbidden\r\nConnection: close'; + socket.end(res + '\r\n\r\n', 'ascii'); + } } + return authenticator.check(request, auth_ok, auth_no); }); // Register secondary WebSocket listener. @@ -81,6 +103,8 @@ module.exports = function(options) { }); function connect(connection, request) { + // save auth from connection requesta + var authorization = request.authorized; // Forward messages to the appropriate endpoint, or close the connection. for (var i = -1, n = endpoints.ws.length, e; ++i < n;) { @@ -98,7 +122,10 @@ module.exports = function(options) { }); connection.on("message", function(message) { - e.dispatch(JSON.parse(message.utf8Data || message), callback); + // staple the authorization back on + var payload = JSON.parse(message.utf8Data || message); + payload.authorized = authorization; + e.dispatch(payload, callback); }); metalog.event('cube_request', { is: 'ws', method: "WebSocket", ip: connection.remoteAddress, path: request.url}, 'minor'); @@ -115,9 +142,18 @@ module.exports = function(options) { // Forward messages to the appropriate endpoint, or 404. for (var i = -1, n = endpoints.http.length, e; ++i < n;) { if ((e = endpoints.http[i]).match(u.pathname, request.method)) { - e.dispatch(request, response); - metalog.event('cube_request', { method: request.method, ip: request.connection.remoteAddress, path: u.pathname }); - return; + + function auth_ok(perms) { + metalog.event('cube_request', { is: 'auth_ok', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: true, user: perms }); + e.dispatch(request, response); + } + function auth_no(reason) { + metalog.event('cube_request', { is: 'auth_no', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: false }); + response.writeHead(403, {"Content-Type": "text/plain"}); + response.end("403 Forbidden"); + } + + return authenticator.check(request, auth_ok, auth_no); } } @@ -151,6 +187,7 @@ module.exports = function(options) { function ready() { server.register(db, endpoints); metalog.putter = require("./event").putter(db); + authenticator = authentication.authenticator(options["authenticator"], db, options); metalog.event("cube_life", { is: 'start_http', port: options["http-port"] }); primary.listen(options["http-port"]); if (endpoints.udp) { diff --git a/package.json b/package.json index 0b0e1890..de8f9966 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,8 @@ "vows": "0.5.11", "node-gyp": "0.6.8", "websocket": "1.0.3", - "websocket-server": "1.4.04" + "websocket-server": "1.4.04", + "cookies": "0.3.1", + "bcrypt": "0.7.1" } } diff --git a/test/authentication-test.js b/test/authentication-test.js new file mode 100644 index 00000000..f04ecb38 --- /dev/null +++ b/test/authentication-test.js @@ -0,0 +1,169 @@ +var vows = require("vows"), + assert = require("assert"), + util = require("util"), + metalog = require("../lib/cube/metalog"), + authentication = require("../lib/cube/authentication"), + test_db = require("./test_db"), + test = require('./test'); + +var suite = vows.describe("authentication"); +suite.options.error = true; + +// +// Test Macros +// + +function successful_auth(req) { + return function(authenticator){ + var that = this; + that.req = req; + authenticator.check(req, + function(arg){ that.callback(null, arg); }, + function(arg){ that.callback(new Error("Auth failed but should have succeeded")); }); + }; +} + +function failed_auth(req) { + return function(authenticator){ + var that = this; + that.req = req; + authenticator.check(req, + function(arg){ that.callback("Auth succeeded but should have failed"); }, + function(arg){ that.callback(null, arg); } ); + }; +} + +// as I'm sure you know: boss_hogg and luke have write access (luke doubly so); +// roscoe can't do much but sit around; and it's as if coy & vance never existed +var test_users = [ + { _id: "boss_hogg", tokens: [{ uid: 'boss_hogg_tok', admin: true, hashed_secret: "$2a$10$3u.CU4pJLnPDM7VwhJtbyuLwGBiOwpQ42q0wFQEDoJZtirAgIrBI6"}] }, + { _id: "luke", tokens: [{ uid: 'luke_tok', admin: true, hashed_secret: "$2a$10$K5NpLr3qrhxsUBW0iCw8iegQzgEINdWDk2n1BrTYe1x1Ay4dU2PlG"}, + { uid: 'luke_2_tok', admin: true, hashed_secret: "$2a$10$0I6KVPSzUIXdlxdY7qPTF.dde4tjPGRahYcja96Fz6ZaakEfdnNGO"}] }, + { _id: "roscoe", tokens: [{ uid: 'roscoe_tok', admin: false, hashed_secret: "$2a$10$BKIqJukrlFtbjFeeLCnEvOwHdLMLDt61iyfMRLiEf9lNeWKD.djrm"}] }, + { _id: "vance", tokens: [] } +]; +var dummy_token = "token_in_cookie"; + +function dummy_request(username, token){ + return({ headers: { cookie: authentication.gen_cookie("_cube_session", username+"_tok", token || dummy_token) } }); +}; + +suite.addBatch(test_db.batch({ + + mongo_cookie: { + topic: test_db.using_objects("test_users", test_users, this.callback), + "": { + topic: function(test_db){ + return authentication.authenticator("mongo_cookie", test_db.db, { collection: "test_users" }); }, + "authenticates": { + "users with good tokens": { + topic: successful_auth(dummy_request("boss_hogg")), + '': function(result){ assert.deepEqual(this.req.authorized, { uid: 'boss_hogg_tok', admin: true }); } + }, + "users with tokens, even if there are many": { + "a": { + topic: successful_auth(dummy_request("luke")), + '': function(result){ assert.deepEqual(this.req.authorized, { uid: 'luke_tok', admin: true }); } + }, + "b": { + topic: successful_auth(dummy_request("luke_2")), + '': function(result){ assert.deepEqual(this.req.authorized, { uid: 'luke_2_tok', admin: true }); } + } + } + }, + "request.authorized": { + "": { + topic: successful_auth(dummy_request("boss_hogg")), + 'is stapled to the request object': function(result){ + assert.isObject(this.req.authorized); + assert.deepEqual(this.req.authorized, { uid: 'boss_hogg_tok', admin: true }); + }, + 'is returned as callback param': function(result){ + assert.deepEqual(result, { uid: 'boss_hogg_tok', admin: true }); + }, + }, + "user with write access": { + topic: successful_auth(dummy_request("boss_hogg")), + 'request.authorized.admin is true': function(result){ + assert.isTrue(this.req.authorized.admin); + } + }, + "user with read-only access": { + topic: successful_auth(dummy_request("roscoe")), + 'authenticates': function(result){ + assert.deepEqual(this.req.authorized, { uid: 'roscoe_tok', admin: false }); }, + 'request.authorized.admin is false': function(result){ + assert.isFalse(this.req.authorized.admin); + } + } + }, + "does not allow": { + "bad token": { + topic: failed_auth(dummy_request("boss_hogg", "bad_token")), + "invokes auth_no callback with reason": function(reason){ + assert.equal(reason, 'bad_token'); + }, + "does not authorize request": function(reason){ assert.isUndefined(this.req.authorized); } + }, + "no token in request": { + topic: failed_auth({ headers: { cookie: "" } }), + "invokes auth_no callback with reason": function(reason){ + assert.equal(reason, 'no_token_in_request'); + }, + "does not authorize request": function(reason){ assert.isUndefined(this.req.authorized); } + }, + "user there, no auth record": { + topic: failed_auth(dummy_request("vance")), + "invokes auth_no callback with reason": function(reason){ + assert.equal(reason, 'missing_user'); + }, + "does not authorize request": function(reason){ assert.isUndefined(this.req.authorized); } + }, + "missing user": { + topic: failed_auth(dummy_request("coy")), + "invokes auth_no callback with reason": function(reason){ + assert.equal(reason, 'missing_user'); + }, + "does not authorize request": function(reason){ assert.isUndefined(this.req.authorized); } + } + } + } + }, + + "allow_all": { + topic: function(test_db){ + return authentication.authenticator("allow_all"); }, + "calls the auth_ok callback immediately" : { + topic: successful_auth({type: "allow_all auth"}), + 'decorates the request object': function(result){ + assert.isObject(this.req.authorized); + }, + 'returns the authorization hash': function(result){ + assert.deepEqual(result, { admin: true }); + }, + 'authorizes writes': function(result){ + assert.deepEqual(this.req.authorized, { admin: true }); + } + } + }, + + "read_only": { + topic: function(test_db){ + return authentication.authenticator("read_only"); }, + "calls the auth_ok callback immediately" : { + topic: successful_auth({type: "read_only auth"}), + 'decorates the request object': function(result){ + assert.isObject(this.req.authorized); + }, + 'returns the authorization hash': function(result){ + assert.deepEqual(result, { admin: false }); + }, + 'authorizes writes': function(result){ + assert.deepEqual(this.req.authorized, { admin: false }); + } + } + } + +})); + +suite.export(module); diff --git a/test/collector-test.js b/test/collector-test.js index 58696389..f887a6e4 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -9,7 +9,8 @@ var port = ++test.port, server = cube.server({ "mongo-host": "localhost", "mongo-port": 27017, "mongo-database": "cube_test", - "http-port": port + "http-port": port, + "authenticator": "allow_all" }); server.register = cube.collector.register; From 8b57375cff6aa8987c43f1db5550c9e95b3e3202 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 27 Aug 2012 20:06:53 -0500 Subject: [PATCH 04/87] applied metalog to rest of code, added comments on major plot lines --- lib/cube/collector.js | 22 ++++++++++++++++++++-- lib/cube/emitter.js | 6 ++++++ lib/cube/endpoint.js | 9 +++++++++ lib/cube/event.js | 26 ++++++++++++++++++++++++++ lib/cube/metric.js | 25 +++++++++++-------------- test/metric-test.js | 11 ++++++++--- test/test.js | 1 - 7 files changed, 80 insertions(+), 20 deletions(-) diff --git a/lib/cube/collector.js b/lib/cube/collector.js index f6a1c2fb..7e9d1c3f 100644 --- a/lib/cube/collector.js +++ b/lib/cube/collector.js @@ -1,11 +1,21 @@ -var endpoint = require("./endpoint"); - // +// collector -- listen for incoming metrics +// + +var endpoint = require("./endpoint"), + metalog = require('./metalog'); + var headers = { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }; +// Register Collector listeners at their appropriate paths: +// +// * putter, handles each isolated event -- see event.js +// * poster, an HTTP listener -- see below +// * collectd, a collectd listener -- see collectd.js +// exports.register = function(db, endpoints) { var putter = require("./event").putter(db), poster = post(putter); @@ -26,6 +36,13 @@ exports.register = function(db, endpoints) { endpoints.udp = putter; }; +// +// Construct HTTP listener +// +// * aggregate content into a complete request +// * JSON-parse the request body +// * dispatch each metric as an event to the putter +// function post(putter) { return function(request, response) { var content = ""; @@ -36,6 +53,7 @@ function post(putter) { try { JSON.parse(content).forEach(putter); } catch (e) { + metalog.event("cube_request", { at: "c", res: "collector_post_error", error: e, code: 400 }); response.writeHead(400, headers); response.end(JSON.stringify({error: e.toString()})); return; diff --git a/lib/cube/emitter.js b/lib/cube/emitter.js index 03e26090..6362f325 100644 --- a/lib/cube/emitter.js +++ b/lib/cube/emitter.js @@ -1,3 +1,9 @@ +// +// emitter - writes events to the collector. +// +// URL's scheme determines UDP, websocket or HTTP emitter, as appropriate. +// + var util = require("util"), url = require("url"), http = require("./emitter-http"), diff --git a/lib/cube/endpoint.js b/lib/cube/endpoint.js index 0e86463e..fb0f1415 100644 --- a/lib/cube/endpoint.js +++ b/lib/cube/endpoint.js @@ -1,3 +1,12 @@ +// +// endpoint -- router for requests. +// +// specify +// * optional HTTP method ("GET", "POST", etc) +// * path, a String or RegExp +// * dispatch, a callback to invoke +// + module.exports = function(method, path, dispatch) { return { match: arguments.length < 3 diff --git a/lib/cube/event.js b/lib/cube/event.js index 4825904e..e47cde20 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -6,6 +6,7 @@ var mongodb = require("mongodb"), tiers = require("./tiers"), types = require("./types"), bisect = require("./bisect"), + metalog = require("./metalog"), ObjectID = mongodb.ObjectID; var type_re = /^[a-z][a-zA-Z0-9_]+$/, @@ -22,6 +23,14 @@ var streamDelayDefault = 5000, // How frequently to invalidate metrics after receiving events. var invalidateInterval = 5000; +// event.putter -- save the event, invalidate any cached metrics impacted by it. +// +// @param request -- +// - id, a unique ID (optional). If included, it will be used as the Mongo record's primary key -- if the collector receives that event multiple times, it will only be stored once. If omitted, Mongo will generate a unique ID for you. +// - time, timestamp for the event (a date-formatted string) +// - type, namespace for the events. A corresponding `foo_events` collection must exist in the DB -- /schema/schema-*.js illustrate how to set up a new event type. +// - data, the event's payload +// exports.putter = function(db) { var collection = types(db), knownByType = {}, @@ -92,6 +101,7 @@ exports.putter = function(db) { // likelihood of a race condition between when the events are read by the // evaluator and when the newly-computed metrics are saved. function save(type, event) { + // metalog.info("cube_request", { is: "ev", at: "save", type: type, event: event }); collection(type).events.save(event, handle); queueInvalidation(type, event); } @@ -125,6 +135,7 @@ exports.putter = function(db) { for (var type in timesToInvalidateByTierByType) { var metrics = collection(type).metrics, timesToInvalidateByTier = timesToInvalidateByTierByType[type]; + metalog.info("cube_compute", { is: "flush", type: type }); for (var tier in tiers) { metrics.update({ i: false, @@ -140,6 +151,18 @@ exports.putter = function(db) { return putter; }; +// +// event.getter - subscribe to event type +// +// if `stop` is not given, does a streaming response, polling for results every +// `streamDelay` (5 seconds). +// +// if `stop` is given, return events from the given interval +// +// * convert the request expression and filters into a MongoDB-ready query +// * Issue the query; +// * if streaming, register the query to be run at a regular interval +// exports.getter = function(db) { var collection = types(db), streamsBySource = {}; @@ -253,6 +276,9 @@ exports.getter = function(db) { } getter.close = function(callback) { + // results or periodic calls may have already been set in motion, but + // trigger in the future; this ensures they quit listening and drop further + // results on the floor. callback.closed = true; }; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 3421aea2..6454a4ab 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -4,7 +4,8 @@ var parser = require("./metric-expression"), tiers = require("./tiers"), types = require("./types"), reduces = require("./reduces"), - event = require("./event"); + event = require("./event"), + metalog = require('./metalog'); var metric_fields = {v: 1}, metric_options = {sort: {"_id.t": 1}, batchSize: 1000}, @@ -14,8 +15,7 @@ var metric_fields = {v: 1}, exports.getter = function(db) { var collection = types(db), Double = db.bson_serializer.Double, - queueByName = {}, - meta = event.putter(db); + queueByName = {}; function getter(request, callback) { var start = new Date(request.start), @@ -95,13 +95,9 @@ exports.getter = function(db) { // Record how long it took us to compute as an event! var time1 = Date.now(); - meta({ - type: "cube_compute", - time: time1, - data: { - expression: expression.source, - ms: time1 - time0 - } + metalog.event("cube_compute", { + expression: expression.source, + ms: time1 - time0 }); } }); @@ -200,8 +196,8 @@ exports.getter = function(db) { function save(time, value) { callback(time, value); - if (value) { - type.metrics.save({ + if (! value) return; + var metric = { _id: { e: expression.source, l: tier.key, @@ -209,8 +205,9 @@ exports.getter = function(db) { }, i: false, v: new Double(value) - }, handle); - } + }; + metalog.info('cube_compute', {is: 'metric_save', metric: metric }); + type.metrics.save(metric, handle); } } } diff --git a/test/metric-test.js b/test/metric-test.js index 318dc0a9..969df026 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -4,6 +4,11 @@ var vows = require("vows"), event = require("../lib/cube/event"), metric = require("../lib/cube/metric"); +// as a hack to get updates to settle, we need to insert delays +// if you see funny heisen-errors in the metrics tests try bumping these +var step_testing_delay = 250, + batch_testing_delay = 500; + var suite = vows.describe("metric"); var steps = { @@ -34,7 +39,7 @@ suite.addBatch(test.batch({ }); } - setTimeout(function() { callback(null, getter); }, 500); + setTimeout(function() { callback(null, getter); }, batch_testing_delay); }, "unary expression": metricTest({ @@ -71,7 +76,7 @@ suite.addBatch(test.batch({ } ), - "compound expression": metricTest({ + "compound expression (sometimes fails due to race condition?)": metricTest({ expression: "max(test(i)) - min(test(i))", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:50:00.000Z", @@ -150,7 +155,7 @@ function metricTest(request, expected) { actual.push(response); } }); - }, depth * 250); + }, depth * step_testing_delay); } }; diff --git a/test/test.js b/test/test.js index ccfdd5fb..e751b883 100644 --- a/test/test.js +++ b/test/test.js @@ -32,5 +32,4 @@ exports.request = function(options, data) { // Disable logging for tests. metalog.loggers.info = metalog.silent; metalog.loggers.minor = metalog.silent; -util.log = function() {}; metalog.send_events = false; From 414d23464b9d35ca9e1acf0b803ea5edc33695cb Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 27 Aug 2012 19:26:09 -0500 Subject: [PATCH 05/87] Log evaluator queries when in verbose mode --- lib/cube/types.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/cube/types.js b/lib/cube/types.js index 251f5e8e..02f90fda 100644 --- a/lib/cube/types.js +++ b/lib/cube/types.js @@ -1,15 +1,26 @@ // Much like db.collection, but caches the result for both events and metrics. // Also, this is synchronous, since we are opening a collection unsafely. var types = module.exports = function(db) { - var collections = {}; + var collections = {}, + metalog = require('./metalog'); return function(type) { var collection = collections[type]; if (!collection) { collection = collections[type] = {}; db.collection(type + "_events", function(error, events) { + var orig_find = events.find; + events.find = function(){ + metalog.minor('cube_query', {is: 'events_find', query: arguments}); + return orig_find.apply(this, arguments); + }; collection.events = events; }); db.collection(type + "_metrics", function(error, metrics) { + var orig_find = metrics.find; + metrics.find = function(){ + metalog.minor('cube_query', {is: 'metrics_find', query: arguments}); + return orig_find.apply(this, arguments); + }; collection.metrics = metrics; }); } From 9889b3318f3f42315d8cc2cdf9401be3217af6b4 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 27 Aug 2012 19:24:36 -0500 Subject: [PATCH 06/87] re-adding visualizer --- bin/evaluator.js | 1 + lib/cube/endpoint.js | 14 +++-- lib/cube/index.js | 1 + lib/cube/visualizer.js | 128 ++++++++++++++++++++++++++++++++++++++++ test/evaluator-test.js | 33 +++++++++++ test/visualizer-test.js | 23 ++++++++ 6 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 lib/cube/visualizer.js create mode 100644 test/evaluator-test.js create mode 100644 test/visualizer-test.js diff --git a/bin/evaluator.js b/bin/evaluator.js index b5da8894..45ba7001 100644 --- a/bin/evaluator.js +++ b/bin/evaluator.js @@ -4,6 +4,7 @@ var options = require("./evaluator-config"), server.register = function(db, endpoints) { cube.evaluator.register(db, endpoints); + cube.visualizer.register(db, endpoints); }; server.start(); diff --git a/lib/cube/endpoint.js b/lib/cube/endpoint.js index fb0f1415..a8aec210 100644 --- a/lib/cube/endpoint.js +++ b/lib/cube/endpoint.js @@ -8,10 +8,14 @@ // module.exports = function(method, path, dispatch) { - return { - match: arguments.length < 3 - ? (dispatch = path, path = method, function(p) { return p == path; }) - : function(p, m) { return m == method && p == path; }, - dispatch: dispatch + if (method instanceof RegExp) { + dispatch = path, path = method; + match = function(p, m) { return path.test(p); }; + } else if (arguments.length < 3) { + dispatch = path, path = method; + match = function(p) { return p == path; }; + } else { // path is a string + match = function(p, m) { return m == method && p == path; }; }; + return { match: match, dispatch: dispatch }; } diff --git a/lib/cube/index.js b/lib/cube/index.js index 76ac57ba..cb8ff7c3 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -4,4 +4,5 @@ exports.emitter = require("./emitter"); exports.server = require("./server"); exports.collector = require("./collector"); exports.evaluator = require("./evaluator"); +exports.visualizer = require("./visualizer"); exports.endpoint = require("./endpoint"); diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js new file mode 100644 index 00000000..b5727cb3 --- /dev/null +++ b/lib/cube/visualizer.js @@ -0,0 +1,128 @@ +var url = require("url"), + path = require("path"), + endpoint = require("./endpoint"), + metalog = require('./metalog'); + +exports.register = function(db, endpoints) { + endpoints.ws.push( + endpoint(/^\/[a-z\-_0-9]+\/boards\/[a-z0-9\-_]+(\/edit)?$/i, viewBoard(db)) + ); +}; + +function viewBoard(db) { + var boards, + boardsByCallback = {}, + callbacksByBoard = {}; + + db.collection("boards", function(error, collection) { + boards = collection; + }); + + function dispatch(request, callback) { + if (request.type != 'ping') metalog.info("cube_request", { is: request.type, bd: request.id, pc: callback.id }); + request.id = require('mongodb').ObjectID(request.id); + + switch (request.type) { + case "load": load(request, callback); break; + case "add": add(request, callback); break; + case "edit": case "move": move(request, callback); break; + case "remove": remove(request, callback); break; + default: callback({type: "error", status: 400}); break; + } + } + + function check_authorization(request, action){ + if (request.authorized.admin) return true; + metalog.info('cube_request', { is: 'denied', action: action, u: request.authorized }); + return false + } + + function add(request, callback) { + if (! check_authorization(request, 'add')) return; + + var boardId = request.id, + callbacks = callbacksByBoard[boardId].filter(function(c) { return c.id != callback.id; }); + boards.update({_id: boardId}, {$push: {pieces: request.piece}}); + if (callbacks.length) emit(callbacks, {type: "add", piece: request.piece}); + } + + function move(request, callback) { + if (! check_authorization(request, 'move')) return; + + var boardId = request.id, + callbacks = callbacksByBoard[boardId].filter(function(c) { return c.id != callback.id; }); + boards.update({_id: boardId, "pieces.id": request.piece.id}, {$set: {"pieces.$": request.piece}}); + if (callbacks.length) emit(callbacks, {type: request.type, piece: request.piece}); + } + + function remove(request, callback) { + if (! check_authorization(request, 'remove')) return; + + var boardId = request.id, + callbacks = callbacksByBoard[boardId].filter(function(c) { return c.id != callback.id; }); + boards.update({_id: boardId}, {$pull: {pieces: {id: request.piece.id}}}); + if (callbacks.length) emit(callbacks, {type: "remove", piece: {id: request.piece.id}}); + } + + function load(request, callback) { + var boardId = boardsByCallback[callback.id], + callbacks; + + // If callback was previously viewing to a different board, remove it. + if (boardId) { + callbacks = callbacksByBoard[boardId]; + callbacks.splice(callbacks.indexOf(callback), 1); + if (callbacks.length) emit(callbacks, {type: "view", count: callbacks.length}); + else delete callbacksByBoard[boardId]; + } + + // Register that we are now viewing the new board. + boardsByCallback[callback.id] = boardId = request.id; + + // If this board has other viewers, notify them. + if (boardId in callbacksByBoard) { + callbacks = callbacksByBoard[boardId]; + callbacks.push(callback); + emit(callbacks, {type: "view", count: callbacks.length}); + } else { + callbacks = callbacksByBoard[boardId] = [callback]; + } + + // Asynchronously load the requested board. + boards.findOne({_id: boardId}, function(error, board) { + if (board != null) { + if (board.pieces) board.pieces.forEach(function(piece) { + callback({type: "add", piece: piece}); + }); + } else { + callback({type: "error", status: 404}); + } + }); + } + + dispatch.close = function(callback) { + var boardId = boardsByCallback[callback.id], + callbacks; + + // If callback was viewing, remove it. + if (boardId) { + callbacks = callbacksByBoard[boardId]; + callbacks.splice(callbacks.indexOf(callback), 1); + if (callbacks.length) emit(callbacks, {type: "view", count: callbacks.length}); + else delete callbacksByBoard[boardId]; + delete boardsByCallback[callback.id]; + } + }; + + return dispatch; +} + +function resolve(file) { + return path.join(__dirname, "../client", file); +} + +function emit(callbacks, event) { + callbacks.forEach(function(callback) { + callback(event); + }); +} diff --git a/test/evaluator-test.js b/test/evaluator-test.js new file mode 100644 index 00000000..1e7d6e9e --- /dev/null +++ b/test/evaluator-test.js @@ -0,0 +1,33 @@ +var vows = require("vows"), + assert = require("assert"), + cube = require("../"), + test = require("./test"); + +var suite = vows.describe("evaluator"); + +var port = ++test.port, server = cube.server({ + "mongo-host": "localhost", + "mongo-port": 27017, + "mongo-database": "cube_test", + "http-port": port, + "authenticator": "allow_all" +}); + +server.register = function(db, endpoints) { + cube.evaluator.register(db, endpoints); + cube.visualizer.register(db, endpoints); +}; + +server.start(); + +// suite.addBatch(test.batch({ +// "GET /1.0/event": { +// topic: test.request({method: "GET", port: port, path: "/1.0/event"}), +// "responds with status 200": function(response) { +// assert.equal(response.statusCode, 200); +// assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); +// } +// } +// })); + +suite.export(module); diff --git a/test/visualizer-test.js b/test/visualizer-test.js new file mode 100644 index 00000000..5b2ca432 --- /dev/null +++ b/test/visualizer-test.js @@ -0,0 +1,23 @@ +var vows = require("vows"), + assert = require("assert"), + cube = require("../"), + test = require("./test"); + +var suite = vows.describe("visualizer"); + +var port = ++test.port, server = cube.server({ + "mongo-host": "localhost", + "mongo-port": 27017, + "mongo-database": "cube_test", + "http-port": port, + "authenticator": "allow_all" +}); + +server.register = function(db, endpoints) { + cube.evaluator.register(db, endpoints); + cube.visualizer.register(db, endpoints); +}; + +server.start(); + +suite.export(module); From 29b5011c9c41929f1d37f2ac9114a8727d23c042 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 27 Aug 2012 19:27:42 -0500 Subject: [PATCH 07/87] Save metrics with zero-counts back to the DB --- lib/cube/metric.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 6454a4ab..44fabe8d 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -196,7 +196,7 @@ exports.getter = function(db) { function save(time, value) { callback(time, value); - if (! value) return; + if ((! value) && (value !== 0)) return; var metric = { _id: { e: expression.source, From 06ac46ded06189900eb77c4fb0201818198f5e7d Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 28 Aug 2012 16:15:39 -0500 Subject: [PATCH 08/87] vows no longer stalls trying to run test.js moved test.js to test_helper.js, renamed it in use. --- test/authentication-test.js | 17 ++++++-------- test/collector-test.js | 30 ++++++++++++------------- test/evaluator-test.js | 14 ++++++------ test/event-expression-test.js | 6 ++--- test/metalog-test.js | 6 ++--- test/metric-expression-test.js | 6 ++--- test/metric-test.js | 24 ++++++++++---------- test/reduces-test.js | 6 ++--- test/test_db.js | 38 +++++++++++++++----------------- test/{test.js => test_helper.js} | 13 +++++------ test/tiers-test.js | 6 ++--- test/types-test.js | 16 +++++++------- test/visualizer-test.js | 10 ++++----- 13 files changed, 92 insertions(+), 100 deletions(-) rename test/{test.js => test_helper.js} (75%) diff --git a/test/authentication-test.js b/test/authentication-test.js index f04ecb38..73afa4c8 100644 --- a/test/authentication-test.js +++ b/test/authentication-test.js @@ -1,10 +1,8 @@ -var vows = require("vows"), - assert = require("assert"), - util = require("util"), - metalog = require("../lib/cube/metalog"), - authentication = require("../lib/cube/authentication"), - test_db = require("./test_db"), - test = require('./test'); +var vows = require("vows"), + assert = require("assert"), + test_helper = require('./test_helper'), + metalog = require("../lib/cube/metalog"), + authentication = require("../lib/cube/authentication"); var suite = vows.describe("authentication"); suite.options.error = true; @@ -48,10 +46,9 @@ function dummy_request(username, token){ return({ headers: { cookie: authentication.gen_cookie("_cube_session", username+"_tok", token || dummy_token) } }); }; -suite.addBatch(test_db.batch({ - +suite.addBatch(test_helper.batch({ mongo_cookie: { - topic: test_db.using_objects("test_users", test_users, this.callback), + topic: function(test_db){ test_db.using_objects("test_users", test_users, this) }, "": { topic: function(test_db){ return authentication.authenticator("mongo_cookie", test_db.db, { collection: "test_users" }); }, diff --git a/test/collector-test.js b/test/collector-test.js index f887a6e4..b617e34d 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -1,11 +1,11 @@ -var vows = require("vows"), - assert = require("assert"), - cube = require("../"), - test = require("./test"); +var vows = require("vows"), + assert = require("assert"), + test_helper = require("./test_helper"), + cube = require("../"); var suite = vows.describe("collector"); -var port = ++test.port, server = cube.server({ +var port = ++test_helper.port, server = cube.server({ "mongo-host": "localhost", "mongo-port": 27017, "mongo-database": "cube_test", @@ -17,9 +17,9 @@ server.register = cube.collector.register; server.start(); -suite.addBatch(test.batch({ +suite.addBatch(test_helper.batch({ "POST /event/put with invalid JSON": { - topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, "This ain't JSON.\n"), + topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, "This ain't JSON.\n"), "responds with status 400": function(response) { assert.equal(response.statusCode, 400); assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); @@ -27,9 +27,9 @@ suite.addBatch(test.batch({ } })); -suite.addBatch(test.batch({ +suite.addBatch(test_helper.batch({ "POST /event/put with a JSON object": { - topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify({ + topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify({ type: "test", time: new Date, data: { @@ -43,9 +43,9 @@ suite.addBatch(test.batch({ } })); -suite.addBatch(test.batch({ +suite.addBatch(test_helper.batch({ "POST /event/put with a JSON array": { - topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ + topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ type: "test", time: new Date, data: { @@ -59,9 +59,9 @@ suite.addBatch(test.batch({ } })); -suite.addBatch(test.batch({ +suite.addBatch(test_helper.batch({ "POST /event/put with a JSON number": { - topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify(42)), + topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify(42)), "responds with status 400": function(response) { assert.equal(response.statusCode, 400); assert.deepEqual(JSON.parse(response.body), {error: "TypeError: Object 42 has no method 'forEach'"}); @@ -69,9 +69,9 @@ suite.addBatch(test.batch({ } })); -suite.addBatch(test.batch({ +suite.addBatch(test_helper.batch({ "POST /event/put without an associated time": { - topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ + topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ type: "test", data: { foo: "bar" diff --git a/test/evaluator-test.js b/test/evaluator-test.js index 1e7d6e9e..4466e3f0 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -1,11 +1,11 @@ -var vows = require("vows"), - assert = require("assert"), - cube = require("../"), - test = require("./test"); +var vows = require("vows"), + assert = require("assert"), + test_helper = require("./test_helper"), + cube = require("../"); var suite = vows.describe("evaluator"); -var port = ++test.port, server = cube.server({ +var port = ++test_helper.port, server = cube.server({ "mongo-host": "localhost", "mongo-port": 27017, "mongo-database": "cube_test", @@ -20,9 +20,9 @@ server.register = function(db, endpoints) { server.start(); -// suite.addBatch(test.batch({ +// suite.addBatch(test_helper.batch({ // "GET /1.0/event": { -// topic: test.request({method: "GET", port: port, path: "/1.0/event"}), +// topic: test_helper.request({method: "GET", port: port, path: "/1.0/event"}), // "responds with status 200": function(response) { // assert.equal(response.statusCode, 200); // assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); diff --git a/test/event-expression-test.js b/test/event-expression-test.js index 9f12a79b..d01acbcb 100644 --- a/test/event-expression-test.js +++ b/test/event-expression-test.js @@ -1,6 +1,6 @@ -var vows = require("vows"), - assert = require("assert"), - parser = require("../lib/cube/event-expression"); +var vows = require("vows"), + assert = require("assert"), + parser = require("../lib/cube/event-expression"); var suite = vows.describe("event-expression"); diff --git a/test/metalog-test.js b/test/metalog-test.js index fee57c54..7fb9c1fa 100644 --- a/test/metalog-test.js +++ b/test/metalog-test.js @@ -1,6 +1,6 @@ -var vows = require("vows"), - assert = require("assert"), - test = require('./test'); +var vows = require("vows"), + assert = require("assert"), + test_helper = require('./test_helper'); var suite = vows.describe("metalog"); diff --git a/test/metric-expression-test.js b/test/metric-expression-test.js index 0fd39844..9da33d89 100644 --- a/test/metric-expression-test.js +++ b/test/metric-expression-test.js @@ -1,6 +1,6 @@ -var vows = require("vows"), - assert = require("assert"), - parser = require("../lib/cube/metric-expression"); +var vows = require("vows"), + assert = require("assert"), + parser = require("../lib/cube/metric-expression"); var suite = vows.describe("metric-expression"); diff --git a/test/metric-test.js b/test/metric-test.js index 969df026..7fd45d25 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -1,11 +1,11 @@ -var vows = require("vows"), - assert = require("assert"), - test = require("./test"), - event = require("../lib/cube/event"), - metric = require("../lib/cube/metric"); - -// as a hack to get updates to settle, we need to insert delays -// if you see funny heisen-errors in the metrics tests try bumping these +var vows = require("vows"), + assert = require("assert"), + test_helper = require("./test_helper"), + event = require("../lib/cube/event"), + metric = require("../lib/cube/metric"); + +// as a hack to get updates to settle, we need to insert delays. +// if you see heisen-errors in the metrics tests, increase these. var step_testing_delay = 250, batch_testing_delay = 500; @@ -25,10 +25,10 @@ steps[3e5].description = "5-minute"; steps[36e5].description = "1-hour"; steps[864e5].description = "1-day"; -suite.addBatch(test.batch({ - topic: function(test) { - var putter = event.putter(test.db), - getter = metric.getter(test.db), +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + var putter = event.putter(test_db.db), + getter = metric.getter(test_db.db), callback = this.callback; for (var i = 0; i < 2500; i++) { diff --git a/test/reduces-test.js b/test/reduces-test.js index 3fa0c62b..2d62585d 100644 --- a/test/reduces-test.js +++ b/test/reduces-test.js @@ -1,6 +1,6 @@ -var vows = require("vows"), - assert = require("assert"), - reduces = require("../lib/cube/reduces"); +var vows = require("vows"), + assert = require("assert"), + reduces = require("../lib/cube/reduces"); var suite = vows.describe("reduces"); diff --git a/test/test_db.js b/test/test_db.js index e3b0a90e..74c642b9 100644 --- a/test/test_db.js +++ b/test/test_db.js @@ -1,33 +1,29 @@ -var mongodb = require("mongodb"), - util = require("util"), - metalog = require("../lib/cube/metalog"); +var mongodb = require("mongodb"), + metalog = require("../lib/cube/metalog"); var test_db = { }; var test_collections = ["test_users", "test_events", "test_metrics"]; -var options = exports.options = { +var options = test_db.options = { "mongo-host": "localhost", "mongo-port": 27017, "mongo-database": "cube_test" }; -exports.using_objects = function (clxn_name, test_objects){ +test_db.using_objects = function (clxn_name, test_objects, that){ metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); - return function(tdb){ - var that = this; - tdb.db.collection(clxn_name, function(err, clxn){ - if (err) throw(err); - that[clxn_name] = clxn; - clxn.remove({ dummy: true }, {safe: true}, function(){ - clxn.insert(test_objects, { safe: true }, function(){ - that.callback(null); - }); }); - }); - }; + test_db.db.collection(clxn_name, function(err, clxn){ + if (err) throw(err); + that[clxn_name] = clxn; + clxn.remove({ dummy: true }, {safe: true}, function(){ + clxn.insert(test_objects, { safe: true }, function(){ + that.callback(null, test_db); + }); }); + }); }; -exports.batch = function(batch) { +test_db.batch = function(batch) { return { "": { topic: function() { @@ -35,9 +31,9 @@ exports.batch = function(batch) { setup_db(this.callback); }, "": batch, - teardown: function(test) { - if (test.client.isConnected()) { - process.nextTick(function(){ test.client.close(); }); + teardown: function(test_db) { + if (test_db.client.isConnected()) { + process.nextTick(function(){ test_db.client.close(); }); }; } } @@ -73,3 +69,5 @@ function drop_collections(cb){ } }); } + +module.exports = test_db; diff --git a/test/test.js b/test/test_helper.js similarity index 75% rename from test/test.js rename to test/test_helper.js index e751b883..0e8e986d 100644 --- a/test/test.js +++ b/test/test_helper.js @@ -1,12 +1,9 @@ -var mongodb = require("mongodb"), - assert = require("assert"), - util = require("util"), - metalog = require("../lib/cube/metalog"), - test_db = require("./test_db"), - http = require("http"); - -exports.port = 1083; +var assert = require("assert"), + http = require("http"), + metalog = require("../lib/cube/metalog"), + test_db = require("./test_db"); +exports.port = 1083; exports.batch = test_db.batch; exports.request = function(options, data) { diff --git a/test/tiers-test.js b/test/tiers-test.js index a638b260..e2a652da 100644 --- a/test/tiers-test.js +++ b/test/tiers-test.js @@ -1,6 +1,6 @@ -var vows = require("vows"), - assert = require("assert"), - tiers = require("../lib/cube/tiers"); +var vows = require("vows"), + assert = require("assert"), + tiers = require("../lib/cube/tiers"); var suite = vows.describe("tiers"); diff --git a/test/types-test.js b/test/types-test.js index 7e211c96..ad33d599 100644 --- a/test/types-test.js +++ b/test/types-test.js @@ -1,14 +1,14 @@ -var vows = require("vows"), - assert = require("assert"), - test = require("./test"), - types = require("../lib/cube/types"), - mongodb = require("mongodb"); +var vows = require("vows"), + assert = require("assert"), + mongodb = require("mongodb"), + test_helper = require("./test_helper"), + types = require("../lib/cube/types"); var suite = vows.describe("types"); -suite.addBatch(test.batch({ - topic: function(test) { - return types(test.db); +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + return types(test_db.db); }, "types": { diff --git a/test/visualizer-test.js b/test/visualizer-test.js index 5b2ca432..74cd8b30 100644 --- a/test/visualizer-test.js +++ b/test/visualizer-test.js @@ -1,11 +1,11 @@ -var vows = require("vows"), - assert = require("assert"), - cube = require("../"), - test = require("./test"); +var vows = require("vows"), + assert = require("assert"), + cube = require("../"), + test_helper = require("./test_helper"); var suite = vows.describe("visualizer"); -var port = ++test.port, server = cube.server({ +var port = ++test_helper.port, server = cube.server({ "mongo-host": "localhost", "mongo-port": 27017, "mongo-database": "cube_test", From 46166ab9c6870e210a2b6f87a00aeeff51291384 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 28 Aug 2012 17:52:45 -0500 Subject: [PATCH 09/87] Fixed the dancing heisen-fails in test/metric-test Some mixture of these changes and the upgraded mongo npm package seem to have made the dancing sometimes-failures in test/metrics-test.js go away. Untangled and commented the code, so if you are still seeing those bugs here maybe is a better stepping-off point. --- lib/cube/metalog.js | 11 ++-- lib/cube/server.js | 2 +- test/metric-test.js | 140 ++++++++++++++++++++++++++++++-------------- test/test_helper.js | 2 + 4 files changed, 107 insertions(+), 48 deletions(-) diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 45b6b993..d1bfca4f 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -17,6 +17,8 @@ metalog.loggers = { // if true, cubify `metalog.event`s metalog.send_events = true; +// -------------------------------------------------------------------------- + // Cubify an event and (optionally) log it. The last parameter specifies the // logger -- 'info' (the default), 'minor' or 'silent'. metalog.event = function(name, hsh, logger){ @@ -37,10 +39,11 @@ metalog.minor = function(name, hsh){ // Dump the 'util.inspect' view of each argument to the console. metalog.inspectify = function(args){ - args = Array.prototype.slice.call(arguments); - args.map(function (arg){ - util.debug(util.inspect(arg)); - }); + var whence = metalog.inspectify.caller.name || '(anon)'; + for (idx in arguments) { + if (whence){ util.print(whence + ' ' + idx + ": "); } + util.print(util.inspect(arguments[idx])+"\n"); + }; }; module.exports = metalog; diff --git a/lib/cube/server.js b/lib/cube/server.js index e344db6f..07ef0038 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -46,7 +46,7 @@ var wsOptions = { // MongoDB driver configuration. var server_options = {auto_reconnect: true}, - db_options = {native_parser: true}; + db_options = { native_parser: true }; module.exports = function(options) { var server = {}, diff --git a/test/metric-test.js b/test/metric-test.js index 7fd45d25..6920a1cb 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -31,6 +31,7 @@ suite.addBatch(test_helper.batch({ getter = metric.getter(test_db.db), callback = this.callback; + // Seed the events table with a simple event: a value going from 0 to 2499 for (var i = 0; i < 2500; i++) { putter({ type: "test", @@ -39,6 +40,7 @@ suite.addBatch(test_helper.batch({ }); } + // So the events can settle in, wait `batch_testing_delay` ms before continuing setTimeout(function() { callback(null, getter); }, batch_testing_delay); }, @@ -123,69 +125,121 @@ suite.addBatch(test_helper.batch({ suite.export(module); +// metricTest -- generates test tree for metrics. +// +// Gets the metric, checks it was calculated correctly from events seeded above; +// then does it again (on a delay) to check that it was cached. +// +// @example given `{ 'unary expression': metricTest({..}, { 60_000: [0, 0, ...], 86_400_000: [82, 2418] })` +// +// { 'unary expression': { +// 'at 1-minute intervals': { +// topic: function get_metrics_with_delay(getter){}, +// 'sum(test)': function metrics_assertions(actual){}, +// '(cached)': { +// topic: function get_metrics_with_delay(_, getter){}, +// 'sum(test)': function metrics_assertions(actual){} } }, +// 'at 1-day intervals': { +// topic: function get_metrics_with_delay(getter){}, +// 'sum(test)': function metrics_assertions(actual){}, +// '(cached)': { +// topic: function get_metrics_with_delay(_, getter){}, +// 'sum(test)': function metrics_assertions(actual){} } } +// } +// } +// function metricTest(request, expected) { - var t = {}, k; - for (k in expected) t["at " + steps[k].description + " intervals"] = testStep(k, expected[k]); - return t; + // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } + var tree = {}, k; + for (step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); + return tree; + // + // { + // topic: get_metrics_with_delay, + // expression: function(){ + // // rounds down the start time (inclusive) + // // formats UTC time in ISO 8601 + // ... + // // returns the expected values + // }, + // '(cached)': { + // topic: get_metrics_with_delay, + // expression: function(){ + // // rounds down the start time (inclusive) + // ... + // } + // } + // } + // function testStep(step, expected) { - var t = testStepDepth(0, step, expected); - t["(cached)"] = testStepDepth(1, step, expected); - return t; - } - - function testStepDepth(depth, step, expected) { var start = new Date(request.start), - stop = new Date(request.stop); - - var test = { - topic: function() { - var actual = [], - timeout = setTimeout(function() { cb("Time's up!"); }, 10000), - cb = this.callback, - req = Object.create(request), - test = arguments[depth]; - req.step = step; - setTimeout(function() { - test(req, function(response) { - if (response.time >= stop) { - clearTimeout(timeout); - cb(null, actual.sort(function(a, b) { return a.time - b.time; })); - } else { - actual.push(response); - } - }); - }, depth * step_testing_delay); - } + stop = new Date(request.stop); + + var subtree = { + topic: get_metrics_with_delay(0), + '(cached)': { + topic: get_metrics_with_delay(1), } }; + subtree[request.expression] = metrics_assertions(); + subtree["(cached)"][request.expression] = metrics_assertions(); + return subtree; - test[request.expression] = function(actual) { + function get_metrics_with_delay(depth){ return function(){ + var actual = [], + timeout = setTimeout(function() { cb("Time's up!"); }, 10000), + cb = this.callback, + req = Object.create(request), + getter = arguments[depth]; + req.step = step; + // Wait long enough for the events to have settled in the db. The + // non-cached (depth=0) round can all start in parallel, making this an + // effective `nextTick`. On the secon + setTimeout(function() { + // ... then invoke the metrics getter. As responses roll in, push them + // on to 'actual'; we're done when the 'stop' time is hit + getter(req, function(response) { + if (response.time >= stop) { + clearTimeout(timeout); + cb(null, actual.sort(function(a, b) { return a.time - b.time; })); + } else { + actual.push(response); + } + }); + }, depth * step_testing_delay); + }}; - // rounds down the start time (inclusive) + function metrics_assertions(){ return { + 'rounds down the start time (inclusive)': function(actual) { var floor = steps[step](start, 0); assert.deepEqual(actual[0].time, floor); + }, - // rounds up the stop time (exclusive) + 'rounds up the stop time (exclusive)': function(actual){ var ceil = steps[step](stop, 0); if (!(ceil - stop)) ceil = steps[step](stop, -1); assert.deepEqual(actual[actual.length - 1].time, ceil); + }, - // formats UTC time in ISO 8601 + 'formats UTC time in ISO 8601': function(actual){ actual.forEach(function(d) { assert.instanceOf(d.time, Date); assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); }); + }, - // returns exactly one value per time + 'returns exactly one value per time': function(actual){ var i = 0, n = actual.length, t = actual[0].time; while (++i < n) assert.isTrue(t < (t = actual[i].time)); + }, - // each metric defines only time and value properties + 'each metric defines only time and value properties': function(actual){ actual.forEach(function(d) { assert.deepEqual(Object.keys(d), ["time", "value"]); }); + }, - // returns the expected times + 'returns the expected times': function(actual){ var floor = steps[step], time = floor(start, 0), times = []; @@ -194,8 +248,9 @@ function metricTest(request, expected) { time = floor(time, 1); } assert.deepEqual(actual.map(function(d) { return d.time; }), times); + }, - // returns the expected values + 'returns the expected values': function(actual){ var actualValues = actual.map(function(d) { return d.value; }); assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); expected.forEach(function(value, i) { @@ -203,9 +258,8 @@ function metricTest(request, expected) { assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); } }); + } - }; - - return test; - } -} + }}; // metric assertions + } // subtree +} // tree diff --git a/test/test_helper.js b/test/test_helper.js index 0e8e986d..b3a476c5 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -6,6 +6,8 @@ var assert = require("assert"), exports.port = 1083; exports.batch = test_db.batch; +exports.inspectify = metalog.inspectify; + exports.request = function(options, data) { return function() { var cb = this.callback; From efd3f51033d59234985980211e0b6daeacca7f66 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 28 Aug 2012 21:17:01 -0500 Subject: [PATCH 10/87] graceful shutdown of server and flushers --- lib/cube/event.js | 14 +++++++++++--- lib/cube/server.js | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/lib/cube/event.js b/lib/cube/event.js index e47cde20..68e15f72 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -23,6 +23,13 @@ var streamDelayDefault = 5000, // How frequently to invalidate metrics after receiving events. var invalidateInterval = 5000; +var flushers = []; +exports.stop = function(){ try{ + flushers.forEach(function(flusher){ + metalog.info('cube_life', {is: 'flusher_unregister' }); + clearInterval(flusher); + }); } catch(err) {}}; + // event.putter -- save the event, invalidate any cached metrics impacted by it. // // @param request -- @@ -31,7 +38,7 @@ var invalidateInterval = 5000; // - type, namespace for the events. A corresponding `foo_events` collection must exist in the DB -- /schema/schema-*.js illustrate how to set up a new event type. // - data, the event's payload // -exports.putter = function(db) { +exports.putter = function(db){ var collection = types(db), knownByType = {}, eventsToSaveByType = {}, @@ -131,7 +138,7 @@ exports.putter = function(db) { // Process any deferred metric invalidations, flushing the queues. Note that // the queue (timesToInvalidateByTierByType) is copied-on-write, so while the // previous batch of events are being invalidated, new events can arrive. - setInterval(function() { + flushers.push(setInterval(function() { for (var type in timesToInvalidateByTierByType) { var metrics = collection(type).metrics, timesToInvalidateByTier = timesToInvalidateByTierByType[type]; @@ -146,8 +153,9 @@ exports.putter = function(db) { flushed = true; } timesToInvalidateByTierByType = {}; // copy-on-write - }, invalidateInterval); + }, invalidateInterval)); + metalog.info('cube_life', {is: 'putter_start'}); return putter; }; diff --git a/lib/cube/server.js b/lib/cube/server.js index 07ef0038..0c43e70f 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -20,6 +20,7 @@ var util = require("util"), static = require("node-static"), mongodb = require("mongodb"), authentication = require("./authentication"), + event = require("./event"), metalog = require("./metalog"); // Don't crash on errors. @@ -53,6 +54,7 @@ module.exports = function(options) { primary = http.createServer(), secondary = websprocket.createServer(), file = new static.Server("static"), + udp, endpoints = {ws: [], http: []}, mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), db = new mongodb.Db(options["mongo-database"], mongo, db_options), @@ -161,15 +163,20 @@ module.exports = function(options) { request.on("end", function() { file.serve(request, response, function(error) { if (error) { - metalog.event('cube_request', { is: 'failed', msg: error, code: error.status, ip: request.connection.remoteAddress, path: u.pathname }); + metalog.event('cube_request', + { is: 'failed', msg: error, code: error.status, ip: request.connection.remoteAddress, path: u.pathname }); response.writeHead(error.status, {"Content-Type": "text/plain"}); response.end(error.status + ""); + } else { + metalog.event('cube_request', + { is: 'static', ip: request.connection.remoteAddress, path: u.pathname }, + 'minor'); } }); }); }); - server.start = function() { + server.start = function(server_start_cb) { // Connect to mongodb. mongo_password = options["mongo-password"]; delete options["mongo-password"]; metalog.info('cube_life', {is: 'mongo_connect', options: options }); @@ -185,22 +192,44 @@ module.exports = function(options) { // Start the server! function ready() { + metalog.putter = event.putter(db); server.register(db, endpoints); - metalog.putter = require("./event").putter(db); authenticator = authentication.authenticator(options["authenticator"], db, options); metalog.event("cube_life", { is: 'start_http', port: options["http-port"] }); primary.listen(options["http-port"]); if (endpoints.udp) { metalog.event("cube_life", { is: 'start_udp', port: options["udp-port"] }); - var udp = dgram.createSocket("udp4"); + udp = dgram.createSocket("udp4"); udp.on("message", function(message) { endpoints.udp(JSON.parse(message.toString("utf8")), ignore); }); udp.bind(options["udp-port"]); } + if (server_start_cb) server_start_cb(null, options); } }; + primary.on( "close", function(){ metalog.info('cube_life', {is: 'http_close' }); }); + secondary.on("close", function(){ metalog.info('cube_life', {is: 'ws_close' }); }); + function try_close(name, obj){ if (obj){ + try { + metalog.info('cube_life', {is:(name+'_stopping'), options: options}); + obj.close( function(){ metalog.info('cube_life', {is:(name+'_stop')}); } ); + } catch(err){}; + }}; + server.stop = function(cb){ + // stop flushing + event.stop(); + // stop serving + try_close('http', primary); + try_close('ws', secondary); + try_close('udp', udp); + // in a short while, stop db'ing + setTimeout(function(){ try_close('mongo', db) }, 100); + // if (cb) setTimeout(function(){ cb(); }, 200); + if (cb) process.nextTick(function(){ cb(); }); + } + return server; }; From 7435bff5edf21442756cfb2551057c77fd4690b7 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 28 Aug 2012 22:08:22 -0500 Subject: [PATCH 11/87] richer tests for server components; test helpers for setting up, starting server. * test_helper.with_server -- starts the server, run tests once server starts, stops server when tests are done * merged test/test_db.js into test/test_helper.js, documentated it good. --- static/semicolon.js | 1 + test/collector-test.js | 44 +++-------- test/server-test.js | 63 ++++++++++++++++ test/test_db.js | 73 ------------------- test/test_helper.js | 162 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 228 insertions(+), 115 deletions(-) create mode 100644 static/semicolon.js create mode 100644 test/server-test.js delete mode 100644 test/test_db.js diff --git a/static/semicolon.js b/static/semicolon.js new file mode 100644 index 00000000..1c8a0e79 --- /dev/null +++ b/static/semicolon.js @@ -0,0 +1 @@ +; \ No newline at end of file diff --git a/test/collector-test.js b/test/collector-test.js index b617e34d..e49cf01e 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -5,31 +5,20 @@ var vows = require("vows"), var suite = vows.describe("collector"); -var port = ++test_helper.port, server = cube.server({ - "mongo-host": "localhost", - "mongo-port": 27017, - "mongo-database": "cube_test", - "http-port": port, - "authenticator": "allow_all" -}); +var server_options = { 'http-port': test_helper.get_port() } -server.register = cube.collector.register; +suite.addBatch( + test_helper.with_server(server_options, cube.collector.register, { -server.start(); - -suite.addBatch(test_helper.batch({ "POST /event/put with invalid JSON": { - topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, "This ain't JSON.\n"), + topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), "responds with status 400": function(response) { assert.equal(response.statusCode, 400); assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); } - } -})); - -suite.addBatch(test_helper.batch({ + }, "POST /event/put with a JSON object": { - topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify({ + topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, JSON.stringify({ type: "test", time: new Date, data: { @@ -40,12 +29,9 @@ suite.addBatch(test_helper.batch({ assert.equal(response.statusCode, 400); assert.deepEqual(JSON.parse(response.body), {error: "TypeError: Object # has no method 'forEach'"}); } - } -})); - -suite.addBatch(test_helper.batch({ + }, "POST /event/put with a JSON array": { - topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ + topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, JSON.stringify([{ type: "test", time: new Date, data: { @@ -56,22 +42,16 @@ suite.addBatch(test_helper.batch({ assert.equal(response.statusCode, 200); assert.deepEqual(JSON.parse(response.body), {}); } - } -})); - -suite.addBatch(test_helper.batch({ + }, "POST /event/put with a JSON number": { - topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify(42)), + topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, JSON.stringify(42)), "responds with status 400": function(response) { assert.equal(response.statusCode, 400); assert.deepEqual(JSON.parse(response.body), {error: "TypeError: Object 42 has no method 'forEach'"}); } - } -})); - -suite.addBatch(test_helper.batch({ + }, "POST /event/put without an associated time": { - topic: test_helper.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ + topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, JSON.stringify([{ type: "test", data: { foo: "bar" diff --git a/test/server-test.js b/test/server-test.js new file mode 100644 index 00000000..89a4a024 --- /dev/null +++ b/test/server-test.js @@ -0,0 +1,63 @@ +var vows = require("vows"), + assert = require("assert"), + http = require("http"), + test_helper = require("./test_helper"), + cube = require("../"); + +var suite = vows.describe("server"); + +var server_options = { 'http-port': test_helper.get_port() } +function frontend_components(db, endpoints) { +}; + +suite.addBatch( + test_helper.with_server(server_options, frontend_components, { + "file": { + "GET": { + topic: test_helper.request({path: "/semicolon.js", method: "GET"}), + "the status should be 200": function(response) { + assert.equal(response.statusCode, 200); + }, + "the expected headers should be set": function(response) { + assert.equal(response.headers["content-type"], "text/javascript"); + assert.equal(response.headers["content-length"], 1); + assert.ok(new Date(response.headers["date"]) > Date.UTC(2011, 0, 1)); + assert.ok(new Date(response.headers["last-modified"]) > Date.UTC(2011, 0, 1)); + }, + "the expected content should be returned": function(response) { + assert.equal(response.body, ";"); + } + }, + "GET If-Modified-Since": { + topic: test_helper.request({path: "/semicolon.js", method: "GET", headers: {"if-modified-since": new Date(2101, 0, 1).toUTCString()}}), + "the status should be 304": function(response) { + assert.equal(response.statusCode, 304); + }, + "the expected headers should be set": function(response) { + assert.ok(!("Content-Length" in response.headers)); + assert.ok(new Date(response.headers["date"]) > Date.UTC(2011, 0, 1)); + assert.ok(new Date(response.headers["last-modified"]) > Date.UTC(2011, 0, 1)); + }, + "no content should be returned": function(response) { + assert.equal(response.body, ""); + } + }, + "HEAD": { + topic: test_helper.request({path: "/semicolon.js", method: "HEAD", headers: {"if-modified-since": new Date(2001, 0, 1).toUTCString()}}), + "the status should be 200": function(response) { + assert.equal(response.statusCode, 200); + }, + "the expected headers should be set": function(response) { + assert.equal(response.headers["content-type"], "text/javascript"); + assert.ok(!("Content-Length" in response.headers)); + assert.ok(new Date(response.headers["date"]) > Date.UTC(2011, 0, 1)); + assert.ok(new Date(response.headers["last-modified"]) > Date.UTC(2011, 0, 1)); + }, + "no content should be returned": function(response) { + assert.equal(response.body, ""); + } + } + } +})); + +suite.export(module); diff --git a/test/test_db.js b/test/test_db.js deleted file mode 100644 index 74c642b9..00000000 --- a/test/test_db.js +++ /dev/null @@ -1,73 +0,0 @@ -var mongodb = require("mongodb"), - metalog = require("../lib/cube/metalog"); - -var test_db = { }; - -var test_collections = ["test_users", "test_events", "test_metrics"]; - -var options = test_db.options = { - "mongo-host": "localhost", - "mongo-port": 27017, - "mongo-database": "cube_test" -}; - -test_db.using_objects = function (clxn_name, test_objects, that){ - metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); - test_db.db.collection(clxn_name, function(err, clxn){ - if (err) throw(err); - that[clxn_name] = clxn; - clxn.remove({ dummy: true }, {safe: true}, function(){ - clxn.insert(test_objects, { safe: true }, function(){ - that.callback(null, test_db); - }); }); - }); -}; - -test_db.batch = function(batch) { - return { - "": { - topic: function() { - connect(); - setup_db(this.callback); - }, - "": batch, - teardown: function(test_db) { - if (test_db.client.isConnected()) { - process.nextTick(function(){ test_db.client.close(); }); - }; - } - } - }; -}; - -// -// db methods -// - -function setup_db(cb){ - drop_collections(cb); -} - -function connect(){ - metalog.minor('cube_testdb', { state: 'connecting to db', options: options }); - test_db.options = options; - test_db.client = new mongodb.Server(options["mongo-host"], options["mongo-port"], {auto_reconnect: true}); - test_db.db = new mongodb.Db(options["mongo-database"], test_db.client, {}); -} - -function drop_collections(cb){ - metalog.minor('cube_testdb', { state: 'dropping test collections', collections: test_collections }); - test_db.db.open(function(error) { - var collectionsRemaining = test_collections.length; - test_collections.forEach(function(collection_name){ - test_db.db.dropCollection(collection_name, collectionReady); - }); - function collectionReady() { - if (!--collectionsRemaining) { - cb(null, test_db); - } - } - }); -} - -module.exports = test_db; diff --git a/test/test_helper.js b/test/test_helper.js index b3a476c5..4433902e 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -1,18 +1,54 @@ var assert = require("assert"), http = require("http"), - metalog = require("../lib/cube/metalog"), - test_db = require("./test_db"); + mongodb = require("mongodb"), + metalog = require("../lib/cube/metalog"); -exports.port = 1083; -exports.batch = test_db.batch; +// ========================================================================== +// +// setup +// -exports.inspectify = metalog.inspectify; +var test_helper = {} +var test_db = {}; +var test_collections = ["test_users", "test_events", "test_metrics"]; +test_helper.inspectify = metalog.inspectify; -exports.request = function(options, data) { +test_helper.settings = { + "mongo-host": "localhost", + "mongo-port": 27017, + "mongo-username": null, + "mongo-password": null, + "mongo-database": "cube_test", + "authenticator": "allow_all" +} + +// Disable logging for tests. +metalog.loggers.info = metalog.silent; +metalog.loggers.minor = metalog.silent; +metalog.send_events = false; + +// ========================================================================== +// +// client / server helpers +// + +var port = 1083; +// test_helper.get_port() -- get a port ID, unique to your batch. +test_helper.get_port = function(){ return ++port; }; + +// test_helper.request -- make an HTTP request. +// +// @param options standard http client options, with these defaults: +// @option host host to contact; default "localhost" +// @option port port to contact; detault `this.http_port` +// @param data request body +// +test_helper.request = function(options, data) { return function() { var cb = this.callback; options.host = "localhost"; + if (! options.port){ options.port = this.http_port }; var request = http.request(options, function(response) { response.body = ""; @@ -28,7 +64,113 @@ exports.request = function(options, data) { }; }; -// Disable logging for tests. -metalog.loggers.info = metalog.silent; -metalog.loggers.minor = metalog.silent; -metalog.send_events = false; +// test_helper.with_server -- +// start server, run tests once server starts, stop server when tests are done +// +// inscribes 'server', 'udp_port' and 'http_port' on the test context -- letting +// you say 'this.server' in your topics, etc. +// +// @param options -- overrides for the settings, above. +// @param components -- passed to server.register() +// @param batch -- the tests to run +test_helper.with_server = function(options, components, batch){ + return { '': { + topic: function(){ start_server(options, components, this); }, + '': batch, + teardown: function(svr){ this.server.stop(this.callback); } + } } +} + +// @see test_helper.with_server +function start_server(options, register, vow){ + for (var key in test_helper.settings){ + if (! options[key]){ options[key] = test_helper.settings[key]; } + } + vow.http_port = options['http-port']; + vow.udp_port = options['udp-port']; + vow.server = require('../lib/cube/server')(options); + vow.server.register = register; + vow.server.start(vow.callback); +} + +// ========================================================================== +// +// db helpers +// + +// test_helper.batch -- +// * connect to db, drop relevant collections +// * run tests once db is ready; +// * close db when tests are done +test_helper.batch = function(batch) { + return { + "": { + topic: function() { + connect(test_helper.settings); + setup_db(this.callback); + }, + "": batch, + teardown: function(test_db) { + if (test_db.client.isConnected()) { + process.nextTick(function(){ test_db.client.close(); }); + }; + } + } + }; +}; + +// test_db.using_objects -- scaffold fixtures into the database, run tests once loaded. +// +// Wrap your tests in test_helper.batch to get the test_db object. +test_db.using_objects = function (clxn_name, test_objects, that){ + metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); + test_db.db.collection(clxn_name, function(err, clxn){ + if (err) throw(err); + that[clxn_name] = clxn; + clxn.remove({ dummy: true }, {safe: true}, function(){ + clxn.insert(test_objects, { safe: true }, function(){ + that.callback(null, test_db); + }); }); + }); +}; + +// ========================================================================== +// +// db methods +// + +// @see test_helper.batch +function setup_db(cb){ + drop_collections(cb); +} + +// @see test_helper.batch +function connect(options){ + metalog.minor('cube_testdb', { state: 'connecting to db', options: options }); + test_db.options = options; + test_db.client = new mongodb.Server(options["mongo-host"], options["mongo-port"], {auto_reconnect: true}); + test_db.db = new mongodb.Db(options["mongo-database"], test_db.client, {}); +} + +// @see test_helper.batch +function drop_collections(cb){ + metalog.minor('cube_testdb', { state: 'dropping test collections', collections: test_collections }); + test_db.db.open(function(error) { + var collectionsRemaining = test_collections.length; + test_collections.forEach(function(collection_name){ + test_db.db.dropCollection(collection_name, collectionReady); + }); + function collectionReady() { + if (!--collectionsRemaining) { + cb(null, test_db); + } + } + }); +} + +// ========================================================================== +// +// fin. +// + +module.exports = test_helper; From e608edc2539f7a02ee0c431e80ae1eff1bf86244 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Wed, 29 Aug 2012 04:19:16 -0500 Subject: [PATCH 12/87] travis.yml --- README.md | 2 ++ travis.yml | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 travis.yml diff --git a/README.md b/README.md index 2d9b2527..264a4630 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Cube +[![build status](https://secure.travis-ci.org/infochimps-labs/cube.png)](http://travis-ci.org/infochimps-labs/cube) + **Cube** is a system for collecting timestamped events and deriving metrics. By collecting events rather than metrics, Cube lets you compute aggregate statistics *post hoc*. It also enables richer analysis, such as quantiles and histograms of arbitrary event sets. Cube is built on [MongoDB](http://www.mongodb.org) and available under the [Apache License](/square/cube/blob/master/LICENSE). Want to learn more? [See the wiki.](/square/cube/wiki) diff --git a/travis.yml b/travis.yml new file mode 100644 index 00000000..895dbd36 --- /dev/null +++ b/travis.yml @@ -0,0 +1,4 @@ +language: node_js +node_js: + - 0.6 + - 0.8 From f64cdb7b9b8a067605b9e1915ff7a1797d57a9f0 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Wed, 29 Aug 2012 04:25:56 -0500 Subject: [PATCH 13/87] added test 'script' to package.json, for make travis-ci happy --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 00006fd2..1b919452 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "bcrypt": "0.7.1" }, "scripts": { - "preinstall": "npm install mongodb --mongodb:native" + "preinstall": "npm install mongodb --mongodb:native", + "test": "vows --spec" } } From 262bfa3bb21e5d8195ea2b41a1b8e0a89b62579a Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Wed, 29 Aug 2012 04:27:13 -0500 Subject: [PATCH 14/87] trying to make travis-ci happy --- travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename travis.yml => .travis.yml (100%) diff --git a/travis.yml b/.travis.yml similarity index 100% rename from travis.yml rename to .travis.yml From ab04068d4c65ef4a1a403e1eeb65c0fd8025e8e5 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Wed, 29 Aug 2012 04:41:01 -0500 Subject: [PATCH 15/87] More tests around server, including UDP, HTTP and static file. --- .travis.yml | 1 - lib/cube/collector.js | 3 ++ lib/cube/event.js | 4 ++- lib/cube/server.js | 3 +- test/evaluator-test.js | 29 ++++++++----------- test/server-test.js | 53 ++++++++++++++++++++++++++++++---- test/test_helper.js | 64 +++++++++++++++++++++++++++++++++++++----- 7 files changed, 122 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index 895dbd36..baa0031d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ language: node_js node_js: - - 0.6 - 0.8 diff --git a/lib/cube/collector.js b/lib/cube/collector.js index 7e9d1c3f..6f3f8489 100644 --- a/lib/cube/collector.js +++ b/lib/cube/collector.js @@ -63,3 +63,6 @@ function post(putter) { }); }; } + +// ignore -- used for testing +exports._post = post; diff --git a/lib/cube/event.js b/lib/cube/event.js index 68e15f72..6a24ac42 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -190,7 +190,9 @@ exports.getter = function(db) { try { expression = parser.parse(request.expression); } catch (error) { - return callback({error: "invalid expression"}), -1; + var resp = { is: "invalid expression", expression: request.expression, error: error } + metalog.info('cube_getter', resp); + return callback(resp), -1; } // Set an optional limit on the number of events to return. diff --git a/lib/cube/server.js b/lib/cube/server.js index 0c43e70f..bd2b8a3f 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -226,8 +226,7 @@ module.exports = function(options) { try_close('udp', udp); // in a short while, stop db'ing setTimeout(function(){ try_close('mongo', db) }, 100); - // if (cb) setTimeout(function(){ cb(); }, 200); - if (cb) process.nextTick(function(){ cb(); }); + if (cb) setTimeout(function(){ cb(); }, 200); } return server; diff --git a/test/evaluator-test.js b/test/evaluator-test.js index 4466e3f0..65e22885 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -5,26 +5,19 @@ var vows = require("vows"), var suite = vows.describe("evaluator"); -var port = ++test_helper.port, server = cube.server({ - "mongo-host": "localhost", - "mongo-port": 27017, - "mongo-database": "cube_test", - "http-port": port, - "authenticator": "allow_all" -}); - -server.register = function(db, endpoints) { - cube.evaluator.register(db, endpoints); - cube.visualizer.register(db, endpoints); +var server_options = { 'http-port': test_helper.get_port() } +function frontend_components() { + cube.evaluator.register.apply(this, arguments); + cube.visualizer.register.apply(this, arguments); }; -server.start(); - -// suite.addBatch(test_helper.batch({ -// "GET /1.0/event": { -// topic: test_helper.request({method: "GET", port: port, path: "/1.0/event"}), -// "responds with status 200": function(response) { -// assert.equal(response.statusCode, 200); +// suite.addBatch( +// test_helper.with_server(server_options, frontend_components, { +// +// "POST /event/put with invalid JSON": { +// topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), +// "responds with status 400": function(response) { +// assert.equal(response.statusCode, 400); // assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); // } // } diff --git a/test/server-test.js b/test/server-test.js index 89a4a024..5f3a2c23 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -1,18 +1,58 @@ var vows = require("vows"), assert = require("assert"), - http = require("http"), test_helper = require("./test_helper"), - cube = require("../"); + cube = require("../"), + endpoint = cube.endpoint, + collector = cube.collector, + metalog = cube.metalog; var suite = vows.describe("server"); -var server_options = { 'http-port': test_helper.get_port() } -function frontend_components(db, endpoints) { +var bucket = { udped: [], httped: [], websocked: [] }; +var now_ish = Date.now(); +var example = { + for_http: { type: "monotreme", time: now_ish, data: { echidnae: 4 } }, + for_udp: { type: "monotreme", time: now_ish, data: { platypodes: 9 } } }; + +var server_options = { + 'http-port': test_helper.get_port(), + 'udp-port': test_helper.get_port(), +} +function dummy_server(db, endpoints){ + endpoints.udp = function(req, cb){ + // metalog.info('rcvd_udp', { req: req }); + bucket.udped.push(req); + }; + endpoints.http.push( + endpoint('POST', '/1.0/test', collector._post(function(req, cb){ + metalog.info('rcvd_http', { req: req }); + bucket.httped.push(req); + }))); }; suite.addBatch( - test_helper.with_server(server_options, frontend_components, { - "file": { + test_helper.with_server(server_options, dummy_server, { + + http: { + topic: test_helper.request({path: "/1.0/test", method: "POST"}, JSON.stringify([example.for_http])), + '': { + topic: test_helper.delaying_topic, + 'sends data to registered putter': function(response){ + assert.deepEqual(bucket.httped.pop(), example.for_http); + assert.isEmpty(bucket.httped); + } + } + }, + + udp: { + topic: test_helper.udp_request(example.for_udp), + '': function(){ + assert.deepEqual(bucket.udped.pop(), example.for_udp); + assert.isEmpty(bucket.udped); + } + }, + + file: { "GET": { topic: test_helper.request({path: "/semicolon.js", method: "GET"}), "the status should be 200": function(response) { @@ -58,6 +98,7 @@ suite.addBatch( } } } + })); suite.export(module); diff --git a/test/test_helper.js b/test/test_helper.js index 4433902e..47198808 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -1,5 +1,6 @@ var assert = require("assert"), http = require("http"), + dgram = require('dgram'), mongodb = require("mongodb"), metalog = require("../lib/cube/metalog"); @@ -19,12 +20,13 @@ test_helper.settings = { "mongo-username": null, "mongo-password": null, "mongo-database": "cube_test", + "host": "localhost", "authenticator": "allow_all" } // Disable logging for tests. -metalog.loggers.info = metalog.silent; -metalog.loggers.minor = metalog.silent; +metalog.loggers.info = metalog.silent; // log +metalog.loggers.minor = metalog.silent; // log metalog.send_events = false; // ========================================================================== @@ -54,7 +56,7 @@ test_helper.request = function(options, data) { response.body = ""; response.setEncoding("utf8"); response.on("data", function(chunk) { response.body += chunk; }); - response.on("end", function() { cb(null, response); }); + response.on("end", function() { cb(null, response); }); }); request.on("error", function(e) { cb(e, null); }); @@ -64,6 +66,54 @@ test_helper.request = function(options, data) { }; }; +// send udp packet, twiddle thumbs briefly, resume tests. +test_helper.udp_request = function (data){ + return function(){ + var udp_client = dgram.createSocket('udp4'); + var buffer = new Buffer(JSON.stringify(data)); + var context = this; + metalog.info('sending_udp', { data: data }); + udp_client.send(buffer, 0, buffer.length, context.udp_port, 'localhost', + function(err, val){ delayed_callback(context)(err, val); udp_client.close(); } ); + }; +} + +// proxies to the test context's callback after a short delay. +// +// @example as a test topic; will get the same data the cb otherwise would have: +// { topic: send_some_data, +// 'a short time later': { +// topic: test_helper.delaying_topic, +// 'is party time': function(arg){ assert.isAwesome(...) } } } +// +function delaying_topic(){ + var args = Array.prototype.slice.apply(arguments); + args.unshift(null); + delayed_callback(this).apply(this, args); +} +test_helper.delaying_topic = delaying_topic; + +// returns a callback that once triggered, delays briefly, then passes the same +// args to the actual context's callback +// +// @example +// // you +// dcb = delayed_callback(this) +// foo.do_something('...', dcb); +// // foo, after do_something'ing, invokes the delayed callback +// dcb(null, 1, 2); +// // 50ms later, dcb does the equivalent of +// this.callback(null, 1, 2); +// +function delayed_callback(context){ + return function(){ + var callback_delay = 100; + args = arguments; + setTimeout(function(){ context.callback.apply(context, args) }, callback_delay) + }; +} +test_helper.delayed_callback = delayed_callback; + // test_helper.with_server -- // start server, run tests once server starts, stop server when tests are done // @@ -75,7 +125,7 @@ test_helper.request = function(options, data) { // @param batch -- the tests to run test_helper.with_server = function(options, components, batch){ return { '': { - topic: function(){ start_server(options, components, this); }, + topic: function(){ start_server(options, components, this); }, '': batch, teardown: function(svr){ this.server.stop(this.callback); } } } @@ -122,14 +172,14 @@ test_helper.batch = function(batch) { // test_db.using_objects -- scaffold fixtures into the database, run tests once loaded. // // Wrap your tests in test_helper.batch to get the test_db object. -test_db.using_objects = function (clxn_name, test_objects, that){ +test_db.using_objects = function (clxn_name, test_objects, context){ metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); test_db.db.collection(clxn_name, function(err, clxn){ if (err) throw(err); - that[clxn_name] = clxn; + context[clxn_name] = clxn; clxn.remove({ dummy: true }, {safe: true}, function(){ clxn.insert(test_objects, { safe: true }, function(){ - that.callback(null, test_db); + context.callback(null, test_db); }); }); }); }; From 8ca223556a277ca6e7fe905d0198cfd53962e65e Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 30 Aug 2012 11:39:29 -0500 Subject: [PATCH 16/87] Temporarily disable complex metric expressions --- lib/cube/metric.js | 4 ++++ lib/cube/reduces.js | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 44fabe8d..6fef6f98 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -25,6 +25,7 @@ exports.getter = function(db) { // Validate the dates. if (isNaN(start)) return callback({error: "invalid start"}), -1; if (isNaN(stop)) return callback({error: "invalid stop"}), -1; + if (request.expression.match(/\(/mg).length > 2) return callback({error: "rejected complex expression"}), -1; // Parse the expression. var expression; @@ -112,6 +113,9 @@ exports.getter = function(db) { reduce = reduces[expression.reduce], filter = {t: {}}, fields = {t: 1}; + + if (!reduce) return callback({error: "invalid reduce operation"}), -1; + // Copy any expression filters into the query object. expression.filter(filter); diff --git a/lib/cube/reduces.js b/lib/cube/reduces.js index 58b25176..607b8160 100644 --- a/lib/cube/reduces.js +++ b/lib/cube/reduces.js @@ -18,21 +18,21 @@ var reduces = module.exports = { return max; }, - distinct: function(values) { - var map = {}, count = 0, i = -1, n = values.length, value; - while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; - return count; - }, - - median: function(values) { - return quantile(values.sort(ascending), .5); - } + // distinct: function(values) { + // var map = {}, count = 0, i = -1, n = values.length, value; + // while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; + // return count; + // }, + // + // median: function(values) { + // return quantile(values.sort(ascending), .5); + // } }; // These metrics have well-defined values for the empty set. reduces.sum.empty = 0; -reduces.distinct.empty = 0; +//reduces.distinct.empty = 0; // These metrics can be computed using pyramidal aggregation. reduces.sum.pyramidal = true; From e0083f10d46260c0debd850f51f184f56b14a8ed Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 30 Aug 2012 17:08:33 -0500 Subject: [PATCH 17/87] Move configs into single config script --- bin/collector-config.js | 11 --------- bin/collector.js | 2 +- bin/evaluator-config.js | 10 -------- bin/evaluator.js | 2 +- config/cube.js | 51 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 23 deletions(-) delete mode 100644 bin/collector-config.js delete mode 100644 bin/evaluator-config.js create mode 100644 config/cube.js diff --git a/bin/collector-config.js b/bin/collector-config.js deleted file mode 100644 index fcbd19ef..00000000 --- a/bin/collector-config.js +++ /dev/null @@ -1,11 +0,0 @@ -// Default configuration for development. -module.exports = { - "mongo-host": "127.0.0.1", - "mongo-port": 27017, - "mongo-database": "cube_development", - "mongo-username": null, - "mongo-password": null, - "http-port": 1080, - "udp-port": 1180, - "authenticator": "allow_all" -}; diff --git a/bin/collector.js b/bin/collector.js index a2b3e80d..bb54cc46 100644 --- a/bin/collector.js +++ b/bin/collector.js @@ -1,4 +1,4 @@ -var options = require("./collector-config"), +var options = require("../config/cube").include("collector"), cube = require("../"), server = cube.server(options); diff --git a/bin/evaluator-config.js b/bin/evaluator-config.js deleted file mode 100644 index 84e544f9..00000000 --- a/bin/evaluator-config.js +++ /dev/null @@ -1,10 +0,0 @@ -// Default configuration for development. -module.exports = { - "mongo-host": "127.0.0.1", - "mongo-port": 27017, - "mongo-database": "cube_development", - "mongo-username": null, - "mongo-password": null, - "http-port": 1081, - "authenticator": "allow_all" -}; diff --git a/bin/evaluator.js b/bin/evaluator.js index 45ba7001..3bdb35c8 100644 --- a/bin/evaluator.js +++ b/bin/evaluator.js @@ -1,4 +1,4 @@ -var options = require("./evaluator-config"), +var options = require("../config/cube").include('evaluator'), cube = require("../"), server = cube.server(options); diff --git a/config/cube.js b/config/cube.js new file mode 100644 index 00000000..1cbf4907 --- /dev/null +++ b/config/cube.js @@ -0,0 +1,51 @@ +var configs = {}; + + +// +//Shared configuration +// +configs.common = { + "mongo-host": "127.0.0.1", + "mongo-port": 27017, + "mongo-database": "dashpot_development", + "mongo-username": null, + "mongo-password": null, + + "mongo-metrics": {capped: true, size: 1e7, autoIndexId: true}, + "mongo-events": {capped: true, size: 1e7, autoIndexId: true}, +}; + + +// +// Collector configuration +// +configs.collector = { + "http-port": 1080, + "udp-port": 1180, + "authenticator": "allow_all" +} + + +// +// Evaluator configuration +// +configs.evaluator = { + "http-port": 1081, + "authenticator": "mongo_cookie" +} + +var options = {}; +Object.defineProperty(options, "include", { + enumerable: false, + value: function() { + for(var config_name in arguments){ + var config = configs[arguments[config_name]]; + for(var prop in config) + if(config.hasOwnProperty(prop)) + options[prop] = config[prop]; + }; + return options; + } +}); + +module.exports = options.include('common'); \ No newline at end of file From 4770ac27e4f3096662928fd0febcbbb8b9919766 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 30 Aug 2012 17:10:44 -0500 Subject: [PATCH 18/87] Add mongo server config to cube config --- config/cube.js | 1 + lib/cube/server.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/cube.js b/config/cube.js index 1cbf4907..475606a0 100644 --- a/config/cube.js +++ b/config/cube.js @@ -10,6 +10,7 @@ configs.common = { "mongo-database": "dashpot_development", "mongo-username": null, "mongo-password": null, + "mongo-server_options": {auto_reconnect: true, poolSize: 8, socketOptions: { noDelay: true }}, "mongo-metrics": {capped: true, size: 1e7, autoIndexId: true}, "mongo-events": {capped: true, size: 1e7, autoIndexId: true}, diff --git a/lib/cube/server.js b/lib/cube/server.js index bd2b8a3f..57d630ac 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -46,7 +46,7 @@ var wsOptions = { }; // MongoDB driver configuration. -var server_options = {auto_reconnect: true}, +var server_options = options["mongo-server_options"], db_options = { native_parser: true }; module.exports = function(options) { From be66b98039e7fd4c4ccaebd3f54b0b073c90a92c Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 30 Aug 2012 17:14:08 -0500 Subject: [PATCH 19/87] Events and metrics collections are created from config file --- lib/cube/event.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/cube/event.js b/lib/cube/event.js index 6a24ac42..1da3c183 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -7,12 +7,14 @@ var mongodb = require("mongodb"), types = require("./types"), bisect = require("./bisect"), metalog = require("./metalog"), + options = require("../../config/cube"), ObjectID = mongodb.ObjectID; var type_re = /^[a-z][a-zA-Z0-9_]+$/, invalidate = {$set: {i: true}}, multi = {multi: true}, - metric_options = {capped: true, size: 1e7, autoIndexId: true}; + metric_options = options["mongo-metrics"], + event_options = options["mongo-events"]; // When streaming events, we should allow a delay for events to arrive, or else // we risk skipping events that arrive after their event.time. This delay can be @@ -78,16 +80,20 @@ exports.putter = function(db){ var events = collection(type).events; if (names.length) return saveEvents(); - // Events are indexed by time. - events.ensureIndex({"t": 1}, handle); - // Create a capped collection for metrics. Three indexes are required: one - // for finding metrics, one (_id) for updating, and one for invalidation. - db.createCollection(type + "_metrics", metric_options, function(error, metrics) { + // Create a collection for events. One index is require, for finding events by time(t) + db.createCollection(type + "_events", event_options, function(error, events){ handle(error); - metrics.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); - metrics.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, handle); - saveEvents(); + events.ensureIndex({"t": 1}, handle); + + // Create a collection for metrics. Three indexes are required: one + // for finding metrics, one (_id) for updating, and one for invalidation. + db.createCollection(type + "_metrics", metric_options, function(error, metrics) { + handle(error); + metrics.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); + metrics.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, handle); + saveEvents(); + }); }); // Save any pending events to the new collection. From 3645464d7eebfef908e97224ce6f4cfff0b8702d Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 30 Aug 2012 17:16:08 -0500 Subject: [PATCH 20/87] Configurable mongo connection --- lib/cube/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cube/server.js b/lib/cube/server.js index 57d630ac..5f809948 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -46,8 +46,7 @@ var wsOptions = { }; // MongoDB driver configuration. -var server_options = options["mongo-server_options"], - db_options = { native_parser: true }; +var db_options = { native_parser: true }; module.exports = function(options) { var server = {}, @@ -56,6 +55,7 @@ module.exports = function(options) { file = new static.Server("static"), udp, endpoints = {ws: [], http: []}, + server_options = options["mongo-server_options"], mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), db = new mongodb.Db(options["mongo-database"], mongo, db_options), id = 0, From aa4fafd892e34bd4a6907e54b8a7a8554eb3f028 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 31 Aug 2012 15:52:01 -0500 Subject: [PATCH 21/87] Add horizons for calculations and invalidations when processing events and metrics --- config/cube.js | 5 +++++ lib/cube/event.js | 3 +++ lib/cube/metric.js | 15 ++++++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/config/cube.js b/config/cube.js index 475606a0..b3d91413 100644 --- a/config/cube.js +++ b/config/cube.js @@ -14,6 +14,11 @@ configs.common = { "mongo-metrics": {capped: true, size: 1e7, autoIndexId: true}, "mongo-events": {capped: true, size: 1e7, autoIndexId: true}, + + "horizons": { + "calculation": 1000 * 60 * 60 * 2, // 2 hours + "invalidation": 1000 * 60 * 60 * 1, // 1 hour + } }; diff --git a/lib/cube/event.js b/lib/cube/event.js index 1da3c183..bfcaf48f 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -53,6 +53,9 @@ exports.putter = function(db){ // Validate the date and type. if (!type_re.test(type)) return callback({error: "invalid type"}), -1; if (isNaN(time)) return callback({error: "invalid time"}), -1; + + // Drop events from before invalidation horizon + if(time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; // If an id is specified, promote it to Mongo's primary key. var event = {t: time, d: request.data}; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 6fef6f98..dc3aa6a5 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -1,11 +1,12 @@ // TODO use expression ids or hashes for more compact storage -var parser = require("./metric-expression"), - tiers = require("./tiers"), - types = require("./types"), +var parser = require("./metric-expression"), + tiers = require("./tiers"), + types = require("./types"), reduces = require("./reduces"), - event = require("./event"), - metalog = require('./metalog'); + event = require("./event"), + metalog = require('./metalog'), + options = require('../../config/cube'); var metric_fields = {v: 1}, metric_options = {sort: {"_id.t": 1}, batchSize: 1000}, @@ -174,6 +175,10 @@ exports.getter = function(db) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { + if(tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); + start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + } filter.t.$gte = start; filter.t.$lt = stop; type.events.find(filter, fields, event_options, function(error, cursor) { From 730ffe12a64301928a9118fb8baad1dc77e80a5c Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 31 Aug 2012 15:52:49 -0500 Subject: [PATCH 22/87] Cascade tiers down to 10 seconds --- lib/cube/tiers.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index f498220a..79312ec3 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -18,14 +18,18 @@ tiers[minute] = { key: minute, floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, ceil: tier_ceil, - step: function(d) { return new Date(+d + minute); } + step: function(d) { return new Date(+d + minute); }, + next: tiers[second10], + size: function() { return 6; } }; tiers[minute5] = { key: minute5, floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, ceil: tier_ceil, - step: function(d) { return new Date(+d + minute5); } + step: function(d) { return new Date(+d + minute5); }, + next: tiers[minute], + size: function() { return 5; } }; tiers[hour] = { From e59272473f7e68533c35200cff1eafde7721b367 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 31 Aug 2012 16:19:49 -0500 Subject: [PATCH 23/87] Add warmer to refresh metrics --- bin/warmer.js | 5 ++++ config/cube.js | 9 +++++++ lib/cube/index.js | 1 + lib/cube/warmer.js | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 bin/warmer.js create mode 100644 lib/cube/warmer.js diff --git a/bin/warmer.js b/bin/warmer.js new file mode 100644 index 00000000..502a29de --- /dev/null +++ b/bin/warmer.js @@ -0,0 +1,5 @@ +var options = require("../config/cube").include('warmer'), + cube = require("../"), + warmer = cube.warmer(options); + +warmer.start(); diff --git a/config/cube.js b/config/cube.js index b3d91413..e21b2f6b 100644 --- a/config/cube.js +++ b/config/cube.js @@ -40,6 +40,15 @@ configs.evaluator = { "authenticator": "mongo_cookie" } + +// +// Warmer configuration +// +configs.warmer = { + "warmer-interval": 1000 * 30, + "warmer-tier": 1000 * 10 +} + var options = {}; Object.defineProperty(options, "include", { enumerable: false, diff --git a/lib/cube/index.js b/lib/cube/index.js index cb8ff7c3..154192ff 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -6,3 +6,4 @@ exports.collector = require("./collector"); exports.evaluator = require("./evaluator"); exports.visualizer = require("./visualizer"); exports.endpoint = require("./endpoint"); +exports.warmer = require("./warmer"); diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js new file mode 100644 index 00000000..3dbae5a4 --- /dev/null +++ b/lib/cube/warmer.js @@ -0,0 +1,64 @@ +var cluster = require('cluster'), + mongodb = require('mongodb'), + metric = require('./metric'), + tiers = require('./tiers'), + metalog = require('./metalog') + +module.exports = function(options){ + var db, mongo, calculate_metric, boards, tier; + + function fetch_metrics(callback){ + var expressions = []; + + if(!boards){ + db.collection("boards", function(error, collection) { boards = collection; fetch_metrics(callback); }); + return; + } + + boards.find({}, {pieces: 1}, function(error, cursor) { + if (error) throw error; + cursor.each(function(error, row) { + if (error) throw error; + if (row) { + expressions.splice.apply(expressions, [0, 0].concat(row.pieces + .map(function(piece){ return piece.query; }) + .filter(function(expression){ return expression && !(expression in expressions); }) + )); + } else { + callback(expressions); + } + }); + }); + } + + function process_metrics(expressions){ + expressions.forEach(function(expression){ + var stop = new Date(), + start = tier.step(tier.floor(new Date(stop - options.horizons.calculation))); + + metalog.info('cube_warm', {is: 'warm_metric', metric: {query: expressions}, start: start, stop: stop }); + + // fake metrics request + calculate_metric({ step: tier.key, expression: expression, start: start, stop: stop }, function(){}); + }); + setTimeout(function(){ fetch_metrics(process_metrics); }, options['warmer-interval']); + } + + return { + start: function(){ + mongo = new mongodb.Server(options['mongo-host'], options['mongo-port']); + db = new mongodb.Db(options["mongo-database"], mongo), + tier = tiers[options['warmer-tier'].toString()]; + + if(typeof tier === "undefined") throw new Error("Undefined warmer tier configured: " + options['warmer-tier']); + + metalog.event("cube_life", { is: 'start_warmer', options: options }); + + db.open(function(error) { + if (error) throw error; + calculate_metric = metric.getter(db); + fetch_metrics(process_metrics); + }); + } + }; +} \ No newline at end of file From 957222f4e5b893771e5be680255dfdbc10feb504 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 31 Aug 2012 17:26:47 -0500 Subject: [PATCH 24/87] Remove paraenthesis limitation --- lib/cube/metric.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/metric.js b/lib/cube/metric.js index dc3aa6a5..ac6ae74b 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -26,7 +26,7 @@ exports.getter = function(db) { // Validate the dates. if (isNaN(start)) return callback({error: "invalid start"}), -1; if (isNaN(stop)) return callback({error: "invalid stop"}), -1; - if (request.expression.match(/\(/mg).length > 2) return callback({error: "rejected complex expression"}), -1; + //if (request.expression.match(/\(/mg).length > 2) return callback({error: "rejected complex expression"}), -1; // Parse the expression. var expression; From 402f68b9cbe2a0fb332343788df6fae5682e1401 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Sun, 2 Sep 2012 20:12:01 -0500 Subject: [PATCH 25/87] use strict mode everywhere --- bin/collector.js | 2 + bin/evaluator.js | 2 + bin/warmer.js | 2 + lib/cube/authentication.js | 6 +- lib/cube/bisect.js | 2 + lib/cube/collectd.js | 2 + lib/cube/collector.js | 2 + lib/cube/emitter-http.js | 2 + lib/cube/emitter-udp.js | 2 + lib/cube/emitter-ws.js | 2 + lib/cube/emitter.js | 2 + lib/cube/endpoint.js | 3 + lib/cube/evaluator.js | 2 + lib/cube/event.js | 11 +- lib/cube/index.js | 2 + lib/cube/metalog.js | 7 +- lib/cube/metric.js | 8 +- lib/cube/models.js | 9 ++ lib/cube/reduces.js | 4 +- lib/cube/server.js | 43 +++--- lib/cube/tiers.js | 2 + lib/cube/types.js | 2 + lib/cube/visualizer.js | 2 + lib/cube/warmer.js | 20 +-- test/authentication-test.js | 2 + test/collector-test.js | 2 + test/evaluator-test.js | 4 +- test/event-expression-test.js | 2 + test/metalog-test.js | 4 +- test/metric-expression-test.js | 2 + test/metric-test.js | 226 ++++++++++++++------------- test/reduces-test.js | 2 + test/server-test.js | 2 + test/test_helper.js | 4 +- test/tiers-test.js | 272 +++++++++++++++++---------------- test/types-test.js | 2 + test/visualizer-test.js | 2 + 37 files changed, 372 insertions(+), 295 deletions(-) create mode 100644 lib/cube/models.js diff --git a/bin/collector.js b/bin/collector.js index bb54cc46..463cd72e 100644 --- a/bin/collector.js +++ b/bin/collector.js @@ -1,3 +1,5 @@ +'use strict'; + var options = require("../config/cube").include("collector"), cube = require("../"), server = cube.server(options); diff --git a/bin/evaluator.js b/bin/evaluator.js index 3bdb35c8..4ba5a9cb 100644 --- a/bin/evaluator.js +++ b/bin/evaluator.js @@ -1,3 +1,5 @@ +'use strict'; + var options = require("../config/cube").include('evaluator'), cube = require("../"), server = cube.server(options); diff --git a/bin/warmer.js b/bin/warmer.js index 502a29de..7c0b5b38 100644 --- a/bin/warmer.js +++ b/bin/warmer.js @@ -1,3 +1,5 @@ +'use strict'; + var options = require("../config/cube").include('warmer'), cube = require("../"), warmer = cube.warmer(options); diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js index 2a34ae91..671b50f6 100644 --- a/lib/cube/authentication.js +++ b/lib/cube/authentication.js @@ -1,3 +1,5 @@ +'use strict'; + // // authentication -- authenticate user identities and authorize their action // @@ -106,8 +108,8 @@ authentication.mongo_cookie = function(db, options){ // base-64 encode a uid and bcrypted secret authentication.gen_cookie = function(session_name, uid, secret){ - encoded_uid = new Buffer(uid, 'utf8').toString('base64'); - encoded_sec = new Buffer(secret, 'utf8').toString('base64'); + var encoded_uid = new Buffer(uid, 'utf8').toString('base64'); + var encoded_sec = new Buffer(secret, 'utf8').toString('base64'); return (session_name+"="+encoded_uid+"--"+encoded_sec+";"); }; diff --git a/lib/cube/bisect.js b/lib/cube/bisect.js index 55fc7c3c..30f3e3de 100644 --- a/lib/cube/bisect.js +++ b/lib/cube/bisect.js @@ -1,3 +1,5 @@ +'use strict'; + module.exports = bisect; function bisect(a, x) { diff --git a/lib/cube/collectd.js b/lib/cube/collectd.js index 1c82717d..654c9ab4 100644 --- a/lib/cube/collectd.js +++ b/lib/cube/collectd.js @@ -1,3 +1,5 @@ +'use strict'; + exports.putter = function(putter) { var valuesByKey = {}; diff --git a/lib/cube/collector.js b/lib/cube/collector.js index 6f3f8489..c58b5a86 100644 --- a/lib/cube/collector.js +++ b/lib/cube/collector.js @@ -1,3 +1,5 @@ +'use strict'; + // // collector -- listen for incoming metrics // diff --git a/lib/cube/emitter-http.js b/lib/cube/emitter-http.js index e54c469e..0cf2fa66 100644 --- a/lib/cube/emitter-http.js +++ b/lib/cube/emitter-http.js @@ -1,3 +1,5 @@ +'use strict'; + var util = require("util"), http = require("http"); diff --git a/lib/cube/emitter-udp.js b/lib/cube/emitter-udp.js index 02b6d494..b9a62380 100644 --- a/lib/cube/emitter-udp.js +++ b/lib/cube/emitter-udp.js @@ -1,3 +1,5 @@ +'use strict'; + var util = require("util"), dgram = require("dgram"); diff --git a/lib/cube/emitter-ws.js b/lib/cube/emitter-ws.js index a3c921b5..ed3f3204 100644 --- a/lib/cube/emitter-ws.js +++ b/lib/cube/emitter-ws.js @@ -1,3 +1,5 @@ +'use strict'; + var util = require("util"), websocket = require("websocket"); diff --git a/lib/cube/emitter.js b/lib/cube/emitter.js index 6362f325..21dbfb77 100644 --- a/lib/cube/emitter.js +++ b/lib/cube/emitter.js @@ -1,3 +1,5 @@ +'use strict'; + // // emitter - writes events to the collector. // diff --git a/lib/cube/endpoint.js b/lib/cube/endpoint.js index a8aec210..03021e32 100644 --- a/lib/cube/endpoint.js +++ b/lib/cube/endpoint.js @@ -1,3 +1,5 @@ +'use strict'; + // // endpoint -- router for requests. // @@ -8,6 +10,7 @@ // module.exports = function(method, path, dispatch) { + var match; if (method instanceof RegExp) { dispatch = path, path = method; match = function(p, m) { return path.test(p); }; diff --git a/lib/cube/evaluator.js b/lib/cube/evaluator.js index 20e911d5..b2b68824 100644 --- a/lib/cube/evaluator.js +++ b/lib/cube/evaluator.js @@ -1,3 +1,5 @@ +'use strict'; + var endpoint = require("./endpoint"), url = require("url"); diff --git a/lib/cube/event.js b/lib/cube/event.js index bfcaf48f..d81b9bed 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -1,3 +1,5 @@ +'use strict'; + // TODO include the event._id (and define a JSON encoding for ObjectId?) // TODO allow the event time to change when updating (fix invalidation) @@ -53,7 +55,7 @@ exports.putter = function(db){ // Validate the date and type. if (!type_re.test(type)) return callback({error: "invalid type"}), -1; if (isNaN(time)) return callback({error: "invalid time"}), -1; - + // Drop events from before invalidation horizon if(time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; @@ -80,6 +82,7 @@ exports.putter = function(db){ // that if you want to customize the size of the capped metrics collection, // or add custom indexes, you can still do all that by hand. db.collectionNames(type + "_events", function(error, names) { + handle(error); var events = collection(type).events; if (names.length) return saveEvents(); @@ -88,7 +91,7 @@ exports.putter = function(db){ db.createCollection(type + "_events", event_options, function(error, events){ handle(error); events.ensureIndex({"t": 1}, handle); - + // Create a collection for metrics. Three indexes are required: one // for finding metrics, one (_id) for updating, and one for invalidation. db.createCollection(type + "_metrics", metric_options, function(error, metrics) { @@ -159,7 +162,6 @@ exports.putter = function(db){ "_id.t": {$in: timesToInvalidateByTier[tier]} }, invalidate, multi); } - flushed = true; } timesToInvalidateByTierByType = {}; // copy-on-write }, invalidateInterval)); @@ -199,7 +201,7 @@ exports.getter = function(db) { try { expression = parser.parse(request.expression); } catch (error) { - var resp = { is: "invalid expression", expression: request.expression, error: error } + var resp = { is: "invalid expression", expression: request.expression, error: error }; metalog.info('cube_getter', resp); return callback(resp), -1; } @@ -305,6 +307,7 @@ exports.getter = function(db) { }; function handle(error) { + metalog.info('cube_request', {is: 'event error', error: error }); if (error) throw error; } diff --git a/lib/cube/index.js b/lib/cube/index.js index 154192ff..737b3ddc 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -1,3 +1,5 @@ +'use strict'; + exports.authentication = require("./authentication"); exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index d1bfca4f..5858dd59 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -1,3 +1,5 @@ +'use strict'; + var util = require("util"); var metalog = { @@ -39,9 +41,8 @@ metalog.minor = function(name, hsh){ // Dump the 'util.inspect' view of each argument to the console. metalog.inspectify = function(args){ - var whence = metalog.inspectify.caller.name || '(anon)'; - for (idx in arguments) { - if (whence){ util.print(whence + ' ' + idx + ": "); } + for (var idx in arguments) { + util.print(idx + ": "); util.print(util.inspect(arguments[idx])+"\n"); }; }; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index dc3aa6a5..dfcf822f 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -1,3 +1,5 @@ +'use strict'; + // TODO use expression ids or hashes for more compact storage var parser = require("./metric-expression"), @@ -114,9 +116,8 @@ exports.getter = function(db) { reduce = reduces[expression.reduce], filter = {t: {}}, fields = {t: 1}; - + if (!reduce) return callback({error: "invalid reduce operation"}), -1; - // Copy any expression filters into the query object. expression.filter(filter); @@ -175,7 +176,7 @@ exports.getter = function(db) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { - if(tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) } @@ -248,5 +249,6 @@ exports.getter = function(db) { }; function handle(error) { + metalog.info('cube_request', {is: 'event error', error: error }); if (error) throw error; } diff --git a/lib/cube/models.js b/lib/cube/models.js new file mode 100644 index 00000000..31ecda82 --- /dev/null +++ b/lib/cube/models.js @@ -0,0 +1,9 @@ + +exports.units = { + second: 1e3, + second10: 10e3, + minute: 60e3, + minute5: 300e3, + hour: 3600e3, + day: 86400e3 +}; diff --git a/lib/cube/reduces.js b/lib/cube/reduces.js index 607b8160..bc12695e 100644 --- a/lib/cube/reduces.js +++ b/lib/cube/reduces.js @@ -1,3 +1,5 @@ +'use strict'; + var reduces = module.exports = { sum: function(values) { @@ -23,7 +25,7 @@ var reduces = module.exports = { // while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; // return count; // }, - // + // // median: function(values) { // return quantile(values.sort(ascending), .5); // } diff --git a/lib/cube/server.js b/lib/cube/server.js index 5f809948..bb81a88c 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -1,3 +1,5 @@ +'use strict'; + // Server -- generic HTTP, UDP and websockets server // // Used by the collector to accept new events via HTTP or websockets @@ -17,7 +19,7 @@ var util = require("util"), dgram = require("dgram"), websocket = require("websocket"), websprocket = require("websocket-server"), - static = require("node-static"), + file_server = require("node-static"), mongodb = require("mongodb"), authentication = require("./authentication"), event = require("./event"), @@ -52,7 +54,7 @@ module.exports = function(options) { var server = {}, primary = http.createServer(), secondary = websprocket.createServer(), - file = new static.Server("static"), + file = new file_server.Server("static"), udp, endpoints = {ws: [], http: []}, server_options = options["mongo-server_options"], @@ -108,26 +110,26 @@ module.exports = function(options) { // save auth from connection requesta var authorization = request.authorized; + function connection_callback(response) { + connection.sendUTF(JSON.stringify(response)); + } + // Forward messages to the appropriate endpoint, or close the connection. for (var i = -1, n = endpoints.ws.length, e; ++i < n;) { if ((e = endpoints.ws[i]).match(request.url)) { - function callback(response) { - connection.sendUTF(JSON.stringify(response)); - } - - callback.id = ++id; + connection_callback.id = ++id; // Listen for socket disconnect. if (e.dispatch.close) connection.socket.on("end", function() { - e.dispatch.close(callback); + e.dispatch.close(connection_callback); }); connection.on("message", function(message) { // staple the authorization back on var payload = JSON.parse(message.utf8Data || message); payload.authorized = authorization; - e.dispatch(payload, callback); + e.dispatch(payload, connection_callback); }); metalog.event('cube_request', { is: 'ws', method: "WebSocket", ip: connection.remoteAddress, path: request.url}, 'minor'); @@ -141,20 +143,19 @@ module.exports = function(options) { primary.on("request", function(request, response) { var u = url.parse(request.url); + function auth_ok(perms) { + metalog.event('cube_request', { is: 'auth_ok', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: true, user: perms }); + e.dispatch(request, response); + } + function auth_no(reason) { + metalog.event('cube_request', { is: 'auth_no', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: false }); + response.writeHead(403, {"Content-Type": "text/plain"}); + response.end("403 Forbidden"); + } + // Forward messages to the appropriate endpoint, or 404. for (var i = -1, n = endpoints.http.length, e; ++i < n;) { if ((e = endpoints.http[i]).match(u.pathname, request.method)) { - - function auth_ok(perms) { - metalog.event('cube_request', { is: 'auth_ok', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: true, user: perms }); - e.dispatch(request, response); - } - function auth_no(reason) { - metalog.event('cube_request', { is: 'auth_no', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: false }); - response.writeHead(403, {"Content-Type": "text/plain"}); - response.end("403 Forbidden"); - } - return authenticator.check(request, auth_ok, auth_no); } } @@ -178,7 +179,7 @@ module.exports = function(options) { server.start = function(server_start_cb) { // Connect to mongodb. - mongo_password = options["mongo-password"]; delete options["mongo-password"]; + var mongo_password = options["mongo-password"]; delete options["mongo-password"]; metalog.info('cube_life', {is: 'mongo_connect', options: options }); db.open(function(error) { if (error) throw error; diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index 79312ec3..00dcabf9 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -1,3 +1,5 @@ +'use strict'; + var tiers = module.exports = {}; var second = 1000, diff --git a/lib/cube/types.js b/lib/cube/types.js index 02f90fda..4ad8b87d 100644 --- a/lib/cube/types.js +++ b/lib/cube/types.js @@ -1,3 +1,5 @@ +'use strict'; + // Much like db.collection, but caches the result for both events and metrics. // Also, this is synchronous, since we are opening a collection unsafely. var types = module.exports = function(db) { diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js index b5727cb3..0c523f57 100644 --- a/lib/cube/visualizer.js +++ b/lib/cube/visualizer.js @@ -1,3 +1,5 @@ +'use strict'; + var url = require("url"), path = require("path"), endpoint = require("./endpoint"), diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index 3dbae5a4..1ed85741 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -1,3 +1,5 @@ +'use strict'; + var cluster = require('cluster'), mongodb = require('mongodb'), metric = require('./metric'), @@ -6,15 +8,15 @@ var cluster = require('cluster'), module.exports = function(options){ var db, mongo, calculate_metric, boards, tier; - + function fetch_metrics(callback){ var expressions = []; - + if(!boards){ db.collection("boards", function(error, collection) { boards = collection; fetch_metrics(callback); }); return; } - + boards.find({}, {pieces: 1}, function(error, cursor) { if (error) throw error; cursor.each(function(error, row) { @@ -30,20 +32,20 @@ module.exports = function(options){ }); }); } - + function process_metrics(expressions){ expressions.forEach(function(expression){ var stop = new Date(), start = tier.step(tier.floor(new Date(stop - options.horizons.calculation))); - + metalog.info('cube_warm', {is: 'warm_metric', metric: {query: expressions}, start: start, stop: stop }); - + // fake metrics request calculate_metric({ step: tier.key, expression: expression, start: start, stop: stop }, function(){}); }); setTimeout(function(){ fetch_metrics(process_metrics); }, options['warmer-interval']); } - + return { start: function(){ mongo = new mongodb.Server(options['mongo-host'], options['mongo-port']); @@ -54,11 +56,11 @@ module.exports = function(options){ metalog.event("cube_life", { is: 'start_warmer', options: options }); - db.open(function(error) { + db.open(function(error) { if (error) throw error; calculate_metric = metric.getter(db); fetch_metrics(process_metrics); }); } }; -} \ No newline at end of file +} diff --git a/test/authentication-test.js b/test/authentication-test.js index 73afa4c8..2b494e16 100644 --- a/test/authentication-test.js +++ b/test/authentication-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), test_helper = require('./test_helper'), diff --git a/test/collector-test.js b/test/collector-test.js index e49cf01e..3846c617 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), diff --git a/test/evaluator-test.js b/test/evaluator-test.js index 65e22885..62cb0b8e 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), @@ -13,7 +15,7 @@ function frontend_components() { // suite.addBatch( // test_helper.with_server(server_options, frontend_components, { -// +// // "POST /event/put with invalid JSON": { // topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), // "responds with status 400": function(response) { diff --git a/test/event-expression-test.js b/test/event-expression-test.js index d01acbcb..92c6f38c 100644 --- a/test/event-expression-test.js +++ b/test/event-expression-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), parser = require("../lib/cube/event-expression"); diff --git a/test/metalog-test.js b/test/metalog-test.js index 7fb9c1fa..dc3ffaaa 100644 --- a/test/metalog-test.js +++ b/test/metalog-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), test_helper = require('./test_helper'); @@ -56,7 +58,7 @@ suite.with_log({ assert.equal(this.logged.infoed.pop(), 'reactor_level\t{"criticality":9,"hemiconducers":"relucting"}'); }, 'writes an event to cube itself': function(metalog){ - event = this.logged.putted.pop(); + var event = this.logged.putted.pop(); event.time = 'whatever'; assert.deepEqual(event, { data: { hemiconducers: 'relucting', criticality: 9 }, diff --git a/test/metric-expression-test.js b/test/metric-expression-test.js index 9da33d89..54b3c45d 100644 --- a/test/metric-expression-test.js +++ b/test/metric-expression-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), parser = require("../lib/cube/metric-expression"); diff --git a/test/metric-test.js b/test/metric-test.js index 6920a1cb..1f832eb0 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -1,6 +1,9 @@ +'use strict' + var vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), + models = require("../lib/cube/models"), units = models.units, event = require("../lib/cube/event"), metric = require("../lib/cube/metric"); @@ -11,25 +14,26 @@ var step_testing_delay = 250, var suite = vows.describe("metric"); +var nowish = Date.now(), nowish10 = (10e3 * Math.floor(nowish/10e3)); var steps = { - 1e4: function(date, n) { return new Date((Math.floor(date / 1e4) + n) * 1e4); }, - 6e4: function(date, n) { return new Date((Math.floor(date / 6e4) + n) * 6e4); }, - 3e5: function(date, n) { return new Date((Math.floor(date / 3e5) + n) * 3e5); }, - 36e5: function(date, n) { return new Date((Math.floor(date / 36e5) + n) * 36e5); }, - 864e5: function(date, n) { return new Date((Math.floor(date / 864e5) + n) * 864e5); } + 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, + 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, + 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, + 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, + 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } }; -steps[1e4].description = "10-second"; -steps[6e4].description = "1-minute"; -steps[3e5].description = "5-minute"; -steps[36e5].description = "1-hour"; -steps[864e5].description = "1-day"; +steps[units.second10].description = "10-second"; +steps[units.minute ].description = "1-minute"; +steps[units.minute5 ].description = "5-minute"; +steps[units.hour ].description = "1-hour"; +steps[units.day ].description = "1-day"; suite.addBatch(test_helper.batch({ topic: function(test_db) { var putter = event.putter(test_db.db), - getter = metric.getter(test_db.db), - callback = this.callback; + getter = metric.getter(test_db.db), + callback = this.callback; // Seed the events table with a simple event: a value going from 0 to 2499 for (var i = 0; i < 2500; i++) { @@ -45,85 +49,77 @@ suite.addBatch(test_helper.batch({ }, "unary expression": metricTest({ - expression: "sum(test)", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", - }, { - 6e4: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - 3e5: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], - 36e5: [82, 2418], - 864e5: [82, 2418] - } - ), + expression: "sum(test)", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z", + }, { + 60e3: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], + 3600e3: [82, 2418], + 86400e3: [82, 2418] + }), "unary expression with data accessor": metricTest({ - expression: "sum(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 3e5: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], - 36e5: [3321, 3120429], - 864e5: [3321, 3120429] - } - ), + expression: "sum(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], + 3600e3: [3321, 3120429], + 86400e3: [3321, 3120429] + }), "unary expression with compound data accessor": metricTest({ - expression: "sum(test(i / 100))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 3e5: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], - 36e5: [33.21, 31204.29], - 864e5: [33.21, 31204.29] - } - ), + expression: "sum(test(i / 100))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], + 3600e3: [33.21, 31204.29], + 86400e3: [33.21, 31204.29] + }), "compound expression (sometimes fails due to race condition?)": metricTest({ - expression: "max(test(i)) - min(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", - }, { - 3e5: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], - 36e5: [81, 2417], - 864e5: [81, 2417] - } - ), + expression: "max(test(i)) - min(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z", + }, { + 300e3: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], + 3600e3: [81, 2417], + 86400e3: [81, 2417] + }), "non-pyramidal expression": metricTest({ - expression: "distinct(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", - }, { - 3e5: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], - 36e5: [82, 2418], - 864e5: [82, 2418] - } - ), + expression: "distinct(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z", + }, { + 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], + 3600e3: [82, 2418], + 86400e3: [82, 2418] + }), "compound pyramidal and non-pyramidal expression": metricTest({ - expression: "sum(test(i)) - median(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", - }, { - 3e5: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], - 36e5: [3280.5, 3119138.5], - 864e5: [3280.5, 3119138.5] - } - ), + expression: "sum(test(i)) - median(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z", + }, { + 300e3: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], + 3600e3: [3280.5, 3119138.5], + 86400e3: [3280.5, 3119138.5] + }), "compound with constant expression": metricTest({ - expression: "-1 + sum(test)", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", - }, { - 3e5: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], - 36e5: [81, 2417], - 864e5: [81, 2417] - } - ) -})); + expression: "-1 + sum(test)", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z", + }, { + 300e3: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], + 3600e3: [81, 2417], + 86400e3: [81, 2417] + }) -suite.export(module); +})); // metricTest -- generates test tree for metrics. // @@ -151,7 +147,7 @@ suite.export(module); function metricTest(request, expected) { // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } var tree = {}, k; - for (step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); + for (var step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); return tree; // @@ -174,7 +170,7 @@ function metricTest(request, expected) { // function testStep(step, expected) { var start = new Date(request.start), - stop = new Date(request.stop); + stop = new Date(request.stop); var subtree = { topic: get_metrics_with_delay(0), @@ -187,11 +183,11 @@ function metricTest(request, expected) { function get_metrics_with_delay(depth){ return function(){ var actual = [], - timeout = setTimeout(function() { cb("Time's up!"); }, 10000), - cb = this.callback, - req = Object.create(request), - getter = arguments[depth]; - req.step = step; + timeout = setTimeout(function() { cb("Time's up!"); }, 10000), + cb = this.callback, + req = Object.create(request), + getter = arguments[depth]; + req.step = step; // Wait long enough for the events to have settled in the db. The // non-cached (depth=0) round can all start in parallel, making this an // effective `nextTick`. On the secon @@ -211,55 +207,57 @@ function metricTest(request, expected) { function metrics_assertions(){ return { 'rounds down the start time (inclusive)': function(actual) { - var floor = steps[step](start, 0); - assert.deepEqual(actual[0].time, floor); + var floor = steps[step](start, 0); + assert.deepEqual(actual[0].time, floor); }, 'rounds up the stop time (exclusive)': function(actual){ - var ceil = steps[step](stop, 0); - if (!(ceil - stop)) ceil = steps[step](stop, -1); - assert.deepEqual(actual[actual.length - 1].time, ceil); + var ceil = steps[step](stop, 0); + if (!(ceil - stop)) ceil = steps[step](stop, -1); + assert.deepEqual(actual[actual.length - 1].time, ceil); }, 'formats UTC time in ISO 8601': function(actual){ - actual.forEach(function(d) { - assert.instanceOf(d.time, Date); - assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); - }); + actual.forEach(function(d) { + assert.instanceOf(d.time, Date); + assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); + }); }, 'returns exactly one value per time': function(actual){ - var i = 0, n = actual.length, t = actual[0].time; - while (++i < n) assert.isTrue(t < (t = actual[i].time)); + var i = 0, n = actual.length, t = actual[0].time; + while (++i < n) assert.isTrue(t < (t = actual[i].time)); }, 'each metric defines only time and value properties': function(actual){ - actual.forEach(function(d) { - assert.deepEqual(Object.keys(d), ["time", "value"]); - }); + actual.forEach(function(d) { + assert.deepEqual(Object.keys(d), ["time", "value"]); + }); }, 'returns the expected times': function(actual){ - var floor = steps[step], - time = floor(start, 0), - times = []; - while (time < stop) { - times.push(time); - time = floor(time, 1); - } - assert.deepEqual(actual.map(function(d) { return d.time; }), times); + var floor = steps[step], + time = floor(start, 0), + times = []; + while (time < stop) { + times.push(time); + time = floor(time, 1); + } + assert.deepEqual(actual.map(function(d) { return d.time; }), times); }, 'returns the expected values': function(actual){ - var actualValues = actual.map(function(d) { return d.value; }); - assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); - expected.forEach(function(value, i) { - if (Math.abs(actual[i].value - value) > 1e-6) { - assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); - } - }); + var actualValues = actual.map(function(d) { return d.value; }); + assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); + expected.forEach(function(value, i) { + if (Math.abs(actual[i].value - value) > 1e-6) { + assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); + } + }); } }}; // metric assertions } // subtree } // tree + +suite.export(module); diff --git a/test/reduces-test.js b/test/reduces-test.js index 2d62585d..2447f9b9 100644 --- a/test/reduces-test.js +++ b/test/reduces-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), reduces = require("../lib/cube/reduces"); diff --git a/test/server-test.js b/test/server-test.js index 5f3a2c23..643c47bc 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), diff --git a/test/test_helper.js b/test/test_helper.js index 47198808..8809497f 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require("assert"), http = require("http"), dgram = require('dgram'), @@ -108,7 +110,7 @@ test_helper.delaying_topic = delaying_topic; function delayed_callback(context){ return function(){ var callback_delay = 100; - args = arguments; + var args = arguments; setTimeout(function(){ context.callback.apply(context, args) }, callback_delay) }; } diff --git a/test/tiers-test.js b/test/tiers-test.js index e2a652da..24f6ff75 100644 --- a/test/tiers-test.js +++ b/test/tiers-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), tiers = require("../lib/cube/tiers"); @@ -31,51 +33,51 @@ suite.addBatch({ "floor": { "rounds down to 10-seconds": function(tier) { - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 20)), utc(2011, 08, 02, 12, 00, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 21)), utc(2011, 08, 02, 12, 00, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 23)), utc(2011, 08, 02, 12, 00, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 39)), utc(2011, 08, 02, 12, 00, 30)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 40)), utc(2011, 08, 02, 12, 00, 40)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 20)), utc(2011, 8, 2, 12, 0, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 21)), utc(2011, 8, 2, 12, 0, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 23)), utc(2011, 8, 2, 12, 0, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 39)), utc(2011, 8, 2, 12, 0, 30)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 40)), utc(2011, 8, 2, 12, 0, 40)); }, "does not modify the passed-in date": function(tier) { - var date = utc(2011, 08, 02, 12, 00, 21); - assert.deepEqual(tier.floor(date), utc(2011, 08, 02, 12, 00, 20)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 00, 21)); + var date = utc(2011, 8, 2, 12, 0, 21); + assert.deepEqual(tier.floor(date), utc(2011, 8, 2, 12, 0, 20)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 0, 21)); } }, "ceil": { "rounds up to 10-seconds": function(tier) { - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 20)), utc(2011, 08, 02, 12, 00, 20)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 21)), utc(2011, 08, 02, 12, 00, 30)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 23)), utc(2011, 08, 02, 12, 00, 30)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 39)), utc(2011, 08, 02, 12, 00, 40)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 40)), utc(2011, 08, 02, 12, 00, 40)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 20)), utc(2011, 8, 2, 12, 0, 20)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 21)), utc(2011, 8, 2, 12, 0, 30)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 23)), utc(2011, 8, 2, 12, 0, 30)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 39)), utc(2011, 8, 2, 12, 0, 40)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 40)), utc(2011, 8, 2, 12, 0, 40)); }, "does not modified the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 00, 21); - assert.deepEqual(tier.ceil(date), utc(2011, 08, 02, 12, 00, 30)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 00, 21)); + var date = utc(2011, 8, 2, 12, 0, 21); + assert.deepEqual(tier.ceil(date), utc(2011, 8, 2, 12, 0, 30)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 0, 21)); } }, "step": { "increments time by ten seconds": function(tier) { - var date = utc(2011, 08, 02, 23, 59, 20); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 59, 30)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 59, 40)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 59, 50)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 00, 10)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 00, 20)); + var date = utc(2011, 8, 2, 23, 59, 20); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 59, 30)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 59, 40)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 59, 50)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 0, 10)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 0, 20)); }, "does not round the specified date": function(tier) { - assert.deepEqual(tier.step(utc(2011, 08, 02, 12, 21, 23)), utc(2011, 08, 02, 12, 21, 33)); + assert.deepEqual(tier.step(utc(2011, 8, 2, 12, 21, 23)), utc(2011, 8, 2, 12, 21, 33)); }, "does not modify the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 20, 00); - assert.deepEqual(tier.step(date), utc(2011, 08, 02, 12, 20, 10)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 20, 00)); + var date = utc(2011, 8, 2, 12, 20, 0); + assert.deepEqual(tier.step(date), utc(2011, 8, 2, 12, 20, 10)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 20, 0)); } } }, @@ -94,52 +96,52 @@ suite.addBatch({ "floor": { "rounds down to minutes": function(tier) { - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 20, 00)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 20, 01)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 12, 21)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 23, 00)), utc(2011, 08, 02, 12, 23)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 24, 59)), utc(2011, 08, 02, 12, 24)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 25, 00)), utc(2011, 08, 02, 12, 25)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 20, 0)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 20, 1)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 12, 21)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 23, 0)), utc(2011, 8, 2, 12, 23)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 24, 59)), utc(2011, 8, 2, 12, 24)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 25, 0)), utc(2011, 8, 2, 12, 25)); }, "does not modify the passed-in date": function(tier) { - var date = utc(2011, 08, 02, 12, 21, 20); - assert.deepEqual(tier.floor(date), utc(2011, 08, 02, 12, 21)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21, 20)); + var date = utc(2011, 8, 2, 12, 21, 20); + assert.deepEqual(tier.floor(date), utc(2011, 8, 2, 12, 21)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21, 20)); } }, "ceil": { "rounds up to minutes": function(tier) { - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 20, 00)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 20, 01)), utc(2011, 08, 02, 12, 21)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 12, 21)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 23, 00)), utc(2011, 08, 02, 12, 23)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 24, 59)), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 25, 00)), utc(2011, 08, 02, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 20, 0)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 20, 1)), utc(2011, 8, 2, 12, 21)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 12, 21)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 23, 0)), utc(2011, 8, 2, 12, 23)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 24, 59)), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 25, 0)), utc(2011, 8, 2, 12, 25)); }, "does not modified the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 21, 20); - assert.deepEqual(tier.ceil(date), utc(2011, 08, 02, 12, 22)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21, 20)); + var date = utc(2011, 8, 2, 12, 21, 20); + assert.deepEqual(tier.ceil(date), utc(2011, 8, 2, 12, 22)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21, 20)); } }, "step": { "increments time by one minute": function(tier) { - var date = utc(2011, 08, 02, 23, 45, 00); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 46)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 47)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 48)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 49)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 50)); + var date = utc(2011, 8, 2, 23, 45, 0); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 46)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 47)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 48)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 49)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 50)); }, "does not round the specified date": function(tier) { - assert.deepEqual(tier.step(utc(2011, 08, 02, 12, 21, 23)), utc(2011, 08, 02, 12, 22, 23)); + assert.deepEqual(tier.step(utc(2011, 8, 2, 12, 21, 23)), utc(2011, 8, 2, 12, 22, 23)); }, "does not modify the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 20); - assert.deepEqual(tier.step(date), utc(2011, 08, 02, 12, 21)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 20)); + var date = utc(2011, 8, 2, 12, 20); + assert.deepEqual(tier.step(date), utc(2011, 8, 2, 12, 21)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 20)); } } }, @@ -158,52 +160,52 @@ suite.addBatch({ "floor": { "rounds down to 5-minutes": function(tier) { - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 20, 00)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 20, 01)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 23, 00)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 24, 59)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 25, 00)), utc(2011, 08, 02, 12, 25)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 20, 0)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 20, 1)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 23, 0)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 24, 59)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 25, 0)), utc(2011, 8, 2, 12, 25)); }, "does not modify the passed-in date": function(tier) { - var date = utc(2011, 08, 02, 12, 21); - assert.deepEqual(tier.floor(date), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21)); + var date = utc(2011, 8, 2, 12, 21); + assert.deepEqual(tier.floor(date), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21)); } }, "ceil": { "rounds up to 5-minutes": function(tier) { - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 20, 00)), utc(2011, 08, 02, 12, 20)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 20, 01)), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 23, 00)), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 24, 59)), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 25, 00)), utc(2011, 08, 02, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 20, 0)), utc(2011, 8, 2, 12, 20)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 20, 1)), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 23, 0)), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 24, 59)), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 25, 0)), utc(2011, 8, 2, 12, 25)); }, "does not modified the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 21, 00); - assert.deepEqual(tier.ceil(date), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21)); + var date = utc(2011, 8, 2, 12, 21, 0); + assert.deepEqual(tier.ceil(date), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21)); } }, "step": { "increments time by five minutes": function(tier) { - var date = utc(2011, 08, 02, 23, 45, 00); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 50)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 55)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 05)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 10)); + var date = utc(2011, 8, 2, 23, 45, 0); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 50)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 55)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 5)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 10)); }, "does not round the specified date": function(tier) { - assert.deepEqual(tier.step(utc(2011, 08, 02, 12, 21, 23)), utc(2011, 08, 02, 12, 26, 23)); + assert.deepEqual(tier.step(utc(2011, 8, 2, 12, 21, 23)), utc(2011, 8, 2, 12, 26, 23)); }, "does not modify the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 20, 00); - assert.deepEqual(tier.step(date), utc(2011, 08, 02, 12, 25)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 20)); + var date = utc(2011, 8, 2, 12, 20, 0); + assert.deepEqual(tier.step(date), utc(2011, 8, 2, 12, 25)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 20)); } } }, @@ -222,50 +224,50 @@ suite.addBatch({ "floor": { "rounds down to hours": function(tier) { - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 00)), utc(2011, 08, 02, 12, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 00, 01)), utc(2011, 08, 02, 12, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 12, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 59, 59)), utc(2011, 08, 02, 12, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 13, 00, 00)), utc(2011, 08, 02, 13, 00)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 0)), utc(2011, 8, 2, 12, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 0, 1)), utc(2011, 8, 2, 12, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 12, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 59, 59)), utc(2011, 8, 2, 12, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 13, 0, 0)), utc(2011, 8, 2, 13, 0)); }, "does not modify the passed-in date": function(tier) { - var date = utc(2011, 08, 02, 12, 21); - assert.deepEqual(tier.floor(date), utc(2011, 08, 02, 12, 00)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21)); + var date = utc(2011, 8, 2, 12, 21); + assert.deepEqual(tier.floor(date), utc(2011, 8, 2, 12, 0)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21)); } }, "ceil": { "rounds up to hours": function(tier) { - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 00)), utc(2011, 08, 02, 12, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 00, 01)), utc(2011, 08, 02, 13, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 13, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 59, 59)), utc(2011, 08, 02, 13, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 13, 00, 00)), utc(2011, 08, 02, 13, 00)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 0)), utc(2011, 8, 2, 12, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 0, 1)), utc(2011, 8, 2, 13, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 13, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 59, 59)), utc(2011, 8, 2, 13, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 13, 0, 0)), utc(2011, 8, 2, 13, 0)); }, "does not modified the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 21, 00); - assert.deepEqual(tier.ceil(date), utc(2011, 08, 02, 13, 00)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21)); + var date = utc(2011, 8, 2, 12, 21, 0); + assert.deepEqual(tier.ceil(date), utc(2011, 8, 2, 13, 0)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21)); } }, "step": { "increments time by one hour": function(tier) { - var date = utc(2011, 08, 02, 22, 00, 00); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 02, 23, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 01, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 02, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 03, 00)); + var date = utc(2011, 8, 2, 22, 0, 0); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 2, 23, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 1, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 2, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 3, 0)); }, "does not round the specified date": function(tier) { - assert.deepEqual(tier.step(utc(2011, 08, 02, 12, 21, 23)), utc(2011, 08, 02, 13, 21, 23)); + assert.deepEqual(tier.step(utc(2011, 8, 2, 12, 21, 23)), utc(2011, 8, 2, 13, 21, 23)); }, "does not modify the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 00, 00); - assert.deepEqual(tier.step(date), utc(2011, 08, 02, 13, 00)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 00)); + var date = utc(2011, 8, 2, 12, 0, 0); + assert.deepEqual(tier.step(date), utc(2011, 8, 2, 13, 0)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 0)); } } }, @@ -284,50 +286,50 @@ suite.addBatch({ "floor": { "rounds down to days": function(tier) { - assert.deepEqual(tier.floor(utc(2011, 08, 02, 00, 00, 00)), utc(2011, 08, 02, 00, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 00, 00, 01)), utc(2011, 08, 02, 00, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 02, 00, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 02, 23, 59, 59)), utc(2011, 08, 02, 00, 00)); - assert.deepEqual(tier.floor(utc(2011, 08, 03, 00, 00, 00)), utc(2011, 08, 03, 00, 00)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 0, 0, 0)), utc(2011, 8, 2, 0, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 0, 0, 1)), utc(2011, 8, 2, 0, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 2, 0, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 2, 23, 59, 59)), utc(2011, 8, 2, 0, 0)); + assert.deepEqual(tier.floor(utc(2011, 8, 3, 0, 0, 0)), utc(2011, 8, 3, 0, 0)); }, "does not modify the passed-in date": function(tier) { - var date = utc(2011, 08, 02, 12, 21); - assert.deepEqual(tier.floor(date), utc(2011, 08, 02, 00, 00)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21)); + var date = utc(2011, 8, 2, 12, 21); + assert.deepEqual(tier.floor(date), utc(2011, 8, 2, 0, 0)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21)); } }, "ceil": { "rounds up to days": function(tier) { - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 00, 00, 00)), utc(2011, 08, 02, 00, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 00, 00, 01)), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 12, 21, 00)), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 02, 23, 59, 59)), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(tier.ceil(utc(2011, 08, 03, 00, 00, 00)), utc(2011, 08, 03, 00, 00)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 0, 0, 0)), utc(2011, 8, 2, 0, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 0, 0, 1)), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 12, 21, 0)), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 2, 23, 59, 59)), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(tier.ceil(utc(2011, 8, 3, 0, 0, 0)), utc(2011, 8, 3, 0, 0)); }, "does not modified the specified date": function(tier) { - var date = utc(2011, 08, 02, 12, 21, 00); - assert.deepEqual(tier.ceil(date), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(date, utc(2011, 08, 02, 12, 21)); + var date = utc(2011, 8, 2, 12, 21, 0); + assert.deepEqual(tier.ceil(date), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(date, utc(2011, 8, 2, 12, 21)); } }, "step": { "increments time by one day": function(tier) { - var date = utc(2011, 08, 02, 00, 00, 00); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 04, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 05, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 06, 00, 00)); - assert.deepEqual(date = tier.step(date), utc(2011, 08, 07, 00, 00)); + var date = utc(2011, 8, 2, 0, 0, 0); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 4, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 5, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 6, 0, 0)); + assert.deepEqual(date = tier.step(date), utc(2011, 8, 7, 0, 0)); }, "does not round the specified date": function(tier) { - assert.deepEqual(tier.step(utc(2011, 08, 02, 12, 21, 23)), utc(2011, 08, 03, 12, 21, 23)); + assert.deepEqual(tier.step(utc(2011, 8, 2, 12, 21, 23)), utc(2011, 8, 3, 12, 21, 23)); }, "does not modify the specified date": function(tier) { - var date = utc(2011, 08, 02, 00, 00, 00); - assert.deepEqual(tier.step(date), utc(2011, 08, 03, 00, 00)); - assert.deepEqual(date, utc(2011, 08, 02, 00, 00)); + var date = utc(2011, 8, 2, 0, 0, 0); + assert.deepEqual(tier.step(date), utc(2011, 8, 3, 0, 0)); + assert.deepEqual(date, utc(2011, 8, 2, 0, 0)); } } } diff --git a/test/types-test.js b/test/types-test.js index ad33d599..c54611f4 100644 --- a/test/types-test.js +++ b/test/types-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), mongodb = require("mongodb"), diff --git a/test/visualizer-test.js b/test/visualizer-test.js index 74cd8b30..52a78def 100644 --- a/test/visualizer-test.js +++ b/test/visualizer-test.js @@ -1,3 +1,5 @@ +'use strict'; + var vows = require("vows"), assert = require("assert"), cube = require("../"), From 740d26c9d038d1c060afbb9cfb59c886a9da48df Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Sun, 2 Sep 2012 20:32:39 -0500 Subject: [PATCH 26/87] doing scale experiments, so reverting the workaround-y stuff (no distinct, all tiers cascade to tensec, horizons) --- config/cube.js | 14 +++++++------- lib/cube/event.js | 7 ++++--- lib/cube/metric.js | 17 +++++++++-------- lib/cube/reduces.js | 20 ++++++++++---------- lib/cube/tiers.js | 8 ++++---- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/config/cube.js b/config/cube.js index e21b2f6b..93306b70 100644 --- a/config/cube.js +++ b/config/cube.js @@ -12,13 +12,13 @@ configs.common = { "mongo-password": null, "mongo-server_options": {auto_reconnect: true, poolSize: 8, socketOptions: { noDelay: true }}, - "mongo-metrics": {capped: true, size: 1e7, autoIndexId: true}, - "mongo-events": {capped: true, size: 1e7, autoIndexId: true}, + "mongo-metrics": {autoIndexId: true, capped: false }, + "mongo-events": {autoIndexId: true, capped: true, size: 1e9 }, - "horizons": { - "calculation": 1000 * 60 * 60 * 2, // 2 hours - "invalidation": 1000 * 60 * 60 * 1, // 1 hour - } + // "horizons": { + // "calculation": 1000 * 60 * 60 * 2, // 2 hours + // "invalidation": 1000 * 60 * 60 * 1, // 1 hour + // } }; @@ -63,4 +63,4 @@ Object.defineProperty(options, "include", { } }); -module.exports = options.include('common'); \ No newline at end of file +module.exports = options.include('common'); diff --git a/lib/cube/event.js b/lib/cube/event.js index d81b9bed..c56a3941 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -56,8 +56,8 @@ exports.putter = function(db){ if (!type_re.test(type)) return callback({error: "invalid type"}), -1; if (isNaN(time)) return callback({error: "invalid time"}), -1; - // Drop events from before invalidation horizon - if(time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; + // // Drop events from before invalidation horizon + // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; // If an id is specified, promote it to Mongo's primary key. var event = {t: time, d: request.data}; @@ -307,8 +307,9 @@ exports.getter = function(db) { }; function handle(error) { + if (!error) return; metalog.info('cube_request', {is: 'event error', error: error }); - if (error) throw error; + throw error; } function open(callback) { diff --git a/lib/cube/metric.js b/lib/cube/metric.js index dfcf822f..a2883e25 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -28,7 +28,7 @@ exports.getter = function(db) { // Validate the dates. if (isNaN(start)) return callback({error: "invalid start"}), -1; if (isNaN(stop)) return callback({error: "invalid stop"}), -1; - if (request.expression.match(/\(/mg).length > 2) return callback({error: "rejected complex expression"}), -1; + // if (request.expression.match(/\(/mg).length > 2) return callback({error: "rejected complex expression"}), -1; // Parse the expression. var expression; @@ -117,7 +117,7 @@ exports.getter = function(db) { filter = {t: {}}, fields = {t: 1}; - if (!reduce) return callback({error: "invalid reduce operation"}), -1; + // if (!reduce) return callback({error: "invalid reduce operation"}), -1; // Copy any expression filters into the query object. expression.filter(filter); @@ -176,10 +176,10 @@ exports.getter = function(db) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { - if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ - metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); - start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) - } + // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); + // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + // } filter.t.$gte = start; filter.t.$lt = stop; type.events.find(filter, fields, event_options, function(error, cursor) { @@ -249,6 +249,7 @@ exports.getter = function(db) { }; function handle(error) { - metalog.info('cube_request', {is: 'event error', error: error }); - if (error) throw error; + if (!error) return; + metalog.info('cube_request', {is: 'metric error', error: error }); + throw error; } diff --git a/lib/cube/reduces.js b/lib/cube/reduces.js index bc12695e..9ef8c055 100644 --- a/lib/cube/reduces.js +++ b/lib/cube/reduces.js @@ -20,21 +20,21 @@ var reduces = module.exports = { return max; }, - // distinct: function(values) { - // var map = {}, count = 0, i = -1, n = values.length, value; - // while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; - // return count; - // }, - // - // median: function(values) { - // return quantile(values.sort(ascending), .5); - // } + distinct: function(values) { + var map = {}, count = 0, i = -1, n = values.length, value; + while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; + return count; + }, + + median: function(values) { + return quantile(values.sort(ascending), .5); + } }; // These metrics have well-defined values for the empty set. reduces.sum.empty = 0; -//reduces.distinct.empty = 0; +reduces.distinct.empty = 0; // These metrics can be computed using pyramidal aggregation. reduces.sum.pyramidal = true; diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index 00dcabf9..322e43bb 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -21,8 +21,8 @@ tiers[minute] = { floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, ceil: tier_ceil, step: function(d) { return new Date(+d + minute); }, - next: tiers[second10], - size: function() { return 6; } + // next: tiers[second10], + // size: function() { return 6; } }; tiers[minute5] = { @@ -30,8 +30,8 @@ tiers[minute5] = { floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, ceil: tier_ceil, step: function(d) { return new Date(+d + minute5); }, - next: tiers[minute], - size: function() { return 5; } + // next: tiers[minute], + // size: function() { return 5; } }; tiers[hour] = { From 458184110764515e57c5227f73e0adeaaf769176 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Sun, 2 Sep 2012 20:34:17 -0500 Subject: [PATCH 27/87] Extracting metric and event code into objects (WIP) --- TODO.md | 270 ++++++++++++++++++++++++++++++++++++++++ config/cube.js | 3 +- lib/cube/evaluator.js | 37 +++++- lib/cube/event.js | 33 +++-- lib/cube/index.js | 4 + lib/cube/metalog.js | 3 +- lib/cube/models.js | 32 +++++ test/metric-test.js | 56 ++++++++- test/visualizer-test.js | 27 ++-- 9 files changed, 433 insertions(+), 32 deletions(-) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..f35974b6 --- /dev/null +++ b/TODO.md @@ -0,0 +1,270 @@ + + +## Scaling Cube and Mongo + + +With some modifications, you can make cube scale to thousands of inserts per second. + +Cube saves two kinds of records: +* Raw individual events. An event stream of 2400 records/s at 1kB each is 200 GB/day, 1.4 TB/week. +* Metrics aggregated over those events. For an event stream of any size, storing a year's worth of aggregates for 100 different metrics is 25 GB. (That's 10-second, 1-minute, 5-minute and hour aggregates, 68 bytes each) + +For an event stream of 2400 records/s @ 1kB/record, + +* network throughput =~ 2.5 MB/s +* disk write =~ 2.5 MB/s +* mongo inserts =~ 2400 insert/s +* queries =~ (100 metrics, re-calculated 5x each) + +* mongo network in =~ 2.5 MB/s +* mongo network out =~ k + +* mongo vsize +* faults +* mongo db locked +* connections + + insert query update delete getmore command flushes mapped vsize res faults locked db idx miss % qr|qw ar|aw netIn netOut conn time + 0 0 0 0 0 1 0 464m 3.33g 45m 0 local:0.0% 0 0|0 0|0 62b 1k 1 17:58:22 + 0 0 0 0 0 1 0 464m 3.33g 45m 0 local:0.0% 0 0|0 0|0 62b 1k 1 17:58:23 + + inserts - # of inserts per second (* means replicated op) + query - # of queries per second + update - # of updates per second + delete - # of deletes per second + getmore - # of get mores (cursor batch) per second + command - # of commands per second, on a slave its local|replicated + flushes - # of fsync flushes per second + mapped - amount of data mmaped (total data size) megabytes + vsize - virtual size of process in megabytes + res - resident size of process in megabytes + faults - # of pages faults per sec + locked - name of and percent time for most locked database + idx miss - percent of btree page misses (sampled) + qr|qw - queue lengths for clients waiting (read|write) + ar|aw - active clients (read|write) + netIn - network traffic in - bits + netOut - network traffic out - bits + conn - number of open connections + + + +### Database patterns of access + +* Collector: insert on events from receiving event + +* Collector: update on metrics for invalidation (flush) + +* Evaluator: find on metrics for metric calculation + +* Evaluator: find on metrics for pyramidal calculation + +* Evaluator: cursored find on events for calculateFlat + +* Evaluator: save (upsert) on metrics from calculateFlat + + +## Cube do's and don'ts + +* ensure time-based / ascending IDs +* Don't send old metrics +* Turn dates into mongo dates, not text strings +* Short field names + +## Metrics persist, Events expire + +* **Capped events, uncapped metrics** + - modify cube to construct uncapped collections for metrics, and capped collections for events + - change invalidation flusher to delete metrics, not update-with-invalid=true. + +* **Calculation horizon** -- Metrics older than the calculation horizon will not be recalculated from events. + +* **Drop stale events on the floor** + - drop events on the floor if they are older than the 'invalidation' horizon (which is shorter than the calculation horizon) + - Warn any time an invalidation extends more than 5 minutes into the past + +* **Save metrics even if their value is zero** + +### Use connection pool + +### Use Mongo 2.2+, and use separate databases for events and metrics + +### Calculation queue + +Currently, the event flush-er does a range-query update setting "i=false"; under heavy load, this is surprisingly hard on the database. We've observed multiple-second pauses for invalidation; I believe mongo has to hold a write lock open while it does a range query for the metrics to update. Metric updates, by contrast, are a reasonably cheap single-row `save()`. (Side note: we tried uncapping the metrics collection and doing a `remove()`; removes also lock the database, and it was no more efficient.) + +Instead, the collector's flush operation should simply trigger a direct recalculation of the metric. This would halve the number of updates and significantly reduce thevariance of load. + +The challenge is to a) not have everybody in sight run off recalculating metrics; b) not complicate the metric calculation code; c) ensure correctness; d) not introduce coupling. + +proposal a: evaluator contains the queue + +* add an api endpoint to the evaluator: + - endpoints: (method `DELETE`, path `1.0/metric`), and method `GET`, path `1.0/metric/refresh`. (I think this is the best match to HTTP semantics, but if anyone has strong opinions please advise.) + - accepts start and stop; this calls deferred method on `metric.getter` and immediately returns `202 (Accepted)`. +* add code to `metric` to handle request +* make the flusher call the evaluator rather than issue an update. + +Good: simple; bad: requires the evaluator be running to handle invalidations. + +proposal b: database contains the queue + +* add a collection to track calculation requests. rather than updating a range of metrics, flusher inserts a single entry specifying a type + time range. +* add code to coalesce all pending calculation requests for a given type and update the metrics appropriately. + + + +## Scaling Mongo + + + +* More ram helps + +* enable `--directoryperdb` -- with the separate +* (??hypothetical enable `noprealloc=true`. Collections that have heavy write loads will fill up very quickly, and cap from there.) +* enable `cpu=true` +* enable `logappend=true` + +* do not enable `smallfiles`. + +logpath=/ephemeral/log/mongod.log +dbpath=/data + +--slowms Specifies the threshold (in milliseconds) above which long-running queries will appear in the log, and in the system.profiler collection if profiling is enabled. + +* `--journalCommitInterval` -- How often to group/batch commit (ms). Default 100ms +* `--syncdelay arg` -- The number of seconds between data file flushes/syncs. Default 60 (seconds). + - The MongoDB journal file flushes almost immediately, but data files are flushed lazily to optimize performance. The default is 60 seconds. A 0 setting means never, but is not recommended and should never be used with journaling enabled. On Linux, longer settings will likely be ineffective unless /proc/sys/vm/dirty_expire_centisecs is adjusted also. Generally, set syncdelay to 4x the desired dirty_expire_centiseconds value. If backgroundFlushing.average_ms is relatively large (>= 10,000 milliseconds), consider increasing this setting as well as dirty_expire_centiseconds. Under extremely high write volume situations, a higher syncdelay value may result in more journal files as they cannot be rotated (deleted) as quickly. + + +* *do* use a journal + +* `--rest` -- while in development, enables the REST interface. + +To consider -- + + +* connection should be `{safe: false, connectTimeoutMS=XX}` + +* mongo uses 1 thread per CPU connection. + +### System Tuning + +Consider *enabling swap*. EC2 + +* ulimit -- Set file descriptor limit (`-n`) and user process limit (`-u`) to 4k+ (see etc/limits and ulimit) +* sysctl + - overcommit + - swappiness +* Do not use large/huge virtual memory pages (the default is small pages which is what you want) +* Use dmesg to see if box is behaving strangely +* Ensure that readahead settings for the block devices that store the dbpath are acceptable Readahead +* `/proc/sys/vm/dirty_expire_centisecs` + +* cube processes + - run with max memory size ulimit (`-m`) set to a finite value + - (??hypothetical -- run other processes with NICE set so that mongo doesn't become target of OOM killer) + + +## Diagnosis + + +* `backgroundFlushing.average_ms` -- if larger than 10_000 ms adjust mongo's `syncdelay` and the `dirty_expire_centiseconds` sysctl. +* virtualbytes - mappedbytes should be small +* (??hypothetical run `db.runCommand({compact:'collectionname'})` to compact?) +* `mongod --repair` -- will put a heavy load on your DB + - you may see some 'yield' warnings when doing a repair; they're harmless + +* Dump stats on all databases and collections: + + db.foo.stats() + var mongo_stats = {} ; db._adminCommand("listDatabases").databases.forEach(function (d) {mdb = db.getSiblingDB(d.name); mongo_stats[d.name] = {} ; mongo_stats[d.name]["_stats"] = mdb.stats(); mdb.getCollectionNames().forEach(function(c) {mongo_stats[d.name][mdb[c].name] = mdb[c].stats(); })}); printjson(mongo_stats); + +* [Database profiling](http://www.mongodb.org/display/DOCS/Database+Profiler) + + db.getProfilingLevel() + db.setProfilingLevel(level) 0=off 1=record slow queries 2= record all queries + mongod --profile=1 --slowms=15 + + # view queries against collection test.foo: + db.system.profile.find( { info: /test.foo/ } ) + # newest info first: + db.system.profile.find().sort({$natural:-1}) + + - `nscanned` is much higher than nreturned, the database is scanning many objects to find the target objects. Consider creating an index to improve this. + - `reslen` A large number of bytes returned (hundreds of kilobytes or more) causes slow performance. Consider passing `find()` a second parameter of the member names you require. + - `fastmodinsert` -- better than `upsert` + - `moved` -- Indicates the update moved the object on disk. bad. + - `key updates` -- had to reindex. bad. + +* [diagnostic console](http://localhost:28017/) + - start with `--rest` -- while in development, enables the REST interface. + + + + +## Storage + + +### Replica Sets + +* Ensure you are using NTP to minimize clock skew. + +#### Sharding + +Note that you **cannot shard** with cube: capped collections do not work in a sharded database. Replica sets are fine; see above. + +## Indexes + +* "Exact values first. Sorted fields next. Ranged fields after that." See [dex](http://blog.mongolab.com/2012/06/introducing-dex-the-index-bot/). +* use hints to force indexes + +For Capped Collections, natural order is guaranteed to be the insertion order, making this a very efficient way to store and retrieve data in insertion order (much faster than say, indexing on a timestamp field). In addition to forward natural order, items may be retrieved in reverse natural order. For example, to return the 50 most recently inserted items (ordered most recent to less recent) from a capped collection, you would invoke: `c=db.cappedCollection.find().sort({$natural:-1}).limit(50)` + +### Storage + +* Use xfs, mounted with `"noatime"`, or ext4 file system, mounted with "noatime,data=writeback,nobarrier" + +* use separate disks for journals, events and metrics +* use the `--directoryperdb` +* use an SSD for metrics. + - don't need to use an SSD for events. +* commodity drives are OK. + +db.runCommand("journalLatencyTest") +You can run this command on an idle system to get a baseline sync time for journaling. In addition, it is safe to run this command on a busy system to see the sync time on a busy system (which may be higher if the journal directory is on the same volume as the data files). + +* see [Baking up MongoDB](http://www.mongodb.org/display/DOCS/Backups) + +### Amazon EC2 scaling + + m1.large 232 7.68 .32 13 23 7.5 4 2 2 850 2 64 500 High + m1.xlarge 465 15.36 .64 13 23 15 8 4 2 1690 4 64 1000 High + m2.xlarge 327 10.80 .45 14 38 17.1 6.5 2 3.25 420 1 64 Moderate + m2.2xlarge 653 21.60 .90 14 38 34.2 13 4 3.25 850 2 64 High + m2.4xlarge 1307 43.20 1.80 14 38 68.4 26 8 3.25 1690 2 64 1000 High + hi1.4xlarge 2265 74.40 3.10 11 20 60.5 35 16 2.2 2048 ssd 2 64 10GB + +* faster cores is better than more cores+more total CPU. + - However I don't think you're CPU-bound so the m1.xlarge with provisioned EBS probably beats the m2.xlarge. + +* Provisioned IOPS for Amazon EBS + +* 8 drives in a [RAID-10](http://en.wikipedia.org/wiki/Nested_RAID_levels#RAID_1.2B0) configuration + - gives you 4x the storage of any one drive + + $ for drive in /dev/md0 /dev/xvdh{1,2,3,4} ; do sudo blockdev --setra 128 $drive ; done + +* Put the oplog on the local drives + - If you put the journal on the local drives, it will not be present if the machine dies. It will persist through a reboot. Still, it may make sense if you are using a replica set. +* do not put anything on the root partition. + +EBS volumes are just fine. You have plenty of network throughput. + +### Diagnostics + +* disk IO: `iostat -txm 5` +* mongostat: `mongostat --host $(hostname)` +* system: `htop` +* network: `ifstat 5` + diff --git a/config/cube.js b/config/cube.js index 93306b70..4034bc4a 100644 --- a/config/cube.js +++ b/config/cube.js @@ -37,7 +37,8 @@ configs.collector = { // configs.evaluator = { "http-port": 1081, - "authenticator": "mongo_cookie" + // "authenticator": "mongo_cookie" + "authenticator": "allow_all" } diff --git a/lib/cube/evaluator.js b/lib/cube/evaluator.js index b2b68824..cd660823 100644 --- a/lib/cube/evaluator.js +++ b/lib/cube/evaluator.js @@ -1,6 +1,7 @@ 'use strict'; var endpoint = require("./endpoint"), + metalog = require("./metalog"), url = require("url"); // To avoid running out of memory, the GET endpoints have a maximum number of @@ -16,6 +17,7 @@ var headers = { exports.register = function(db, endpoints) { var event = require("./event").getter(db), + event_putter = require("./event").putter(db), metric = require("./metric").getter(db), types = require("./types").getter(db); @@ -28,14 +30,37 @@ exports.register = function(db, endpoints) { // endpoints.http.push( - endpoint("GET", "/1.0/event", eventGet), - endpoint("GET", "/1.0/event/get", eventGet), - endpoint("GET", "/1.0/metric", metricGet), - endpoint("GET", "/1.0/metric/get", metricGet), - endpoint("GET", "/1.0/types", typesGet), - endpoint("GET", "/1.0/types/get", typesGet) + endpoint("GET", "/1.0/event", eventGet), + endpoint("GET", "/1.0/event/get", eventGet), + endpoint("GET", "/1.0/metric", metricGet), + endpoint("GET", "/1.0/metric/get", metricGet), + endpoint("DELETE", "/1.0/metric", metricRefresh), + endpoint("GET", "/1.0/metric/refresh", metricRefresh), + endpoint("GET", "/1.0/types", typesGet), + endpoint("GET", "/1.0/types/get", typesGet) ); + function metricRefresh(request, response) { + request = url.parse(request.url, true).query; + + if (! (request.start && request.stop && request.type)) { + metalog.info('cube_request', {is: 'error', error: 'specify type, start and stop'}); + response.writeHead(400, headers); + response.end('{}'); + } else { + var type = request.type, + start = new Date(request.start), + stop = new Date(request.stop); + + metalog.info('cube_request', {is: 'refresh', type: type, start: start, stop: stop}); + + event_putter.invalidate_range(type, start, stop); + + response.writeHead(202, headers); + response.end('{}'); + } + } + function eventGet(request, response) { request = url.parse(request.url, true).query; diff --git a/lib/cube/event.js b/lib/cube/event.js index c56a3941..fef6be13 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -4,9 +4,10 @@ // TODO allow the event time to change when updating (fix invalidation) var mongodb = require("mongodb"), - parser = require("./event-expression"), tiers = require("./tiers"), types = require("./types"), + models = require("./models"), Event = models.Event, Metric = models.Metric, + parser = require("./event-expression"), bisect = require("./bisect"), metalog = require("./metalog"), options = require("../../config/cube"), @@ -59,9 +60,13 @@ exports.putter = function(db){ // // Drop events from before invalidation horizon // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; - // If an id is specified, promote it to Mongo's primary key. - var event = {t: time, d: request.data}; - if ("id" in request) event._id = request.id; + // // Drop events from before invalidation horizon + // if ((! request.force) && (time < new Date(new Date() - options.horizons.invalidation))) { + // metric.info('cube_compute', {error: "event before invalidation horizon"}); + // return callback({error: "event before invalidation horizon"}), -1; + // } + + var event = new Event(type, time, request.data, request.id); // If this is a known event type, save immediately. if (type in knownByType) return save(type, event); @@ -86,7 +91,6 @@ exports.putter = function(db){ var events = collection(type).events; if (names.length) return saveEvents(); - // Create a collection for events. One index is require, for finding events by time(t) db.createCollection(type + "_events", event_options, function(error, events){ handle(error); @@ -120,7 +124,7 @@ exports.putter = function(db){ // likelihood of a race condition between when the events are read by the // evaluator and when the newly-computed metrics are saved. function save(type, event) { - // metalog.info("cube_request", { is: "ev", at: "save", type: type, event: event }); + // metalog.minor("cube_request", { is: "ev", at: "save", type: type, event: event }); collection(type).events.save(event, handle); queueInvalidation(type, event); } @@ -147,6 +151,21 @@ exports.putter = function(db){ } } + function invalidate_range(type, start, stop) { + var metrics = collection(type).metrics; + for (var tier in tiers) { + tierStart = tiers[tier].floor(start); + tierStop = tiers[tier].floor(stop); + metalog.info("cube_compute", { is: "inval", type: type, tier: tier, times: [tierStart, tierStop], start: start, stop: stop }); + metrics.update({ + i: false, + "_id.l": +tier, + "_id.t": {$gte: tierStart, $lt: tierStop}, + }, invalidate, multi); + } + } + putter.invalidate_range = invalidate_range; + // Process any deferred metric invalidations, flushing the queues. Note that // the queue (timesToInvalidateByTierByType) is copied-on-write, so while the // previous batch of events are being invalidated, new events can arrive. @@ -154,8 +173,8 @@ exports.putter = function(db){ for (var type in timesToInvalidateByTierByType) { var metrics = collection(type).metrics, timesToInvalidateByTier = timesToInvalidateByTierByType[type]; - metalog.info("cube_compute", { is: "flush", type: type }); for (var tier in tiers) { + metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: timesToInvalidateByTier[tier] }); metrics.update({ i: false, "_id.l": +tier, diff --git a/lib/cube/index.js b/lib/cube/index.js index 737b3ddc..1404b1cd 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -1,5 +1,9 @@ 'use strict'; +process.env.TZ = 'UTC'; +exports.models = require("./models"); +exports.Event = exports.models.Event; +exports.Metric = exports.models.Metric; exports.authentication = require("./authentication"); exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 5858dd59..66b6f250 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -40,11 +40,12 @@ metalog.minor = function(name, hsh){ }; // Dump the 'util.inspect' view of each argument to the console. -metalog.inspectify = function(args){ +metalog.inspectify = function inspectify(args){ for (var idx in arguments) { util.print(idx + ": "); util.print(util.inspect(arguments[idx])+"\n"); }; + util.print('----\n'); }; module.exports = metalog; diff --git a/lib/cube/models.js b/lib/cube/models.js index 31ecda82..f29f3c10 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -1,3 +1,35 @@ +'use strict'; + +function Event(type, time, data, id){ + this.type = function(){ return type; } + this.t = time; + this.d = data; + if (id) this._id = id; +} + +exports.Event = Event; + +function Metric(type, time, tier, expression, value){ + + // this.type = type; + // this.time = time; + // this.tier = tier; + // this.expression = expression; + // if (id) this._id = id; + + this.to_wire = function to_wire(){ + return { + _id: { + e: expression.source, + l: tier.key, + t: time + }, + i: false, + v: value + } + }; +} +exports.Metric = Metric; exports.units = { second: 1e3, diff --git a/test/metric-test.js b/test/metric-test.js index 1f832eb0..e4e69eb6 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -29,6 +29,52 @@ steps[units.minute5 ].description = "5-minute"; steps[units.hour ].description = "1-hour"; steps[units.day ].description = "1-day"; +function gen_request(attrs){ + var req = { start: nowish, stop: nowish, step: units.second10, expression: 'sum(test)'} + for (var key in attrs){ req[key] = attrs[key]; }; + return req; +} + +function assert_invalid_request(req, expected_err) { + return { + topic: function(getter){ this.ret = getter(gen_request(req), this.callback); }, + 'fails': function(err, _){ assert.deepEqual(err, expected_err); }, + 'returns -1': function(err, _){ assert.equal(this.ret, -1) } + }; +} + +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + return metric.getter(test_db.db); + }, + 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), + 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), + 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), + 'invalid expression': assert_invalid_request({expression: 'DANCE'}, {error: "invalid expression"}), + + 'with request id' : { + topic: function(getter){ this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(1))'}), this.callback); }, + 'fires callback with id': function w_req_vow(result, _){ + assert.equal(result.id, 'joe'); + assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); + assert.include([nowish10, 10000+nowish10], +result.time); + } + }, +})).addBatch(test_helper.batch({ + topic: function(test_db) { + return metric.getter(test_db.db); + }, + 'no request id' : { + topic: function(getter){ this.ret = getter(gen_request({}), this.callback); }, + 'fires callback with no id': function no_req_vow(result, _){ + assert.isFalse("id" in result); + assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); + assert.include([nowish10, 10000+nowish10], +result.time); + } + } + +})); + suite.addBatch(test_helper.batch({ topic: function(test_db) { var putter = event.putter(test_db.db), @@ -170,7 +216,7 @@ function metricTest(request, expected) { // function testStep(step, expected) { var start = new Date(request.start), - stop = new Date(request.stop); + stop = new Date(request.stop); var subtree = { topic: get_metrics_with_delay(0), @@ -183,10 +229,10 @@ function metricTest(request, expected) { function get_metrics_with_delay(depth){ return function(){ var actual = [], - timeout = setTimeout(function() { cb("Time's up!"); }, 10000), - cb = this.callback, - req = Object.create(request), - getter = arguments[depth]; + timeout = setTimeout(function() { cb("Time's up!"); }, 10000), + cb = this.callback, + req = Object.create(request), + getter = arguments[depth]; req.step = step; // Wait long enough for the events to have settled in the db. The // non-cached (depth=0) round can all start in parallel, making this an diff --git a/test/visualizer-test.js b/test/visualizer-test.js index 52a78def..3e907597 100644 --- a/test/visualizer-test.js +++ b/test/visualizer-test.js @@ -7,19 +7,22 @@ var vows = require("vows"), var suite = vows.describe("visualizer"); -var port = ++test_helper.port, server = cube.server({ - "mongo-host": "localhost", - "mongo-port": 27017, - "mongo-database": "cube_test", - "http-port": port, - "authenticator": "allow_all" -}); - -server.register = function(db, endpoints) { - cube.evaluator.register(db, endpoints); - cube.visualizer.register(db, endpoints); +var server_options = { 'http-port': test_helper.get_port() } +function frontend_components() { + cube.evaluator.register.apply(this, arguments); + cube.visualizer.register.apply(this, arguments); }; -server.start(); +// suite.addBatch( +// test_helper.with_server(server_options, frontend_components, { +// +// "POST /event/put with invalid JSON": { +// topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), +// "responds with status 400": function(response) { +// assert.equal(response.statusCode, 400); +// assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); +// } +// } +// })); suite.export(module); From 2635db0b1da763c00f966e574f50efc7e41b6b1c Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 3 Sep 2012 03:13:45 -0500 Subject: [PATCH 28/87] adding underscore lib --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 1b919452..c8e1e032 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "websocket": "1.0.7", "websocket-server": "1.4.04", "cookies": "0.3.1", - "bcrypt": "0.7.1" + "bcrypt": "0.7.1", + "underscore": "1.3.3" }, "scripts": { "preinstall": "npm install mongodb --mongodb:native", From 7881f54b2e037d6f0853f413360ff746e97639bd Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 3 Sep 2012 03:15:36 -0500 Subject: [PATCH 29/87] separated invalidation logic from putter (preparation for implementating calculator queue) --- lib/cube/event.js | 131 +++++++++++++++++++------------------------- lib/cube/models.js | 69 +++++++++++++++++++---- lib/cube/tiers.js | 5 ++ test/event-test.js | 38 +++++++++++++ test/metric-test.js | 12 ++-- 5 files changed, 164 insertions(+), 91 deletions(-) create mode 100644 test/event-test.js diff --git a/lib/cube/event.js b/lib/cube/event.js index fef6be13..098e97dd 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -3,7 +3,8 @@ // TODO include the event._id (and define a JSON encoding for ObjectId?) // TODO allow the event time to change when updating (fix invalidation) -var mongodb = require("mongodb"), +var _ = require("underscore"), + mongodb = require("mongodb"), tiers = require("./tiers"), types = require("./types"), models = require("./models"), Event = models.Event, Metric = models.Metric, @@ -13,10 +14,7 @@ var mongodb = require("mongodb"), options = require("../../config/cube"), ObjectID = mongodb.ObjectID; -var type_re = /^[a-z][a-zA-Z0-9_]+$/, - invalidate = {$set: {i: true}}, - multi = {multi: true}, - metric_options = options["mongo-metrics"], +var metric_options = options["mongo-metrics"], event_options = options["mongo-events"]; // When streaming events, we should allow a delay for events to arrive, or else @@ -28,12 +26,47 @@ var streamDelayDefault = 5000, // How frequently to invalidate metrics after receiving events. var invalidateInterval = 5000; -var flushers = []; -exports.stop = function(){ try{ - flushers.forEach(function(flusher){ - metalog.info('cube_life', {is: 'flusher_unregister' }); - clearInterval(flusher); - }); } catch(err) {}}; +exports.stop = function(){ Invalidator.stop_flushers(); }; + +// -------------------------------------------------------------------------- + +// Schedule deferred invalidation of metrics by type and tier. +function Invalidator(){ + var type_tsets = {}, + invalidate = { $set: {i: true} }, + multi = { multi: true }; + + this.add = function(type, ev){ + var tt = type_tset(type); + for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true } + } + + this.flush = function(collection){ + _.each(type_tsets, function(type_tset, type){ + var metrics = collection(type).metrics; + _.each(type_tset, function(tset, tier){ + var times = dateify(tset); + metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: times }); + metrics.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); + }); + }); + } + + this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }) }; + + function type_tset(type){ + if (! (type in type_tsets)) type_tsets[type] = empty_tsets(); + return type_tsets[type]; + } + function empty_tsets(){ return _.mapHash(tiers, function(){ return {}; }); }; + function dateify(tset){ return _.map(_.keys(tset), function(time){ return new Date(+time); }).sort(function(aa,bb){return aa-bb;}); } +} +Invalidator.flushers = []; +Invalidator.start_flusher = function(cb){ Invalidator.flushers.push(setInterval(cb, invalidateInterval)); } +Invalidator.stop_flushers = function(){ Invalidator.flushers.forEach(clearInterval); Invalidator.flushers = []; }; + +var invalidator = new Invalidator(); +exports.invalidator = function(){ return invalidator }; // event.putter -- save the event, invalidate any cached metrics impacted by it. // @@ -46,17 +79,12 @@ exports.stop = function(){ try{ exports.putter = function(db){ var collection = types(db), knownByType = {}, - eventsToSaveByType = {}, - timesToInvalidateByTierByType = {}; + eventsToSaveByType = {}; function putter(request, callback) { var time = "time" in request ? new Date(request.time) : new Date(), type = request.type; - // Validate the date and type. - if (!type_re.test(type)) return callback({error: "invalid type"}), -1; - if (isNaN(time)) return callback({error: "invalid time"}), -1; - // // Drop events from before invalidation horizon // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; @@ -67,6 +95,9 @@ exports.putter = function(db){ // } var event = new Event(type, time, request.data, request.id); + try{ event.validate(); } catch(err) { return callback({error: err}), -1; } + + event.on_save = callback; // If this is a known event type, save immediately. if (type in knownByType) return save(type, event); @@ -126,64 +157,17 @@ exports.putter = function(db){ function save(type, event) { // metalog.minor("cube_request", { is: "ev", at: "save", type: type, event: event }); collection(type).events.save(event, handle); - queueInvalidation(type, event); - } - - // Schedule deferred invalidation of metrics for this type. - // For each type and tier, track the metric times to invalidate. - // The times are kept in sorted order for bisection. - function queueInvalidation(type, event) { - var timesToInvalidateByTier = timesToInvalidateByTierByType[type], - time = event.t; - if (timesToInvalidateByTier) { - for (var tier in tiers) { - var tierTimes = timesToInvalidateByTier[tier], - tierTime = tiers[tier].floor(time), - i = bisect(tierTimes, tierTime); - if (i >= tierTimes.length) tierTimes.push(tierTime); - else if (tierTimes[i] > tierTime) tierTimes.splice(i, 0, tierTime); - } - } else { - timesToInvalidateByTier = timesToInvalidateByTierByType[type] = {}; - for (var tier in tiers) { - timesToInvalidateByTier[tier] = [tiers[tier].floor(time)]; - } - } - } - - function invalidate_range(type, start, stop) { - var metrics = collection(type).metrics; - for (var tier in tiers) { - tierStart = tiers[tier].floor(start); - tierStop = tiers[tier].floor(stop); - metalog.info("cube_compute", { is: "inval", type: type, tier: tier, times: [tierStart, tierStop], start: start, stop: stop }); - metrics.update({ - i: false, - "_id.l": +tier, - "_id.t": {$gte: tierStart, $lt: tierStop}, - }, invalidate, multi); - } + invalidator.add(type, event); + if (event.on_save) event.on_save(null); } - putter.invalidate_range = invalidate_range; // Process any deferred metric invalidations, flushing the queues. Note that // the queue (timesToInvalidateByTierByType) is copied-on-write, so while the // previous batch of events are being invalidated, new events can arrive. - flushers.push(setInterval(function() { - for (var type in timesToInvalidateByTierByType) { - var metrics = collection(type).metrics, - timesToInvalidateByTier = timesToInvalidateByTierByType[type]; - for (var tier in tiers) { - metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: timesToInvalidateByTier[tier] }); - metrics.update({ - i: false, - "_id.l": +tier, - "_id.t": {$in: timesToInvalidateByTier[tier]} - }, invalidate, multi); - } - } - timesToInvalidateByTierByType = {}; // copy-on-write - }, invalidateInterval)); + Invalidator.start_flusher(function(){ + invalidator.flush(collection); + invalidator = new Invalidator(); // copy-on-write + }); metalog.info('cube_life', {is: 'putter_start'}); return putter; @@ -213,7 +197,7 @@ exports.getter = function(db) { // Validate the dates. if (isNaN(start)) return callback({error: "invalid start"}), -1; - if (isNaN(stop)) return callback({error: "invalid stop"}), -1; + if (isNaN(stop)) return callback({error: "invalid stop"}), -1; // Parse the expression. var expression; @@ -316,9 +300,8 @@ exports.getter = function(db) { } getter.close = function(callback) { - // results or periodic calls may have already been set in motion, but - // trigger in the future; this ensures they quit listening and drop further - // results on the floor. + // as results or periodic calls trigger in the future, ensure that they quit + // listening and drop further results on the floor. callback.closed = true; }; diff --git a/lib/cube/models.js b/lib/cube/models.js index f29f3c10..52d302ac 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -1,12 +1,68 @@ 'use strict'; +var _ = require("underscore"), + metalog = require('./metalog'); + +var second = 1e3, + second10 = 10e3, + minute = 60e3, + minute5 = 300e3, + hour = 3600e3, + day = 86400e3; +exports.units = { second: second, second10: second10, minute: minute, minute5: minute5, hour: hour, day: day }; + +var tiers = require("./tiers"), + tensec = tiers[second10], + type_re = /^[a-z][a-zA-Z0-9_]+$/; + +_.mapHash = function(obj, func){ + var res = {}; + _.each(obj, function(val, key){ res[key] = func(val, key, res); }); + return res; +} + + function Event(type, time, data, id){ - this.type = function(){ return type; } this.t = time; this.d = data; if (id) this._id = id; -} + this._type = function(){ return type; }; + + this.bin = function(tr){ return tiers[tr].bin(this.t); }; + + this.day_bin = function(){ return tiers[day ].bin(this.t); }; + this.m05_bin = function(){ return tiers[minute5 ].bin(this.t); }; + this.s10_bin = function(){ return tiers[second10].bin(this.t); }; + + this.day_ago = function day_ago(){ return Math.floor((Date.now() - this.t) / day); } + this.m05_ago = function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); } + this.s10_ago = function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); } + + this.bins = function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; } + this.agos = function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; } + this.report = function report(){ + return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; + } + + this.validate = function(){ + // Validate the date and type. + if (!type_re.test(type)) throw("invalid type"); + if (isNaN(time)) throw("invalid time"); + } + + this.to_wire = function(){ + var ev = { t: this.t, d: this.d }; + if (id) ev._id = this._id; + return ev; + } + this.to_request = function(attrs){ + var ev = { time: this.t, data: this.d, type: type }; + if (id) ev.id = this._id; + for (var key in attrs){ ev[key] = attrs[key]; }; + return ev; + }; +} exports.Event = Event; function Metric(type, time, tier, expression, value){ @@ -30,12 +86,3 @@ function Metric(type, time, tier, expression, value){ }; } exports.Metric = Metric; - -exports.units = { - second: 1e3, - second10: 10e3, - minute: 60e3, - minute5: 300e3, - hour: 3600e3, - day: 86400e3 -}; diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index 322e43bb..13d38b0f 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -12,6 +12,7 @@ var second = 1000, tiers[second10] = { key: second10, floor: function(d) { return new Date(Math.floor(d / second10) * second10); }, + bin: function(d) { return Math.floor(d / second10); }, ceil: tier_ceil, step: function(d) { return new Date(+d + second10); } }; @@ -19,6 +20,7 @@ tiers[second10] = { tiers[minute] = { key: minute, floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, + bin: function(d) { return Math.floor(d / minute); }, ceil: tier_ceil, step: function(d) { return new Date(+d + minute); }, // next: tiers[second10], @@ -28,6 +30,7 @@ tiers[minute] = { tiers[minute5] = { key: minute5, floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, + bin: function(d) { return Math.floor(d / minute5); }, ceil: tier_ceil, step: function(d) { return new Date(+d + minute5); }, // next: tiers[minute], @@ -37,6 +40,7 @@ tiers[minute5] = { tiers[hour] = { key: hour, floor: function(d) { return new Date(Math.floor(d / hour) * hour); }, + bin: function(d) { return Math.floor(d / hour); }, ceil: tier_ceil, step: function(d) { return new Date(+d + hour); }, next: tiers[minute5], @@ -46,6 +50,7 @@ tiers[hour] = { tiers[day] = { key: day, floor: function(d) { return new Date(Math.floor(d / day) * day); }, + bin: function(d) { return Math.floor(d / day); }, ceil: tier_ceil, step: function(d) { return new Date(+d + day); }, next: tiers[hour], diff --git a/test/event-test.js b/test/event-test.js new file mode 100644 index 00000000..ae14875a --- /dev/null +++ b/test/event-test.js @@ -0,0 +1,38 @@ +'use strict'; + +var vows = require("vows"), + assert = require("assert"), + test_helper = require("./test_helper"), + models = require("../lib/cube/models"), units = models.units, Event = models.Event, + event = require("../lib/cube/event"); + +var suite = vows.describe("event"); + +var ice_cubes_good_day = Date.UTC(1992, 1, 20, 1, 8, 7), + fuck_wit_dre_day = Date.UTC(1993, 2, 18, 8,44, 54) + + +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + return event.putter(test_db.db); + }, + 'invalidates': { + topic: function(putter){ + var ctxt = this; + putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ + putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback) }); + }, + 'heckya': function(){ + var ts = event.invalidator().tsets(); + assert.deepEqual(ts, { 'test': { + 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], + 60e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:00Z') ], + 300e3: [new Date('1992-02-20T01:05:00Z'), new Date('1993-03-18T08:40:00Z') ], + 3600e3: [new Date('1992-02-20T01:00:00Z'), new Date('1993-03-18T08:00:00Z') ], + 86400e3: [new Date('1992-02-20T00:00:00Z'), new Date('1993-03-18T00:00:00Z') ] + }}); + } + } +})); + +suite.export(module); diff --git a/test/metric-test.js b/test/metric-test.js index e4e69eb6..3ac59a7f 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -38,8 +38,8 @@ function gen_request(attrs){ function assert_invalid_request(req, expected_err) { return { topic: function(getter){ this.ret = getter(gen_request(req), this.callback); }, - 'fails': function(err, _){ assert.deepEqual(err, expected_err); }, - 'returns -1': function(err, _){ assert.equal(this.ret, -1) } + 'fails': function(err, val){ assert.deepEqual(err, expected_err); }, + 'returns -1': function(err, val){ assert.equal(this.ret, -1) } }; } @@ -54,7 +54,7 @@ suite.addBatch(test_helper.batch({ 'with request id' : { topic: function(getter){ this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(1))'}), this.callback); }, - 'fires callback with id': function w_req_vow(result, _){ + 'fires callback with id': function(result, j_){ assert.equal(result.id, 'joe'); assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); assert.include([nowish10, 10000+nowish10], +result.time); @@ -66,7 +66,7 @@ suite.addBatch(test_helper.batch({ }, 'no request id' : { topic: function(getter){ this.ret = getter(gen_request({}), this.callback); }, - 'fires callback with no id': function no_req_vow(result, _){ + 'fires callback with no id': function(result, j_){ assert.isFalse("id" in result); assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); assert.include([nowish10, 10000+nowish10], +result.time); @@ -179,13 +179,13 @@ suite.addBatch(test_helper.batch({ // topic: function get_metrics_with_delay(getter){}, // 'sum(test)': function metrics_assertions(actual){}, // '(cached)': { -// topic: function get_metrics_with_delay(_, getter){}, +// topic: function get_metrics_with_delay(err, getter){}, // 'sum(test)': function metrics_assertions(actual){} } }, // 'at 1-day intervals': { // topic: function get_metrics_with_delay(getter){}, // 'sum(test)': function metrics_assertions(actual){}, // '(cached)': { -// topic: function get_metrics_with_delay(_, getter){}, +// topic: function get_metrics_with_delay(err, getter){}, // 'sum(test)': function metrics_assertions(actual){} } } // } // } From 3b7999586ffd0386fed586011f937ce4964340bf Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 3 Sep 2012 05:06:20 -0500 Subject: [PATCH 30/87] JSLinted files, groomed away the lint --- .jslint.conf | 144 +++++++++++++++++++++++++++++++++ lib/cube/authentication.js | 14 ++-- lib/cube/endpoint.js | 4 +- lib/cube/evaluator.js | 2 +- lib/cube/event.js | 14 ++-- lib/cube/metalog.js | 6 +- lib/cube/metric.js | 2 +- lib/cube/models.js | 23 +++--- lib/cube/reduces.js | 8 +- lib/cube/server.js | 17 ++-- lib/cube/tiers.js | 4 +- lib/cube/visualizer.js | 4 +- lib/cube/warmer.js | 10 +-- test/authentication-test.js | 6 +- test/collector-test.js | 2 +- test/evaluator-test.js | 4 +- test/event-test.js | 13 ++- test/metalog-test.js | 4 +- test/metric-expression-test.js | 6 +- test/metric-test.js | 30 +++---- test/server-test.js | 6 +- test/test_helper.js | 16 ++-- test/visualizer-test.js | 4 +- 23 files changed, 243 insertions(+), 100 deletions(-) create mode 100644 .jslint.conf diff --git a/.jslint.conf b/.jslint.conf new file mode 100644 index 00000000..ff0a5967 --- /dev/null +++ b/.jslint.conf @@ -0,0 +1,144 @@ +# +# Configuration File for JavaScript Lint 0.3.0 +# Developed by Matthias Miller (http://www.JavaScriptLint.com) +# +# This configuration file can be used to lint a collection of scripts, or to enable +# or disable warnings for scripts that are linted via the command line. +# + +### Warnings +# Enable or disable warnings based on requirements. +# Use "+WarningName" to display or "-WarningName" to suppress. +# +-no_return_value ## function {0} does not always return a value ++duplicate_formal # duplicate formal argument {0} ++equal_as_assign # test for equality (==) mistyped as assignment (=)?{0} ++var_hides_arg # variable {0} hides argument ++redeclared_var # redeclaration of {0} {1} +-anon_no_return_value # anonymous function does not always return a value ++missing_semicolon # missing semicolon ++meaningless_block # meaningless block; curly braces have no impact +-comma_separated_stmts ## multiple statements separated by commas (use semicolons?) +-unreachable_code ## unreachable code ++missing_break # missing break statement ++missing_break_for_last_case # missing break statement for last case in switch ++comparison_type_conv # comparisons against null, 0, true, false, or an empty string allowing implicit type conversion (use === or !==) +-inc_dec_within_stmt ## increment (++) and decrement (--) operators used as part of greater statement ++useless_void # use of the void type may be unnecessary (void is always undefined) ++multiple_plus_minus # unknown order of operations for successive plus (e.g. x+++y) or minus (e.g. x---y) signs ++use_of_label # use of label +-block_without_braces ## block statement without curly braces ++leading_decimal_point # leading decimal point may indicate a number or an object member ++trailing_decimal_point # trailing decimal point may indicate a number or an object member ++octal_number # leading zeros make an octal number ++nested_comment # nested comment ++misplaced_regex # regular expressions should be preceded by a left parenthesis, assignment, colon, or comma +-ambiguous_newline ## unexpected end of line; it is ambiguous whether these lines are part of the same statement ++empty_statement # empty statement or extra semicolon +-missing_option_explicit ## the "option explicit" control comment is missing ++partial_option_explicit # the "option explicit" control comment, if used, must be in the first script tag ++dup_option_explicit # duplicate "option explicit" control comment ++useless_assign # useless assignment ++ambiguous_nested_stmt # block statements containing block statements should use curly braces to resolve ambiguity ++ambiguous_else_stmt # the else statement could be matched with one of multiple if statements (use curly braces to indicate intent) +-missing_default_case ## missing default case in switch statement ++duplicate_case_in_switch # duplicate case in switch statements ++default_not_at_end # the default case is not at the end of the switch statement ++legacy_cc_not_understood # couldn't understand control comment using /*@keyword@*/ syntax ++jsl_cc_not_understood # couldn't understand control comment using /*jsl:keyword*/ syntax ++useless_comparison # useless comparison; comparing identical expressions ++with_statement # with statement hides undeclared variables; use temporary variable instead ++trailing_comma_in_array # extra comma is not recommended in array initializers ++assign_to_function_call # assignment to a function call ++parseint_missing_radix # parseInt missing radix parameter + +### Output format +# Customize the format of the error message. +# __FILE__ indicates current file path +# __FILENAME__ indicates current file name +# __LINE__ indicates current line +# __ERROR__ indicates error message +# +# Visual Studio syntax (default): ++output-format __FILE__(__LINE__): __ERROR__ +# Alternative syntax: +#+output-format __FILE__:__LINE__: __ERROR__ + + +### Context +# Show the in-line position of the error. +# Use "+context" to display or "-context" to suppress. +# ++context + +### Semicolons +# By default, assignments of an anonymous function to a variable or +# property (such as a function prototype) must be followed by a semicolon. +# ++lambda_assign_requires_semicolon + +### Control Comments +# Both JavaScript Lint and the JScript interpreter confuse each other with the syntax for +# the /*@keyword@*/ control comments and JScript conditional comments. (The latter is +# enabled in JScript with @cc_on@). The /*jsl:keyword*/ syntax is preferred for this reason, +# although legacy control comments are enabled by default for backward compatibility. +# ++legacy_control_comments + + +### JScript Function Extensions +# JScript allows member functions to be defined like this: +# function MyObj() { /*constructor*/ } +# function MyObj.prototype.go() { /*member function*/ } +# +# It also allows events to be attached like this: +# function window::onload() { /*init page*/ } +# +# This is a Microsoft-only JavaScript extension. Enable this setting to allow them. +# +-jscript_function_extensions + + +### Defining identifiers +# By default, "option explicit" is enabled on a per-file basis. +# To enable this for all files, use "+always_use_option_explicit" +-always_use_option_explicit + +# Define certain identifiers of which the lint is not aware. +# (Use this in conjunction with the "undeclared identifier" warning.) +# +# Common uses for webpages might be: +#+define window +#+define document + + +### Files +# Specify which files to lint +# Use "+recurse" to enable recursion (disabled by default). +# To add a set of files, use "+process FileName", "+process Folder\Path\*.js", +# or "+process Folder\Path\*.htm". +# ++process lib/cube/authentication.js ++process lib/cube/bisect.js ++process lib/cube/collectd.js ++process lib/cube/collector.js ++process lib/cube/emitter-http.js ++process lib/cube/emitter-udp.js ++process lib/cube/emitter-ws.js ++process lib/cube/emitter.js ++process lib/cube/endpoint.js ++process lib/cube/evaluator.js ++process lib/cube/event.js ++process lib/cube/index.js ++process lib/cube/metalog.js ++process lib/cube/metric.js ++process lib/cube/models.js ++process lib/cube/reduces.js ++process lib/cube/server.js ++process lib/cube/tiers.js ++process lib/cube/types.js ++process lib/cube/visualizer.js ++process lib/cube/warmer.js + ++process test/*.js ++process bin/*.js diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js index 671b50f6..9b59447b 100644 --- a/lib/cube/authentication.js +++ b/lib/cube/authentication.js @@ -29,14 +29,14 @@ var authentication = {}; authentication.authenticator = function(strategy, db, options){ metalog.minor('cube_auth', { strategy: strategy }); return authentication[strategy](db, options); -} +}; authentication.allow_all = function(){ function check(request, auth_ok, auth_no) { metalog.event('cube_auth', { authenticator: 'allow_all', path: request.url }, 'minor'); request.authorized = { admin: true }; return auth_ok(request.authorized); - }; + } return { check: check }; }; @@ -45,7 +45,7 @@ authentication.read_only = function(){ metalog.event('cube_auth', { authenticator: 'read_only', path: request.url }, 'minor'); request.authorized = { admin: false }; return auth_ok(request.authorized); - }; + } return { check: check }; }; @@ -70,7 +70,7 @@ authentication.mongo_cookie = function(db, options){ metalog.event('cube_auth', { is: 'no', r: 'no_token_in_request' }); auth_no('no_token_in_request'); return; - }; + } validate_token({"tokens.uid": token_uid}, auth_ok, auth_no); @@ -99,10 +99,10 @@ authentication.mongo_cookie = function(db, options){ } else { metalog.event('cube_auth', { is: 'no', r: 'missing_user', tu: token_uid }); auth_no('missing_user'); - }; + } }); - }; - }; + } + } return { check: check }; }; diff --git a/lib/cube/endpoint.js b/lib/cube/endpoint.js index 03021e32..98490a02 100644 --- a/lib/cube/endpoint.js +++ b/lib/cube/endpoint.js @@ -19,6 +19,6 @@ module.exports = function(method, path, dispatch) { match = function(p) { return p == path; }; } else { // path is a string match = function(p, m) { return m == method && p == path; }; - }; + } return { match: match, dispatch: dispatch }; -} +}; diff --git a/lib/cube/evaluator.js b/lib/cube/evaluator.js index cd660823..9cfbf7c0 100644 --- a/lib/cube/evaluator.js +++ b/lib/cube/evaluator.js @@ -80,7 +80,7 @@ exports.register = function(db, endpoints) { } function callback(d) { - if (d == null) response.end(JSON.stringify(data.reverse())); + if (d === null) response.end(JSON.stringify(data.reverse())); else data.push(d); } } diff --git a/lib/cube/event.js b/lib/cube/event.js index 098e97dd..01767fb1 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -38,8 +38,8 @@ function Invalidator(){ this.add = function(type, ev){ var tt = type_tset(type); - for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true } - } + for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true; } + }; this.flush = function(collection){ _.each(type_tsets, function(type_tset, type){ @@ -50,23 +50,23 @@ function Invalidator(){ metrics.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); }); }); - } + }; - this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }) }; + this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }); }; function type_tset(type){ if (! (type in type_tsets)) type_tsets[type] = empty_tsets(); return type_tsets[type]; } - function empty_tsets(){ return _.mapHash(tiers, function(){ return {}; }); }; + function empty_tsets(){ return _.mapHash(tiers, function(){ return {}; }); } function dateify(tset){ return _.map(_.keys(tset), function(time){ return new Date(+time); }).sort(function(aa,bb){return aa-bb;}); } } Invalidator.flushers = []; -Invalidator.start_flusher = function(cb){ Invalidator.flushers.push(setInterval(cb, invalidateInterval)); } +Invalidator.start_flusher = function(cb){ Invalidator.flushers.push(setInterval(cb, invalidateInterval)); }; Invalidator.stop_flushers = function(){ Invalidator.flushers.forEach(clearInterval); Invalidator.flushers = []; }; var invalidator = new Invalidator(); -exports.invalidator = function(){ return invalidator }; +exports.invalidator = function(){ return invalidator; }; // event.putter -- save the event, invalidate any cached metrics impacted by it. // diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 66b6f250..c127a111 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -6,7 +6,7 @@ var metalog = { putter: null, log: util.log, silent: function(){ } -} +}; // adjust verboseness by reassigning `metalog.loggers.{level}` // @example: quiet all logs @@ -14,7 +14,7 @@ var metalog = { metalog.loggers = { info: metalog.log, minor: metalog.silent -} +}; // if true, cubify `metalog.event`s metalog.send_events = true; @@ -44,7 +44,7 @@ metalog.inspectify = function inspectify(args){ for (var idx in arguments) { util.print(idx + ": "); util.print(util.inspect(arguments[idx])+"\n"); - }; + } util.print('----\n'); }; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index a2883e25..5e03a5ab 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -79,7 +79,7 @@ exports.getter = function(db) { step = tier.key; // Compute the expected number of values. - while (time < stop) ++remaining, time = tier.step(time); + while (time < stop){ ++remaining; time = tier.step(time); } // If no results were requested, return immediately. if (!remaining) return callback(stop); diff --git a/lib/cube/models.js b/lib/cube/models.js index 52d302ac..25f0ccbc 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -19,8 +19,7 @@ _.mapHash = function(obj, func){ var res = {}; _.each(obj, function(val, key){ res[key] = func(val, key, res); }); return res; -} - +}; function Event(type, time, data, id){ this.t = time; @@ -35,31 +34,31 @@ function Event(type, time, data, id){ this.m05_bin = function(){ return tiers[minute5 ].bin(this.t); }; this.s10_bin = function(){ return tiers[second10].bin(this.t); }; - this.day_ago = function day_ago(){ return Math.floor((Date.now() - this.t) / day); } - this.m05_ago = function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); } - this.s10_ago = function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); } + this.day_ago = function day_ago(){ return Math.floor((Date.now() - this.t) / day); }; + this.m05_ago = function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }; + this.s10_ago = function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }; - this.bins = function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; } - this.agos = function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; } + this.bins = function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }; + this.agos = function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }; this.report = function report(){ return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; - } + }; this.validate = function(){ // Validate the date and type. if (!type_re.test(type)) throw("invalid type"); if (isNaN(time)) throw("invalid time"); - } + }; this.to_wire = function(){ var ev = { t: this.t, d: this.d }; if (id) ev._id = this._id; return ev; - } + }; this.to_request = function(attrs){ var ev = { time: this.t, data: this.d, type: type }; if (id) ev.id = this._id; - for (var key in attrs){ ev[key] = attrs[key]; }; + for (var key in attrs){ ev[key] = attrs[key]; } return ev; }; } @@ -82,7 +81,7 @@ function Metric(type, time, tier, expression, value){ }, i: false, v: value - } + }; }; } exports.Metric = Metric; diff --git a/lib/cube/reduces.js b/lib/cube/reduces.js index 9ef8c055..bacfdd0e 100644 --- a/lib/cube/reduces.js +++ b/lib/cube/reduces.js @@ -10,24 +10,24 @@ var reduces = module.exports = { min: function(values) { var i = -1, n = values.length, min = Infinity, value; - while (++i < n) if ((value = values[i]) < min) min = value; + while (++i < n){ if ((value = values[i]) < min){ min = value; } } return min; }, max: function(values) { var i = -1, n = values.length, max = -Infinity, value; - while (++i < n) if ((value = values[i]) > max) max = value; + while (++i < n){ if ((value = values[i]) > max){ max = value; } } return max; }, distinct: function(values) { var map = {}, count = 0, i = -1, n = values.length, value; - while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; + while (++i < n){ if (!((value = values[i]) in map)){ map[value] = ++count; } } return count; }, median: function(values) { - return quantile(values.sort(ascending), .5); + return quantile(values.sort(ascending), 0.5); } }; diff --git a/lib/cube/server.js b/lib/cube/server.js index bb81a88c..f6fc319f 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -69,9 +69,9 @@ module.exports = function(options) { return ("sec-websocket-version" in request.headers); } function is_ws_initiation(request){ - return (request.method === "GET" - && /^websocket$/i.test(request.headers.upgrade) - && /^upgrade$/i.test(request.headers.connection)); + return (request.method === "GET" && + (/^websocket$/i).test(request.headers.upgrade) && + (/^upgrade$/i).test(request.headers.connection) ); } // Register primary WebSocket listener with fallback. @@ -183,7 +183,7 @@ module.exports = function(options) { metalog.info('cube_life', {is: 'mongo_connect', options: options }); db.open(function(error) { if (error) throw error; - if (options["mongo-username"] == null) return ready(); + if (! options["mongo-username"]) return ready(); db.authenticate(options["mongo-username"], mongo_password, function(error, success) { if (error) throw error; if (!success) throw new Error("authentication failed"); @@ -216,8 +216,8 @@ module.exports = function(options) { try { metalog.info('cube_life', {is:(name+'_stopping'), options: options}); obj.close( function(){ metalog.info('cube_life', {is:(name+'_stop')}); } ); - } catch(err){}; - }}; + } catch(err){} + } } server.stop = function(cb){ // stop flushing event.stop(); @@ -226,9 +226,10 @@ module.exports = function(options) { try_close('ws', secondary); try_close('udp', udp); // in a short while, stop db'ing - setTimeout(function(){ try_close('mongo', db) }, 100); + setTimeout(function(){ try_close('mongo', db); }, 100); + // finally, holler at your boys if (cb) setTimeout(function(){ cb(); }, 200); - } + }; return server; }; diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index 13d38b0f..f45232a0 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -22,7 +22,7 @@ tiers[minute] = { floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, bin: function(d) { return Math.floor(d / minute); }, ceil: tier_ceil, - step: function(d) { return new Date(+d + minute); }, + step: function(d) { return new Date(+d + minute); } // next: tiers[second10], // size: function() { return 6; } }; @@ -32,7 +32,7 @@ tiers[minute5] = { floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, bin: function(d) { return Math.floor(d / minute5); }, ceil: tier_ceil, - step: function(d) { return new Date(+d + minute5); }, + step: function(d) { return new Date(+d + minute5); } // next: tiers[minute], // size: function() { return 5; } }; diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js index 0c523f57..f6f1c2a9 100644 --- a/lib/cube/visualizer.js +++ b/lib/cube/visualizer.js @@ -36,7 +36,7 @@ function viewBoard(db) { function check_authorization(request, action){ if (request.authorized.admin) return true; metalog.info('cube_request', { is: 'denied', action: action, u: request.authorized }); - return false + return false; } function add(request, callback) { @@ -92,7 +92,7 @@ function viewBoard(db) { // Asynchronously load the requested board. boards.findOne({_id: boardId}, function(error, board) { - if (board != null) { + if (board !== null) { if (board.pieces) board.pieces.forEach(function(piece) { callback({type: "add", piece: piece}); }); diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index 1ed85741..b47ef02d 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -4,7 +4,7 @@ var cluster = require('cluster'), mongodb = require('mongodb'), metric = require('./metric'), tiers = require('./tiers'), - metalog = require('./metalog') + metalog = require('./metalog'); module.exports = function(options){ var db, mongo, calculate_metric, boards, tier; @@ -48,9 +48,9 @@ module.exports = function(options){ return { start: function(){ - mongo = new mongodb.Server(options['mongo-host'], options['mongo-port']); - db = new mongodb.Db(options["mongo-database"], mongo), - tier = tiers[options['warmer-tier'].toString()]; + var mongo = new mongodb.Server(options['mongo-host'], options['mongo-port']), + db = new mongodb.Db(options["mongo-database"], mongo), + tier = tiers[options['warmer-tier'].toString()]; if(typeof tier === "undefined") throw new Error("Undefined warmer tier configured: " + options['warmer-tier']); @@ -63,4 +63,4 @@ module.exports = function(options){ }); } }; -} +}; diff --git a/test/authentication-test.js b/test/authentication-test.js index 2b494e16..727ddf2a 100644 --- a/test/authentication-test.js +++ b/test/authentication-test.js @@ -46,11 +46,11 @@ var dummy_token = "token_in_cookie"; function dummy_request(username, token){ return({ headers: { cookie: authentication.gen_cookie("_cube_session", username+"_tok", token || dummy_token) } }); -}; +} suite.addBatch(test_helper.batch({ mongo_cookie: { - topic: function(test_db){ test_db.using_objects("test_users", test_users, this) }, + topic: function(test_db){ test_db.using_objects("test_users", test_users, this); }, "": { topic: function(test_db){ return authentication.authenticator("mongo_cookie", test_db.db, { collection: "test_users" }); }, @@ -79,7 +79,7 @@ suite.addBatch(test_helper.batch({ }, 'is returned as callback param': function(result){ assert.deepEqual(result, { uid: 'boss_hogg_tok', admin: true }); - }, + } }, "user with write access": { topic: successful_auth(dummy_request("boss_hogg")), diff --git a/test/collector-test.js b/test/collector-test.js index 3846c617..8eed18d0 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -7,7 +7,7 @@ var vows = require("vows"), var suite = vows.describe("collector"); -var server_options = { 'http-port': test_helper.get_port() } +var server_options = { 'http-port': test_helper.get_port() }; suite.addBatch( test_helper.with_server(server_options, cube.collector.register, { diff --git a/test/evaluator-test.js b/test/evaluator-test.js index 62cb0b8e..63787bff 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -7,11 +7,11 @@ var vows = require("vows"), var suite = vows.describe("evaluator"); -var server_options = { 'http-port': test_helper.get_port() } +var server_options = { 'http-port': test_helper.get_port() }; function frontend_components() { cube.evaluator.register.apply(this, arguments); cube.visualizer.register.apply(this, arguments); -}; +} // suite.addBatch( // test_helper.with_server(server_options, frontend_components, { diff --git a/test/event-test.js b/test/event-test.js index ae14875a..2cd9345b 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -8,9 +8,8 @@ var vows = require("vows"), var suite = vows.describe("event"); -var ice_cubes_good_day = Date.UTC(1992, 1, 20, 1, 8, 7), - fuck_wit_dre_day = Date.UTC(1993, 2, 18, 8,44, 54) - +var ice_cubes_good_day = Date.UTC(1992, 1, 20, 1, 8, 7), + fuck_wit_dre_day = Date.UTC(1993, 2, 18, 8, 44, 54); suite.addBatch(test_helper.batch({ topic: function(test_db) { @@ -19,12 +18,12 @@ suite.addBatch(test_helper.batch({ 'invalidates': { topic: function(putter){ var ctxt = this; - putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ - putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback) }); + putter((new Event('test2', ice_cubes_good_day, {value: 3})).to_request(), function(){ + putter((new Event('test2', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); }, 'heckya': function(){ var ts = event.invalidator().tsets(); - assert.deepEqual(ts, { 'test': { + assert.deepEqual(ts, { 'test2': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], 60e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:00Z') ], 300e3: [new Date('1992-02-20T01:05:00Z'), new Date('1993-03-18T08:40:00Z') ], @@ -32,7 +31,7 @@ suite.addBatch(test_helper.batch({ 86400e3: [new Date('1992-02-20T00:00:00Z'), new Date('1993-03-18T00:00:00Z') ] }}); } - } + } })); suite.export(module); diff --git a/test/metalog-test.js b/test/metalog-test.js index dc3ffaaa..29988b23 100644 --- a/test/metalog-test.js +++ b/test/metalog-test.js @@ -28,7 +28,7 @@ suite.with_log = function(batch){ } }); return suite; -} +}; suite.with_log({ '.info': { @@ -103,7 +103,7 @@ suite.with_log({ } }); -function dummy_logger(arg){}; +function dummy_logger(arg){} suite.addBatch({ 'metalog':{ diff --git a/test/metric-expression-test.js b/test/metric-expression-test.js index 54b3c45d..d1246286 100644 --- a/test/metric-expression-test.js +++ b/test/metric-expression-test.js @@ -207,7 +207,7 @@ suite.addBatch({ assert.isUndefined(e.source); }, "computes the specified value expression": function(e) { - assert.equal(e.op(2, 3), 5) + assert.equal(e.op(2, 3), 5); } }, @@ -225,7 +225,7 @@ suite.addBatch({ "a negated unary expression": { topic: parser.parse("-sum(foo)"), "negates the specified value expression": function(e) { - assert.equal(e.value(), -1) + assert.equal(e.value(), -1); }, "has the expected source": function(e) { assert.equal(e.source, "-sum(foo)"); @@ -235,7 +235,7 @@ suite.addBatch({ "constant expressions": { topic: parser.parse("-4"), "has a constant value": function(e) { - assert.equal(e.value(), -4) + assert.equal(e.value(), -4); }, "does not have a source": function(e) { assert.isUndefined(e.source); diff --git a/test/metric-test.js b/test/metric-test.js index 3ac59a7f..aded0f80 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -1,4 +1,4 @@ -'use strict' +'use strict'; var vows = require("vows"), assert = require("assert"), @@ -30,8 +30,8 @@ steps[units.hour ].description = "1-hour"; steps[units.day ].description = "1-day"; function gen_request(attrs){ - var req = { start: nowish, stop: nowish, step: units.second10, expression: 'sum(test)'} - for (var key in attrs){ req[key] = attrs[key]; }; + var req = { start: nowish, stop: nowish, step: units.second10, expression: 'sum(test)'}; + for (var key in attrs){ req[key] = attrs[key]; } return req; } @@ -39,7 +39,7 @@ function assert_invalid_request(req, expected_err) { return { topic: function(getter){ this.ret = getter(gen_request(req), this.callback); }, 'fails': function(err, val){ assert.deepEqual(err, expected_err); }, - 'returns -1': function(err, val){ assert.equal(this.ret, -1) } + 'returns -1': function(err, val){ assert.equal(this.ret, -1); } }; } @@ -59,7 +59,8 @@ suite.addBatch(test_helper.batch({ assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); assert.include([nowish10, 10000+nowish10], +result.time); } - }, + } + })).addBatch(test_helper.batch({ topic: function(test_db) { return metric.getter(test_db.db); @@ -97,7 +98,7 @@ suite.addBatch(test_helper.batch({ "unary expression": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", + stop: "2011-07-18T00:50:00.000Z" }, { 60e3: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], @@ -128,7 +129,7 @@ suite.addBatch(test_helper.batch({ "compound expression (sometimes fails due to race condition?)": metricTest({ expression: "max(test(i)) - min(test(i))", start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", + stop: "2011-07-18T00:50:00.000Z" }, { 300e3: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], 3600e3: [81, 2417], @@ -138,7 +139,7 @@ suite.addBatch(test_helper.batch({ "non-pyramidal expression": metricTest({ expression: "distinct(test(i))", start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", + stop: "2011-07-18T00:50:00.000Z" }, { 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], 3600e3: [82, 2418], @@ -148,7 +149,7 @@ suite.addBatch(test_helper.batch({ "compound pyramidal and non-pyramidal expression": metricTest({ expression: "sum(test(i)) - median(test(i))", start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", + stop: "2011-07-18T00:50:00.000Z" }, { 300e3: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], 3600e3: [3280.5, 3119138.5], @@ -158,7 +159,7 @@ suite.addBatch(test_helper.batch({ "compound with constant expression": metricTest({ expression: "-1 + sum(test)", start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z", + stop: "2011-07-18T00:50:00.000Z" }, { 300e3: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], 3600e3: [81, 2417], @@ -219,9 +220,8 @@ function metricTest(request, expected) { stop = new Date(request.stop); var subtree = { - topic: get_metrics_with_delay(0), - '(cached)': { - topic: get_metrics_with_delay(1), } + topic: get_metrics_with_delay(0), + '(cached)': { topic: get_metrics_with_delay(1) } }; subtree[request.expression] = metrics_assertions(); subtree["(cached)"][request.expression] = metrics_assertions(); @@ -249,7 +249,7 @@ function metricTest(request, expected) { } }); }, depth * step_testing_delay); - }}; + };} function metrics_assertions(){ return { 'rounds down the start time (inclusive)': function(actual) { @@ -302,7 +302,7 @@ function metricTest(request, expected) { }); } - }}; // metric assertions + };} // metric assertions } // subtree } // tree diff --git a/test/server-test.js b/test/server-test.js index 643c47bc..5ec63b5b 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -18,8 +18,8 @@ var example = { var server_options = { 'http-port': test_helper.get_port(), - 'udp-port': test_helper.get_port(), -} + 'udp-port': test_helper.get_port() +}; function dummy_server(db, endpoints){ endpoints.udp = function(req, cb){ // metalog.info('rcvd_udp', { req: req }); @@ -30,7 +30,7 @@ function dummy_server(db, endpoints){ metalog.info('rcvd_http', { req: req }); bucket.httped.push(req); }))); -}; +} suite.addBatch( test_helper.with_server(server_options, dummy_server, { diff --git a/test/test_helper.js b/test/test_helper.js index 8809497f..6589a3d5 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -11,7 +11,7 @@ var assert = require("assert"), // setup // -var test_helper = {} +var test_helper = {}; var test_db = {}; var test_collections = ["test_users", "test_events", "test_metrics"]; test_helper.inspectify = metalog.inspectify; @@ -24,7 +24,7 @@ test_helper.settings = { "mongo-database": "cube_test", "host": "localhost", "authenticator": "allow_all" -} +}; // Disable logging for tests. metalog.loggers.info = metalog.silent; // log @@ -52,7 +52,7 @@ test_helper.request = function(options, data) { var cb = this.callback; options.host = "localhost"; - if (! options.port){ options.port = this.http_port }; + if (! options.port){ options.port = this.http_port; } var request = http.request(options, function(response) { response.body = ""; @@ -78,7 +78,7 @@ test_helper.udp_request = function (data){ udp_client.send(buffer, 0, buffer.length, context.udp_port, 'localhost', function(err, val){ delayed_callback(context)(err, val); udp_client.close(); } ); }; -} +}; // proxies to the test context's callback after a short delay. // @@ -111,7 +111,7 @@ function delayed_callback(context){ return function(){ var callback_delay = 100; var args = arguments; - setTimeout(function(){ context.callback.apply(context, args) }, callback_delay) + setTimeout(function(){ context.callback.apply(context, args); }, callback_delay); }; } test_helper.delayed_callback = delayed_callback; @@ -130,8 +130,8 @@ test_helper.with_server = function(options, components, batch){ topic: function(){ start_server(options, components, this); }, '': batch, teardown: function(svr){ this.server.stop(this.callback); } - } } -} + } }; +}; // @see test_helper.with_server function start_server(options, register, vow){ @@ -165,7 +165,7 @@ test_helper.batch = function(batch) { teardown: function(test_db) { if (test_db.client.isConnected()) { process.nextTick(function(){ test_db.client.close(); }); - }; + } } } }; diff --git a/test/visualizer-test.js b/test/visualizer-test.js index 3e907597..81af59b4 100644 --- a/test/visualizer-test.js +++ b/test/visualizer-test.js @@ -7,11 +7,11 @@ var vows = require("vows"), var suite = vows.describe("visualizer"); -var server_options = { 'http-port': test_helper.get_port() } +var server_options = { 'http-port': test_helper.get_port() }; function frontend_components() { cube.evaluator.register.apply(this, arguments); cube.visualizer.register.apply(this, arguments); -}; +} // suite.addBatch( // test_helper.with_server(server_options, frontend_components, { From 865f340e9170e673786821a1d4efefb1aeb50a89 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 3 Sep 2012 05:22:39 -0500 Subject: [PATCH 31/87] DB shouldn't reconnect in test suite --- test/event-test.js | 6 +++--- test/test_helper.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/event-test.js b/test/event-test.js index 2cd9345b..8fca40cf 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -18,12 +18,12 @@ suite.addBatch(test_helper.batch({ 'invalidates': { topic: function(putter){ var ctxt = this; - putter((new Event('test2', ice_cubes_good_day, {value: 3})).to_request(), function(){ - putter((new Event('test2', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); + putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ + putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); }, 'heckya': function(){ var ts = event.invalidator().tsets(); - assert.deepEqual(ts, { 'test2': { + assert.deepEqual(ts, { 'test': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], 60e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:00Z') ], 300e3: [new Date('1992-02-20T01:05:00Z'), new Date('1993-03-18T08:40:00Z') ], diff --git a/test/test_helper.js b/test/test_helper.js index 6589a3d5..3278923d 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -200,7 +200,7 @@ function setup_db(cb){ function connect(options){ metalog.minor('cube_testdb', { state: 'connecting to db', options: options }); test_db.options = options; - test_db.client = new mongodb.Server(options["mongo-host"], options["mongo-port"], {auto_reconnect: true}); + test_db.client = new mongodb.Server(options["mongo-host"], options["mongo-port"], {auto_reconnect: false}); test_db.db = new mongodb.Db(options["mongo-database"], test_db.client, {}); } From 6578fe24c1a95236e4bf54bec346981db9074d24 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 3 Sep 2012 16:07:28 -0500 Subject: [PATCH 32/87] Reworking metric code -- objectified measurements into separate methods. WIP --- lib/cube/metric.js | 118 ++++++++++++++++++++++++++------------------ test/metric-test.js | 41 ++++++++------- 2 files changed, 92 insertions(+), 67 deletions(-) diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 5e03a5ab..641a90f4 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -21,33 +21,43 @@ exports.getter = function(db) { queueByName = {}; function getter(request, callback) { - var start = new Date(request.start), - stop = new Date(request.stop), - id = request.id; - - // Validate the dates. - if (isNaN(start)) return callback({error: "invalid start"}), -1; - if (isNaN(stop)) return callback({error: "invalid stop"}), -1; - // if (request.expression.match(/\(/mg).length > 2) return callback({error: "rejected complex expression"}), -1; - - // Parse the expression. - var expression; + var measurement, expression, + tier = tiers[+request.step], + start = new Date(request.start), + stop = new Date(request.stop); try { - expression = parser.parse(request.expression); - } catch (e) { - return callback({error: "invalid expression"}), -1; - } - - // Round start and stop to the appropriate time step. - var tier = tiers[+request.step]; - if (!tier) return callback({error: "invalid step"}), -1; - start = tier.floor(start); - stop = tier.ceil(stop); + if (!tier) throw "invalid step"; + if (isNaN(start)) throw "invalid start"; + if (isNaN(stop)) throw "invalid stop"; + // if (request.expression && (request.expression.match(/\(/mg).length > 2)){ throw("rejected complex expression"); } + + // Round start and stop to the appropriate time step. + start = tier.floor(start); + stop = tier.ceil(stop); + expression = parser.parse(request.expression); + measurement = new Measurement(expression, start, stop, tier); + } catch(err) { return callback({error: err}), -1; }; // Compute the request metric! - measure(expression, start, stop, tier, "id" in request - ? function(time, value) { callback({time: time, value: value, id: id}); } - : function(time, value) { callback({time: time, value: value}); }); + measurement.measure((("id" in request) ? + function(time, value) { callback({time: time, value: value, id: request.id}); } : + function(time, value) { callback({time: time, value: value}); })); + }; + + function Measurement(expression, start, stop, tier){ + // Round the start/stop to the tier edges + this.start = start; + this.stop = stop; + this.tier = tier; + this.expression = expression; + } + + // execute cb on each interval from t1 to t2 + function walk(t1, t2, tier, cb){ + while (t1 < t2) { + cb(t1, t2); + t1 = tier.step(t1); + } } // Computes the metric for the given expression for the time interval from @@ -55,31 +65,39 @@ exports.getter = function(db) { // by the specified tier, such as daily or hourly. The callback is invoked // repeatedly for each metric value, being passed two arguments: the time and // the value. The values may be out of order due to partial cache hits. - function measure(expression, start, stop, tier, callback) { - (expression.op ? binary : expression.type ? unary : constant)(expression, start, stop, tier, callback); + Measurement.prototype.measure = function measure(callback) { + switch(this.flavor()){ + case 'binary': this.binary(callback); break; + case 'unary': this.unary(callback); break; + case 'constant': this.constant(callback); break; + default: throw('cannot measure '+this.inspect()); + } } // Computes a constant expression; - function constant(expression, start, stop, tier, callback) { - var value = expression.value(); - while (start < stop) { - callback(start, value); - start = tier.step(start); - } - callback(stop); + Measurement.prototype.constant = function constant(callback) { + var value = this.expression.value(); + walk(this.start, this.stop, this.tier, function(time){ callback(time, value); }); + callback(this.stop); + } + + Measurement.prototype.flavor = function(){ return this.expression.op ? 'binary' : (this.expression.type ? 'unary' : 'constant'); }; + Measurement.prototype.inspect = function(){ + return { start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), tier: this.tier.key, flavor: this.flavor(), expr: this.expression.source }; } // Serializes a unary expression for computation. - function unary(expression, start, stop, tier, callback) { + Measurement.prototype.unary = function unary(callback) { + var expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; + // metalog.inspectify(this.inspect()); var remaining = 0, time0 = Date.now(), - time = start, name = expression.source, queue = queueByName[name], step = tier.key; // Compute the expected number of values. - while (time < stop){ ++remaining; time = tier.step(time); } + walk(start, stop, tier, function(time){ ++remaining; }); // If no results were requested, return immediately. if (!remaining) return callback(stop); @@ -223,24 +241,26 @@ exports.getter = function(db) { } // Computes a binary expression by merging two subexpressions. - function binary(expression, start, stop, tier, callback) { - var left = {}, right = {}; - - measure(expression.left, start, stop, tier, function(t, l) { - if (t in right) { - callback(t, t < stop ? expression.op(l, right[t]) : l); - delete right[t]; + Measurement.prototype.binary = function binary(callback) { + var expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; + var left = new Measurement(expression.left, start, stop, tier), + right = new Measurement(expression.right, start, stop, tier); + + left.measure(function(time, vall) { + if (time in right) { + callback(time, time < stop ? expression.op(vall, right[time]) : vall); + delete right[time]; } else { - left[t] = l; + left[time] = vall; } }); - measure(expression.right, start, stop, tier, function(t, r) { - if (t in left) { - callback(t, t < stop ? expression.op(left[t], r) : r); - delete left[t]; + right.measure(function(time, valr) { + if (time in left) { + callback(time, time < stop ? expression.op(left[time], valr) : valr); + delete left[time]; } else { - right[t] = r; + right[time] = valr; } }); } diff --git a/test/metric-test.js b/test/metric-test.js index aded0f80..c65275e5 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -15,19 +15,7 @@ var step_testing_delay = 250, var suite = vows.describe("metric"); var nowish = Date.now(), nowish10 = (10e3 * Math.floor(nowish/10e3)); -var steps = { - 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, - 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, - 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, - 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, - 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } -}; - -steps[units.second10].description = "10-second"; -steps[units.minute ].description = "1-minute"; -steps[units.minute5 ].description = "5-minute"; -steps[units.hour ].description = "1-hour"; -steps[units.day ].description = "1-day"; +var invalid_expression_error = { error: { message: 'Expected "(", "-", "distinct", "max", "median", "min", "sum" or number but "D" found.', column: 1, line: 1, name: 'SyntaxError' }}; function gen_request(attrs){ var req = { start: nowish, stop: nowish, step: units.second10, expression: 'sum(test)'}; @@ -50,17 +38,17 @@ suite.addBatch(test_helper.batch({ 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), - 'invalid expression': assert_invalid_request({expression: 'DANCE'}, {error: "invalid expression"}), + 'invalid expression': assert_invalid_request({expression: 'DANCE'}, invalid_expression_error), 'with request id' : { topic: function(getter){ this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(1))'}), this.callback); }, 'fires callback with id': function(result, j_){ assert.equal(result.id, 'joe'); assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); - assert.include([nowish10, 10000+nowish10], +result.time); + assert.include([nowish10, 10e3+nowish10], +result.time); } } - + })).addBatch(test_helper.batch({ topic: function(test_db) { return metric.getter(test_db.db); @@ -70,12 +58,26 @@ suite.addBatch(test_helper.batch({ 'fires callback with no id': function(result, j_){ assert.isFalse("id" in result); assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); - assert.include([nowish10, 10000+nowish10], +result.time); + assert.include([nowish10, 10e3+nowish10], +result.time); } } - })); +function skip(){ // FIXME: remove ------------------------------------------------------------ + +var steps = { + 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, + 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, + 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, + 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, + 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } +}; +steps[units.second10].description = "10-second"; +steps[units.minute ].description = "1-minute"; +steps[units.minute5 ].description = "5-minute"; +steps[units.hour ].description = "1-hour"; +steps[units.day ].description = "1-day"; + suite.addBatch(test_helper.batch({ topic: function(test_db) { var putter = event.putter(test_db.db), @@ -306,4 +308,7 @@ function metricTest(request, expected) { } // subtree } // tree +}; +skip(); + suite.export(module); From 50f759aaffd7c01508b24ef15ddfd8725213f793 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 4 Sep 2012 13:45:15 -0500 Subject: [PATCH 33/87] Made lint shut up about 'export' keyword --- .jslint.conf | 4 ++-- config/cube.js | 4 +++- lib/cube/authentication.js | 3 +-- package.json | 2 +- test/authentication-test.js | 2 +- test/collector-test.js | 2 +- test/evaluator-test.js | 2 +- test/event-expression-test.js | 2 +- test/event-test.js | 2 +- test/metalog-test.js | 2 +- test/metric-expression-test.js | 2 +- test/reduces-test.js | 2 +- test/server-test.js | 2 +- test/tiers-test.js | 2 +- test/types-test.js | 2 +- test/visualizer-test.js | 2 +- 16 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.jslint.conf b/.jslint.conf index ff0a5967..2976be24 100644 --- a/.jslint.conf +++ b/.jslint.conf @@ -15,7 +15,7 @@ +equal_as_assign # test for equality (==) mistyped as assignment (=)?{0} +var_hides_arg # variable {0} hides argument +redeclared_var # redeclaration of {0} {1} --anon_no_return_value # anonymous function does not always return a value +-anon_no_return_value ## anonymous function does not always return a value +missing_semicolon # missing semicolon +meaningless_block # meaningless block; curly braces have no impact -comma_separated_stmts ## multiple statements separated by commas (use semicolons?) @@ -108,7 +108,7 @@ # (Use this in conjunction with the "undeclared identifier" warning.) # # Common uses for webpages might be: -#+define window ++define exports #+define document diff --git a/config/cube.js b/config/cube.js index 4034bc4a..6758b618 100644 --- a/config/cube.js +++ b/config/cube.js @@ -1,5 +1,7 @@ -var configs = {}; +var configs = {}, + metalog = require('../lib/cube/metalog'); +metalog.send_events = false; // //Shared configuration diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js index 9b59447b..78eed24c 100644 --- a/lib/cube/authentication.js +++ b/lib/cube/authentication.js @@ -21,8 +21,7 @@ var mongodb = require("mongodb"), cookies = require("cookies"), bcrypt = require("bcrypt"), - metalog = require('./metalog') - ; + metalog = require('./metalog'); var authentication = {}; diff --git a/package.json b/package.json index c8e1e032..69f39dd5 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,6 @@ }, "scripts": { "preinstall": "npm install mongodb --mongodb:native", - "test": "vows --spec" + "test": "vows --isolate --spec" } } diff --git a/test/authentication-test.js b/test/authentication-test.js index 727ddf2a..c208e863 100644 --- a/test/authentication-test.js +++ b/test/authentication-test.js @@ -165,4 +165,4 @@ suite.addBatch(test_helper.batch({ })); -suite.export(module); +suite['export'](module); diff --git a/test/collector-test.js b/test/collector-test.js index 8eed18d0..908f2ef7 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -66,4 +66,4 @@ suite.addBatch( } })); -suite.export(module); +suite['export'](module); diff --git a/test/evaluator-test.js b/test/evaluator-test.js index 63787bff..aa3f92ed 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -25,4 +25,4 @@ function frontend_components() { // } // })); -suite.export(module); +suite['export'](module); diff --git a/test/event-expression-test.js b/test/event-expression-test.js index 92c6f38c..73f15758 100644 --- a/test/event-expression-test.js +++ b/test/event-expression-test.js @@ -242,4 +242,4 @@ suite.addBatch({ }); -suite.export(module); +suite['export'](module); diff --git a/test/event-test.js b/test/event-test.js index 8fca40cf..09d9c9b8 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -34,4 +34,4 @@ suite.addBatch(test_helper.batch({ } })); -suite.export(module); +suite['export'](module); diff --git a/test/metalog-test.js b/test/metalog-test.js index 29988b23..6f405da4 100644 --- a/test/metalog-test.js +++ b/test/metalog-test.js @@ -121,4 +121,4 @@ suite.addBatch({ } }}); -suite.export(module); +suite['export'](module); diff --git a/test/metric-expression-test.js b/test/metric-expression-test.js index d1246286..a1550aee 100644 --- a/test/metric-expression-test.js +++ b/test/metric-expression-test.js @@ -302,4 +302,4 @@ suite.addBatch({ }); -suite.export(module); +suite['export'](module); diff --git a/test/reduces-test.js b/test/reduces-test.js index 2447f9b9..1955aa18 100644 --- a/test/reduces-test.js +++ b/test/reduces-test.js @@ -137,4 +137,4 @@ suite.addBatch({ } }); -suite.export(module); +suite['export'](module); diff --git a/test/server-test.js b/test/server-test.js index 5ec63b5b..feec4ecb 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -103,4 +103,4 @@ suite.addBatch( })); -suite.export(module); +suite['export'](module); diff --git a/test/tiers-test.js b/test/tiers-test.js index 24f6ff75..0ebea835 100644 --- a/test/tiers-test.js +++ b/test/tiers-test.js @@ -340,4 +340,4 @@ function utc(year, month, day, hours, minutes, seconds) { return new Date(Date.UTC(year, month, day, hours || 0, minutes || 0, seconds || 0)); } -suite.export(module); +suite['export'](module); diff --git a/test/types-test.js b/test/types-test.js index c54611f4..98acb950 100644 --- a/test/types-test.js +++ b/test/types-test.js @@ -36,4 +36,4 @@ suite.addBatch(test_helper.batch({ } })); -suite.export(module); +suite['export'](module); diff --git a/test/visualizer-test.js b/test/visualizer-test.js index 81af59b4..327a04af 100644 --- a/test/visualizer-test.js +++ b/test/visualizer-test.js @@ -25,4 +25,4 @@ function frontend_components() { // } // })); -suite.export(module); +suite['export'](module); From b51881ae40ea02404266f64f6159177bebbf0452 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Tue, 4 Sep 2012 13:46:24 -0500 Subject: [PATCH 34/87] added a 'spy' helper to metalog -- wraps a callback, shows the call and args on the way back --- lib/cube/metalog.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index c127a111..c34bbac2 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -29,6 +29,11 @@ metalog.event = function(name, hsh, logger){ metalog.putter({ type: name, time: Date.now(), data: hsh }); }; +// Always goes thru to metalog.log +metalog.warn = function(name, hsh){ + metalog.log(name + "\t" + JSON.stringify(hsh)); +}; + // Events important enough for the production log file. Does not cubify. metalog.info = function(name, hsh){ metalog.loggers.info(name + "\t" + JSON.stringify(hsh)); @@ -48,4 +53,19 @@ metalog.inspectify = function inspectify(args){ util.print('----\n'); }; +// wraps a callback, inspectifies its contents as it goes by +// @example +// metalog.spy(callback, 'unary', this); +// @example you don't have to supply anything but the callback +// metalog.spy(function(err, val){ ... }) +metalog.spy = function spy(callback, name, ctxt){ + if (! name) name = 'spyable'; + if (! ctxt) ctxt = null; + return function(){ + util.print(name+': ', callback, ' called with ', util.inspect(arguments), '\n'); + // metalog.inspectify(callback, arguments); + return callback.apply(ctxt, arguments); + }; +}; + module.exports = metalog; From 84c47e4b29a93a805395d31071fd037bbb82608c Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Wed, 5 Sep 2012 14:41:11 -0500 Subject: [PATCH 35/87] Added mbox queue, to better schedule queries under load * the Broker lets you defer tasks into the queue under a chosen mailbox name. * the Broker has Workers that handle deferred jobs. Workers each handle a non-overlapping set of mailboxes. * when a worker is ready, it fires off the task with lowest mbox name, attaching its own callback. The task is responsible for pinging back the supplied proxy callback * The proxy callback will re-issue the call to all the registered on_complete callbacks. Unfortunately, this doesn't yet solve the problem -- but it's a spike in the ground --- .gitignore | 1 + lib/cube/broker.js | 159 ++++++++++++++++++++++++++++++++++++++++++++ lib/cube/metalog.js | 9 ++- test/broker-test.js | 136 +++++++++++++++++++++++++++++++++++++ test/test_helper.js | 21 +++--- 5 files changed, 313 insertions(+), 13 deletions(-) create mode 100644 lib/cube/broker.js create mode 100644 test/broker-test.js diff --git a/.gitignore b/.gitignore index 3c3629e6..d52b66ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +away/* diff --git a/lib/cube/broker.js b/lib/cube/broker.js new file mode 100644 index 00000000..7759ec02 --- /dev/null +++ b/lib/cube/broker.js @@ -0,0 +1,159 @@ +'use strict'; + +var _ = require('underscore'), + util = require('util'), + metalog = require('./metalog'); + +// milliseconds to sleep, if all jobs are satisfied, until checking again +var worker_sleep_ms = 100; + +// ---- Broker ---- + +function Broker(name, interval){ + var workers = []; + add_worker(); + this.name = name; + + // `deferProxy(name, perform, *args, on_complete)` -- Issue a request with + // controlled concurrency; send its results to all interested listeners. We + // use it so that when multiple clients are interested in a metric, we only + // issue the query once, yet share the good news immediately. + // + // Worker will at some time invoke perform with the supplied args, tacking on + // its own callback (`perform(*args, worker_cb)`, essentially). The task must, + // success or failure, eventually invoke the proxy callback. + // + // When the task triggers the proxy callback with the result, every + // `on_complete` handler waiting on that name is invoked with that result. + // For the second and further tasks `defer`ing to a given mbox, the `perform` + // function is ignored -- tasks with the same name must be interchangeable. + // + // @param [String] name -- handle for the query + // @param [Function] perform(*args) -- function for worker to dispatch + // @param [Array] *args -- args sent to `perform` when dispatched + // @param [Function] complete_cb -- called when the task completes + // + this.deferProxy = function deferProxy(){ // + var args = _.toArray(arguments), + name = args.shift(), perform = args.shift(), on_complete = args.pop(); + if (! (name && perform && on_complete)) throw new TypeError('you must supply a name, perform callback and on-complete handler: got ' + [name, perform, on_complete]); + worker_for(name).add(name, perform, args, on_complete); + return this; + }; + + function worker_for(name){ + return workers[0]; + } + + function add_worker(){ + var worker = new Worker((name+'-'+workers.length), interval); + workers.push(worker); + worker.start(); + return worker; + } + + function stop(){ _.each(workers, function(worker){ worker.stop() }) }; + + function report(){ return { workers: _.map(workers, function(worker){ return worker.report() }) }; } + function toString(){ return util.inspect(this.report()); }; + + Object.defineProperties(this, { + stop: {value: stop}, + report: {value: report}, toString: {value: toString}, + }); +} + +// ---- Worker ---- + +// A worker executes a set of tasks with parallelism 1 +function Worker(qname, interval){ + var queue = Object.create(null), + active = null, + self = this, + clock; + this.qname = qname; + + function add(mbox, perform, args, on_complete) { + var job = ((active && (active.mbox === mbox)) ? active : queue[mbox]); + if (! job){ job = queue[mbox] = new Job(mbox, perform, args); } + // + job.listen(on_complete); + metalog.minor('q_worker', {is: 'add', mbox: job.mbox, am: self.report(), job: job.report() }); + return job; + } + + function invoke(job) { + // move this job to be active + active = job; + delete queue[job.mbox]; + // add our callback as last arg. when triggered, it takes the arguments it + // was triggered with and has job fire that at all its `on_complete`s, and + // clears the active job (letting the next task start). + job.args.push(function _completed(){ + var result = arguments; + metalog.warn('q_worker', {is: '<-!', mbox: job.mbox, am: self.report(), result: util.inspect(result).slice(0,80) }); + if (job !== active) metalog.warn('q_worker', {is: 'ERR', am: qname, error: 'job was missing when callback triggered', self: self.report() }); + active = null; + job.complete(result); + }); + // start the task + metalog.warn('q_worker', {is: '?->', mbox: job.mbox, am: self.report(), perform_args: util.inspect(job.args).slice(0,80) }); + try{ job.perform.apply(null, job.args); } catch(err){ metalog.warn('q_worker', {is: 'ERR', am: qname, as: 'performing job '+job.mbox, error: err.message }) }; + } + + function start(){ + if (clock){ return metalog.warn('q_worker', {is: 'ERR', am: self.report(), error: 'tried to start an already-running worker' }); } + clock = setInterval(self.work, interval); + } + function stop(){ + metalog.info('q_worker', {is: 'stp'}); + if (! clock){ return metalog.warn('q_worker', {is: 'ERR', am: self.report(), error: 'tried to stop an already-stopped worker' }); } + clearInterval(clock); + clock = null; + } + + function work (){ + var job; + if (active) { self.onWait(); } + else if (job = next()){ self.invoke(job); } + else { self.onIdle(); } + }; + + function size(){ return _.size(queue); } + function next(){ return queue[ _.keys(queue).sort()[0] ]; } + + // function onIdle(){ util.print(' '+self.qname+'!'); }; + // function onWait(){ util.print(' '+self.qname+'@'); }; + function onIdle(){ }; + function onWait(){ }; + + function report(){ return { qname: this.qname, size: this.size, queue: _.keys(this.queue) }; }; + function toString(){ return util.inspect(this.report()); }; + + Object.defineProperties(this, { + add: {value: add}, size: {get: size}, + start: {value: start}, stop: {value: stop}, + report: {value: report}, toString: {value: toString}, + onWait: {value: onWait}, onIdle: {value: onIdle} + }); +} + +// ---- Job ---- + +function Job(mbox, perform, args){ + _.extend(this, { mbox: mbox, perform: perform, args: args, on_completes: [] }); +} +Job.prototype.complete = function(result){ + for (var ii in this.on_completes){ + process.nextTick(function(){ this.on_completes[ii].apply(null, result); }); + }; +} +Job.prototype.listen = function(on_complete){ + this.on_completes.push(on_complete); +}; +Job.prototype.toString = function (){ return util.inspect(this.report()); }; +Job.prototype.report = function (){ return this; } + +// ---- + +module.exports = { Broker: Broker, Job: Job, Worker: Worker }; diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index c34bbac2..327a0ed7 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -1,6 +1,7 @@ 'use strict'; -var util = require("util"); +var util = require("util"), + _ = require("underscore"); var metalog = { putter: null, @@ -31,7 +32,7 @@ metalog.event = function(name, hsh, logger){ // Always goes thru to metalog.log metalog.warn = function(name, hsh){ - metalog.log(name + "\t" + JSON.stringify(hsh)); + metalog.log(name + "\t" + (+Date.now()) + "\t" + JSON.stringify(hsh)); }; // Events important enough for the production log file. Does not cubify. @@ -48,7 +49,9 @@ metalog.minor = function(name, hsh){ metalog.inspectify = function inspectify(args){ for (var idx in arguments) { util.print(idx + ": "); - util.print(util.inspect(arguments[idx])+"\n"); + var val = arguments[idx]; + if (_.isFunction(val)) val = val.toString().slice(0, 80); + util.print(util.inspect(val, false, 2, true)+"\n"); // , null, true } util.print('----\n'); }; diff --git a/test/broker-test.js b/test/broker-test.js new file mode 100644 index 00000000..0f092728 --- /dev/null +++ b/test/broker-test.js @@ -0,0 +1,136 @@ +'use strict'; + +var _ = require('underscore'), + util = require("util"), + vows = require("vows"), + assert = require("assert"), + test_helper = require("./test_helper"), + broker = require("../lib/cube/broker"), + Job = broker.Job, Broker = broker.Broker, Worker = broker.Worker, + metalog = require('../lib/cube/metalog'); + +var suite = vows.describe("broker"); + +var squarer = function(ii, cb){ cb(null, ii*ii, 'squarer'); }; + +assert.isCalledTimes = function(ctxt, reps){ + var results = []; + setTimeout(function(){ ctxt.callback(new Error('timeout: need '+reps+' results only have '+util.inspect(results))); }, 2000); + return function _is_called_checker(){ + results.push(_.toArray(arguments)); + if (results.length >= reps) ctxt.callback(null, results); + }; +} +assert.isNotCalled = function(name){ + return function(){ throw new Error(name + ' should not have been called, but was'); }; +}; + +function example_worker(paused){ + var worker = new Worker('worker', 50); + // worker.idle = _.identity; + if (! paused) worker.start(); + return worker; +} + + +function example_job(){ return (new Job('smurf', squarer, [7])); }; + +suite.addBatch({ + // 'Worker': { + // topic: example_worker, + // // '.new': { + // // '': function(worker){ + // // test_helper.inspectify('new worker', worker, worker); + // // worker.add('smurfette', squarer, [3], metalog.inspectify); + // // }, + // // }, + // '.invoke': { + // topic: function(worker){ + // var ctxt = this; + // ctxt.checker = assert.isCalledTimes(ctxt, 3); + // ctxt.performances = 0; + // // shortly, worker will invoke perfom (once). 200 ms later, `perform` + // // will call `worker`'s proxy callback, which invokes all 3 callbacks. + // var perform = function(cb){ ctxt.performances++; setTimeout(function(){cb('hi')}, 200); }; + // worker.add('thrice', perform, [], ctxt.checker); + // worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); + // worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); + // this.worker = worker; + // }, + // 'calls perform exactly once': function(){ assert.equal(this.performances, 1); }, + // 'calls all registered callbacks': function(results){ assert.deepEqual(results, [['hi'], ['hi'], ['hi']]) }, + // teardown: function(){ + // this.worker.stop(); + // } + // } + // }, + + // 'Job': { + // '.new': { + // topic: example_job, + // '': function(job){ + // assert.deepEqual(job, {name: 'smurf', perform: squarer, args: [7], on_completes: []}); + // }, + // }, + // '.listen': { + // topic: function(){ + // var job = example_job(); + // job.listen(squarer); + // return job; + // }, + // '': function(job){ + // test_helper.inspectify(job, job.toString()); + // } + // }, + // // '.add': { + // // topic: example_job, + // // '': function(job){ + // // } + // // }, + // }, + + 'Broker': { + 'handles interleaved jobs': { + topic: function(){ + var ctxt = this, + broker = this.broker = new Broker('test', 10), + ignored = assert.isNotCalled('perform'); + ctxt.perfs = {a: 0, b: 0, c:0}; + ctxt.checker = assert.isCalledTimes(ctxt, 8); + var task_a = function(ii, a2, cb){ ctxt.perfs.a++; setTimeout(function(){cb('result_a', ii*ii, a2)}, 10); }; + var task_b = function(ii, cb){ ctxt.perfs.b++; setTimeout(function(){cb('result_b', ii*ii )}, 20); }; + var task_c = function(ii, cb){ ctxt.perfs.c++; setTimeout(function(){cb('result_c', ii*ii )}, 300); }; + // will go second: jobs are sorted + broker.deferProxy('task_b', task_b, 1, ctxt.checker); + broker.deferProxy('task_b', ignored, '<>', ctxt.checker); + // will go first + broker.deferProxy('task_a', task_a, 0, '?', ctxt.checker); + broker.deferProxy('task_a', ignored, '<>', ctxt.checker); + // will go third + broker.deferProxy('task_c', task_c, 2, ctxt.checker); + // a & b will be done; c (takes 300ms) will still be running. + setTimeout(function(){ + broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); + broker.deferProxy('task_c', ignored, '<>', ctxt.checker); + broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); + }, 200); + }, + 'calls perform exactly once': function(){ assert.deepEqual(this.perfs, {a: 2, b: 1, c: 1}); }, + 'calls all registered callbacks': function(results){ + assert.deepEqual(results, [ + ['result_a', 0, '?'], ['result_a', 0, '?'], + ['result_b', 1], ['result_b', 1], + ['result_c', 4], ['result_c', 4], + ['result_a', 9, '!'], ['result_a', 9, '!'] + ]) + }, + teardown: function(){ + this.broker.stop(); + } + } + }, +}); + + + +suite['export'](module); diff --git a/test/test_helper.js b/test/test_helper.js index 3278923d..e0ef243c 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -73,16 +73,16 @@ test_helper.udp_request = function (data){ return function(){ var udp_client = dgram.createSocket('udp4'); var buffer = new Buffer(JSON.stringify(data)); - var context = this; + var ctxt = this; metalog.info('sending_udp', { data: data }); - udp_client.send(buffer, 0, buffer.length, context.udp_port, 'localhost', - function(err, val){ delayed_callback(context)(err, val); udp_client.close(); } ); + udp_client.send(buffer, 0, buffer.length, ctxt.udp_port, 'localhost', + function(err, val){ delay(ctxt.callback, ctxt)(err, val); udp_client.close(); } ); }; }; // proxies to the test context's callback after a short delay. // -// @example as a test topic; will get the same data the cb otherwise would have: +// @example the test topic introduces a delay; the 'is party time' vow gets the same data the cb otherwise would have: // { topic: send_some_data, // 'a short time later': { // topic: test_helper.delaying_topic, @@ -91,7 +91,7 @@ test_helper.udp_request = function (data){ function delaying_topic(){ var args = Array.prototype.slice.apply(arguments); args.unshift(null); - delayed_callback(this).apply(this, args); + delay(this.callback, this).apply(this, args); } test_helper.delaying_topic = delaying_topic; @@ -100,21 +100,22 @@ test_helper.delaying_topic = delaying_topic; // // @example // // you -// dcb = delayed_callback(this) +// dcb = delay(this) // foo.do_something('...', dcb); // // foo, after do_something'ing, invokes the delayed callback // dcb(null, 1, 2); // // 50ms later, dcb does the equivalent of // this.callback(null, 1, 2); // -function delayed_callback(context){ +function delay(orig_cb, ctxt, ms){ + ctxt = ctxt || null; + ms = ms || 100; return function(){ - var callback_delay = 100; var args = arguments; - setTimeout(function(){ context.callback.apply(context, args); }, callback_delay); + setTimeout(function(){ orig_cb.apply(ctxt, args); }, ms); }; } -test_helper.delayed_callback = delayed_callback; +test_helper.delay = delay; // test_helper.with_server -- // start server, run tests once server starts, stop server when tests are done From 4151aabb0023b65180f0554ccb4e334ba34e66e9 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 6 Sep 2012 14:49:47 -0500 Subject: [PATCH 36/87] Extract database interactions into separate file. Add config to allow splitting metrics and event into separate databases --- config/cube.js | 14 +-- lib/cube/authentication.js | 5 +- lib/cube/db.js | 169 +++++++++++++++++++++++++++++++++++++ lib/cube/evaluator.js | 6 +- lib/cube/event.js | 131 ++++++++++++---------------- lib/cube/metalog.js | 2 +- lib/cube/metric.js | 62 ++++++++------ lib/cube/models.js | 12 ++- lib/cube/server.js | 42 +++------ lib/cube/types.js | 50 ----------- lib/cube/visualizer.js | 21 +++-- lib/cube/warmer.js | 43 +++++----- test/test_helper.js | 47 ++++++----- test/types-test.js | 39 --------- 14 files changed, 352 insertions(+), 291 deletions(-) create mode 100644 lib/cube/db.js delete mode 100644 lib/cube/types.js delete mode 100644 test/types-test.js diff --git a/config/cube.js b/config/cube.js index 6758b618..5948b01a 100644 --- a/config/cube.js +++ b/config/cube.js @@ -13,14 +13,16 @@ configs.common = { "mongo-username": null, "mongo-password": null, "mongo-server_options": {auto_reconnect: true, poolSize: 8, socketOptions: { noDelay: true }}, - + "mongo-metrics": {autoIndexId: true, capped: false }, "mongo-events": {autoIndexId: true, capped: true, size: 1e9 }, - - // "horizons": { - // "calculation": 1000 * 60 * 60 * 2, // 2 hours - // "invalidation": 1000 * 60 * 60 * 1, // 1 hour - // } + + "separate-events-database": true, + + "horizons": { + "calculation": 1000 * 60 * 60 * 2, // 2 hours + "invalidation": 1000 * 60 * 60 * 1, // 1 hour + } }; diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js index 78eed24c..a0b5c750 100644 --- a/lib/cube/authentication.js +++ b/lib/cube/authentication.js @@ -21,9 +21,8 @@ var mongodb = require("mongodb"), cookies = require("cookies"), bcrypt = require("bcrypt"), - metalog = require('./metalog'); - -var authentication = {}; + metalog = require('./metalog'), + authentication = {}; authentication.authenticator = function(strategy, db, options){ metalog.minor('cube_auth', { strategy: strategy }); diff --git a/lib/cube/db.js b/lib/cube/db.js new file mode 100644 index 00000000..05937cc3 --- /dev/null +++ b/lib/cube/db.js @@ -0,0 +1,169 @@ +'use strict'; + +var mongodb = require("mongodb"), + metalog = require("./metalog"), + metric_options, event_options, + database, events_db, metrics_db; + +module.exports = { + open: open, + isConnected: isConnected +} + + +// +// Connect to mongodb. +// + +function open(options, callback){ + var server_options = options["mongo-server_options"], // MongoDB server configuration + db_options = { native_parser: true }, // MongoDB driver configuration. + mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), + database_name = options["mongo-database"], + mongo_password = options["mongo-password"]; + + metric_options = options["mongo-metrics"], + event_options = options["mongo-events"], + database = new mongodb.Db(database_name, mongo, db_options); + + delete options["mongo-password"]; + metalog.info('cube_life', {is: 'mongo_connect', options: options }); + + database.open(function(error){ + if (error) return callback(error); + + // Open separate events database if configured + if (options["separate-events-database"]) events_db = database.db(database_name + '-events'); + else events_db = database; + + // Open separate metrics database if configured + if (options["separate-metrics-database"]) metrics_db = database.db(database_name + '-metrics'); + else metrics_db = database; + + module.exports.metrics = metrics(metrics_db); + module.exports.events = events(events_db); + module.exports.types = types(events_db); + module.exports.collection = collection(database); + module.exports.close = close; + + if (! options["mongo-username"]) return callback(null, module.exports); + database.authenticate(options["mongo-username"], mongo_password, function(error, success) { + if (error) return callback(error); + if (!success) return callback(new Error("authentication failed")); + callback(null, module.exports); + }); + }); +} + + +// +// Close connection to mongodb. +// + +function close(callback){ + delete module.exports.metrics, + delete module.exports.events, + delete module.exports.types, + delete module.exports.collection, + delete module.exports.close; + + database.close(callback); +} + +function isConnected(){ + try { + return database.serverConfig.isConnected(); + } catch (e){ + return false; + } +} + + +// Much like db.collection, but caches the result for both events and metrics. +// Also, this is synchronous, since we are opening a collection unsafely. +function metrics(database) { + var collections = {}; + + return function(name, callback){ + if(collections[name]) return callback(null, collections[name]); + + var collection_name = name + "_metrics", + _this = this; + database.createCollection(collection_name, metric_options||{safe: false}, function(error, metrics) { + + if (error && error.errmsg == "collection already exists") return _this.metrics(name, callback); + if (error) return callback(error); + + metrics.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); + metrics.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, handle); + + metrics = overwrite_find(metrics, 'metrics'); + collections[name] = metrics; + return callback(null, metrics); + }); + } +} + +function events(database) { + var collections = {}; + + return function(name, callback){ + if (collections[name]) return callback(null, collections[name]); + + var collection_name = name + "_events"; + + // Create a collection for events. One index is require, for finding events by time(t) + database.createCollection(collection_name, event_options||{safe: false}, function(error, events){ + + if (error && error.errmsg == "collection already exists") return _this.metrics(name, callback); + if(error) return callback(error); + + events.ensureIndex({"t": 1}, handle); + + events = overwrite_find(events, 'events'); + collections[name] = events; + return callback(null, events); + }); + } +} + +function collection(database){ + var collections = {}; + + return function(name, callback){ + if (collections[name]) return callback(null, collections[name]); + database.collection.apply(database, arguments); + } +} + +function overwrite_find(collection, type){ + var orig_find = collection.find; + collection.find = function(){ + metalog.minor('cube_query', {is: type + '_find', query: arguments}); + return orig_find.apply(this, arguments); + }; + return collection; +} + +var eventRe = /_events$/; + +function types(database) { + return function(request, callback) { + database.collectionNames(function(error, names) { + handle(error); + callback(names + .map(function(d) { return d.name.split(".")[1]; }) + .filter(function(d) { return eventRe.test(d); }) + .map(function(d) { return d.substring(0, d.length - 7); }) + .sort()); + }); + }; +}; + +function handle(error){ + if (!error) return; + metalog.info('cube_request', {is: 'db error', error: error }); + throw error; +} + +function noop(){}; diff --git a/lib/cube/evaluator.js b/lib/cube/evaluator.js index 9cfbf7c0..fa53675e 100644 --- a/lib/cube/evaluator.js +++ b/lib/cube/evaluator.js @@ -16,10 +16,10 @@ var headers = { }; exports.register = function(db, endpoints) { - var event = require("./event").getter(db), + var event = require("./event").getter(db), event_putter = require("./event").putter(db), - metric = require("./metric").getter(db), - types = require("./types").getter(db); + metric = require("./metric").getter(db), + types = db.types; // endpoints.ws.push( diff --git a/lib/cube/event.js b/lib/cube/event.js index 01767fb1..ab982275 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -4,19 +4,15 @@ // TODO allow the event time to change when updating (fix invalidation) var _ = require("underscore"), - mongodb = require("mongodb"), - tiers = require("./tiers"), - types = require("./types"), - models = require("./models"), Event = models.Event, Metric = models.Metric, - parser = require("./event-expression"), - bisect = require("./bisect"), - metalog = require("./metalog"), - options = require("../../config/cube"), + tiers = require("./tiers"), + models = require("./models"), Event = models.Event, Metric = models.Metric, + parser = require("./event-expression"), + bisect = require("./bisect"), + metalog = require("./metalog"), + mongodb = require("mongodb"), + //options = require("../../config/cube"), ObjectID = mongodb.ObjectID; -var metric_options = options["mongo-metrics"], - event_options = options["mongo-events"]; - // When streaming events, we should allow a delay for events to arrive, or else // we risk skipping events that arrive after their event.time. This delay can be // customized by specifying a `delay` property as part of the request. @@ -41,13 +37,16 @@ function Invalidator(){ for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true; } }; - this.flush = function(collection){ + this.flush = function(db){ _.each(type_tsets, function(type_tset, type){ - var metrics = collection(type).metrics; - _.each(type_tset, function(tset, tier){ - var times = dateify(tset); - metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: times }); - metrics.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); + db.metrics(type, function(error, collection){ + handle(error); + + _.each(type_tset, function(tset, tier){ + var times = dateify(tset); + metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: times }); + collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); + }); }); }); }; @@ -77,8 +76,7 @@ exports.invalidator = function(){ return invalidator; }; // - data, the event's payload // exports.putter = function(db){ - var collection = types(db), - knownByType = {}, + var knownByType = {}, eventsToSaveByType = {}; function putter(request, callback) { @@ -97,10 +95,8 @@ exports.putter = function(db){ var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } - event.on_save = callback; - // If this is a known event type, save immediately. - if (type in knownByType) return save(type, event); + if (type in knownByType) return event.save(after_save); // If someone is already creating the event collection for this new type, // then append this event to the queue for later save. @@ -112,60 +108,35 @@ exports.putter = function(db){ // First add the new event to the queue. eventsToSaveByType[type] = [event]; + event.save(function(error, event){ + after_save(error, event); // trigger normal callback - // If the events collection exists, then we assume the metrics & indexes do - // too. Otherwise, we must create the required collections and indexes. Note - // that if you want to customize the size of the capped metrics collection, - // or add custom indexes, you can still do all that by hand. - db.collectionNames(type + "_events", function(error, names) { - handle(error); - var events = collection(type).events; - if (names.length) return saveEvents(); - - // Create a collection for events. One index is require, for finding events by time(t) - db.createCollection(type + "_events", event_options, function(error, events){ - handle(error); - events.ensureIndex({"t": 1}, handle); - - // Create a collection for metrics. Three indexes are required: one - // for finding metrics, one (_id) for updating, and one for invalidation. - db.createCollection(type + "_metrics", metric_options, function(error, metrics) { - handle(error); - metrics.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); - metrics.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, handle); - saveEvents(); - }); - }); - - // Save any pending events to the new collection. - function saveEvents() { - knownByType[type] = true; - eventsToSaveByType[type].forEach(function(event) { save(type, event); }); - delete eventsToSaveByType[type]; - } + knownByType[type] = true; + eventsToSaveByType[type].forEach(function(event) { event.save(after_save); }); + delete eventsToSaveByType[type]; }); - } - // Save the event of the specified type, and queue invalidation of any cached - // metrics associated with this event type and time. - // - // We don't invalidate the events immediately. This would cause many redundant - // updates when many events are received simultaneously. Also, having a short - // delay between saving the event and invalidating the metrics reduces the - // likelihood of a race condition between when the events are read by the - // evaluator and when the newly-computed metrics are saved. - function save(type, event) { - // metalog.minor("cube_request", { is: "ev", at: "save", type: type, event: event }); - collection(type).events.save(event, handle); - invalidator.add(type, event); - if (event.on_save) event.on_save(null); + // After saving the event, queue invalidation of any cached + // metrics associated with this event. + // + // We don't invalidate the events immediately. This would cause many redundant + // updates when many events are received simultaneously. Also, having a short + // delay between saving the event and invalidating the metrics reduces the + // likelihood of a race condition between when the events are read by the + // evaluator and when the newly-computed metrics are saved. + function after_save(error, event) { + // metalog.minor("cube_request", { is: "ev", at: "save", type: type, event: event }); + handle(error); + invalidator.add(event._type(), event); + if(callback) callback(null, event); + } } // Process any deferred metric invalidations, flushing the queues. Note that // the queue (timesToInvalidateByTierByType) is copied-on-write, so while the // previous batch of events are being invalidated, new events can arrive. Invalidator.start_flusher(function(){ - invalidator.flush(collection); + invalidator.flush(db); invalidator = new Invalidator(); // copy-on-write }); @@ -186,8 +157,7 @@ exports.putter = function(db){ // * if streaming, register the query to be run at a regular interval // exports.getter = function(db) { - var collection = types(db), - streamsBySource = {}; + var streamsBySource = {}; function getter(request, callback) { var stream = !("stop" in request), @@ -223,20 +193,23 @@ exports.getter = function(db) { // Query for the desired events. function query(callback) { - collection(expression.type).events.find(filter, fields, options, function(error, cursor) { + db.events(expression.type, function(error, collection){ handle(error); - cursor.each(function(error, event) { + collection.find(filter, fields, options, function(error, cursor) { + handle(error); + cursor.each(function(error, event) { - // If the callback is closed (i.e., if the WebSocket connection was - // closed), then abort the query. Note that closing the cursor mid- - // loop causes an error, which we subsequently ignore! - if (callback.closed) return cursor.close(); + // If the callback is closed (i.e., if the WebSocket connection was + // closed), then abort the query. Note that closing the cursor mid- + // loop causes an error, which we subsequently ignore! + if (callback.closed) return cursor.close(); - handle(error); + handle(error); - // A null event indicates that there are no more results. - if (event) callback({id: event._id instanceof ObjectID ? undefined : event._id, time: event.t, data: event.d}); - else callback(null); + // A null event indicates that there are no more results. + if (event) callback({id: event._id instanceof ObjectID ? undefined : event._id, time: event.t, data: event.d}); + else callback(null); + }); }); }); } diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 327a0ed7..40edf0c7 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -14,7 +14,7 @@ var metalog = { // metalog.loggers.info = metalog.silent(); metalog.loggers = { info: metalog.log, - minor: metalog.silent + minor: metalog.log }; // if true, cubify `metalog.event`s diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 641a90f4..13b1294d 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -4,7 +4,8 @@ var parser = require("./metric-expression"), tiers = require("./tiers"), - types = require("./types"), + db = require("./db"), + mongodb = require("mongodb"), reduces = require("./reduces"), event = require("./event"), metalog = require('./metalog'), @@ -16,8 +17,7 @@ var metric_fields = {v: 1}, // Query for metrics. exports.getter = function(db) { - var collection = types(db), - Double = db.bson_serializer.Double, + var Double = mongodb.Double, queueByName = {}; function getter(request, callback) { @@ -30,7 +30,7 @@ exports.getter = function(db) { if (isNaN(start)) throw "invalid start"; if (isNaN(stop)) throw "invalid stop"; // if (request.expression && (request.expression.match(/\(/mg).length > 2)){ throw("rejected complex expression"); } - + // Round start and stop to the appropriate time step. start = tier.floor(start); stop = tier.ceil(stop); @@ -128,12 +128,12 @@ exports.getter = function(db) { // Finds or computes a unary (primary) expression. function findOrComputeUnary(expression, start, stop, tier, callback) { - var name = expression.type, - type = collection(name), - map = expression.value, + var name = expression.type, + map = expression.value, reduce = reduces[expression.reduce], filter = {t: {}}, - fields = {t: 1}; + fields = {t: 1}, + metrics, events; // if (!reduce) return callback({error: "invalid reduce operation"}), -1; @@ -143,7 +143,12 @@ exports.getter = function(db) { // Request any needed fields. expression.fields(fields); - find(start, stop, tier, callback); + db.metrics(name, function(error, collection){ + handle(error); + + metrics = collection; + find(start, stop, tier, callback); + }); // The metric is computed recursively, reusing the above variables. function find(start, stop, tier, callback) { @@ -151,7 +156,7 @@ exports.getter = function(db) { step = tier.key; // Query for the desired metric in the cache. - type.metrics.find({ + metrics.find({ i: false, "_id.e": expression.source, "_id.l": tier.key, @@ -200,26 +205,31 @@ exports.getter = function(db) { // } filter.t.$gte = start; filter.t.$lt = stop; - type.events.find(filter, fields, event_options, function(error, cursor) { + + db.events(name, function(error, collection){ handle(error); - var time = start, values = []; - cursor.each(function(error, row) { + + collection.find(filter, fields, event_options, function(error, cursor) { handle(error); - if (row) { - var then = tier.floor(row.t); - if (time < then) { - save(time, values.length ? reduce(values) : reduce.empty); - while ((time = tier.step(time)) < then) save(time, reduce.empty); - values = [map(row)]; + var time = start, values = []; + cursor.each(function(error, row) { + handle(error); + if (row) { + var then = tier.floor(row.t); + if (time < then) { + save(time, values.length ? reduce(values) : reduce.empty); + while ((time = tier.step(time)) < then) save(time, reduce.empty); + values = [map(row)]; + } else { + values.push(map(row)); + } } else { - values.push(map(row)); + save(time, values.length ? reduce(values) : reduce.empty); + while ((time = tier.step(time)) < stop) save(time, reduce.empty); } - } else { - save(time, values.length ? reduce(values) : reduce.empty); - while ((time = tier.step(time)) < stop) save(time, reduce.empty); - } + }); }); - }); + }) } function save(time, value) { @@ -235,7 +245,7 @@ exports.getter = function(db) { v: new Double(value) }; metalog.info('cube_compute', {is: 'metric_save', metric: metric }); - type.metrics.save(metric, handle); + metrics.save(metric, handle); } } } diff --git a/lib/cube/models.js b/lib/cube/models.js index 25f0ccbc..79608b2e 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -1,7 +1,8 @@ 'use strict'; var _ = require("underscore"), - metalog = require('./metalog'); + metalog = require('./metalog'), + db = require('./db'); var second = 1e3, second10 = 10e3, @@ -55,12 +56,21 @@ function Event(type, time, data, id){ if (id) ev._id = this._id; return ev; }; + this.to_request = function(attrs){ var ev = { time: this.t, data: this.d, type: type }; if (id) ev.id = this._id; for (var key in attrs){ ev[key] = attrs[key]; } return ev; }; + + this.save = function(callback){ + var _this = this; + db.events(type, function(error, collection){ + if (error) return callback(error); + collection.save(_this.to_wire(), function(error){ return callback(error, _this); }); + }); + }; } exports.Event = Event; diff --git a/lib/cube/server.js b/lib/cube/server.js index f6fc319f..5534d28d 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -13,17 +13,17 @@ // * the UDP listener connection // -var util = require("util"), - url = require("url"), - http = require("http"), - dgram = require("dgram"), - websocket = require("websocket"), - websprocket = require("websocket-server"), - file_server = require("node-static"), - mongodb = require("mongodb"), +var util = require("util"), + url = require("url"), + http = require("http"), + dgram = require("dgram"), + websocket = require("websocket"), + websprocket = require("websocket-server"), + file_server = require("node-static"), authentication = require("./authentication"), - event = require("./event"), - metalog = require("./metalog"); + event = require("./event"), + metalog = require("./metalog"), + db = require("./db"); // Don't crash on errors. process.on("uncaughtException", function(error) { @@ -47,9 +47,6 @@ var wsOptions = { closeTimeout: 5000 }; -// MongoDB driver configuration. -var db_options = { native_parser: true }; - module.exports = function(options) { var server = {}, primary = http.createServer(), @@ -57,9 +54,6 @@ module.exports = function(options) { file = new file_server.Server("static"), udp, endpoints = {ws: [], http: []}, - server_options = options["mongo-server_options"], - mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), - db = new mongodb.Db(options["mongo-database"], mongo, db_options), id = 0, authenticator; @@ -178,21 +172,13 @@ module.exports = function(options) { }); server.start = function(server_start_cb) { - // Connect to mongodb. - var mongo_password = options["mongo-password"]; delete options["mongo-password"]; - metalog.info('cube_life', {is: 'mongo_connect', options: options }); - db.open(function(error) { - if (error) throw error; - if (! options["mongo-username"]) return ready(); - db.authenticate(options["mongo-username"], mongo_password, function(error, success) { - if (error) throw error; - if (!success) throw new Error("authentication failed"); - ready(); - }); + db.open(options, function(error, db){ + if(error) throw error; + ready(db); }); // Start the server! - function ready() { + function ready(db) { metalog.putter = event.putter(db); server.register(db, endpoints); authenticator = authentication.authenticator(options["authenticator"], db, options); diff --git a/lib/cube/types.js b/lib/cube/types.js deleted file mode 100644 index 4ad8b87d..00000000 --- a/lib/cube/types.js +++ /dev/null @@ -1,50 +0,0 @@ -'use strict'; - -// Much like db.collection, but caches the result for both events and metrics. -// Also, this is synchronous, since we are opening a collection unsafely. -var types = module.exports = function(db) { - var collections = {}, - metalog = require('./metalog'); - return function(type) { - var collection = collections[type]; - if (!collection) { - collection = collections[type] = {}; - db.collection(type + "_events", function(error, events) { - var orig_find = events.find; - events.find = function(){ - metalog.minor('cube_query', {is: 'events_find', query: arguments}); - return orig_find.apply(this, arguments); - }; - collection.events = events; - }); - db.collection(type + "_metrics", function(error, metrics) { - var orig_find = metrics.find; - metrics.find = function(){ - metalog.minor('cube_query', {is: 'metrics_find', query: arguments}); - return orig_find.apply(this, arguments); - }; - collection.metrics = metrics; - }); - } - return collection; - }; -}; - -var eventRe = /_events$/; - -types.getter = function(db) { - return function(request, callback) { - db.collectionNames(function(error, names) { - handle(error); - callback(names - .map(function(d) { return d.name.split(".")[1]; }) - .filter(function(d) { return eventRe.test(d); }) - .map(function(d) { return d.substring(0, d.length - 7); }) - .sort()); - }); - }; -}; - -function handle(error) { - if (error) throw error; -} diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js index f6f1c2a9..baa5f795 100644 --- a/lib/cube/visualizer.js +++ b/lib/cube/visualizer.js @@ -16,21 +16,20 @@ function viewBoard(db) { boardsByCallback = {}, callbacksByBoard = {}; - db.collection("boards", function(error, collection) { - boards = collection; - }); - function dispatch(request, callback) { if (request.type != 'ping') metalog.info("cube_request", { is: request.type, bd: request.id, pc: callback.id }); request.id = require('mongodb').ObjectID(request.id); - switch (request.type) { - case "load": load(request, callback); break; - case "add": add(request, callback); break; - case "edit": case "move": move(request, callback); break; - case "remove": remove(request, callback); break; - default: callback({type: "error", status: 400}); break; - } + db.collection("boards", function(error, collection) { + boards = collection; + switch (request.type) { + case "load": load(request, callback); break; + case "add": add(request, callback); break; + case "edit": case "move": move(request, callback); break; + case "remove": remove(request, callback); break; + default: callback({type: "error", status: 400}); break; + } + }); } function check_authorization(request, action){ diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index b47ef02d..f8c82c49 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -1,34 +1,31 @@ 'use strict'; var cluster = require('cluster'), - mongodb = require('mongodb'), metric = require('./metric'), + db = require('./db'), tiers = require('./tiers'), metalog = require('./metalog'); module.exports = function(options){ - var db, mongo, calculate_metric, boards, tier; + var calculate_metric, tier; function fetch_metrics(callback){ var expressions = []; - if(!boards){ - db.collection("boards", function(error, collection) { boards = collection; fetch_metrics(callback); }); - return; - } - - boards.find({}, {pieces: 1}, function(error, cursor) { - if (error) throw error; - cursor.each(function(error, row) { + db.collection("boards", function(error, collection){ + collection.find({}, {pieces: 1}, function(error, cursor) { if (error) throw error; - if (row) { - expressions.splice.apply(expressions, [0, 0].concat(row.pieces - .map(function(piece){ return piece.query; }) - .filter(function(expression){ return expression && !(expression in expressions); }) - )); - } else { - callback(expressions); - } + cursor.each(function(error, row) { + if (error) throw error; + if (row) { + expressions.splice.apply(expressions, [0, 0].concat(row.pieces + .map(function(piece){ return piece.query; }) + .filter(function(expression){ return expression && !(expression in expressions); }) + )); + } else { + callback(expressions); + } + }); }); }); } @@ -41,22 +38,22 @@ module.exports = function(options){ metalog.info('cube_warm', {is: 'warm_metric', metric: {query: expressions}, start: start, stop: stop }); // fake metrics request - calculate_metric({ step: tier.key, expression: expression, start: start, stop: stop }, function(){}); + calculate_metric({ step: tier.key, expression: expression, start: start, stop: stop }, noop); }); setTimeout(function(){ fetch_metrics(process_metrics); }, options['warmer-interval']); } + function noop(){}; + return { start: function(){ - var mongo = new mongodb.Server(options['mongo-host'], options['mongo-port']), - db = new mongodb.Db(options["mongo-database"], mongo), - tier = tiers[options['warmer-tier'].toString()]; + tier = tiers[options['warmer-tier'].toString()]; if(typeof tier === "undefined") throw new Error("Undefined warmer tier configured: " + options['warmer-tier']); metalog.event("cube_life", { is: 'start_warmer', options: options }); - db.open(function(error) { + db.open(options, function(error) { if (error) throw error; calculate_metric = metric.getter(db); fetch_metrics(process_metrics); diff --git a/test/test_helper.js b/test/test_helper.js index e0ef243c..d6a71b29 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -3,7 +3,7 @@ var assert = require("assert"), http = require("http"), dgram = require('dgram'), - mongodb = require("mongodb"), + db = require("../lib/cube/db"), metalog = require("../lib/cube/metalog"); // ========================================================================== @@ -29,6 +29,7 @@ test_helper.settings = { // Disable logging for tests. metalog.loggers.info = metalog.silent; // log metalog.loggers.minor = metalog.silent; // log +metalog.loggers.warn = metalog.silent; // log metalog.send_events = false; // ========================================================================== @@ -159,13 +160,15 @@ test_helper.batch = function(batch) { return { "": { topic: function() { - connect(test_helper.settings); - setup_db(this.callback); + var _this = this; + connect(test_helper.settings, function(error, db){ + setup_db(_this.callback); + }); }, "": batch, teardown: function(test_db) { - if (test_db.client.isConnected()) { - process.nextTick(function(){ test_db.client.close(); }); + if (test_db.db.isConnected()) { + process.nextTick(function(){ test_db.db.close(); }); } } } @@ -180,8 +183,8 @@ test_db.using_objects = function (clxn_name, test_objects, context){ test_db.db.collection(clxn_name, function(err, clxn){ if (err) throw(err); context[clxn_name] = clxn; - clxn.remove({ dummy: true }, {safe: true}, function(){ - clxn.insert(test_objects, { safe: true }, function(){ + clxn.remove({ dummy: true }, function(){ + clxn.insert(test_objects, function(){ context.callback(null, test_db); }); }); }); @@ -198,27 +201,29 @@ function setup_db(cb){ } // @see test_helper.batch -function connect(options){ +function connect(options, callback){ metalog.minor('cube_testdb', { state: 'connecting to db', options: options }); - test_db.options = options; - test_db.client = new mongodb.Server(options["mongo-host"], options["mongo-port"], {auto_reconnect: false}); - test_db.db = new mongodb.Db(options["mongo-database"], test_db.client, {}); + test_db.options = options, + test_db.db = db; + + test_db.db.open(options, callback); } // @see test_helper.batch function drop_collections(cb){ metalog.minor('cube_testdb', { state: 'dropping test collections', collections: test_collections }); - test_db.db.open(function(error) { - var collectionsRemaining = test_collections.length; - test_collections.forEach(function(collection_name){ - test_db.db.dropCollection(collection_name, collectionReady); - }); - function collectionReady() { - if (!--collectionsRemaining) { - cb(null, test_db); - } - } + + var collectionsRemaining = test_collections.length; + test_collections.forEach(function(collection_name){ + test_db.db.collection(collection_name, function(error, collection){ + collection.drop(collectionReady); + }) }); + function collectionReady() { + if (!--collectionsRemaining) { + cb(null, test_db); + } + } } // ========================================================================== diff --git a/test/types-test.js b/test/types-test.js deleted file mode 100644 index 98acb950..00000000 --- a/test/types-test.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict'; - -var vows = require("vows"), - assert = require("assert"), - mongodb = require("mongodb"), - test_helper = require("./test_helper"), - types = require("../lib/cube/types"); - -var suite = vows.describe("types"); - -suite.addBatch(test_helper.batch({ - topic: function(test_db) { - return types(test_db.db); - }, - - "types": { - "returns collection cache for a given database": function(types) { - assert.equal(typeof types, "function"); - }, - "each typed collection has events and metrics": function(types) { - var collection = types("random"), - keys = []; - for (var key in collection) { - keys.push(key); - } - keys.sort(); - assert.deepEqual(keys, ["events", "metrics"]); - assert.isTrue(collection.events instanceof mongodb.Collection); - assert.isTrue(collection.metrics instanceof mongodb.Collection); - assert.equal(collection.events.collectionName, "random_events"); - assert.equal(collection.metrics.collectionName, "random_metrics"); - }, - "memoizes cached collections": function(types) { - assert.strictEqual(types("random"), types("random")); - } - } -})); - -suite['export'](module); From f6967f56bb0e6ab074a2e459ada0f1098d89bfe7 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Fri, 7 Sep 2012 00:48:27 -0500 Subject: [PATCH 37/87] cleaned up broker and metrics in advance of merging new DB code --- lib/cube/broker.js | 42 +++++++------- lib/cube/metric.js | 98 ++++++++++++++++++-------------- test/broker-test.js | 134 +++++++++++++++++++++----------------------- test/event-test.js | 2 +- test/metric-test.js | 81 +++++++++++++++++++++++--- 5 files changed, 215 insertions(+), 142 deletions(-) diff --git a/lib/cube/broker.js b/lib/cube/broker.js index 7759ec02..7c0871e0 100644 --- a/lib/cube/broker.js +++ b/lib/cube/broker.js @@ -12,7 +12,7 @@ var worker_sleep_ms = 100; function Broker(name, interval){ var workers = []; add_worker(); - this.name = name; + this.name = name; // `deferProxy(name, perform, *args, on_complete)` -- Issue a request with // controlled concurrency; send its results to all interested listeners. We @@ -53,7 +53,7 @@ function Broker(name, interval){ } function stop(){ _.each(workers, function(worker){ worker.stop() }) }; - + function report(){ return { workers: _.map(workers, function(worker){ return worker.report() }) }; } function toString(){ return util.inspect(this.report()); }; @@ -67,20 +67,20 @@ function Broker(name, interval){ // A worker executes a set of tasks with parallelism 1 function Worker(qname, interval){ - var queue = Object.create(null), + var queue = {}, active = null, self = this, - clock; + clock = null; this.qname = qname; - function add(mbox, perform, args, on_complete) { + this.add = function add(mbox, perform, args, on_complete) { var job = ((active && (active.mbox === mbox)) ? active : queue[mbox]); if (! job){ job = queue[mbox] = new Job(mbox, perform, args); } // job.listen(on_complete); metalog.minor('q_worker', {is: 'add', mbox: job.mbox, am: self.report(), job: job.report() }); return job; - } + }; function invoke(job) { // move this job to be active @@ -91,21 +91,21 @@ function Worker(qname, interval){ // clears the active job (letting the next task start). job.args.push(function _completed(){ var result = arguments; - metalog.warn('q_worker', {is: '<-!', mbox: job.mbox, am: self.report(), result: util.inspect(result).slice(0,80) }); + metalog.minor('q_worker', {is: '<-!', mbox: job.mbox, am: self.report(), result: util.inspect(result).slice(0,80) }); if (job !== active) metalog.warn('q_worker', {is: 'ERR', am: qname, error: 'job was missing when callback triggered', self: self.report() }); active = null; job.complete(result); }); // start the task - metalog.warn('q_worker', {is: '?->', mbox: job.mbox, am: self.report(), perform_args: util.inspect(job.args).slice(0,80) }); + metalog.minor('q_worker', {is: '?->', mbox: job.mbox, am: self.report(), perform_args: util.inspect(job.args).slice(0,80) }); try{ job.perform.apply(null, job.args); } catch(err){ metalog.warn('q_worker', {is: 'ERR', am: qname, as: 'performing job '+job.mbox, error: err.message }) }; } - function start(){ + this.start = function start(){ if (clock){ return metalog.warn('q_worker', {is: 'ERR', am: self.report(), error: 'tried to start an already-running worker' }); } - clock = setInterval(self.work, interval); + clock = setInterval(work.bind(self), interval); } - function stop(){ + this.stop = function stop(){ metalog.info('q_worker', {is: 'stp'}); if (! clock){ return metalog.warn('q_worker', {is: 'ERR', am: self.report(), error: 'tried to stop an already-stopped worker' }); } clearInterval(clock); @@ -115,24 +115,25 @@ function Worker(qname, interval){ function work (){ var job; if (active) { self.onWait(); } - else if (job = next()){ self.invoke(job); } + else if (job = next()){ invoke(job); } else { self.onIdle(); } }; - function size(){ return _.size(queue); } - function next(){ return queue[ _.keys(queue).sort()[0] ]; } + + function size(){ return _.size(queue); } + function state(){ return ((clock === null) ? 'stop' : (active ? 'busy' : 'idle')); } + function next(){ return queue[ _.keys(queue).sort()[0] ]; } // function onIdle(){ util.print(' '+self.qname+'!'); }; // function onWait(){ util.print(' '+self.qname+'@'); }; function onIdle(){ }; function onWait(){ }; - function report(){ return { qname: this.qname, size: this.size, queue: _.keys(this.queue) }; }; + function report(){ return { qname: this.qname, size: this.size, state: this.state, queue: _.keys(queue) }; }; function toString(){ return util.inspect(this.report()); }; Object.defineProperties(this, { - add: {value: add}, size: {get: size}, - start: {value: start}, stop: {value: stop}, + size: {get: size}, state: {get: state}, report: {value: report}, toString: {value: toString}, onWait: {value: onWait}, onIdle: {value: onIdle} }); @@ -144,15 +145,16 @@ function Job(mbox, perform, args){ _.extend(this, { mbox: mbox, perform: perform, args: args, on_completes: [] }); } Job.prototype.complete = function(result){ - for (var ii in this.on_completes){ - process.nextTick(function(){ this.on_completes[ii].apply(null, result); }); + var on_completes = this.on_completes; + for (var ii in on_completes){ + process.nextTick(function(){ on_completes[ii].apply(null, result); }); }; } Job.prototype.listen = function(on_complete){ this.on_completes.push(on_complete); }; Job.prototype.toString = function (){ return util.inspect(this.report()); }; -Job.prototype.report = function (){ return this; } +Job.prototype.report = function (){ return { mbox: this.mbox, args: this.args, on_completes: this.on_completes.length }; } // ---- diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 13b1294d..3cb105ad 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -17,8 +17,10 @@ var metric_fields = {v: 1}, // Query for metrics. exports.getter = function(db) { - var Double = mongodb.Double, - queueByName = {}; + var Double = mongodb.Double, + queueByName = {}, + async_queues = {}, + qcount = 0; function getter(request, callback) { var measurement, expression, @@ -36,13 +38,14 @@ exports.getter = function(db) { stop = tier.ceil(stop); expression = parser.parse(request.expression); measurement = new Measurement(expression, start, stop, tier); - } catch(err) { return callback({error: err}), -1; }; + } catch(err) { return callback({error: err}), -1; } // Compute the request metric! - measurement.measure((("id" in request) ? - function(time, value) { callback({time: time, value: value, id: request.id}); } : - function(time, value) { callback({time: time, value: value}); })); - }; + measurement.measure( + (("id" in request) ? + function(time, value){ callback({time: time, value: value, id: request.id}); } : + function(time, value){ callback({time: time, value: value}); })); + } function Measurement(expression, start, stop, tier){ // Round the start/stop to the tier edges @@ -52,13 +55,10 @@ exports.getter = function(db) { this.expression = expression; } - // execute cb on each interval from t1 to t2 - function walk(t1, t2, tier, cb){ - while (t1 < t2) { - cb(t1, t2); - t1 = tier.step(t1); - } - } + Measurement.prototype.flavor = function(){ return this.expression.op ? 'binary' : (this.expression.type ? 'unary' : 'constant'); }; + Measurement.prototype.inspect = function(){ + return { flavor: this.flavor(), tier: this.tier.key, start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), expr: this.expression.source }; + }; // Computes the metric for the given expression for the time interval from // start (inclusive) to stop (exclusive). The time granularity is determined @@ -70,61 +70,53 @@ exports.getter = function(db) { case 'binary': this.binary(callback); break; case 'unary': this.unary(callback); break; case 'constant': this.constant(callback); break; - default: throw('cannot measure '+this.inspect()); } - } + }; - // Computes a constant expression; + // Computes a constant expression like the "7" in "x * 7" Measurement.prototype.constant = function constant(callback) { var value = this.expression.value(); walk(this.start, this.stop, this.tier, function(time){ callback(time, value); }); callback(this.stop); - } - - Measurement.prototype.flavor = function(){ return this.expression.op ? 'binary' : (this.expression.type ? 'unary' : 'constant'); }; - Measurement.prototype.inspect = function(){ - return { start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), tier: this.tier.key, flavor: this.flavor(), expr: this.expression.source }; - } + }; // Serializes a unary expression for computation. Measurement.prototype.unary = function unary(callback) { - var expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; - // metalog.inspectify(this.inspect()); + var measurement = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; var remaining = 0, time0 = Date.now(), + time = start, name = expression.source, queue = queueByName[name], step = tier.key; + metalog.minor('unary start', measurement.inspect(), start); // Compute the expected number of values. - walk(start, stop, tier, function(time){ ++remaining; }); - + walk(start, stop, tier, function(time){ remaining++; }); // If no results were requested, return immediately. if (!remaining) return callback(stop); // Add this task to the appropriate queue. + task.qcount = qcount++; if (queue) queue.next = task; else process.nextTick(task); queueByName[name] = task; function task() { - findOrComputeUnary(expression, start, stop, tier, function(time, value) { + findOrComputeUnary(expression, start, stop, tier, function(time, value){ callback(time, value); + metalog.minor('unary result', { meas: measurement.inspect(), qcount: qcount, next: (task.next && task.next.qcount) }); if (!--remaining) { callback(stop); if (task.next) process.nextTick(task.next); else delete queueByName[name]; - // Record how long it took us to compute as an event! - var time1 = Date.now(); - metalog.event("cube_compute", { - expression: expression.source, - ms: time1 - time0 - }); + metalog.minor("cube_compute", { is: 'metric', at: 'done', meas: measurement.inspect(), ms: Date.now() - time0}); } }); } - } + return null; + }; // Finds or computes a unary (primary) expression. function findOrComputeUnary(expression, start, stop, tier, callback) { @@ -135,6 +127,8 @@ exports.getter = function(db) { fields = {t: 1}, metrics, events; + metalog.minor('findOrComputeUnary', expression.source, start, stop, tier.key, callback.toString()); + // if (!reduce) return callback({error: "invalid reduce operation"}), -1; // Copy any expression filters into the query object. @@ -152,8 +146,8 @@ exports.getter = function(db) { // The metric is computed recursively, reusing the above variables. function find(start, stop, tier, callback) { - var compute = tier.next && reduce.pyramidal ? computePyramidal : computeFlat, - step = tier.key; + var compute = ((tier.next && reduce.pyramidal) ? computePyramidal : computeFlat), + step = tier.key; // Query for the desired metric in the cache. metrics.find({ @@ -169,9 +163,11 @@ exports.getter = function(db) { // Immediately report back whatever we have. If any values are missing, // merge them into contiguous intervals and asynchronously compute them. function foundMetrics(error, cursor) { + metalog.minor('foundMetrics', expression.source, start, stop, tier.key, callback.toString()); handle(error); var time = start; cursor.each(function(error, row) { + handle(error); if (row) { callback(row._id.t, row.v); @@ -185,6 +181,7 @@ exports.getter = function(db) { // Group metrics from the next tier. function computePyramidal(start, stop) { + metalog.minor('computePyramidal', expression.source, start, stop, tier.key, callback.toString()); var bins = {}; find(start, stop, tier.next, function(time, value) { var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); @@ -195,6 +192,10 @@ exports.getter = function(db) { }); } + function get_mbox(meas){ + return [meas.tier.key, meas.tier.bin(meas.start), meas.tier.bin(meas.stop), meas.expression.source, meas.fields].join('~'); + } + // Group raw events. Unlike the pyramidal computation, here we can control // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. @@ -206,7 +207,7 @@ exports.getter = function(db) { filter.t.$gte = start; filter.t.$lt = stop; - db.events(name, function(error, collection){ + db.events(name, function _computeFlatOnComplete(error, collection) { handle(error); collection.find(filter, fields, event_options, function(error, cursor) { @@ -250,17 +251,22 @@ exports.getter = function(db) { } } - // Computes a binary expression by merging two subexpressions. + // Computes a binary expression by merging two subexpressions + // + // "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and + // unary "sum(resp)". We don't know what order they'll show up in, so if say + // the value for left appears first, it parks that value as left[time], where + // the result for right will eventually find it. Measurement.prototype.binary = function binary(callback) { var expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; var left = new Measurement(expression.left, start, stop, tier), right = new Measurement(expression.right, start, stop, tier); left.measure(function(time, vall) { - if (time in right) { + if (time in right) { // right val already appeared; get a result callback(time, time < stop ? expression.op(vall, right[time]) : vall); delete right[time]; - } else { + } else { // right val still on the way; stash the value left[time] = vall; } }); @@ -273,6 +279,14 @@ exports.getter = function(db) { right[time] = valr; } }); + }; + + // execute cb on each interval from t1 to t2 + function walk(t1, t2, tier, cb){ + while (t1 < t2) { + cb(t1, t2); + t1 = tier.step(t1); + } } return getter; @@ -280,6 +294,6 @@ exports.getter = function(db) { function handle(error) { if (!error) return; - metalog.info('cube_request', {is: 'metric error', error: error }); + metalog.warn('cube_request', {is: 'metric error', error: error }); throw error; } diff --git a/test/broker-test.js b/test/broker-test.js index 0f092728..bbd94937 100644 --- a/test/broker-test.js +++ b/test/broker-test.js @@ -14,81 +14,72 @@ var suite = vows.describe("broker"); var squarer = function(ii, cb){ cb(null, ii*ii, 'squarer'); }; assert.isCalledTimes = function(ctxt, reps){ - var results = []; - setTimeout(function(){ ctxt.callback(new Error('timeout: need '+reps+' results only have '+util.inspect(results))); }, 2000); + var results = [], finished = false; + setTimeout(function(){ if (! finished){ ctxt.callback(new Error('timeout: need '+reps+' results only have '+util.inspect(results))); } }, 2000); return function _is_called_checker(){ results.push(_.toArray(arguments)); - if (results.length >= reps) ctxt.callback(null, results); + if (results.length >= reps){ finished = true; ctxt.callback(null, results); } }; -} +}; + assert.isNotCalled = function(name){ return function(){ throw new Error(name + ' should not have been called, but was'); }; }; -function example_worker(paused){ - var worker = new Worker('worker', 50); - // worker.idle = _.identity; - if (! paused) worker.start(); - return worker; -} +function example_worker(){ var worker = new Worker('test', 50); worker.start(); return worker; } +function example_job(){ return (new Job('smurf', squarer, [7])); } +suite.addBatch({ + 'Worker': { + '.new': { + topic: function(){ return new Worker('test', 50); }, + 'has an empty queue': function(worker){ assert.deepEqual(worker.size, 0); }, + 'is stopped': function(worker){ assert.equal(worker.state, 'stop'); } + }, + '.add': { + topic: example_worker, + 'pushes a job onto the queue': function(worker){ + worker.add('smurfette', function(){ setTimeout(_.identity, 50) }, [3], _.identity); + assert.deepEqual(worker.report(), { qname: 'test', size: 1, state: 'idle', queue: [ 'smurfette' ] }); + } + }, + '.invoke': { + topic: function(){ + var worker = this.worker = example_worker(); + var ctxt = this; + ctxt.checker = assert.isCalledTimes(ctxt, 3); + ctxt.performances = 0; + // shortly, worker will invoke perfom (once). 200 ms later, `perform` + // will call `worker`'s proxy callback, which invokes all 3 callbacks. + var perform = function(cb){ ctxt.performances++; setTimeout(function(){cb('hi');}, 20); }; + worker.add('thrice', perform, [], ctxt.checker); + worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); + worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); + }, + 'calls perform exactly once': function(){ assert.equal(this.performances, 1); }, + 'calls all registered callbacks': function(results){ assert.deepEqual(results, [['hi'], ['hi'], ['hi']]); }, + teardown: function(){ + this.worker.stop(); + } + } + }, -function example_job(){ return (new Job('smurf', squarer, [7])); }; + 'Job': { + '.new': { + topic: example_job, + '': function(job){ + assert.deepEqual(job, {mbox: 'smurf', perform: squarer, args: [7], on_completes: []}); + } + }, + '.listen': { + topic: function(){ + var job = example_job(); + job.listen(squarer); + return job; + } + } + }, -suite.addBatch({ - // 'Worker': { - // topic: example_worker, - // // '.new': { - // // '': function(worker){ - // // test_helper.inspectify('new worker', worker, worker); - // // worker.add('smurfette', squarer, [3], metalog.inspectify); - // // }, - // // }, - // '.invoke': { - // topic: function(worker){ - // var ctxt = this; - // ctxt.checker = assert.isCalledTimes(ctxt, 3); - // ctxt.performances = 0; - // // shortly, worker will invoke perfom (once). 200 ms later, `perform` - // // will call `worker`'s proxy callback, which invokes all 3 callbacks. - // var perform = function(cb){ ctxt.performances++; setTimeout(function(){cb('hi')}, 200); }; - // worker.add('thrice', perform, [], ctxt.checker); - // worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); - // worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); - // this.worker = worker; - // }, - // 'calls perform exactly once': function(){ assert.equal(this.performances, 1); }, - // 'calls all registered callbacks': function(results){ assert.deepEqual(results, [['hi'], ['hi'], ['hi']]) }, - // teardown: function(){ - // this.worker.stop(); - // } - // } - // }, - - // 'Job': { - // '.new': { - // topic: example_job, - // '': function(job){ - // assert.deepEqual(job, {name: 'smurf', perform: squarer, args: [7], on_completes: []}); - // }, - // }, - // '.listen': { - // topic: function(){ - // var job = example_job(); - // job.listen(squarer); - // return job; - // }, - // '': function(job){ - // test_helper.inspectify(job, job.toString()); - // } - // }, - // // '.add': { - // // topic: example_job, - // // '': function(job){ - // // } - // // }, - // }, - 'Broker': { 'handles interleaved jobs': { topic: function(){ @@ -97,9 +88,9 @@ suite.addBatch({ ignored = assert.isNotCalled('perform'); ctxt.perfs = {a: 0, b: 0, c:0}; ctxt.checker = assert.isCalledTimes(ctxt, 8); - var task_a = function(ii, a2, cb){ ctxt.perfs.a++; setTimeout(function(){cb('result_a', ii*ii, a2)}, 10); }; - var task_b = function(ii, cb){ ctxt.perfs.b++; setTimeout(function(){cb('result_b', ii*ii )}, 20); }; - var task_c = function(ii, cb){ ctxt.perfs.c++; setTimeout(function(){cb('result_c', ii*ii )}, 300); }; + var task_a = function(ii, a2, cb){ ctxt.perfs.a++; setTimeout(function(){cb('result_a', ii*ii, a2);}, 10); }; + var task_b = function(ii, cb){ ctxt.perfs.b++; setTimeout(function(){cb('result_b', ii*ii );}, 20); }; + var task_c = function(ii, cb){ ctxt.perfs.c++; setTimeout(function(){cb('result_c', ii*ii );}, 300); }; // will go second: jobs are sorted broker.deferProxy('task_b', task_b, 1, ctxt.checker); broker.deferProxy('task_b', ignored, '<>', ctxt.checker); @@ -122,15 +113,16 @@ suite.addBatch({ ['result_b', 1], ['result_b', 1], ['result_c', 4], ['result_c', 4], ['result_a', 9, '!'], ['result_a', 9, '!'] - ]) + ]); }, teardown: function(){ this.broker.stop(); } } - }, + } // broker + }); - + suite['export'](module); diff --git a/test/event-test.js b/test/event-test.js index 09d9c9b8..eff7a1bb 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -21,7 +21,7 @@ suite.addBatch(test_helper.batch({ putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); }, - 'heckya': function(){ + 'correct tiers': function(){ var ts = event.invalidator().tsets(); assert.deepEqual(ts, { 'test': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], diff --git a/test/metric-test.js b/test/metric-test.js index c65275e5..397dd3cf 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -31,6 +31,40 @@ function assert_invalid_request(req, expected_err) { }; } +var thenish = Date.UTC(2011, 6, 18, 0, 0, 0); +function gen_date(sec){ + return new Date(thenish + sec*units.second); +} + +var t1 = gen_date(2), + t2 = gen_date(71); + +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + var putter = event.putter(test_db.db), + getter = metric.getter(test_db.db), + callback = this.callback; + + // Seed the events table with a simple event: a value going from 0 to 2499 + for (var i = 0; i < 250; i++) { + putter({ + type: "test", + time: gen_date(i * 3).toISOString(), + data: {i: 3 * i} + }); + } + + // So the events can settle in, wait `batch_testing_delay` ms before continuing + setTimeout(function() { callback(null, getter); }, batch_testing_delay); + }, + 'hi': { + topic: function(getter){ + this.ret = getter(gen_request({start: t1, stop: t2, expression: 'sum(test)'}), this.callback); }, + 'fires callback with id': function(result, j_){ + } + } +})); + suite.addBatch(test_helper.batch({ topic: function(test_db) { return metric.getter(test_db.db); @@ -48,8 +82,8 @@ suite.addBatch(test_helper.batch({ assert.include([nowish10, 10e3+nowish10], +result.time); } } - })).addBatch(test_helper.batch({ + topic: function(test_db) { return metric.getter(test_db.db); }, @@ -64,7 +98,7 @@ suite.addBatch(test_helper.batch({ })); function skip(){ // FIXME: remove ------------------------------------------------------------ - + var steps = { 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, @@ -97,6 +131,36 @@ suite.addBatch(test_helper.batch({ setTimeout(function() { callback(null, getter); }, batch_testing_delay); }, + // FIXME: ---- remove below ------------------------------------ + + "unary expression a": metricTest({ + expression: "sum(test)", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:02:00.000Z" + }, { + 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] + }), + + "unary expression b": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17 ] }), + "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), + "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), + "unary expression e": metricTest({ expression: "sum(test)", start: "2011-07-17T23:50:00.000Z", stop: "2011-07-18T00:03:00.000Z"}, { 60e3: [ 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25 ] }), + + "unary expression f": metricTest({ + expression: "sum(test)", + start: "2011-07-17T23:57:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 60e3: [13, 15, 17, 39, 23, 25, 27, 29, 31, 33, + 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, + 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, + 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, + 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0] + }), + + // FIXME: ---- remove above ------------------------------------ + "unary expression": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", @@ -197,7 +261,6 @@ function metricTest(request, expected) { // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } var tree = {}, k; for (var step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); - return tree; // // { @@ -227,7 +290,6 @@ function metricTest(request, expected) { }; subtree[request.expression] = metrics_assertions(); subtree["(cached)"][request.expression] = metrics_assertions(); - return subtree; function get_metrics_with_delay(depth){ return function(){ var actual = [], @@ -303,12 +365,15 @@ function metricTest(request, expected) { } }); } - };} // metric assertions + + return subtree; } // subtree + return tree; } // tree -}; +} skip(); - -suite.export(module); + +suite['export'](module); + From c3a5870d952cae227ef8316f632fc23327a8ed2b Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Sun, 9 Sep 2012 01:25:25 -0500 Subject: [PATCH 38/87] adding tracing and queuing (WIP) --- .gitmodules | 3 + examples/helper.js | 49 ++ examples/random-emitter/cromulator.js | 70 ++ examples/random-emitter/random-emitter.js | 1 + examples/random-emitter/random-streamer.js | 43 ++ lib/cube/db.js | 156 +++-- lib/cube/emitter-ws.js | 25 +- lib/cube/event.js | 160 ++--- lib/cube/metalog.js | 99 ++- lib/cube/metric.js | 270 ++++---- lib/cube/models.js | 146 +++-- lib/cube/server.js | 53 +- lib/cube/tiers.js | 4 +- test/authentication-test.js | 2 +- test/broker-test.js | 13 - test/event-test.js | 6 +- test/metalog-test.js | 4 +- test/metric-test.js | 707 +++++++++++---------- test/test_helper.js | 50 +- 19 files changed, 1111 insertions(+), 750 deletions(-) create mode 100644 .gitmodules create mode 100644 examples/helper.js create mode 100644 examples/random-emitter/cromulator.js create mode 100644 examples/random-emitter/random-streamer.js diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..a554f13a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "lib/queue-async"] + path = lib/queue-async + url = https://github.com/mbostock/queue.git diff --git a/examples/helper.js b/examples/helper.js new file mode 100644 index 00000000..f46d34ba --- /dev/null +++ b/examples/helper.js @@ -0,0 +1,49 @@ +'use strict'; +process.env.TZ = 'UTC'; + +var cube = require("../"), + metalog = cube.metalog, + cromulator = require("./random-emitter/cromulator"), + un = cromulator.un, + event_mod = require("../lib/cube/event"), + models = require("../lib/cube/models"), Event = models.Event; + +var options = require("../config/cube").include('evaluator'), + mongodb = require("mongodb"), + mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], options["mongo-server_options"]), + db = new mongodb.Db(options["mongo-database"], mongo, { native_parser: true }), + putter, getter; + +var type = 'doh'; + +db.open(function(error) { + putter = event_mod.putter(db); + getter = event_mod.getter(db); + + var emitter = cube.emitter("ws://127.0.0.1:1080"); + + var helper = { + emitter: emitter + }; + + // helper.invalidate = function(){ + // helper.emitter.invalidate_range(type, Date.now - (20 * un.min), Date.now - (12 * un.min)); + // } + + metalog.info('emitter', {is: 'starting'}); + + var thenish = 1346629570000; + + var ev = new Event(type, thenish, {value: 3}); + // emitter.send(ev.to_request()); + + for (var ii = 0; ii <= 3; ii++){ + ev = new Event(type, thenish - ii * ii * 40000, {value: ii}); + putter(ev.to_request()); + } + setTimeout(function(){ metalog.inspectify(event_mod.invalidator().tsets()); }, 200); + setTimeout(function(){ event_mod.stop(); }, 1200); + + metalog.info('emitter', {is: 'stopping'}); + helper.emitter.close(); +}); diff --git a/examples/random-emitter/cromulator.js b/examples/random-emitter/cromulator.js new file mode 100644 index 00000000..4082a70c --- /dev/null +++ b/examples/random-emitter/cromulator.js @@ -0,0 +1,70 @@ +var metalog = require("../../lib/cube/metalog"), + models = require("../../lib/cube/models"); + +var un = {}; un.sec = 1000; un.min = 60 * un.sec; un.hr = 60 * un.min; + +var cromulator = { + count: 0, + step: un.sec * 1, + start: Date.now(), + jitter: 2 * un.sec, // spread in event times + visit_rate: 10.0, + un: un +}; + +var ev = { + ramp: 0.0, + walk: 0.0, + characters: { + homer: 324, bart: 263, lisa: 203, marge: 142, scratchy: 79, itchy: 79, maggie: 51, mr_burns: 49, + ned_flanders: 39, milhouse: 38, skinner: 37, sideshow_mel: 31, willie: 30, quimby: 25, moe: 25, + krusty: 24, nelson: 23, wiggum: 22, grampa: 22, frink: 19, apu: 16, sideshow_bob: 14, + selma: 14, patty: 14, barney: 13, mrs_krabappel: 12, comic_book_guy: 12, martin: 10, + dr_hibbert: 10, smithers: 9, ralph: 9, rev_lovejoy: 8, lionel_hutz: 8, fat_tony: 8, chalmers: 8, + snake: 7, otto: 6, dr_nick: 6, cletus: 5, troy_mcclure: 4, todd: 3, rodd: 3, kent_brockman: 3 }, + characters_pool: [] +}; +for (var ch in ev.characters) { for (i=0; i < ev.characters[ch]; i++){ ev.characters_pool.push(ch) } }; + +// random element from a list +function rand_element(list){ return list[Math.floor(Math.random() * list.length)]; } +// Fuzzy sine wave +function sine(since, period, fuzz){ + return (Math.sin(Math.PI * since / period) + normal(0, fuzz)); +} +// Normally-distributed variate with given average and standard deviation -- http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform +function normal(avg, stdev){ return avg + (stdev * Math.sqrt(-2 * Math.log(Math.random())) * Math.cos(2 * Math.PI * Math.random())); } +// Exponential variate; 0 < lambda, the rate of events -- http://en.wikipedia.org/wiki/Exponential_distribution#Generating_exponential_variates +function exprand(lambda){ return (- Math.log(Math.random()) / lambda); } +// Geometric variate; 0 < p < 1, the probability an event will happen +function geomrand(p){ return Math.floor(exprand( -Math.log(1-p) )); } + +cromulator.report = function(stage, time){ + var stop = cromulator.stop || Date.now(); + return { is: stage, + start: new Date(cromulator.start), + stop: new Date(stop), + per_sec: (1000.0 / cromulator.step), + qty: (stop - cromulator.start) / cromulator.step, + secs: (stop - cromulator.start) / 1000, + count: cromulator.count, + ago: (Date.now() - time)/1000 }; +}; + +cromulator.data_at = function(time){ + var stop = cromulator.stop || Date.now(); + var data = { + sine_45m: sine(time - cromulator.start, 45 * un.min, 0.2), + sine_5m: sine(time - cromulator.start, 5 * un.min, 0.1), + walk: ev.walk += (Math.random() - 0.5), + ramp: ev.ramp += (Math.random() * 20 * cromulator.step / (stop - cromulator.start)), + visits: exprand( un.sec * cromulator.visit_rate / cromulator.step ), + who: rand_element(ev.characters_pool) + }; + ++cromulator.count; + return data; +}; + +cromulator.spread_time = function(time){ return new Date(time + (cromulator.jitter * (Math.random()-0.5)) ) }; + +module.exports = cromulator; diff --git a/examples/random-emitter/random-emitter.js b/examples/random-emitter/random-emitter.js index a9ce70f9..9b4cf45c 100644 --- a/examples/random-emitter/random-emitter.js +++ b/examples/random-emitter/random-emitter.js @@ -23,6 +23,7 @@ while (start < stop) { }); start += step; ++count; + if (count > 3) break; } util.log("sent " + count + " events"); diff --git a/examples/random-emitter/random-streamer.js b/examples/random-emitter/random-streamer.js new file mode 100644 index 00000000..18a4c0b7 --- /dev/null +++ b/examples/random-emitter/random-streamer.js @@ -0,0 +1,43 @@ +process.env.TZ = 'UTC'; + +var cube = require("../../"), // replace with require("cube") + metalog = cube.metalog, + options = require("./random-config"), + cromulator = require("./cromulator"), + models = require("../../lib/cube/models"), Event = models.Event; + +var options = { + "collector": "ws://127.0.0.1:1080", + + // The offset and duration to backfill, in milliseconds. + // For example, if the offset is minus four hours, then the first event that + // the random emitter sends will be four hours old. It will then generate more + // recent events based on the step interval, all the way up to the duration. + "offset": -0.49 * 60 * 60 * 1000, + "duration": 0.49 * 60 * 60 * 1000, + + // The time between random events. + "step": 1000 * 2, + + event_type: "doh" +}; + +var emitter = cube.emitter(options["collector"]); + +cromulator.start = Date.now() + options.offset; +cromulator.stop = cromulator.start + options.duration; +cromulator.step = options.step; + +metalog.info('emitter', cromulator.report('starting')); + +var time = cromulator.start; +while (time < cromulator.stop) { + var event = new Event(options.event_type, cromulator.spread_time(time), cromulator.data_at(time)); + if (cromulator.count % 1000 == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); + event.force = true; + emitter.send(event.to_request()); + time += cromulator.step; +} + +metalog.info('emitter', cromulator.report('stopping', time)); +emitter.close(); diff --git a/lib/cube/db.js b/lib/cube/db.js index 05937cc3..6aa7721e 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -1,15 +1,17 @@ 'use strict'; -var mongodb = require("mongodb"), +var util = require("util"), + mongodb = require("mongodb"), metalog = require("./metalog"), metric_options, event_options, database, events_db, metrics_db; -module.exports = { +var db = { open: open, isConnected: isConnected } +var collections = {}; // // Connect to mongodb. @@ -40,17 +42,18 @@ function open(options, callback){ if (options["separate-metrics-database"]) metrics_db = database.db(database_name + '-metrics'); else metrics_db = database; - module.exports.metrics = metrics(metrics_db); - module.exports.events = events(events_db); - module.exports.types = types(events_db); - module.exports.collection = collection(database); - module.exports.close = close; + db.metrics = metrics_collection_factory(metrics_db); + db.events = events_collection_factory(events_db); + db.types = types(events_db); + db.collection = collection(database); + db.close = close; - if (! options["mongo-username"]) return callback(null, module.exports); + if (! options["mongo-username"]) return callback(null, db); + database.authenticate(options["mongo-username"], mongo_password, function(error, success) { if (error) return callback(error); if (!success) return callback(new Error("authentication failed")); - callback(null, module.exports); + callback(null, db); }); }); } @@ -61,13 +64,13 @@ function open(options, callback){ // function close(callback){ - delete module.exports.metrics, - delete module.exports.events, - delete module.exports.types, - delete module.exports.collection, - delete module.exports.close; - - database.close(callback); + collections = {}; + delete db.metrics; + delete db.events; + delete db.types; + delete db.collection; + delete db.close; + if (isConnected()){ database.close(callback); } else { callback(null); } } function isConnected(){ @@ -78,68 +81,99 @@ function isConnected(){ } } - -// Much like db.collection, but caches the result for both events and metrics. -// Also, this is synchronous, since we are opening a collection unsafely. -function metrics(database) { - var collections = {}; - - return function(name, callback){ - if(collections[name]) return callback(null, collections[name]); - - var collection_name = name + "_metrics", - _this = this; - database.createCollection(collection_name, metric_options||{safe: false}, function(error, metrics) { - - if (error && error.errmsg == "collection already exists") return _this.metrics(name, callback); - if (error) return callback(error); - - metrics.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); - metrics.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, handle); - - metrics = overwrite_find(metrics, 'metrics'); - collections[name] = metrics; - return callback(null, metrics); - }); - } +// // Much like db.collection, but caches the result for both events and metrics. +// // Also, this is synchronous, since we are opening a collection unsafely. +// function metrics(database) { +// var m_collections = {}; +// +// return function(name, callback){ +// if(m_collections[name]) return callback(null, m_collections[name]); +// +// var collection_name = name + "_metrics", +// _this = this; +// database.createCollection(collection_name, metric_options||{safe: false}, function(error, metrics) { +// +// if (error && error.errmsg == "collection already exists") return _this.metrics(name, callback); +// if (error) return callback(error); +// +// metrics = logging_clxn(metrics, 'metrics'); +// m_collections[name] = metrics; +// return callback(null, metrics); +// }); +// } +// } + +function metrics_collection_factory(database){ + return function metrics(name, on_collection, tr){ + return clxn_for(database, (name + "_metrics"), (metric_options||{safe: false}), on_collection, on_m_create, tr); + function on_m_create(clxn, oc_cb){ + clxn.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); + clxn.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, function(error){ oc_cb(error, clxn); }); + } + }; } -function events(database) { - var collections = {}; - - return function(name, callback){ - if (collections[name]) return callback(null, collections[name]); +function events_collection_factory(database){ + return function events(name, on_collection, tr){ + return clxn_for(database, (name + "_events"), (event_options||{safe: false}), on_collection, on_e_create, tr); + function on_e_create(clxn, oc_cb){ + db.metrics(name, function(error){ + if (error) return oc_cb(error); + clxn.ensureIndex({"t": 1}, function(error){ oc_cb(error, clxn); }); + }); + } + }; +} - var collection_name = name + "_events"; +var pending_callbacks = {}; - // Create a collection for events. One index is require, for finding events by time(t) - database.createCollection(collection_name, event_options||{safe: false}, function(error, events){ +function clxn_for(database, clxnname, clxnopts, on_collection, on_create, tr){ + on_collection = on_collection || function(){}; + // If we've cached the collection, call back immediately + if (collections[clxnname]) return on_collection(null, collections[clxnname]); - if (error && error.errmsg == "collection already exists") return _this.metrics(name, callback); - if(error) return callback(error); + // If someone is already creating the collection for this new type, + // then append the callback to the queue for later save. + if (clxnname in pending_callbacks){ + metalog.trace('clxnQ', tr); + return pending_callbacks[clxnname].push(on_collection); + } - events.ensureIndex({"t": 1}, handle); + // Otherwise, it's up to us to create the corresponding collection, then save + // any requests that have queued up in the interim! + + // First add the new event to the queue. + pending_callbacks[clxnname] = [on_collection]; + + // Create collection, and issue callback for index creation, etc + database.createCollection(clxnname, clxnopts, function(error, clxn){ + metalog.minor('create collection', {name: clxnname}); + if (error && (error.errmsg == "collection already exists")) return db.collection(clxnname, adopt_collection); + if (!error) return on_create(clxn, adopt_collection); + if (error) return adopt_collection(error); + }); - events = overwrite_find(events, 'events'); - collections[name] = events; - return callback(null, events); - }); + function adopt_collection(error, clxn){ + if (! error){ + clxn = logging_clxn(clxn, clxnname); + collections[clxnname] = clxn; + } + pending_callbacks[clxnname].forEach(function(cb){ cb(error, clxn) }); + delete pending_callbacks[clxnname]; } } function collection(database){ - var collections = {}; - return function(name, callback){ if (collections[name]) return callback(null, collections[name]); database.collection.apply(database, arguments); } } -function overwrite_find(collection, type){ +function logging_clxn(collection, type){ var orig_find = collection.find; collection.find = function(){ - metalog.minor('cube_query', {is: type + '_find', query: arguments}); + //metalog.minor((type + '_find'), {query: arguments}); return orig_find.apply(this, arguments); }; return collection; @@ -162,8 +196,10 @@ function types(database) { function handle(error){ if (!error) return; - metalog.info('cube_request', {is: 'db error', error: error }); + metalog.error('db error', error); throw error; } function noop(){}; + +module.exports = db; diff --git a/lib/cube/emitter-ws.js b/lib/cube/emitter-ws.js index ed3f3204..b8463f06 100644 --- a/lib/cube/emitter-ws.js +++ b/lib/cube/emitter-ws.js @@ -1,19 +1,20 @@ 'use strict'; -var util = require("util"), - websocket = require("websocket"); +var util = require("util"), + websocket = require("websocket"), + metalog = require("./metalog"); module.exports = function(protocol, host, port) { var emitter = {}, queue = [], - url = protocol + "//" + host + ":" + port + "/1.0/event/put", + url = protocol + "\/\/" + host + ":" + port + "/1.0/event/put", socket, timeout, closing; function close() { + metalog.warn('cube_emitter', {is: 'closing socket', emitter: emitter.report()}); if (socket) { - util.log("closing socket"); socket.removeListener("error", reopen); socket.removeListener("close", reopen); socket.close(); @@ -24,6 +25,7 @@ module.exports = function(protocol, host, port) { function closeWhenDone() { closing = true; if (socket) { + metalog.warn('cube_emitter', {is: 'closing when done', emitter: emitter.report()}); if (!socket.bytesWaitingToFlush) close(); else setTimeout(closeWhenDone, 1000); } @@ -32,7 +34,7 @@ module.exports = function(protocol, host, port) { function open() { timeout = 0; close(); - util.log("opening socket: " + url); + metalog.warn('cube_emitter', {is: 'opening socket', url: url}); var client = new websocket.client(); client.on("connect", function(connection) { socket = connection; @@ -49,7 +51,7 @@ module.exports = function(protocol, host, port) { function reopen() { if (!timeout && !closing) { - util.log("reopening soon"); + metalog.warn('cube_emitter', {is: 'reopening soon', delay: 1000}); timeout = setTimeout(open, 1000); } } @@ -59,8 +61,8 @@ module.exports = function(protocol, host, port) { while (event = queue.pop()) { try { socket.sendUTF(JSON.stringify(event)); - } catch (e) { - util.log(e.stack); + } catch (err) { + metalog.warn('cube_emitter', {is: 'error', error: err.message, stack: er.stack}); reopen(); return queue.push(event); } @@ -68,10 +70,15 @@ module.exports = function(protocol, host, port) { } function log(message) { - util.log(message.utf8Data); + metalog.minor('cube_emitter', {is: 'response', message: message.utf8Data, emitter: emitter.report()}); + } + + emitter.report = function report(){ + return {pending: (socket && socket.bytesWaitingToFlush), socket: (socket ? 'open' : 'closed'), url: url}; } emitter.send = function(event) { + metalog.trace('emitter_send', event); queue.push(event); if (socket) flush(); return emitter; diff --git a/lib/cube/event.js b/lib/cube/event.js index ab982275..158fd72a 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -4,14 +4,14 @@ // TODO allow the event time to change when updating (fix invalidation) var _ = require("underscore"), + mongodb = require("mongodb"), + ObjectID = mongodb.ObjectID, + util = require("util"), tiers = require("./tiers"), models = require("./models"), Event = models.Event, Metric = models.Metric, parser = require("./event-expression"), bisect = require("./bisect"), - metalog = require("./metalog"), - mongodb = require("mongodb"), - //options = require("../../config/cube"), - ObjectID = mongodb.ObjectID; + metalog = require("./metalog"); // When streaming events, we should allow a delay for events to arrive, or else // we risk skipping events that arrive after their event.time. This delay can be @@ -24,46 +24,6 @@ var invalidateInterval = 5000; exports.stop = function(){ Invalidator.stop_flushers(); }; -// -------------------------------------------------------------------------- - -// Schedule deferred invalidation of metrics by type and tier. -function Invalidator(){ - var type_tsets = {}, - invalidate = { $set: {i: true} }, - multi = { multi: true }; - - this.add = function(type, ev){ - var tt = type_tset(type); - for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true; } - }; - - this.flush = function(db){ - _.each(type_tsets, function(type_tset, type){ - db.metrics(type, function(error, collection){ - handle(error); - - _.each(type_tset, function(tset, tier){ - var times = dateify(tset); - metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: times }); - collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); - }); - }); - }); - }; - - this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }); }; - - function type_tset(type){ - if (! (type in type_tsets)) type_tsets[type] = empty_tsets(); - return type_tsets[type]; - } - function empty_tsets(){ return _.mapHash(tiers, function(){ return {}; }); } - function dateify(tset){ return _.map(_.keys(tset), function(time){ return new Date(+time); }).sort(function(aa,bb){return aa-bb;}); } -} -Invalidator.flushers = []; -Invalidator.start_flusher = function(cb){ Invalidator.flushers.push(setInterval(cb, invalidateInterval)); }; -Invalidator.stop_flushers = function(){ Invalidator.flushers.forEach(clearInterval); Invalidator.flushers = []; }; - var invalidator = new Invalidator(); exports.invalidator = function(){ return invalidator; }; @@ -76,12 +36,12 @@ exports.invalidator = function(){ return invalidator; }; // - data, the event's payload // exports.putter = function(db){ - var knownByType = {}, - eventsToSaveByType = {}; - function putter(request, callback) { + function putter(request, callback){ var time = "time" in request ? new Date(request.time) : new Date(), type = request.type; + callback = callback || function(){}; + // metalog.trace('putter', request); // // Drop events from before invalidation horizon // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; @@ -94,42 +54,19 @@ exports.putter = function(db){ var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } - - // If this is a known event type, save immediately. - if (type in knownByType) return event.save(after_save); - - // If someone is already creating the event collection for this new type, - // then append this event to the queue for later save. - if (type in eventsToSaveByType) return eventsToSaveByType[type].push(event); - - // Otherwise, it's up to us to see if the collection exists, verify the - // associated indexes, create the corresponding metrics collection, and save - // any events that have queued up in the interim! - - // First add the new event to the queue. - eventsToSaveByType[type] = [event]; - event.save(function(error, event){ - after_save(error, event); // trigger normal callback - - knownByType[type] = true; - eventsToSaveByType[type].forEach(function(event) { event.save(after_save); }); - delete eventsToSaveByType[type]; - }); - - // After saving the event, queue invalidation of any cached - // metrics associated with this event. + // metalog.trace('ev0', event, { using: request }); + + // Save the event, then queue invalidation of its associated cached metrics. // - // We don't invalidate the events immediately. This would cause many redundant - // updates when many events are received simultaneously. Also, having a short - // delay between saving the event and invalidating the metrics reduces the - // likelihood of a race condition between when the events are read by the - // evaluator and when the newly-computed metrics are saved. - function after_save(error, event) { - // metalog.minor("cube_request", { is: "ev", at: "save", type: type, event: event }); - handle(error); - invalidator.add(event._type(), event); - if(callback) callback(null, event); - } + // We don't invalidate the events immediately. This would cause redundant + // updates when many events are received simultaneously. Also, having a + // short delay between saving the event and invalidating the metrics reduces + // the likelihood of a race condition between when the events are read by + // the evaluator and when the newly-computed metrics are saved. + event.save(function after_save(error, event){ + if (event) invalidator.add(event.type, event); + callback(error, event); + }); } // Process any deferred metric invalidations, flushing the queues. Note that @@ -140,10 +77,52 @@ exports.putter = function(db){ invalidator = new Invalidator(); // copy-on-write }); - metalog.info('cube_life', {is: 'putter_start'}); + metalog.info('putter_start'); return putter; }; + + +// -------------------------------------------------------------------------- + +// Schedule deferred invalidation of metrics by type and tier. +function Invalidator(){ + var type_tsets = {}, + invalidate = { $set: {i: true} }, + multi = { multi: true }; + + this.add = function(type, ev){ + var tt = type_tset(type); + for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true; } + }; + + this.flush = function(db){ + _.each(type_tsets, function(type_tset, type){ + db.metrics(type, function(error, collection){ + handle(error); + + _.each(type_tset, function(tset, tier){ + var times = dateify(tset); + metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: times }); + collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); + }); + }); + }); + }; + + this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }); }; + + function type_tset(type){ + if (! (type in type_tsets)) type_tsets[type] = empty_tsets(); + return type_tsets[type]; + } + function empty_tsets(){ return _.mapHash(tiers, function(){ return {}; }); } + function dateify(tset){ return _.map(_.keys(tset), function(time){ return new Date(+time); }).sort(function(aa,bb){return aa-bb;}); } +} +Invalidator.flushers = []; +Invalidator.start_flusher = function(cb){ Invalidator.flushers.push(setInterval(cb, invalidateInterval)); }; +Invalidator.stop_flushers = function(){ Invalidator.flushers.forEach(clearInterval); Invalidator.flushers = []; }; + // // event.getter - subscribe to event type // @@ -182,11 +161,9 @@ exports.getter = function(db) { // Set an optional limit on the number of events to return. var options = {sort: {t: -1}, batchSize: 1000}; if ("limit" in request) options.limit = +request.limit; - // Copy any expression filters into the query object. var filter = {t: {$gte: start, $lt: stop}}; expression.filter(filter); - // Request any needed fields. var fields = {t: 1}; expression.fields(fields); @@ -198,7 +175,6 @@ exports.getter = function(db) { collection.find(filter, fields, options, function(error, cursor) { handle(error); cursor.each(function(error, event) { - // If the callback is closed (i.e., if the WebSocket connection was // closed), then abort the query. Note that closing the cursor mid- // loop causes an error, which we subsequently ignore! @@ -208,7 +184,7 @@ exports.getter = function(db) { // A null event indicates that there are no more results. if (event) callback({id: event._id instanceof ObjectID ? undefined : event._id, time: event.t, data: event.d}); - else callback(null); + else callback(null); }); }); }); @@ -281,12 +257,12 @@ exports.getter = function(db) { return getter; }; +function open(callback) { + return !callback.closed; +} + function handle(error) { if (!error) return; - metalog.info('cube_request', {is: 'event error', error: error }); + metalog.error('event', error); throw error; } - -function open(callback) { - return !callback.closed; -} diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 40edf0c7..5a09fac1 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -1,5 +1,11 @@ 'use strict'; +// metalog -- log, trace and cubify internal processing stages +// +// * log progress to disk +// * trace a request through the process stack +// * + var util = require("util"), _ = require("underscore"); @@ -9,12 +15,15 @@ var metalog = { silent: function(){ } }; +var tracestack = {}, // container for traces + tracecap = 10000; // max traces to track + // adjust verboseness by reassigning `metalog.loggers.{level}` // @example: quiet all logs // metalog.loggers.info = metalog.silent(); metalog.loggers = { info: metalog.log, - minor: metalog.log + minor: metalog.silent }; // if true, cubify `metalog.event`s @@ -24,25 +33,83 @@ metalog.send_events = true; // Cubify an event and (optionally) log it. The last parameter specifies the // logger -- 'info' (the default), 'minor' or 'silent'. -metalog.event = function(name, hsh, logger){ - metalog[logger||"info"](name, hsh); - if ((! metalog.send_events) || (! metalog.putter)) return; - metalog.putter({ type: name, time: Date.now(), data: hsh }); +metalog.event = function(label, hsh, logger){ + metalog[logger||"info"](label, hsh); + metalog.cubify(label, hsh); }; +metalog.cubify = function(label, hsh){ + if ((! metalog.send_events) || (! metalog.putter)) return; + hsh.at = label; + metalog.putter({ type: 'cube', time: Date.now(), data: hsh }); +} + +metalog.logify = function logify(label, hsh, logger){ + hsh = hsh || {}; + try{ + logger(label + "\t" + JSON.stringify(hsh)); + } catch(error) { + logger(label + "\t" + util.inspect(hsh)); + } +} + +// -------------------------------------------------------------------------- + // Always goes thru to metalog.log -metalog.warn = function(name, hsh){ - metalog.log(name + "\t" + (+Date.now()) + "\t" + JSON.stringify(hsh)); -}; +metalog.warn = function(label, hsh){ metalog.logify(label, hsh, metalog.log); }; // Events important enough for the production log file. Does not cubify. -metalog.info = function(name, hsh){ - metalog.loggers.info(name + "\t" + JSON.stringify(hsh)); -}; +metalog.info = function(label, hsh){ metalog.logify(label, hsh, metalog.loggers.info); }; // Debug-level statements; loggers.minor is typically mapped to 'silent'. -metalog.minor = function(name, hsh){ - metalog.loggers.minor(name + "\t" + JSON.stringify(hsh)); +metalog.minor = function(label, hsh){ metalog.logify(label, hsh, metalog.loggers.minor); }; + +// log an error; always goes thru to metalog.log +metalog.error = function(at, error, info){ + info = _.extend((info||{}), { at: at, error: error.message, stack: error.stack, code: error.status }); + metalog.logify('error', info, metalog.log); +}; + +// -------------------------------------------------------------------------- + +var trace_id = 1; + +metalog.trace = function(label, item, hsh){ + item = item || {}; hsh = hsh || {}; + var using = (hsh.using||{}); delete hsh.using; + if (! item._trace) item._trace = { beg: +(new Date()) }; + // + item._trace = _.extend(item._trace, using._trace||{}, hsh); + // + item._trace[label] = ((new Date()) - item._trace['beg']); + if (! item._trace.tid) item._trace.tid = trace_id++; + // if (value) item._trace['val'] = (Math.round(100*value)/100.0); + // } catch(err){ metalog.log(err); } + + return item; +}; + +var dump_keys = {tid: 4, beg: 15, m_get: 3, m_run: 3, m_res: 3, tier: 8, start: 9, stop: 9, bin: 9}; + +function dump(hsh){ + var fields = _.map(dump_keys, function(len, key){ return (hsh[key]+' ').slice(0,len); }) + metalog.log(fields.join('|') + "\t" + (JSON.stringify(hsh).slice(0,150))); +} + +function dump_header(){ + metalog.log(_.map(dump_keys, function(len, key){ return (key+' ').slice(0,len); }).join('|')); +} + +metalog.dump_trace = function(label, item, hsh){ + var item = metalog.trace(label, item, hsh); + var tr = item._trace; + function p(v,l){ var str = (v ? v.toString() : ''); return (v+'................').slice(0,l); } + try{ + item._trace['end'] = +(new Date()); + if (Math.random() < 0.2) dump_header(); + dump(item._trace); + } catch(err){ metalog.log(err); metalog.log(err.stack); } + return item; }; // Dump the 'util.inspect' view of each argument to the console. @@ -61,11 +128,11 @@ metalog.inspectify = function inspectify(args){ // metalog.spy(callback, 'unary', this); // @example you don't have to supply anything but the callback // metalog.spy(function(err, val){ ... }) -metalog.spy = function spy(callback, name, ctxt){ - if (! name) name = 'spyable'; +metalog.spy = function spy(callback, label, ctxt){ + if (! label) label = 'spyable'; if (! ctxt) ctxt = null; return function(){ - util.print(name+': ', callback, ' called with ', util.inspect(arguments), '\n'); + util.print(label+': ', callback, ' called with ', util.inspect(arguments), '\n'); // metalog.inspectify(callback, arguments); return callback.apply(ctxt, arguments); }; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 3cb105ad..2c31717c 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -2,31 +2,38 @@ // TODO use expression ids or hashes for more compact storage -var parser = require("./metric-expression"), - tiers = require("./tiers"), - db = require("./db"), +var _ = require("underscore"), mongodb = require("mongodb"), + util = require("util"), + queuer = require("../queue-async/queue"), + db = require("./db"), + parser = require("./metric-expression"), + tiers = require("./tiers"), reduces = require("./reduces"), + models = require("./models"), Metric = models.Metric, event = require("./event"), metalog = require('./metalog'), options = require('../../config/cube'); var metric_fields = {v: 1}, metric_options = {sort: {"_id.t": 1}, batchSize: 1000}, - event_options = {sort: {t: 1}, batchSize: 1000}; + event_options = {sort: {t: 1}, batchSize: 1000}, + metric_parallelism = 5; // Query for metrics. -exports.getter = function(db) { +exports.getter = function(db){ var Double = mongodb.Double, queueByName = {}, - async_queues = {}, - qcount = 0; + request_queues = {}, + qcount = 0; function getter(request, callback) { var measurement, expression, tier = tiers[+request.step], start = new Date(request.start), stop = new Date(request.stop); + metalog.trace('m_get', request); + try { if (!tier) throw "invalid step"; if (isNaN(start)) throw "invalid start"; @@ -37,89 +44,162 @@ exports.getter = function(db) { start = tier.floor(start); stop = tier.ceil(stop); expression = parser.parse(request.expression); - measurement = new Measurement(expression, start, stop, tier); - } catch(err) { return callback({error: err}), -1; } + measurement = new Measurement(expression, start, stop, tier, request.id, callback); + } catch(error) { + metalog.error('m_get', error, { info: util.inspect([start, stop, tier, expression] )}); + return callback({error: error, _trace: request._trace}), -1; + } // Compute the request metric! - measurement.measure( - (("id" in request) ? - function(time, value){ callback({time: time, value: value, id: request.id}); } : - function(time, value){ callback({time: time, value: value}); })); + metalog.trace('m_run', measurement, _.extend({ using: request }, measurement.report())); + measurement.measure(); } - function Measurement(expression, start, stop, tier){ + function Measurement(expression, start, stop, tier, id, sender){ // Round the start/stop to the tier edges + this.expression = expression; this.start = start; this.stop = stop; this.tier = tier; - this.expression = expression; + this.id = id; + this.sender = sender; } Measurement.prototype.flavor = function(){ return this.expression.op ? 'binary' : (this.expression.type ? 'unary' : 'constant'); }; - Measurement.prototype.inspect = function(){ + Measurement.prototype.report = function(){ return { flavor: this.flavor(), tier: this.tier.key, start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), expr: this.expression.source }; }; + Measurement.prototype.send_result = function(time, value){ + var resp = new Metric(time, value, this.id, this); + metalog.dump_trace('m_res', resp, {using: this, bin: resp.bin }); + this.sender(resp); + }; + Measurement.prototype.complete = function(){ this.send_result(this.stop, null); }; + + // Computes the metric for the given expression for the time interval from // start (inclusive) to stop (exclusive). The time granularity is determined // by the specified tier, such as daily or hourly. The callback is invoked // repeatedly for each metric value, being passed two arguments: the time and // the value. The values may be out of order due to partial cache hits. - Measurement.prototype.measure = function measure(callback) { + Measurement.prototype.measure = function measure() { switch(this.flavor()){ - case 'binary': this.binary(callback); break; - case 'unary': this.unary(callback); break; - case 'constant': this.constant(callback); break; + case 'binary': this.binary(this.sender); break; + case 'unary': this.unary(this.sender); break; + case 'constant': this.constant(); break; } }; // Computes a constant expression like the "7" in "x * 7" - Measurement.prototype.constant = function constant(callback) { - var value = this.expression.value(); - walk(this.start, this.stop, this.tier, function(time){ callback(time, value); }); - callback(this.stop); + Measurement.prototype.constant = function constant() { + var self = this, value = this.expression.value(); + walk(this.start, this.stop, this.tier, function(time){ self.send_result(time, value); }); + self.complete(); }; + // Computes a binary expression by merging two subexpressions + // + // "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and + // unary "sum(resp)". We don't know what order they'll show up in, so if say + // the value for left appears first, it parks that value as left[time], where + // the result for right will eventually find it. + Measurement.prototype.binary = function binary(callback) { + var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier, value; + var left = new Measurement(expression.left, start, stop, tier), + right = new Measurement(expression.right, start, stop, tier); + metalog.trace('msbl', left, {using: self}); metalog.trace('msbr', right, {using: self}); + + left.measure(function(time, vall, tr) { + if (time in right) { // right val already appeared; get a result + value = (time < stop ? expression.op(vall, right[time]) : vall); + this.send_result(time, value); + delete right[time]; + } else { // right val still on the way; stash the value + left[time] = vall; + } + }); + + right.measure(function(time, valr, tr) { + if (time in left) { + value = (time < stop ? expression.op(left[time], valr) : valr) + this.send_result(time, value); + delete left[time]; + } else { + right[time] = valr; + } + }); + }; + + Measurement.prototype.unary = unary_new + + function get_queue(name){ + if (! request_queues[name]) request_queues[name] = queuer(metric_parallelism); + return request_queues[name]; + } + // Serializes a unary expression for computation. - Measurement.prototype.unary = function unary(callback) { - var measurement = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; - var remaining = 0, - time0 = Date.now(), - time = start, - name = expression.source, - queue = queueByName[name], - step = tier.key; - metalog.minor('unary start', measurement.inspect(), start); - - // Compute the expected number of values. - walk(start, stop, tier, function(time){ remaining++; }); - // If no results were requested, return immediately. - if (!remaining) return callback(stop); + function unary_new(callback) { + var self = this, + remaining = 0, + queue = get_queue(this.expression.source); + + // Compute the expected number of values; if no results were requested, return immediately. + walk(self.start, self.stop, self.tier, function(time){ remaining++; }); + if (! remaining) return this.complete(); // Add this task to the appropriate queue. - task.qcount = qcount++; - if (queue) queue.next = task; - else process.nextTick(task); - queueByName[name] = task; - - function task() { - findOrComputeUnary(expression, start, stop, tier, function(time, value){ - callback(time, value); - metalog.minor('unary result', { meas: measurement.inspect(), qcount: qcount, next: (task.next && task.next.qcount) }); + queue.defer(function task(q_callback){ + self.findOrComputeUnary(function(time, value, tr){ + self.send_result(time, value); if (!--remaining) { - callback(stop); - if (task.next) process.nextTick(task.next); - else delete queueByName[name]; - // Record how long it took us to compute as an event! - metalog.minor("cube_compute", { is: 'metric', at: 'done', meas: measurement.inspect(), ms: Date.now() - time0}); + process.nextTick(function(){ q_callback(null, [time, value]); }); + self.complete(); } }); - } - return null; - }; + }).await(_.identity); + } + + // // Serializes a unary expression for computation. + // function unary_old(callback) { + // var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; + // var remaining = 0, + // time0 = Date.now(), + // time = start, + // name = expression.source, + // queue = queueByName[name], + // step = tier.key; + // + // // Compute the expected number of values. + // walk(start, stop, tier, function(time){ remaining++; }); + // // If no results were requested, return immediately. + // if (!remaining) return callback(stop, null, metalog.trace('msu', {}, { using: self})); + // + // // Add this task to the appropriate queue. + // task.qcount = qcount++; + // if (queue){ queue.next = task; queue.next.qindex = queue.qindex + 1; } + // else { task.qindex = 0 ; process.nextTick(task); } + // queueByName[name] = task; + // + // function task() { + // findOrComputeUnary(expression, start, stop, tier, function(time, value){ + // callback(time, value); + // // metalog.warn('unary result', { time: time, meas: self.report(), qcount: qcount, next: (task.next && task.next.qcount) }); + // if (!--remaining) { + // callback(stop); + // if (task.next) process.nextTick(task.next); + // else delete queueByName[name]; + // // Record how long it took us to compute as an event! + // // metalog.warn("cube_compute", { is: 'metric', at: 'done', meas: self.report(), ms: Date.now() - time0}); + // } + // }); + // } + // return null; + // } // Finds or computes a unary (primary) expression. - function findOrComputeUnary(expression, start, stop, tier, callback) { + Measurement.prototype.findOrComputeUnary = function findOrComputeUnary(callback) { + var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; var name = expression.type, map = expression.value, reduce = reduces[expression.reduce], @@ -127,28 +207,26 @@ exports.getter = function(db) { fields = {t: 1}, metrics, events; - metalog.minor('findOrComputeUnary', expression.source, start, stop, tier.key, callback.toString()); - - // if (!reduce) return callback({error: "invalid reduce operation"}), -1; + metalog.trace('focu', this); // Copy any expression filters into the query object. expression.filter(filter); - // Request any needed fields. expression.fields(fields); db.metrics(name, function(error, collection){ handle(error); - metrics = collection; - find(start, stop, tier, callback); + find(start, stop, tier, self, callback); }); // The metric is computed recursively, reusing the above variables. - function find(start, stop, tier, callback) { + function find(start, stop, tier, tr, callback) { var compute = ((tier.next && reduce.pyramidal) ? computePyramidal : computeFlat), step = tier.key; + metalog.trace('m_f0', tr); + // Query for the desired metric in the cache. metrics.find({ i: false, @@ -163,27 +241,27 @@ exports.getter = function(db) { // Immediately report back whatever we have. If any values are missing, // merge them into contiguous intervals and asynchronously compute them. function foundMetrics(error, cursor) { - metalog.minor('foundMetrics', expression.source, start, stop, tier.key, callback.toString()); handle(error); var time = start; cursor.each(function(error, row) { + var fmtr = metalog.trace('m_f1', {}, { using: tr }); handle(error); if (row) { - callback(row._id.t, row.v); - if (time < row._id.t) compute(time, row._id.t); - time = tier.step(row._id.t); + callback(row._id.t, row.v, fmtr); // send back value for this timeslot + if (time < row._id.t) compute(time, row._id.t, fmtr); // recurse from last value seen up to this timeslot + time = tier.step(row._id.t); // update the last-observed timeslot } else { - if (time < stop) compute(time, stop); + if (time < stop) compute(time, stop, fmtr); // once last row is seen, compute rest of range } }); } // Group metrics from the next tier. - function computePyramidal(start, stop) { - metalog.minor('computePyramidal', expression.source, start, stop, tier.key, callback.toString()); + function computePyramidal(start, stop, tr) { + // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); var bins = {}; - find(start, stop, tier.next, function(time, value) { + find(start, stop, tier.next, tr, function(time, value, tr) { var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); if (bin.values.push(value) === bin.size) { save(time, reduce(bin.values)); @@ -199,7 +277,8 @@ exports.getter = function(db) { // Group raw events. Unlike the pyramidal computation, here we can control // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. - function computeFlat(start, stop) { + function computeFlat(start, stop, tr) { + metalog.trace('m_cf', tr, { expr: expression.source, start: start, stop: stop, tier: tier.key }); // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) @@ -214,27 +293,28 @@ exports.getter = function(db) { handle(error); var time = start, values = []; cursor.each(function(error, row) { + var res_tr = metalog.trace('m_cfn', {}, { using: tr }); handle(error); if (row) { var then = tier.floor(row.t); if (time < then) { - save(time, values.length ? reduce(values) : reduce.empty); - while ((time = tier.step(time)) < then) save(time, reduce.empty); + save(time, values.length ? reduce(values) : reduce.empty, res_tr); + while ((time = tier.step(time)) < then) save(time, reduce.empty, res_tr); values = [map(row)]; } else { values.push(map(row)); } } else { - save(time, values.length ? reduce(values) : reduce.empty); - while ((time = tier.step(time)) < stop) save(time, reduce.empty); + save(time, values.length ? reduce(values) : reduce.empty, res_tr); + while ((time = tier.step(time)) < stop) save(time, reduce.empty, res_tr); } }); }); }) } - function save(time, value) { - callback(time, value); + function save(time, value, tr) { + callback(time, value, metalog.trace('m_sv', {}, {using: tr})); if ((! value) && (value !== 0)) return; var metric = { _id: { @@ -245,42 +325,12 @@ exports.getter = function(db) { i: false, v: new Double(value) }; - metalog.info('cube_compute', {is: 'metric_save', metric: metric }); + // metalog.trace('cube_compute', {is: 'metric_save', metric: metric }); metrics.save(metric, handle); } } } - // Computes a binary expression by merging two subexpressions - // - // "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and - // unary "sum(resp)". We don't know what order they'll show up in, so if say - // the value for left appears first, it parks that value as left[time], where - // the result for right will eventually find it. - Measurement.prototype.binary = function binary(callback) { - var expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; - var left = new Measurement(expression.left, start, stop, tier), - right = new Measurement(expression.right, start, stop, tier); - - left.measure(function(time, vall) { - if (time in right) { // right val already appeared; get a result - callback(time, time < stop ? expression.op(vall, right[time]) : vall); - delete right[time]; - } else { // right val still on the way; stash the value - left[time] = vall; - } - }); - - right.measure(function(time, valr) { - if (time in left) { - callback(time, time < stop ? expression.op(left[time], valr) : valr); - delete left[time]; - } else { - right[time] = valr; - } - }); - }; - // execute cb on each interval from t1 to t2 function walk(t1, t2, tier, cb){ while (t1 < t2) { @@ -294,6 +344,6 @@ exports.getter = function(db) { function handle(error) { if (!error) return; - metalog.warn('cube_request', {is: 'metric error', error: error }); + metalog.error('metric', error); throw error; } diff --git a/lib/cube/models.js b/lib/cube/models.js index 79608b2e..5db735bc 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -25,73 +25,99 @@ _.mapHash = function(obj, func){ function Event(type, time, data, id){ this.t = time; this.d = data; + this.type = type; if (id) this._id = id; - this._type = function(){ return type; }; + Object.defineProperties(this, { + _trace: { value: null, enumerable: false, writable: true } + }); +} +Event.prototype = { + bin: function(tr){ return tiers[tr].bin(this.t); }, + day_bin: function(){ return tiers[day ].bin(this.t); }, + m05_bin: function(){ return tiers[minute5 ].bin(this.t); }, + s10_bin: function(){ return tiers[second10].bin(this.t); }, + day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, + m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, + s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, + bins: function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }, + agos: function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, + report: function report(){ + return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; + } +}; - this.bin = function(tr){ return tiers[tr].bin(this.t); }; +// Validate the date and type. +Event.prototype.validate = function(){ + if (!type_re.test(this.type)) throw("invalid type"); + if (isNaN(this.t)) throw("invalid time"); +}; - this.day_bin = function(){ return tiers[day ].bin(this.t); }; - this.m05_bin = function(){ return tiers[minute5 ].bin(this.t); }; - this.s10_bin = function(){ return tiers[second10].bin(this.t); }; +Event.prototype.to_wire = function(){ + if (this._id) return { t: this.t, d: this.d, _id: this._id }; + else return { t: this.t, d: this.d }; +}; - this.day_ago = function day_ago(){ return Math.floor((Date.now() - this.t) / day); }; - this.m05_ago = function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }; - this.s10_ago = function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }; +Event.prototype.to_request = function(attrs){ + var ev = { time: this.t, data: this.d, type: this.type }; + if (this._id) ev.id = this._id; + for (var key in attrs){ ev[key] = attrs[key]; } + metalog.trace('new_event', ev); + return ev; +}; + +Event.prototype.save = function(callback){ + var self = this; + db.events(self.type, function(error, collection){ + if (error) return callback(error); + metalog.trace('eSvc', self); + collection.save(self.to_wire(), function saver(error){ + callback(error, self + , metalog.trace('eSvd', self) + ); }); + }, metalog.trace('eSva', self) + ); +}; - this.bins = function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }; - this.agos = function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }; - this.report = function report(){ - return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; - }; - - this.validate = function(){ - // Validate the date and type. - if (!type_re.test(type)) throw("invalid type"); - if (isNaN(time)) throw("invalid time"); - }; - - this.to_wire = function(){ - var ev = { t: this.t, d: this.d }; - if (id) ev._id = this._id; - return ev; - }; - - this.to_request = function(attrs){ - var ev = { time: this.t, data: this.d, type: type }; - if (id) ev.id = this._id; - for (var key in attrs){ ev[key] = attrs[key]; } - return ev; - }; - - this.save = function(callback){ - var _this = this; - db.events(type, function(error, collection){ - if (error) return callback(error); - collection.save(_this.to_wire(), function(error){ return callback(error, _this); }); - }); - }; -} exports.Event = Event; -function Metric(type, time, tier, expression, value){ - - // this.type = type; - // this.time = time; - // this.tier = tier; - // this.expression = expression; - // if (id) this._id = id; - - this.to_wire = function to_wire(){ - return { - _id: { - e: expression.source, - l: tier.key, - t: time - }, - i: false, - v: value - }; - }; +// -------------------------------------------------------------------------- + +function Metric(time, value, id, measurement){ + this.value = value; + this.time = time; + if (id) this.id = id; + + Object.defineProperties(this, { + e: { value: measurement.expression.source, enumerable: false, writable: false }, + measurement: { value: measurement, enumerable: false, writable: false }, + _trace: { value: null, enumerable: false, writable: true }, + }); } +Object.defineProperties(Metric.prototype, { + tier: { get: function(){ return this.measurement.tier } }, + bin: { get: function(){ return this.tier.bin(this.time); } }, + // day_bin: function(){ return tiers[day ].bin(this.t); }, + // m05_bin: function(){ return tiers[minute5 ].bin(this.t); }, + // s10_bin: function(){ return tiers[second10].bin(this.t); }, + // day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, + // m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, + // s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, + // bins: function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }, + // agos: function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, + // report: function report(){ + // return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; + // } +}); + +Metric.prototype.to_wire = function to_wire(){ + return { i: false, v: this.value, _id: { e: this.e, l: this.tier.key, t: time } }; +}; + +Metric.prototype.report = function report(){ + var hsh = { time: this.time, value: this.value }; + if (this.id) hsh.id = this.id; + return hsh; +}; + exports.Metric = Metric; diff --git a/lib/cube/server.js b/lib/cube/server.js index 5534d28d..0692cf1e 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -27,8 +27,7 @@ var util = require("util"), // Don't crash on errors. process.on("uncaughtException", function(error) { - util.log("uncaught exception: " + error); - util.log(error.stack); + metalog.error('server', error); }); // And then this happened: @@ -105,6 +104,7 @@ module.exports = function(options) { var authorization = request.authorized; function connection_callback(response) { + metalog.dump_trace('resp', response); delete response._trace; connection.sendUTF(JSON.stringify(response)); } @@ -120,13 +120,14 @@ module.exports = function(options) { }); connection.on("message", function(message) { - // staple the authorization back on + // parse, staple the authorization on, then process var payload = JSON.parse(message.utf8Data || message); payload.authorized = authorization; + metalog.trace('req', payload); e.dispatch(payload, connection_callback); }); - metalog.event('cube_request', { is: 'ws', method: "WebSocket", ip: connection.remoteAddress, path: request.url}, 'minor'); + metalog.event('connect', { method: 'ws', ip: connection.remoteAddress, path: request.url}, 'minor'); return; } } @@ -136,13 +137,14 @@ module.exports = function(options) { // Register HTTP listener. primary.on("request", function(request, response) { var u = url.parse(request.url); + metalog.trace('http', request); function auth_ok(perms) { - metalog.event('cube_request', { is: 'auth_ok', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: true, user: perms }); + metalog.trace('auth_ok', request, { method: request.method, ip: request.connection.remoteAddress, path: u.pathname }); e.dispatch(request, response); } function auth_no(reason) { - metalog.event('cube_request', { is: 'auth_no', method: request.method, ip: request.connection.remoteAddress, path: u.pathname, auth: false }); + metalog.dump_trace('auth_no', request, { method: request.method, ip: request.connection.remoteAddress, path: u.pathname }); response.writeHead(403, {"Content-Type": "text/plain"}); response.end("403 Forbidden"); } @@ -158,14 +160,11 @@ module.exports = function(options) { request.on("end", function() { file.serve(request, response, function(error) { if (error) { - metalog.event('cube_request', - { is: 'failed', msg: error, code: error.status, ip: request.connection.remoteAddress, path: u.pathname }); + metalog.error('req_file', error, { ip: request.connection.remoteAddress, path: u.pathname }); response.writeHead(error.status, {"Content-Type": "text/plain"}); response.end(error.status + ""); } else { - metalog.event('cube_request', - { is: 'static', ip: request.connection.remoteAddress, path: u.pathname }, - 'minor'); + metalog.trace('req_file', request, { ip: request.connection.remoteAddress, path: u.pathname }); } }); }); @@ -173,7 +172,7 @@ module.exports = function(options) { server.start = function(server_start_cb) { db.open(options, function(error, db){ - if(error) throw error; + handle(error); ready(db); }); @@ -182,10 +181,10 @@ module.exports = function(options) { metalog.putter = event.putter(db); server.register(db, endpoints); authenticator = authentication.authenticator(options["authenticator"], db, options); - metalog.event("cube_life", { is: 'start_http', port: options["http-port"] }); + metalog.event('start_http', { port: options["http-port"] }); primary.listen(options["http-port"]); if (endpoints.udp) { - metalog.event("cube_life", { is: 'start_udp', port: options["udp-port"] }); + metalog.event('start_udp', { port: options["udp-port"] }); udp = dgram.createSocket("udp4"); udp.on("message", function(message) { endpoints.udp(JSON.parse(message.toString("utf8")), ignore); @@ -196,24 +195,20 @@ module.exports = function(options) { } }; - primary.on( "close", function(){ metalog.info('cube_life', {is: 'http_close' }); }); - secondary.on("close", function(){ metalog.info('cube_life', {is: 'ws_close' }); }); + primary.on( "close", function(){ metalog.info('http_close'); }); + secondary.on("close", function(){ metalog.info('ws_close' ); }); function try_close(name, obj){ if (obj){ try { - metalog.info('cube_life', {is:(name+'_stopping'), options: options}); - obj.close( function(){ metalog.info('cube_life', {is:(name+'_stop')}); } ); - } catch(err){} + metalog.info(name+'_stopping', options); + obj.close( function(){ metalog.info(name+'_stop'); } ); + } catch(error){} } } server.stop = function(cb){ - // stop flushing - event.stop(); - // stop serving - try_close('http', primary); + event.stop(); // stop flushing + try_close('http', primary); // stop serving try_close('ws', secondary); try_close('udp', udp); - // in a short while, stop db'ing - setTimeout(function(){ try_close('mongo', db); }, 100); - // finally, holler at your boys + setTimeout(function(){try_close('mongo', db);},100); // stop db'ing if (cb) setTimeout(function(){ cb(); }, 200); }; @@ -223,3 +218,9 @@ module.exports = function(options) { function ignore() { // Responses for UDP are ignored; there's nowhere for them to go! } + +function handle(error) { + if (!error) return; + metalog.error('server', error); + throw error; +} diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index f45232a0..13d38b0f 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -22,7 +22,7 @@ tiers[minute] = { floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, bin: function(d) { return Math.floor(d / minute); }, ceil: tier_ceil, - step: function(d) { return new Date(+d + minute); } + step: function(d) { return new Date(+d + minute); }, // next: tiers[second10], // size: function() { return 6; } }; @@ -32,7 +32,7 @@ tiers[minute5] = { floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, bin: function(d) { return Math.floor(d / minute5); }, ceil: tier_ceil, - step: function(d) { return new Date(+d + minute5); } + step: function(d) { return new Date(+d + minute5); }, // next: tiers[minute], // size: function() { return 5; } }; diff --git a/test/authentication-test.js b/test/authentication-test.js index c208e863..3743b0c4 100644 --- a/test/authentication-test.js +++ b/test/authentication-test.js @@ -53,7 +53,7 @@ suite.addBatch(test_helper.batch({ topic: function(test_db){ test_db.using_objects("test_users", test_users, this); }, "": { topic: function(test_db){ - return authentication.authenticator("mongo_cookie", test_db.db, { collection: "test_users" }); }, + return authentication.authenticator("mongo_cookie", test_db, { collection: "test_users" }); }, "authenticates": { "users with good tokens": { topic: successful_auth(dummy_request("boss_hogg")), diff --git a/test/broker-test.js b/test/broker-test.js index bbd94937..dcc29fe6 100644 --- a/test/broker-test.js +++ b/test/broker-test.js @@ -13,19 +13,6 @@ var suite = vows.describe("broker"); var squarer = function(ii, cb){ cb(null, ii*ii, 'squarer'); }; -assert.isCalledTimes = function(ctxt, reps){ - var results = [], finished = false; - setTimeout(function(){ if (! finished){ ctxt.callback(new Error('timeout: need '+reps+' results only have '+util.inspect(results))); } }, 2000); - return function _is_called_checker(){ - results.push(_.toArray(arguments)); - if (results.length >= reps){ finished = true; ctxt.callback(null, results); } - }; -}; - -assert.isNotCalled = function(name){ - return function(){ throw new Error(name + ' should not have been called, but was'); }; -}; - function example_worker(){ var worker = new Worker('test', 50); worker.start(); return worker; } function example_job(){ return (new Job('smurf', squarer, [7])); } diff --git a/test/event-test.js b/test/event-test.js index eff7a1bb..3f0675db 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -13,15 +13,19 @@ var ice_cubes_good_day = Date.UTC(1992, 1, 20, 1, 8, 7), suite.addBatch(test_helper.batch({ topic: function(test_db) { - return event.putter(test_db.db); + test_helper.inspectify(test_db); + return event.putter(test_db); }, 'invalidates': { topic: function(putter){ var ctxt = this; + test_helper.inspectify('a'); putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ + test_helper.inspectify('b'); putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); }, 'correct tiers': function(){ + test_helper.inspectify('invalidates putter tiers', arguments); var ts = event.invalidator().tsets(); assert.deepEqual(ts, { 'test': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], diff --git a/test/metalog-test.js b/test/metalog-test.js index 6f405da4..27b9639a 100644 --- a/test/metalog-test.js +++ b/test/metalog-test.js @@ -61,8 +61,8 @@ suite.with_log({ var event = this.logged.putted.pop(); event.time = 'whatever'; assert.deepEqual(event, { - data: { hemiconducers: 'relucting', criticality: 9 }, - type: 'reactor_level', + data: { hemiconducers: 'relucting', criticality: 9, at: "reactor_level" }, + type: 'cube', time: 'whatever' }); } diff --git a/test/metric-test.js b/test/metric-test.js index 397dd3cf..4f324dd4 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -1,8 +1,12 @@ 'use strict'; -var vows = require("vows"), +var util = require("util"), metalog = require('../lib/cube/metalog'); + +var _ = require("underscore"), + vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), + queuer = require("../lib/queue-async/queue"), models = require("../lib/cube/models"), units = models.units, event = require("../lib/cube/event"), metric = require("../lib/cube/metric"); @@ -10,15 +14,26 @@ var vows = require("vows"), // as a hack to get updates to settle, we need to insert delays. // if you see heisen-errors in the metrics tests, increase these. var step_testing_delay = 250, - batch_testing_delay = 500; + batch_testing_delay = 500, + put_queue = queuer(10); var suite = vows.describe("metric"); -var nowish = Date.now(), nowish10 = (10e3 * Math.floor(nowish/10e3)); +var nowish = Date.now(), + nowish_floor = (10e3 * Math.floor(nowish/10e3)), + nowish_stop = nowish_floor + 30e3, + thenish = Date.UTC(2011, 6, 18, 0, 0, 0); var invalid_expression_error = { error: { message: 'Expected "(", "-", "distinct", "max", "median", "min", "sum" or number but "D" found.', column: 1, line: 1, name: 'SyntaxError' }}; +function gen_date(sec){ + return new Date(thenish + sec*units.second); +} + +var t1 = gen_date(3), t1_10s = new Date(10e3 * Math.floor(t1/10e3)), + t2 = gen_date(35), t2_10s = new Date(10e3 * Math.floor(t2/10e3)); + function gen_request(attrs){ - var req = { start: nowish, stop: nowish, step: units.second10, expression: 'sum(test)'}; + var req = { start: t1, stop: t2, step: units.second10, expression: 'sum(test)'}; for (var key in attrs){ req[key] = attrs[key]; } return req; } @@ -31,349 +46,373 @@ function assert_invalid_request(req, expected_err) { }; } -var thenish = Date.UTC(2011, 6, 18, 0, 0, 0); -function gen_date(sec){ - return new Date(thenish + sec*units.second); -} - -var t1 = gen_date(2), - t2 = gen_date(71); - suite.addBatch(test_helper.batch({ - topic: function(test_db) { - var putter = event.putter(test_db.db), - getter = metric.getter(test_db.db), - callback = this.callback; - + topic: function(test_db){ + var putter = event.putter(test_db), + getter = metric.getter(test_db), + callback = this.callback; // Seed the events table with a simple event: a value going from 0 to 2499 - for (var i = 0; i < 250; i++) { - putter({ - type: "test", - time: gen_date(i * 3).toISOString(), - data: {i: 3 * i} + for (var i = 0; i < 250; i++){ + put_queue.defer(function(cb){ + putter({ type: "test", time: gen_date(i*10).toISOString(), data: {i: i}}, function(){ cb(null, null); }); }); } - - // So the events can settle in, wait `batch_testing_delay` ms before continuing - setTimeout(function() { callback(null, getter); }, batch_testing_delay); - }, - 'hi': { - topic: function(getter){ - this.ret = getter(gen_request({start: t1, stop: t2, expression: 'sum(test)'}), this.callback); }, - 'fires callback with id': function(result, j_){ - } - } -})); - -suite.addBatch(test_helper.batch({ - topic: function(test_db) { - return metric.getter(test_db.db); + put_queue.await(function(){ callback(null, getter) }); }, - 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), - 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), - 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), - 'invalid expression': assert_invalid_request({expression: 'DANCE'}, invalid_expression_error), + // 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), + // 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), + // 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), + // 'invalid expression': assert_invalid_request({expression: 'DANCE'}, invalid_expression_error), 'with request id' : { - topic: function(getter){ this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(1))'}), this.callback); }, - 'fires callback with id': function(result, j_){ - assert.equal(result.id, 'joe'); - assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); - assert.include([nowish10, 10e3+nowish10], +result.time); - } - } -})).addBatch(test_helper.batch({ - - topic: function(test_db) { - return metric.getter(test_db.db); + topic: function(getter){ + var checker = assert.isCalledTimes(this, 5); + this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(i))'}), checker); + }, + 'includes id in result': function(results){ + metalog.inspectify(results); + _.each([0,10,20,30], function(step, idx){ + assert.deepEqual(results[idx][0].report(), { id: 'joe', time: gen_date(step), value: idx }); + }); + }, + 'sends a null metric for the end slot': function(results){ assert.deepEqual(results[4][0].report(), {id: 'joe', time: gen_date(40), value: null}); } }, - 'no request id' : { - topic: function(getter){ this.ret = getter(gen_request({}), this.callback); }, - 'fires callback with no id': function(result, j_){ - assert.isFalse("id" in result); - assert.equal(result.value, (result.time > nowish10 ? undefined : 0)); - assert.include([nowish10, 10e3+nowish10], +result.time); - } - } -})); - -function skip(){ // FIXME: remove ------------------------------------------------------------ - -var steps = { - 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, - 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, - 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, - 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, - 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } -}; -steps[units.second10].description = "10-second"; -steps[units.minute ].description = "1-minute"; -steps[units.minute5 ].description = "5-minute"; -steps[units.hour ].description = "1-hour"; -steps[units.day ].description = "1-day"; - -suite.addBatch(test_helper.batch({ - topic: function(test_db) { - var putter = event.putter(test_db.db), - getter = metric.getter(test_db.db), - callback = this.callback; - - // Seed the events table with a simple event: a value going from 0 to 2499 - for (var i = 0; i < 2500; i++) { - putter({ - type: "test", - time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(i) - 10)).toISOString(), - data: {i: i} + 'simple constant' : { + topic: function(getter){ + var checker = assert.isCalledTimes(this, 5); + getter(gen_request({expression: '1'}), checker); + }, + 'gets a metric for each time slot': function(results){ + _.each([0,10,20,30], function(step, idx){ + assert.deepEqual(results[idx][0].report(), {time: gen_date(step), value: 1}); }); - } - - // So the events can settle in, wait `batch_testing_delay` ms before continuing - setTimeout(function() { callback(null, getter); }, batch_testing_delay); + }, + 'sends a null metric for the end slot': function(results){ assert.deepEqual(results[4][0].report(), {time: gen_date(40), value: null}); } }, - - // FIXME: ---- remove below ------------------------------------ - - "unary expression a": metricTest({ - expression: "sum(test)", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:02:00.000Z" - }, { - 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] - }), - - "unary expression b": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17 ] }), - "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), - "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), - "unary expression e": metricTest({ expression: "sum(test)", start: "2011-07-17T23:50:00.000Z", stop: "2011-07-18T00:03:00.000Z"}, { 60e3: [ 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25 ] }), - - "unary expression f": metricTest({ - expression: "sum(test)", - start: "2011-07-17T23:57:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 60e3: [13, 15, 17, 39, 23, 25, 27, 29, 31, 33, - 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, - 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, - 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, - 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0] - }), - - // FIXME: ---- remove above ------------------------------------ - - "unary expression": metricTest({ - expression: "sum(test)", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 60e3: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], - 3600e3: [82, 2418], - 86400e3: [82, 2418] - }), - - "unary expression with data accessor": metricTest({ - expression: "sum(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 300e3: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], - 3600e3: [3321, 3120429], - 86400e3: [3321, 3120429] - }), - - "unary expression with compound data accessor": metricTest({ - expression: "sum(test(i / 100))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 300e3: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], - 3600e3: [33.21, 31204.29], - 86400e3: [33.21, 31204.29] - }), - - "compound expression (sometimes fails due to race condition?)": metricTest({ - expression: "max(test(i)) - min(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 300e3: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], - 3600e3: [81, 2417], - 86400e3: [81, 2417] - }), - - "non-pyramidal expression": metricTest({ - expression: "distinct(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], - 3600e3: [82, 2418], - 86400e3: [82, 2418] - }), - - "compound pyramidal and non-pyramidal expression": metricTest({ - expression: "sum(test(i)) - median(test(i))", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 300e3: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], - 3600e3: [3280.5, 3119138.5], - 86400e3: [3280.5, 3119138.5] - }), - - "compound with constant expression": metricTest({ - expression: "-1 + sum(test)", - start: "2011-07-17T23:47:00.000Z", - stop: "2011-07-18T00:50:00.000Z" - }, { - 300e3: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], - 3600e3: [81, 2417], - 86400e3: [81, 2417] - }) - })); -// metricTest -- generates test tree for metrics. -// -// Gets the metric, checks it was calculated correctly from events seeded above; -// then does it again (on a delay) to check that it was cached. -// -// @example given `{ 'unary expression': metricTest({..}, { 60_000: [0, 0, ...], 86_400_000: [82, 2418] })` -// -// { 'unary expression': { -// 'at 1-minute intervals': { -// topic: function get_metrics_with_delay(getter){}, -// 'sum(test)': function metrics_assertions(actual){}, -// '(cached)': { -// topic: function get_metrics_with_delay(err, getter){}, -// 'sum(test)': function metrics_assertions(actual){} } }, -// 'at 1-day intervals': { -// topic: function get_metrics_with_delay(getter){}, -// 'sum(test)': function metrics_assertions(actual){}, -// '(cached)': { -// topic: function get_metrics_with_delay(err, getter){}, -// 'sum(test)': function metrics_assertions(actual){} } } -// } -// } -// -function metricTest(request, expected) { - // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } - var tree = {}, k; - for (var step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); - - // - // { - // topic: get_metrics_with_delay, - // expression: function(){ - // // rounds down the start time (inclusive) - // // formats UTC time in ISO 8601 - // ... - // // returns the expected values - // }, - // '(cached)': { - // topic: get_metrics_with_delay, - // expression: function(){ - // // rounds down the start time (inclusive) - // ... - // } - // } - // } - // - function testStep(step, expected) { - var start = new Date(request.start), - stop = new Date(request.stop); - - var subtree = { - topic: get_metrics_with_delay(0), - '(cached)': { topic: get_metrics_with_delay(1) } - }; - subtree[request.expression] = metrics_assertions(); - subtree["(cached)"][request.expression] = metrics_assertions(); - - function get_metrics_with_delay(depth){ return function(){ - var actual = [], - timeout = setTimeout(function() { cb("Time's up!"); }, 10000), - cb = this.callback, - req = Object.create(request), - getter = arguments[depth]; - req.step = step; - // Wait long enough for the events to have settled in the db. The - // non-cached (depth=0) round can all start in parallel, making this an - // effective `nextTick`. On the secon - setTimeout(function() { - // ... then invoke the metrics getter. As responses roll in, push them - // on to 'actual'; we're done when the 'stop' time is hit - getter(req, function(response) { - if (response.time >= stop) { - clearTimeout(timeout); - cb(null, actual.sort(function(a, b) { return a.time - b.time; })); - } else { - actual.push(response); - } - }); - }, depth * step_testing_delay); - };} - - function metrics_assertions(){ return { - 'rounds down the start time (inclusive)': function(actual) { - var floor = steps[step](start, 0); - assert.deepEqual(actual[0].time, floor); - }, - - 'rounds up the stop time (exclusive)': function(actual){ - var ceil = steps[step](stop, 0); - if (!(ceil - stop)) ceil = steps[step](stop, -1); - assert.deepEqual(actual[actual.length - 1].time, ceil); - }, - 'formats UTC time in ISO 8601': function(actual){ - actual.forEach(function(d) { - assert.instanceOf(d.time, Date); - assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); - }); - }, - - 'returns exactly one value per time': function(actual){ - var i = 0, n = actual.length, t = actual[0].time; - while (++i < n) assert.isTrue(t < (t = actual[i].time)); - }, - - 'each metric defines only time and value properties': function(actual){ - actual.forEach(function(d) { - assert.deepEqual(Object.keys(d), ["time", "value"]); - }); - }, - - 'returns the expected times': function(actual){ - var floor = steps[step], - time = floor(start, 0), - times = []; - while (time < stop) { - times.push(time); - time = floor(time, 1); - } - assert.deepEqual(actual.map(function(d) { return d.time; }), times); - }, - - 'returns the expected values': function(actual){ - var actualValues = actual.map(function(d) { return d.value; }); - assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); - expected.forEach(function(value, i) { - if (Math.abs(actual[i].value - value) > 1e-6) { - assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); - } - }); - } - };} // metric assertions - - return subtree; - } // subtree - return tree; -} // tree - -} -skip(); +// suite.addBatch(test_helper.batch({ +// topic: function(test_db){ +// var putter = event.putter(test_db), +// getter = metric.getter(test_db), +// callback = this.callback; +// +// // Seed the events table with a simple event: a value going from 0 to 2499 +// for (var i = 0; i < 250; i++){ +// put_queue.defer(function(cb){ +// putter({ type: "test", time: gen_date(i * 3).toISOString(), data: {i: 3 * i}}, function(){ cb(null, null); }); +// }); +// } +// test_helper.inspectify('d', arguments); +// put_queue.await(function(){ callback(null, getter) }); +// }, +// 'hi': { +// topic: function(getter){ +// test_helper.inspectify('3', arguments); +// this.ret = getter(gen_request({start: t1, stop: t2, expression: 'sum(test)'}), this.callback); +// }, +// 'fires callback with id': function(result, j_){ +// test_helper.inspectify('f', arguments); +// // assert.equal(result.value, (result.time > nowish_floor ? undefined : 0)); +// // assert.include([nowish_floor, 10e3+nowish_floor], +result.time); +// // assert.isFalse("id" in result); +// } +// } +// })); + +// suite.addBatch(test_helper.batch({ +// topic: function(test_db) { +// return metric.getter(test_db); +// }, +// 'no request id' : { +// topic: function(getter){ this.ret = getter(gen_request({}), this.callback); }, +// 'fires callback with no id': function(result, j_){ +// assert.equal(result.value, (result.time > nowish_floor ? undefined : 0)); +// assert.include([nowish_floor, 10e3+nowish_floor], +result.time); +// assert.isFalse("id" in result); +// } +// } +// })); + +// function skip(){ // FIXME: remove ------------------------------------------------------------ +// +// var steps = { +// 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, +// 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, +// 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, +// 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, +// 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } +// }; +// steps[units.second10].description = "10-second"; +// steps[units.minute ].description = "1-minute"; +// steps[units.minute5 ].description = "5-minute"; +// steps[units.hour ].description = "1-hour"; +// steps[units.day ].description = "1-day"; +// +// suite.addBatch(test_helper.batch({ +// topic: function(test_db) { +// var putter = event.putter(test_db), +// getter = metric.getter(test_db), +// callback = this.callback; +// +// // Seed the events table with a simple event: a value going from 0 to 2499 +// for (var i = 0; i < 2500; i++) { +// putter({ +// type: "test", +// time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(i) - 10)).toISOString(), +// data: {i: i} +// }); +// } +// +// // So the events can settle in, wait `batch_testing_delay` ms before continuing +// setTimeout(function() { callback(null, getter); }, batch_testing_delay); +// }, +// +// // FIXME: ---- remove below ------------------------------------ +// +// "unary expression a": metricTest({ +// expression: "sum(test)", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:02:00.000Z" +// }, { +// 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] +// }), +// +// "unary expression b": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17 ] }), +// "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), +// "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), +// "unary expression e": metricTest({ expression: "sum(test)", start: "2011-07-17T23:50:00.000Z", stop: "2011-07-18T00:03:00.000Z"}, { 60e3: [ 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25 ] }), +// +// "unary expression f": metricTest({ +// expression: "sum(test)", +// start: "2011-07-17T23:57:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 60e3: [13, 15, 17, 39, 23, 25, 27, 29, 31, 33, +// 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, +// 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, +// 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, +// 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, +// 0, 0, 0] +// }), +// +// // FIXME: ---- remove above ------------------------------------ +// +// "unary expression": metricTest({ +// expression: "sum(test)", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 60e3: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], +// 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], +// 3600e3: [82, 2418], +// 86400e3: [82, 2418] +// }), +// +// "unary expression with data accessor": metricTest({ +// expression: "sum(test(i))", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 300e3: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], +// 3600e3: [3321, 3120429], +// 86400e3: [3321, 3120429] +// }), +// +// "unary expression with compound data accessor": metricTest({ +// expression: "sum(test(i / 100))", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 300e3: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], +// 3600e3: [33.21, 31204.29], +// 86400e3: [33.21, 31204.29] +// }), +// +// "compound expression (sometimes fails due to race condition?)": metricTest({ +// expression: "max(test(i)) - min(test(i))", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 300e3: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], +// 3600e3: [81, 2417], +// 86400e3: [81, 2417] +// }), +// +// "non-pyramidal expression": metricTest({ +// expression: "distinct(test(i))", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], +// 3600e3: [82, 2418], +// 86400e3: [82, 2418] +// }), +// +// "compound pyramidal and non-pyramidal expression": metricTest({ +// expression: "sum(test(i)) - median(test(i))", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 300e3: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], +// 3600e3: [3280.5, 3119138.5], +// 86400e3: [3280.5, 3119138.5] +// }), +// +// "compound with constant expression": metricTest({ +// expression: "-1 + sum(test)", +// start: "2011-07-17T23:47:00.000Z", +// stop: "2011-07-18T00:50:00.000Z" +// }, { +// 300e3: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], +// 3600e3: [81, 2417], +// 86400e3: [81, 2417] +// }) +// +// })); +// +// // metricTest -- generates test tree for metrics. +// // +// // Gets the metric, checks it was calculated correctly from events seeded above; +// // then does it again (on a delay) to check that it was cached. +// // +// // @example given `{ 'unary expression': metricTest({..}, { 60_000: [0, 0, ...], 86_400_000: [82, 2418] })` +// // +// // { 'unary expression': { +// // 'at 1-minute intervals': { +// // topic: function get_metrics_with_delay(getter){}, +// // 'sum(test)': function metrics_assertions(actual){}, +// // '(cached)': { +// // topic: function get_metrics_with_delay(err, getter){}, +// // 'sum(test)': function metrics_assertions(actual){} } }, +// // 'at 1-day intervals': { +// // topic: function get_metrics_with_delay(getter){}, +// // 'sum(test)': function metrics_assertions(actual){}, +// // '(cached)': { +// // topic: function get_metrics_with_delay(err, getter){}, +// // 'sum(test)': function metrics_assertions(actual){} } } +// // } +// // } +// // +// function metricTest(request, expected) { +// // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } +// var tree = {}, k; +// for (var step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); +// +// // +// // { +// // topic: get_metrics_with_delay, +// // expression: function(){ +// // // rounds down the start time (inclusive) +// // // formats UTC time in ISO 8601 +// // ... +// // // returns the expected values +// // }, +// // '(cached)': { +// // topic: get_metrics_with_delay, +// // expression: function(){ +// // // rounds down the start time (inclusive) +// // ... +// // } +// // } +// // } +// // +// function testStep(step, expected) { +// var start = new Date(request.start), +// stop = new Date(request.stop); +// +// var subtree = { +// topic: get_metrics_with_delay(0), +// '(cached)': { topic: get_metrics_with_delay(1) } +// }; +// subtree[request.expression] = metrics_assertions(); +// subtree["(cached)"][request.expression] = metrics_assertions(); +// +// function get_metrics_with_delay(depth){ return function(){ +// var actual = [], +// timeout = setTimeout(function() { cb("Time's up!"); }, 10000), +// cb = this.callback, +// req = Object.create(request), +// getter = arguments[depth]; +// req.step = step; +// // Wait long enough for the events to have settled in the db. The +// // non-cached (depth=0) round can all start in parallel, making this an +// // effective `nextTick`. On the secon +// setTimeout(function() { +// // ... then invoke the metrics getter. As responses roll in, push them +// // on to 'actual'; we're done when the 'stop' time is hit +// getter(req, function(response){ +// if (response.time >= stop) { +// clearTimeout(timeout); +// cb(null, actual.sort(function(a, b) { return a.time - b.time; })); +// } else { +// actual.push(response); +// } +// }); +// }, depth * step_testing_delay); +// };} +// +// function metrics_assertions(){ return { +// 'rounds down the start time (inclusive)': function(actual) { +// var floor = steps[step](start, 0); +// assert.deepEqual(actual[0].time, floor); +// }, +// +// 'rounds up the stop time (exclusive)': function(actual){ +// var ceil = steps[step](stop, 0); +// if (!(ceil - stop)) ceil = steps[step](stop, -1); +// assert.deepEqual(actual[actual.length - 1].time, ceil); +// }, +// +// 'formats UTC time in ISO 8601': function(actual){ +// actual.forEach(function(d) { +// assert.instanceOf(d.time, Date); +// assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); +// }); +// }, +// +// 'returns exactly one value per time': function(actual){ +// var i = 0, n = actual.length, t = actual[0].time; +// while (++i < n) assert.isTrue(t < (t = actual[i].time)); +// }, +// +// 'each metric defines only time and value properties': function(actual){ +// actual.forEach(function(d) { +// delete d._trace; +// assert.deepEqual(Object.keys(d), ["time", "value"]); +// }); +// }, +// +// 'returns the expected times': function(actual){ +// var floor = steps[step], +// time = floor(start, 0), +// times = []; +// while (time < stop) { +// times.push(time); +// time = floor(time, 1); +// } +// assert.deepEqual(actual.map(function(d) { return d.time; }), times); +// }, +// +// 'returns the expected values': function(actual){ +// var actualValues = actual.map(function(d) { return d.value; }); +// assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); +// expected.forEach(function(value, i) { +// if (Math.abs(actual[i].value - value) > 1e-6) { +// assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); +// } +// }); +// } +// };} // metric assertions +// +// return subtree; +// } // subtree +// return tree; +// } // tree +// +// } +// skip(); suite['export'](module); diff --git a/test/test_helper.js b/test/test_helper.js index d6a71b29..0d09e908 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -1,9 +1,10 @@ 'use strict'; -var assert = require("assert"), +var _ = require("underscore"), + assert = require("assert"), http = require("http"), dgram = require('dgram'), - db = require("../lib/cube/db"), + test_db = require("../lib/cube/db"), metalog = require("../lib/cube/metalog"); // ========================================================================== @@ -12,7 +13,6 @@ var assert = require("assert"), // var test_helper = {}; -var test_db = {}; var test_collections = ["test_users", "test_events", "test_metrics"]; test_helper.inspectify = metalog.inspectify; @@ -161,15 +161,13 @@ test_helper.batch = function(batch) { "": { topic: function() { var _this = this; - connect(test_helper.settings, function(error, db){ - setup_db(_this.callback); + test_db.open(test_helper.settings, function(error){ + drop_collections(_this.callback); }); }, "": batch, teardown: function(test_db) { - if (test_db.db.isConnected()) { - process.nextTick(function(){ test_db.db.close(); }); - } + test_db.close(this.callback); } } }; @@ -180,7 +178,7 @@ test_helper.batch = function(batch) { // Wrap your tests in test_helper.batch to get the test_db object. test_db.using_objects = function (clxn_name, test_objects, context){ metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); - test_db.db.collection(clxn_name, function(err, clxn){ + test_db.collection(clxn_name, function(err, clxn){ if (err) throw(err); context[clxn_name] = clxn; clxn.remove({ dummy: true }, function(){ @@ -195,27 +193,13 @@ test_db.using_objects = function (clxn_name, test_objects, context){ // db methods // -// @see test_helper.batch -function setup_db(cb){ - drop_collections(cb); -} - -// @see test_helper.batch -function connect(options, callback){ - metalog.minor('cube_testdb', { state: 'connecting to db', options: options }); - test_db.options = options, - test_db.db = db; - - test_db.db.open(options, callback); -} - // @see test_helper.batch function drop_collections(cb){ metalog.minor('cube_testdb', { state: 'dropping test collections', collections: test_collections }); var collectionsRemaining = test_collections.length; test_collections.forEach(function(collection_name){ - test_db.db.collection(collection_name, function(error, collection){ + test_db.collection(collection_name, function(error, collection){ collection.drop(collectionReady); }) }); @@ -226,6 +210,24 @@ function drop_collections(cb){ } } +// ========================================================================== +// +// assertions +// + +assert.isCalledTimes = function(ctxt, reps){ + var results = [], finished = false; + setTimeout(function(){ if (! finished){ ctxt.callback(new Error('timeout: need '+reps+' results only have '+util.inspect(results))); } }, 2000); + return function _is_called_checker(){ + results.push(_.toArray(arguments)); + if (results.length >= reps){ finished = true; ctxt.callback(null, results); } + }; +}; + +assert.isNotCalled = function(name){ + return function(){ throw new Error(name + ' should not have been called, but was'); }; +}; + // ========================================================================== // // fin. From 81086f086e75e1ceddd67bcc2f9ccce66b3b2f88 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Sun, 9 Sep 2012 21:13:45 -0500 Subject: [PATCH 39/87] Added a trace system -- tracks internal progress through whole call stack. Moving things over to @mbostock's async-queue --- lib/cube/index.js | 18 +- lib/cube/metalog.js | 82 ++++-- lib/cube/metric.js | 291 +++++++++--------- lib/cube/models.js | 40 +-- lib/queue-async | 1 + test/metric-test.js | 700 ++++++++++++++++++++++---------------------- test/test_helper.js | 1 + 7 files changed, 575 insertions(+), 558 deletions(-) create mode 160000 lib/queue-async diff --git a/lib/cube/index.js b/lib/cube/index.js index 1404b1cd..78313693 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -3,13 +3,13 @@ process.env.TZ = 'UTC'; exports.models = require("./models"); exports.Event = exports.models.Event; -exports.Metric = exports.models.Metric; +// exports.Metric = require("./metric").Metric; exports.authentication = require("./authentication"); -exports.metalog = require("./metalog"); -exports.emitter = require("./emitter"); -exports.server = require("./server"); -exports.collector = require("./collector"); -exports.evaluator = require("./evaluator"); -exports.visualizer = require("./visualizer"); -exports.endpoint = require("./endpoint"); -exports.warmer = require("./warmer"); +exports.metalog = require("./metalog"); +exports.emitter = require("./emitter"); +exports.server = require("./server"); +exports.collector = require("./collector"); +exports.evaluator = require("./evaluator"); +exports.visualizer = require("./visualizer"); +exports.endpoint = require("./endpoint"); +exports.warmer = require("./warmer"); diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 5a09fac1..41df7358 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -72,45 +72,67 @@ metalog.error = function(at, error, info){ // -------------------------------------------------------------------------- -var trace_id = 1; - -metalog.trace = function(label, item, hsh){ - item = item || {}; hsh = hsh || {}; - var using = (hsh.using||{}); delete hsh.using; - if (! item._trace) item._trace = { beg: +(new Date()) }; - // - item._trace = _.extend(item._trace, using._trace||{}, hsh); - // - item._trace[label] = ((new Date()) - item._trace['beg']); - if (! item._trace.tid) item._trace.tid = trace_id++; - // if (value) item._trace['val'] = (Math.round(100*value)/100.0); - // } catch(err){ metalog.log(err); } - - return item; -}; +var trace_id = 1, boot = +(new Date()); -var dump_keys = {tid: 4, beg: 15, m_get: 3, m_run: 3, m_res: 3, tier: 8, start: 9, stop: 9, bin: 9}; +var dump_keys = {tid: 4, beg: 15, boot: 4, + tier: 8, start: 9, stop: 9, bin: 9, + mget: 4, find: 4, mf0: 4, mf1: 4, msb: 4, + mflt: 4, mfl0: 4, mfly: 4, msav: 4, mres: 4, resp: 4, + expr: 12}; function dump(hsh){ - var fields = _.map(dump_keys, function(len, key){ return (hsh[key]+' ').slice(0,len); }) - metalog.log(fields.join('|') + "\t" + (JSON.stringify(hsh).slice(0,150))); + var value = hsh.val; + var fields = _.map(dump_keys, function(len, key){ return ((hsh[key] === undefined ? '' : hsh[key])+' ').slice(0,len); }); + var extras = {}; for (var key in hsh){ if (!(key in dump_keys)) extras[key] = hsh[key]; } + if (_.isNumber(value)) fields.push((value*100)/100); + metalog.log(fields.join('|') + "\t" + (JSON.stringify(extras).slice(0,150))); } function dump_header(){ metalog.log(_.map(dump_keys, function(len, key){ return (key+' ').slice(0,len); }).join('|')); } -metalog.dump_trace = function(label, item, hsh){ - var item = metalog.trace(label, item, hsh); - var tr = item._trace; - function p(v,l){ var str = (v ? v.toString() : ''); return (v+'................').slice(0,l); } - try{ - item._trace['end'] = +(new Date()); - if (Math.random() < 0.2) dump_header(); - dump(item._trace); - } catch(err){ metalog.log(err); metalog.log(err.stack); } - return item; -}; +var tracing = false; + +if (tracing){ + + metalog.trace = function(label, item, hsh){ + + // try{ throw new Error("Trace"); } catch(e){ console.log(e.stack); } + + item = item || {}; hsh = hsh || {}; + var using = (hsh.using||{}); delete hsh.using; + if (! item._trace) item._trace = { beg: +(new Date()), boot: ((new Date()) - boot) }; + // + item._trace = _.extend(item._trace, (using._trace||{}), hsh); + // + item._trace[label] = ((new Date()) - item._trace['beg']); + if (! item._trace.tid) item._trace.tid = trace_id++; + // if (value) item._trace['val'] = (Math.round(100*value)/100.0); + // } catch(err){ metalog.log(err); } + + // metalog.warn('tr_'+label, item); + // if (label == 'msbl') metalog.inspectify(label, item, using, hsh) + + return item; + }; + + metalog.dump_trace = function(label, item, hsh){ + var item = metalog.trace(label, item, hsh); + var tr = item._trace; + function p(v,l){ var str = (v ? v.toString() : ''); return (v+'................').slice(0,l); } + try{ + item._trace['end'] = +(new Date()); + if (Math.random() < 0.3) dump_header(); + dump(item._trace); + } catch(err){ metalog.log(err); metalog.log(err.stack); } + return item; + }; +} else { + metalog.trace = function(){}; + metalog.dump_trace = function(){}; +} + // Dump the 'util.inspect' view of each argument to the console. metalog.inspectify = function inspectify(args){ diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 2c31717c..5e813b20 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -18,7 +18,7 @@ var _ = require("underscore"), var metric_fields = {v: 1}, metric_options = {sort: {"_id.t": 1}, batchSize: 1000}, event_options = {sort: {t: 1}, batchSize: 1000}, - metric_parallelism = 5; + queue_parallelism = 1; // Query for metrics. exports.getter = function(db){ @@ -32,8 +32,7 @@ exports.getter = function(db){ tier = tiers[+request.step], start = new Date(request.start), stop = new Date(request.stop); - metalog.trace('m_get', request); - + try { if (!tier) throw "invalid step"; if (isNaN(start)) throw "invalid start"; @@ -44,39 +43,44 @@ exports.getter = function(db){ start = tier.floor(start); stop = tier.ceil(stop); expression = parser.parse(request.expression); - measurement = new Measurement(expression, start, stop, tier, request.id, callback); + measurement = new Measurement(expression, start, stop, tier, send_response); } catch(error) { - metalog.error('m_get', error, { info: util.inspect([start, stop, tier, expression] )}); + metalog.error('mget', error, { info: util.inspect([start, stop, tier, expression] )}); return callback({error: error, _trace: request._trace}), -1; } + function send_response(time, value, tr){ + var resp = new Metric(time, value, request.id, measurement); + metalog.dump_trace('resp', resp, {using: tr, bin: resp.bin }); + callback(resp); + } + // Compute the request metric! - metalog.trace('m_run', measurement, _.extend({ using: request }, measurement.report())); + metalog.trace('mget', measurement, { using: request }); measurement.measure(); } - function Measurement(expression, start, stop, tier, id, sender){ + function Measurement(expression, start, stop, tier, sender){ // Round the start/stop to the tier edges this.expression = expression; this.start = start; this.stop = stop; this.tier = tier; - this.id = id; this.sender = sender; - } + this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); - Measurement.prototype.flavor = function(){ return this.expression.op ? 'binary' : (this.expression.type ? 'unary' : 'constant'); }; + // Object.defineProperties(this, { + // }); + } Measurement.prototype.report = function(){ - return { flavor: this.flavor(), tier: this.tier.key, start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), expr: this.expression.source }; + return { flavor: this.flavor, tier: this.tier.key, start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), expr: (this.expression.op||this.expression.source||this.expression.value()) }; }; - Measurement.prototype.send_result = function(time, value){ - var resp = new Metric(time, value, this.id, this); - metalog.dump_trace('m_res', resp, {using: this, bin: resp.bin }); - this.sender(resp); + Measurement.prototype.send_result = function(time, value, tr){ + var ret_tr = metalog.dump_trace('mres', tr, { using: this, bin: this.tier.bin(time), val: value }); + this.sender(time, value, ret_tr); }; - Measurement.prototype.complete = function(){ this.send_result(this.stop, null); }; - + Measurement.prototype.complete = function(tr){ this.send_result(this.stop, null, tr); }; // Computes the metric for the given expression for the time interval from // start (inclusive) to stop (exclusive). The time granularity is determined @@ -84,11 +88,8 @@ exports.getter = function(db){ // repeatedly for each metric value, being passed two arguments: the time and // the value. The values may be out of order due to partial cache hits. Measurement.prototype.measure = function measure() { - switch(this.flavor()){ - case 'binary': this.binary(this.sender); break; - case 'unary': this.unary(this.sender); break; - case 'constant': this.constant(); break; - } + metalog.trace('meas', this, this.report()); + this[this.flavor](this.sender); }; // Computes a constant expression like the "7" in "x * 7" @@ -98,116 +99,77 @@ exports.getter = function(db){ self.complete(); }; - // Computes a binary expression by merging two subexpressions - // - // "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and - // unary "sum(resp)". We don't know what order they'll show up in, so if say - // the value for left appears first, it parks that value as left[time], where - // the result for right will eventually find it. - Measurement.prototype.binary = function binary(callback) { - var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier, value; - var left = new Measurement(expression.left, start, stop, tier), - right = new Measurement(expression.right, start, stop, tier); - metalog.trace('msbl', left, {using: self}); metalog.trace('msbr', right, {using: self}); - - left.measure(function(time, vall, tr) { - if (time in right) { // right val already appeared; get a result - value = (time < stop ? expression.op(vall, right[time]) : vall); - this.send_result(time, value); - delete right[time]; - } else { // right val still on the way; stash the value - left[time] = vall; - } - }); - - right.measure(function(time, valr, tr) { - if (time in left) { - value = (time < stop ? expression.op(left[time], valr) : valr) - this.send_result(time, value); - delete left[time]; - } else { - right[time] = valr; - } - }); - }; - - Measurement.prototype.unary = unary_new - - function get_queue(name){ - if (! request_queues[name]) request_queues[name] = queuer(metric_parallelism); - return request_queues[name]; - } - - // Serializes a unary expression for computation. - function unary_new(callback) { - var self = this, - remaining = 0, - queue = get_queue(this.expression.source); - - // Compute the expected number of values; if no results were requested, return immediately. - walk(self.start, self.stop, self.tier, function(time){ remaining++; }); - if (! remaining) return this.complete(); - - // Add this task to the appropriate queue. - queue.defer(function task(q_callback){ - self.findOrComputeUnary(function(time, value, tr){ - self.send_result(time, value); - if (!--remaining) { - process.nextTick(function(){ q_callback(null, [time, value]); }); - self.complete(); - } - }); - }).await(_.identity); - } - // // Serializes a unary expression for computation. - // function unary_old(callback) { - // var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; - // var remaining = 0, - // time0 = Date.now(), - // time = start, - // name = expression.source, - // queue = queueByName[name], - // step = tier.key; + // Measurement.prototype.unary = function unary() { + // var self = this, + // remaining = 0, + // queue = get_queue(this.expression.source); // - // // Compute the expected number of values. - // walk(start, stop, tier, function(time){ remaining++; }); - // // If no results were requested, return immediately. - // if (!remaining) return callback(stop, null, metalog.trace('msu', {}, { using: self})); + // // Compute the expected number of values; if no results were requested, return immediately. + // walk(self.start, self.stop, self.tier, function(time){ remaining++; }); + // if (! remaining) return this.complete(); // // // Add this task to the appropriate queue. - // task.qcount = qcount++; - // if (queue){ queue.next = task; queue.next.qindex = queue.qindex + 1; } - // else { task.qindex = 0 ; process.nextTick(task); } - // queueByName[name] = task; - // - // function task() { - // findOrComputeUnary(expression, start, stop, tier, function(time, value){ - // callback(time, value); - // // metalog.warn('unary result', { time: time, meas: self.report(), qcount: qcount, next: (task.next && task.next.qcount) }); + // queue.defer(function task(q_callback){ + // self.run_unary(function(time, value, tr){ + // self.send_result(time, value); // if (!--remaining) { - // callback(stop); - // if (task.next) process.nextTick(task.next); - // else delete queueByName[name]; - // // Record how long it took us to compute as an event! - // // metalog.warn("cube_compute", { is: 'metric', at: 'done', meas: self.report(), ms: Date.now() - time0}); + // process.nextTick(function(){ q_callback(null, [time, value]); }); + // self.complete(); // } // }); - // } - // return null; + // }).await(_.identity); // } - // Finds or computes a unary (primary) expression. - Measurement.prototype.findOrComputeUnary = function findOrComputeUnary(callback) { + // Serializes a unary expression for computation. + Measurement.prototype.unary = function unary(callback) { var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; + var remaining = 0, + time0 = Date.now(), + name = expression.source, + queue = queueByName[name], + step = tier.key; + + // Compute the expected number of values. + walk(start, stop, tier, function(time){ ++remaining; }); + + // If no results were requested, return immediately. + if (!remaining) return callback(stop); + + // Add this task to the appropriate queue. + if (queue) queue.next = task; + else process.nextTick(task); + queueByName[name] = task; + + function task() { + findOrComputeUnary(expression, start, stop, tier, function(time, value, tr) { + self.send_result(time, value, tr); + if (!--remaining) { + self.complete(tr); + if (task.next) process.nextTick(task.next); + else delete queueByName[name]; + + // Record how long it took us to compute as an event! + var time1 = Date.now(); + metalog.event("cube_compute", { + expression: expression.source, + ms: time1 - time0 + }); + } + }, self); + } + } + + // Finds or computes a unary (primary) expression. + function findOrComputeUnary(expression, start, stop, tier, callback, tr) { var name = expression.type, - map = expression.value, - reduce = reduces[expression.reduce], - filter = {t: {}}, - fields = {t: 1}, - metrics, events; + map = expression.value, + reduce = reduces[expression.reduce], + filter = {t: {}}, + fields = {t: 1}, + metrics, events; - metalog.trace('focu', this); + metalog.trace('find', this); // Copy any expression filters into the query object. expression.filter(filter); @@ -217,16 +179,16 @@ exports.getter = function(db){ db.metrics(name, function(error, collection){ handle(error); metrics = collection; - find(start, stop, tier, self, callback); + find(start, stop, tier, tr, callback); }); // The metric is computed recursively, reusing the above variables. function find(start, stop, tier, tr, callback) { var compute = ((tier.next && reduce.pyramidal) ? computePyramidal : computeFlat), - step = tier.key; + step = tier.key; + + metalog.trace('mf0', tr); - metalog.trace('m_f0', tr); - // Query for the desired metric in the cache. metrics.find({ i: false, @@ -244,23 +206,25 @@ exports.getter = function(db){ handle(error); var time = start; cursor.each(function(error, row) { - var fmtr = metalog.trace('m_f1', {}, { using: tr }); + var mftr = metalog.trace('mf1', {}, { using: tr }); handle(error); if (row) { - callback(row._id.t, row.v, fmtr); // send back value for this timeslot - if (time < row._id.t) compute(time, row._id.t, fmtr); // recurse from last value seen up to this timeslot - time = tier.step(row._id.t); // update the last-observed timeslot + callback(row._id.t, row.v, metalog.trace('mfy', mftr)); // send back value for this timeslot + if (time < row._id.t) compute(time, row._id.t, mftr); // recurse from last value seen up to this timeslot + time = tier.step(row._id.t); // update the last-observed timeslot } else { - if (time < stop) compute(time, stop, fmtr); // once last row is seen, compute rest of range + if (time < stop) compute(time, stop, mftr); // once last row is seen, compute rest of range } }); } // Group metrics from the next tier. function computePyramidal(start, stop, tr) { - // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); var bins = {}; + // + // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); + metalog.trace('mfl0', tr, { start: start, stop: stop }); find(start, stop, tier.next, tr, function(time, value, tr) { var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); if (bin.values.push(value) === bin.size) { @@ -270,15 +234,13 @@ exports.getter = function(db){ }); } - function get_mbox(meas){ - return [meas.tier.key, meas.tier.bin(meas.start), meas.tier.bin(meas.stop), meas.expression.source, meas.fields].join('~'); - } - // Group raw events. Unlike the pyramidal computation, here we can control // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop, tr) { - metalog.trace('m_cf', tr, { expr: expression.source, start: start, stop: stop, tier: tier.key }); + // metalog.warn('computeFlat', { expr: expression.source, start: start, stop: stop, tier: tier.key }); + metalog.trace('mfl0', tr, { start: start, stop: stop }); + // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) @@ -286,14 +248,14 @@ exports.getter = function(db){ filter.t.$gte = start; filter.t.$lt = stop; - db.events(name, function _computeFlatOnComplete(error, collection) { + db.events(name, function (error, collection) { handle(error); collection.find(filter, fields, event_options, function(error, cursor) { handle(error); var time = start, values = []; cursor.each(function(error, row) { - var res_tr = metalog.trace('m_cfn', {}, { using: tr }); + var res_tr = metalog.trace('mflt', {}, { using: tr }); handle(error); if (row) { var then = tier.floor(row.t); @@ -314,16 +276,16 @@ exports.getter = function(db){ } function save(time, value, tr) { - callback(time, value, metalog.trace('m_sv', {}, {using: tr})); + callback(time, value, metalog.trace('msav', tr)); if ((! value) && (value !== 0)) return; var metric = { - _id: { - e: expression.source, - l: tier.key, - t: time - }, - i: false, - v: new Double(value) + _id: { + e: expression.source, + l: tier.key, + t: time + }, + i: false, + v: new Double(value) }; // metalog.trace('cube_compute', {is: 'metric_save', metric: metric }); metrics.save(metric, handle); @@ -331,6 +293,45 @@ exports.getter = function(db){ } } + // Computes a binary expression by merging two subexpressions + // + // "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and + // unary "sum(resp)". We don't know what order they'll show up in, so if say + // the value for left appears first, it parks that value as left[time], where + // the result for right will eventually find it. + Measurement.prototype.binary = function binary() { + var self = this, expression = this.expression, value; + var left = new Measurement(expression.left, this.start, this.stop, this.tier, this.id), + right = new Measurement(expression.right, this.start, this.stop, this.tier, this.id); + metalog.trace('msb0', left, {using: self}); metalog.trace('msb0', right, {using: self}); + + left.sender = function(time, vall, tr) { + if (time in right) { // right val already appeared; get a result + self.send_result(time, (time < self.stop ? expression.op(vall, right[time]) : vall), metalog.trace('msb', tr)); + delete right[time]; + } else { // right val still on the way; stash the value + left[time] = vall; + } + }; + + right.sender = function(time, valr, tr) { + if (time in left) { + self.send_result(time, (time < self.stop ? expression.op(left[time], valr) : valr), metalog.trace('msb', tr)); + delete left[time]; + } else { + right[time] = valr; + } + }; + + left.measure(); + right.measure(); + }; + + function get_queue(name){ + if (! request_queues[name]) request_queues[name] = queuer(queue_parallelism); + return request_queues[name]; + } + // execute cb on each interval from t1 to t2 function walk(t1, t2, tier, cb){ while (t1 < t2) { diff --git a/lib/cube/models.js b/lib/cube/models.js index 5db735bc..4f1ee43e 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -28,6 +28,11 @@ function Event(type, time, data, id){ this.type = type; if (id) this._id = id; + var self = this; + + if (this._id) Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, _id: _id }, enumerable: false, writable: false }); + else Object.defineProperty(this, 'to_wire', { value: { t: time, d: data }, enumerable: false, writable: false }); + Object.defineProperties(this, { _trace: { value: null, enumerable: false, writable: true } }); @@ -47,17 +52,24 @@ Event.prototype = { } }; +Event.prototype.save = function(callback){ + var self=this + db.events(self.type, function(error, collection){ + if (error) return callback(error); + metalog.trace('eSvc', self); + collection.save(self.to_wire, function saver(error){ + callback(error, self + // , metalog.trace('eSvd', self) + ); }); + }, metalog.trace('eSva', self)); +}; + // Validate the date and type. Event.prototype.validate = function(){ if (!type_re.test(this.type)) throw("invalid type"); if (isNaN(this.t)) throw("invalid time"); }; -Event.prototype.to_wire = function(){ - if (this._id) return { t: this.t, d: this.d, _id: this._id }; - else return { t: this.t, d: this.d }; -}; - Event.prototype.to_request = function(attrs){ var ev = { time: this.t, data: this.d, type: this.type }; if (this._id) ev.id = this._id; @@ -66,30 +78,18 @@ Event.prototype.to_request = function(attrs){ return ev; }; -Event.prototype.save = function(callback){ - var self = this; - db.events(self.type, function(error, collection){ - if (error) return callback(error); - metalog.trace('eSvc', self); - collection.save(self.to_wire(), function saver(error){ - callback(error, self - , metalog.trace('eSvd', self) - ); }); - }, metalog.trace('eSva', self) - ); -}; - exports.Event = Event; // -------------------------------------------------------------------------- function Metric(time, value, id, measurement){ - this.value = value; this.time = time; + this.value = value; if (id) this.id = id; Object.defineProperties(this, { e: { value: measurement.expression.source, enumerable: false, writable: false }, + l: { value: measurement.tier.key, enumerable: false, writable: false }, measurement: { value: measurement, enumerable: false, writable: false }, _trace: { value: null, enumerable: false, writable: true }, }); @@ -111,7 +111,7 @@ Object.defineProperties(Metric.prototype, { }); Metric.prototype.to_wire = function to_wire(){ - return { i: false, v: this.value, _id: { e: this.e, l: this.tier.key, t: time } }; + return { i: false, v: this.value, _id: { e: this.e, l: this.l, t: time } }; }; Metric.prototype.report = function report(){ diff --git a/lib/queue-async b/lib/queue-async new file mode 160000 index 00000000..5edd2096 --- /dev/null +++ b/lib/queue-async @@ -0,0 +1 @@ +Subproject commit 5edd2096b64954c3927ac0185974949499decb59 diff --git a/test/metric-test.js b/test/metric-test.js index 4f324dd4..e085d6ba 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -14,8 +14,7 @@ var _ = require("underscore"), // as a hack to get updates to settle, we need to insert delays. // if you see heisen-errors in the metrics tests, increase these. var step_testing_delay = 250, - batch_testing_delay = 500, - put_queue = queuer(10); + batch_testing_delay = 500; var suite = vows.describe("metric"); @@ -33,7 +32,7 @@ var t1 = gen_date(3), t1_10s = new Date(10e3 * Math.floor(t1/10e3)), t2 = gen_date(35), t2_10s = new Date(10e3 * Math.floor(t2/10e3)); function gen_request(attrs){ - var req = { start: t1, stop: t2, step: units.second10, expression: 'sum(test)'}; + var req = { start: t1, stop: t2, step: units.second10, expression: 'max(test(i))'}; for (var key in attrs){ req[key] = attrs[key]; } return req; } @@ -46,373 +45,366 @@ function assert_invalid_request(req, expected_err) { }; } -suite.addBatch(test_helper.batch({ - topic: function(test_db){ - var putter = event.putter(test_db), - getter = metric.getter(test_db), - callback = this.callback; - // Seed the events table with a simple event: a value going from 0 to 2499 - for (var i = 0; i < 250; i++){ - put_queue.defer(function(cb){ - putter({ type: "test", time: gen_date(i*10).toISOString(), data: {i: i}}, function(){ cb(null, null); }); - }); - } - put_queue.await(function(){ callback(null, getter) }); - }, - // 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), - // 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), - // 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), - // 'invalid expression': assert_invalid_request({expression: 'DANCE'}, invalid_expression_error), - - 'with request id' : { - topic: function(getter){ - var checker = assert.isCalledTimes(this, 5); - this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(i))'}), checker); - }, - 'includes id in result': function(results){ - metalog.inspectify(results); - _.each([0,10,20,30], function(step, idx){ - assert.deepEqual(results[idx][0].report(), { id: 'joe', time: gen_date(step), value: idx }); - }); - }, - 'sends a null metric for the end slot': function(results){ assert.deepEqual(results[4][0].report(), {id: 'joe', time: gen_date(40), value: null}); } - }, - 'simple constant' : { - topic: function(getter){ - var checker = assert.isCalledTimes(this, 5); - getter(gen_request({expression: '1'}), checker); - }, - 'gets a metric for each time slot': function(results){ - _.each([0,10,20,30], function(step, idx){ - assert.deepEqual(results[idx][0].report(), {time: gen_date(step), value: 1}); - }); - }, - 'sends a null metric for the end slot': function(results){ assert.deepEqual(results[4][0].report(), {time: gen_date(40), value: null}); } - }, -})); - - // suite.addBatch(test_helper.batch({ // topic: function(test_db){ // var putter = event.putter(test_db), // getter = metric.getter(test_db), -// callback = this.callback; -// +// callback = this.callback, +// put_queue = queuer(10); // // Seed the events table with a simple event: a value going from 0 to 2499 // for (var i = 0; i < 250; i++){ -// put_queue.defer(function(cb){ -// putter({ type: "test", time: gen_date(i * 3).toISOString(), data: {i: 3 * i}}, function(){ cb(null, null); }); -// }); +// put_queue.defer(function(num, dt, cb){ +// putter({ type: "test", time: dt, data: {i: num}}, function(){ cb(null, null); }); +// }, i, gen_date(i*10).toISOString()); // } -// test_helper.inspectify('d', arguments); // put_queue.await(function(){ callback(null, getter) }); // }, -// 'hi': { -// topic: function(getter){ -// test_helper.inspectify('3', arguments); -// this.ret = getter(gen_request({start: t1, stop: t2, expression: 'sum(test)'}), this.callback); +// // 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), +// // 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), +// // 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), +// // 'invalid expression': assert_invalid_request({expression: 'DANCE'}, invalid_expression_error), +// +// 'simple constant' : { +// topic: function(getter){ +// var checker = assert.isCalledTimes(this, 5); +// getter(gen_request({expression: '1'}), checker); // }, -// 'fires callback with id': function(result, j_){ -// test_helper.inspectify('f', arguments); -// // assert.equal(result.value, (result.time > nowish_floor ? undefined : 0)); -// // assert.include([nowish_floor, 10e3+nowish_floor], +result.time); -// // assert.isFalse("id" in result); -// } -// } -// })); - -// suite.addBatch(test_helper.batch({ -// topic: function(test_db) { -// return metric.getter(test_db); +// 'gets a metric for each time slot': function(results){ +// _.each([0,10,20,30], function(step, idx){ +// assert.deepEqual(results[idx][0].report(), {time: gen_date(step), value: 1}); +// }); +// }, +// 'sends a null metric for the end slot': function(results){ assert.deepEqual(results[4][0].report(), {time: gen_date(40), value: null}); } // }, +// // 'no request id' : { -// topic: function(getter){ this.ret = getter(gen_request({}), this.callback); }, -// 'fires callback with no id': function(result, j_){ -// assert.equal(result.value, (result.time > nowish_floor ? undefined : 0)); -// assert.include([nowish_floor, 10e3+nowish_floor], +result.time); -// assert.isFalse("id" in result); -// } -// } -// })); - -// function skip(){ // FIXME: remove ------------------------------------------------------------ -// -// var steps = { -// 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, -// 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, -// 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, -// 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, -// 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } -// }; -// steps[units.second10].description = "10-second"; -// steps[units.minute ].description = "1-minute"; -// steps[units.minute5 ].description = "5-minute"; -// steps[units.hour ].description = "1-hour"; -// steps[units.day ].description = "1-day"; -// -// suite.addBatch(test_helper.batch({ -// topic: function(test_db) { -// var putter = event.putter(test_db), -// getter = metric.getter(test_db), -// callback = this.callback; -// -// // Seed the events table with a simple event: a value going from 0 to 2499 -// for (var i = 0; i < 2500; i++) { -// putter({ -// type: "test", -// time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(i) - 10)).toISOString(), -// data: {i: i} +// topic: function(getter){ +// var checker = assert.isCalledTimes(this, 5); +// this.ret = getter(gen_request({}), checker); +// }, +// 'does not have id in result': function(results){ +// test_helper.inspectify(results) +// _.each([0,10,20,30], function(step, idx){ +// assert.isFalse("id" in results[idx][0]); +// assert.deepEqual(results[idx][0].report(), { time: gen_date(step), value: idx }); // }); +// }, +// 'sends a null metric for the end slot': function(results){ +// assert.deepEqual(results[4][0].report(), {time: gen_date(40), value: null}); // } -// -// // So the events can settle in, wait `batch_testing_delay` ms before continuing -// setTimeout(function() { callback(null, getter); }, batch_testing_delay); // }, -// -// // FIXME: ---- remove below ------------------------------------ -// -// "unary expression a": metricTest({ -// expression: "sum(test)", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:02:00.000Z" -// }, { -// 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] -// }), -// -// "unary expression b": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17 ] }), -// "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), -// "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), -// "unary expression e": metricTest({ expression: "sum(test)", start: "2011-07-17T23:50:00.000Z", stop: "2011-07-18T00:03:00.000Z"}, { 60e3: [ 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25 ] }), -// -// "unary expression f": metricTest({ -// expression: "sum(test)", -// start: "2011-07-17T23:57:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 60e3: [13, 15, 17, 39, 23, 25, 27, 29, 31, 33, -// 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, -// 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, -// 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, -// 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, -// 0, 0, 0] -// }), -// -// // FIXME: ---- remove above ------------------------------------ -// -// "unary expression": metricTest({ -// expression: "sum(test)", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 60e3: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], -// 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], -// 3600e3: [82, 2418], -// 86400e3: [82, 2418] -// }), -// -// "unary expression with data accessor": metricTest({ -// expression: "sum(test(i))", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 300e3: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], -// 3600e3: [3321, 3120429], -// 86400e3: [3321, 3120429] -// }), -// -// "unary expression with compound data accessor": metricTest({ -// expression: "sum(test(i / 100))", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 300e3: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], -// 3600e3: [33.21, 31204.29], -// 86400e3: [33.21, 31204.29] -// }), -// -// "compound expression (sometimes fails due to race condition?)": metricTest({ -// expression: "max(test(i)) - min(test(i))", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 300e3: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], -// 3600e3: [81, 2417], -// 86400e3: [81, 2417] -// }), -// -// "non-pyramidal expression": metricTest({ -// expression: "distinct(test(i))", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], -// 3600e3: [82, 2418], -// 86400e3: [82, 2418] -// }), -// -// "compound pyramidal and non-pyramidal expression": metricTest({ -// expression: "sum(test(i)) - median(test(i))", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 300e3: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], -// 3600e3: [3280.5, 3119138.5], -// 86400e3: [3280.5, 3119138.5] -// }), -// -// "compound with constant expression": metricTest({ -// expression: "-1 + sum(test)", -// start: "2011-07-17T23:47:00.000Z", -// stop: "2011-07-18T00:50:00.000Z" -// }, { -// 300e3: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], -// 3600e3: [81, 2417], -// 86400e3: [81, 2417] -// }) -// +// +// 'with request id' : { +// topic: function(getter){ +// var checker = assert.isCalledTimes(this, 5); +// this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(i))'}), checker); +// }, +// 'includes id in result': function(results){ +// metalog.inspectify(results); +// _.each([0,10,20,30], function(step, idx){ +// assert.deepEqual(results[idx][0].report(), { id: 'joe', time: gen_date(step), value: idx }); +// }); +// }, +// 'sends a null metric for the end slot': function(results){ +// assert.deepEqual(results[4][0].report(), {id: 'joe', time: gen_date(40), value: null}); +// } +// } +// // })); -// -// // metricTest -- generates test tree for metrics. -// // -// // Gets the metric, checks it was calculated correctly from events seeded above; -// // then does it again (on a delay) to check that it was cached. -// // -// // @example given `{ 'unary expression': metricTest({..}, { 60_000: [0, 0, ...], 86_400_000: [82, 2418] })` -// // -// // { 'unary expression': { -// // 'at 1-minute intervals': { -// // topic: function get_metrics_with_delay(getter){}, -// // 'sum(test)': function metrics_assertions(actual){}, -// // '(cached)': { -// // topic: function get_metrics_with_delay(err, getter){}, -// // 'sum(test)': function metrics_assertions(actual){} } }, -// // 'at 1-day intervals': { -// // topic: function get_metrics_with_delay(getter){}, -// // 'sum(test)': function metrics_assertions(actual){}, -// // '(cached)': { -// // topic: function get_metrics_with_delay(err, getter){}, -// // 'sum(test)': function metrics_assertions(actual){} } } -// // } -// // } -// // -// function metricTest(request, expected) { -// // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } -// var tree = {}, k; -// for (var step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); -// -// // -// // { -// // topic: get_metrics_with_delay, -// // expression: function(){ -// // // rounds down the start time (inclusive) -// // // formats UTC time in ISO 8601 -// // ... -// // // returns the expected values -// // }, -// // '(cached)': { -// // topic: get_metrics_with_delay, -// // expression: function(){ -// // // rounds down the start time (inclusive) -// // ... -// // } -// // } -// // } -// // -// function testStep(step, expected) { -// var start = new Date(request.start), -// stop = new Date(request.stop); -// -// var subtree = { -// topic: get_metrics_with_delay(0), -// '(cached)': { topic: get_metrics_with_delay(1) } -// }; -// subtree[request.expression] = metrics_assertions(); -// subtree["(cached)"][request.expression] = metrics_assertions(); -// -// function get_metrics_with_delay(depth){ return function(){ -// var actual = [], -// timeout = setTimeout(function() { cb("Time's up!"); }, 10000), -// cb = this.callback, -// req = Object.create(request), -// getter = arguments[depth]; -// req.step = step; -// // Wait long enough for the events to have settled in the db. The -// // non-cached (depth=0) round can all start in parallel, making this an -// // effective `nextTick`. On the secon -// setTimeout(function() { -// // ... then invoke the metrics getter. As responses roll in, push them -// // on to 'actual'; we're done when the 'stop' time is hit -// getter(req, function(response){ -// if (response.time >= stop) { -// clearTimeout(timeout); -// cb(null, actual.sort(function(a, b) { return a.time - b.time; })); -// } else { -// actual.push(response); -// } -// }); -// }, depth * step_testing_delay); -// };} -// -// function metrics_assertions(){ return { -// 'rounds down the start time (inclusive)': function(actual) { -// var floor = steps[step](start, 0); -// assert.deepEqual(actual[0].time, floor); -// }, -// -// 'rounds up the stop time (exclusive)': function(actual){ -// var ceil = steps[step](stop, 0); -// if (!(ceil - stop)) ceil = steps[step](stop, -1); -// assert.deepEqual(actual[actual.length - 1].time, ceil); -// }, -// -// 'formats UTC time in ISO 8601': function(actual){ -// actual.forEach(function(d) { -// assert.instanceOf(d.time, Date); -// assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); -// }); -// }, -// -// 'returns exactly one value per time': function(actual){ -// var i = 0, n = actual.length, t = actual[0].time; -// while (++i < n) assert.isTrue(t < (t = actual[i].time)); -// }, -// -// 'each metric defines only time and value properties': function(actual){ -// actual.forEach(function(d) { -// delete d._trace; -// assert.deepEqual(Object.keys(d), ["time", "value"]); -// }); -// }, -// -// 'returns the expected times': function(actual){ -// var floor = steps[step], -// time = floor(start, 0), -// times = []; -// while (time < stop) { -// times.push(time); -// time = floor(time, 1); -// } -// assert.deepEqual(actual.map(function(d) { return d.time; }), times); -// }, -// -// 'returns the expected values': function(actual){ -// var actualValues = actual.map(function(d) { return d.value; }); -// assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); -// expected.forEach(function(value, i) { -// if (Math.abs(actual[i].value - value) > 1e-6) { -// assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); -// } -// }); -// } -// };} // metric assertions -// -// return subtree; -// } // subtree -// return tree; -// } // tree -// -// } -// skip(); + +function skip(){ // FIXME: remove ------------------------------------------------------------ + +var steps = { + 10e3: function(date, n) { return new Date((Math.floor(date / units.second10) + n) * units.second10); }, + 60e3: function(date, n) { return new Date((Math.floor(date / units.minute) + n) * units.minute); }, + 300e3: function(date, n) { return new Date((Math.floor(date / units.minute5) + n) * units.minute5); }, + 3600e3: function(date, n) { return new Date((Math.floor(date / units.hour) + n) * units.hour); }, + 86400e3: function(date, n) { return new Date((Math.floor(date / units.day) + n) * units.day); } +}; +steps[units.second10].description = "10-second"; +steps[units.minute ].description = "1-minute"; +steps[units.minute5 ].description = "5-minute"; +steps[units.hour ].description = "1-hour"; +steps[units.day ].description = "1-day"; + +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + var putter = event.putter(test_db), + getter = metric.getter(test_db), + callback = this.callback, + put_queue = queuer(10); + + // Seed the events table with a simple event: a value going from 0 to 2499 + for (var i = 0; i < 2500; i++){ + put_queue.defer(function(num, cb){ + putter({ + type: "test", + time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(num) - 10)).toISOString(), + data: {i: num} + }, function(){ cb(null, null); }); + }, i); + } + // continue when queue clears + put_queue.await(function(){ callback(null, getter) }); + + // // Seed the events table with a simple event: a value going from 0 to 2499 + // for (var i = 0; i < 2500; i++) { + // putter({ + // type: "test", + // time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(i) - 10)).toISOString(), + // data: {i: i} + // }); + // } + + // // So the events can settle in, wait `batch_testing_delay` ms before continuing + // setTimeout(function() { callback(null, getter); }, batch_testing_delay); + }, + + // FIXME: ---- remove below ------------------------------------ + + "constant expression": metricTest({ expression: "1", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] }), + + "unary expression a": metricTest({ + expression: "sum(test)", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:02:00.000Z" + }, { + 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] + }), + + "unary expression b": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17 ] }), + "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), + "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), + "unary expression e": metricTest({ expression: "sum(test)", start: "2011-07-17T23:50:00.000Z", stop: "2011-07-18T00:03:00.000Z"}, { 60e3: [ 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25 ] }), + + "unary expression f": metricTest({ + expression: "sum(test)", + start: "2011-07-17T23:57:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 60e3: [13, 15, 17, 39, 23, 25, 27, 29, 31, 33, + 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, + 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, + 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, + 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0] + }), + + // FIXME: ---- remove above ------------------------------------ + + "unary expression": metricTest({ + expression: "sum(test)", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 60e3: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], + 3600e3: [82, 2418], + 86400e3: [82, 2418] + }), + + "unary expression with data accessor": metricTest({ + expression: "sum(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], + 3600e3: [3321, 3120429], + 86400e3: [3321, 3120429] + }), + + "unary expression with compound data accessor": metricTest({ + expression: "sum(test(i / 100))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], + 3600e3: [33.21, 31204.29], + 86400e3: [33.21, 31204.29] + }), + + "compound expression (sometimes fails due to race condition?)": metricTest({ + expression: "max(test(i)) - min(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], + 3600e3: [81, 2417], + 86400e3: [81, 2417] + }), + + "non-pyramidal expression": metricTest({ + expression: "distinct(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], + 3600e3: [82, 2418], + 86400e3: [82, 2418] + }), + + "compound pyramidal and non-pyramidal expression": metricTest({ + expression: "sum(test(i)) - median(test(i))", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], + 3600e3: [3280.5, 3119138.5], + 86400e3: [3280.5, 3119138.5] + }), + + "compound with constant expression": metricTest({ + expression: "-1 + sum(test)", + start: "2011-07-17T23:47:00.000Z", + stop: "2011-07-18T00:50:00.000Z" + }, { + 300e3: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], + 3600e3: [81, 2417], + 86400e3: [81, 2417] + }) + +})); + +// metricTest -- generates test tree for metrics. +// +// Gets the metric, checks it was calculated correctly from events seeded above; +// then does it again (on a delay) to check that it was cached. +// +// @example given `{ 'unary expression': metricTest({..}, { 60_000: [0, 0, ...], 86_400_000: [82, 2418] })` +// +// { 'unary expression': { +// 'at 1-minute intervals': { +// topic: function get_metrics_with_delay(getter){}, +// 'sum(test)': function metrics_assertions(actual){}, +// '(cached)': { +// topic: function get_metrics_with_delay(err, getter){}, +// 'sum(test)': function metrics_assertions(actual){} } }, +// 'at 1-day intervals': { +// topic: function get_metrics_with_delay(getter){}, +// 'sum(test)': function metrics_assertions(actual){}, +// '(cached)': { +// topic: function get_metrics_with_delay(err, getter){}, +// 'sum(test)': function metrics_assertions(actual){} } } +// } +// } +// +function metricTest(request, expected) { + // { 'at 1-minute intervals': { }, 'at 1-day intervals': { } } + var tree = {}, k; + for (var step in expected) tree["at " + steps[step].description + " intervals"] = testStep(step, expected[step]); + + // + // { + // topic: get_metrics_with_delay, + // expression: function(){ + // // rounds down the start time (inclusive) + // // formats UTC time in ISO 8601 + // ... + // // returns the expected values + // }, + // '(cached)': { + // topic: get_metrics_with_delay, + // expression: function(){ + // // rounds down the start time (inclusive) + // ... + // } + // } + // } + // + function testStep(step, expected) { + var start = new Date(request.start), + stop = new Date(request.stop); + + var subtree = { + topic: get_metrics_with_delay(0), + '(cached)': { topic: get_metrics_with_delay(1) } + }; + subtree[request.expression] = metrics_assertions(); + subtree["(cached)"][request.expression] = metrics_assertions(); + + function get_metrics_with_delay(depth){ return function(){ + var actual = [], + timeout = setTimeout(function() { cb(new Error("Time's up!")); }, 20000), + cb = this.callback, + req = Object.create(request), + getter = arguments[depth]; + req.step = step; + // Wait long enough for the events to have settled in the db. The + // non-cached (depth=0) round can all start in parallel, making this an + // effective `nextTick`. On the secon + setTimeout(function() { + // ... then invoke the metrics getter. As responses roll in, push them + // on to 'actual'; we're done when the 'stop' time is hit + getter(req, function(response){ + if (response.time >= stop) { + clearTimeout(timeout); + cb(null, actual.sort(function(a, b) { return a.time - b.time; })); + } else { + actual.push(response); + } + }); + }, depth * step_testing_delay); + };} + + function metrics_assertions(){ return { + 'rounds down the start time (inclusive)': function(actual) { + var floor = steps[step](start, 0); + assert.deepEqual(actual[0].time, floor); + }, + + 'rounds up the stop time (exclusive)': function(actual){ + var ceil = steps[step](stop, 0); + if (!(ceil - stop)) ceil = steps[step](stop, -1); + assert.deepEqual(actual[actual.length - 1].time, ceil); + }, + + 'formats UTC time in ISO 8601': function(actual){ + actual.forEach(function(d) { + assert.instanceOf(d.time, Date); + assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); + }); + }, + + 'returns exactly one value per time': function(actual){ + var i = 0, n = actual.length, t = actual[0].time; + while (++i < n) assert.isTrue(t < (t = actual[i].time)); + }, + + 'each metric defines only time and value properties': function(actual){ + actual.forEach(function(d) { + // if ('_trace' in d) delete d._trace; + assert.deepEqual(Object.keys(d), ["time", "value"]); + }); + }, + + 'returns the expected times': function(actual){ + var floor = steps[step], + time = floor(start, 0), + times = []; + while (time < stop) { + times.push(time); + time = floor(time, 1); + } + assert.deepEqual(actual.map(function(d) { return d.time; }), times); + }, + + 'returns the expected values': function(actual){ + var actualValues = actual.map(function(d) { return d.value; }); + assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); + expected.forEach(function(value, i) { + if (Math.abs(actual[i].value - value) > 1e-6) { + assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); + } + }); + } + };} // metric assertions + + return subtree; + } // subtree + return tree; +} // tree +} +skip(); suite['export'](module); diff --git a/test/test_helper.js b/test/test_helper.js index 0d09e908..324a9121 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -1,6 +1,7 @@ 'use strict'; var _ = require("underscore"), + util = require("util"), assert = require("assert"), http = require("http"), dgram = require('dgram'), From f1c9715e5dc3c2fa9d4d7da3b36b0c1af04f7191 Mon Sep 17 00:00:00 2001 From: "Philip (flip) Kromer" Date: Mon, 10 Sep 2012 02:12:56 -0500 Subject: [PATCH 40/87] getting tests to run non-isolated --- lib/cube/collector.js | 2 +- lib/cube/db.js | 327 +++++++++++++++++++++-------------------- lib/cube/evaluator.js | 6 +- lib/cube/event.js | 52 ++++--- lib/cube/metalog.js | 30 ++-- lib/cube/metric.js | 4 +- lib/cube/models.js | 19 ++- lib/cube/server.js | 12 +- lib/cube/visualizer.js | 4 +- lib/cube/warmer.js | 2 +- test/broker-test.js | 186 +++++++++++------------ test/evaluator-test.js | 22 +-- test/event-test.js | 14 +- test/metalog-test.js | 34 ++--- test/metric-test.js | 2 + test/test_helper.js | 91 +++++++----- 16 files changed, 427 insertions(+), 380 deletions(-) diff --git a/lib/cube/collector.js b/lib/cube/collector.js index c58b5a86..ea2dbb99 100644 --- a/lib/cube/collector.js +++ b/lib/cube/collector.js @@ -18,7 +18,7 @@ var headers = { // * poster, an HTTP listener -- see below // * collectd, a collectd listener -- see collectd.js // -exports.register = function(db, endpoints) { +exports.register = function collector(db, endpoints) { var putter = require("./event").putter(db), poster = post(putter); diff --git a/lib/cube/db.js b/lib/cube/db.js index 6aa7721e..25e80a5e 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -1,198 +1,203 @@ 'use strict'; -var util = require("util"), - mongodb = require("mongodb"), - metalog = require("./metalog"), - metric_options, event_options, - database, events_db, metrics_db; - -var db = { - open: open, - isConnected: isConnected -} +var util = require("util"), + mongodb = require("mongodb"), + metalog = require("./metalog"), + db_index = 0; -var collections = {}; +function Db(){ + var metric_options, event_options, db_client, events_client, metrics_client; + var db_id = db_index++; -// -// Connect to mongodb. -// + var db = this; + db.open = open; + db.close = close; + db.isConnected = isConnected; -function open(options, callback){ - var server_options = options["mongo-server_options"], // MongoDB server configuration - db_options = { native_parser: true }, // MongoDB driver configuration. - mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), - database_name = options["mongo-database"], - mongo_password = options["mongo-password"]; + var collections = {}; // cache of collection handles + var pending_callbacks = {}; // callbacks waiting for collections to be found + var collection_prefix = ''; - metric_options = options["mongo-metrics"], - event_options = options["mongo-events"], - database = new mongodb.Db(database_name, mongo, db_options); + this.report = function report(){ return { idx: db_id, isHalted: db.isHalted }; }; - delete options["mongo-password"]; - metalog.info('cube_life', {is: 'mongo_connect', options: options }); + // + // Connect to mongodb. + // - database.open(function(error){ - if (error) return callback(error); + function open(options, callback){ + var server_options = options["mongo-server_options"], // MongoDB server configuration + db_options = { native_parser: true }, // MongoDB driver configuration. + mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), + database_name = options["mongo-database"], + mongo_password = options["mongo-password"]; - // Open separate events database if configured - if (options["separate-events-database"]) events_db = database.db(database_name + '-events'); - else events_db = database; + if (db_client) { metalog.minor('mongo_already_open', db.report()); return callback(null, db); } - // Open separate metrics database if configured - if (options["separate-metrics-database"]) metrics_db = database.db(database_name + '-metrics'); - else metrics_db = database; + collection_prefix = options["collection_prefix"] || ''; + metric_options = options["mongo-metrics"], + event_options = options["mongo-events"], + db_client = new mongodb.Db(database_name, mongo, db_options); - db.metrics = metrics_collection_factory(metrics_db); - db.events = events_collection_factory(events_db); - db.types = types(events_db); - db.collection = collection(database); - db.close = close; + delete options["mongo-password"]; + metalog.info('mongo_connect', db.report()); - if (! options["mongo-username"]) return callback(null, db); - - database.authenticate(options["mongo-username"], mongo_password, function(error, success) { + db_client.open(function(error){ if (error) return callback(error); - if (!success) return callback(new Error("authentication failed")); - callback(null, db); + + // Open separate events database if configured + if (options["separate-events-database"]) events_client = db_client.db(database_name + '-events'); + else events_client = db_client; + + // Open separate metrics database if configured + if (options["separate-metrics-database"]) metrics_client = db_client.db(database_name + '-metrics'); + else metrics_client = db_client; + + db.metrics = metrics_collection_factory(metrics_client); + db.events = events_collection_factory(events_client); + db.types = types(events_client); + db.collection = collection(db_client); + + if (! options["mongo-username"]) return callback(null, db); + + db_client.authenticate(options["mongo-username"], mongo_password, function(error, success) { + if (error) return callback(error); + if (!success) return callback(new Error("authentication failed")); + callback(null, db); + }); }); - }); -} + } -// -// Close connection to mongodb. -// + // + // Close connection to mongodb. + // -function close(callback){ - collections = {}; - delete db.metrics; - delete db.events; - delete db.types; - delete db.collection; - delete db.close; - if (isConnected()){ database.close(callback); } else { callback(null); } -} + function close(callback){ + metalog.trace('mongo_closing', db.report()); + if (! db_client) return metalog.minor('mongo_already_closed'); + collections = {}; + delete db.metrics; + delete db.events; + delete db.types; + delete db.collection; + if (isConnected()){ db_client.close(); } + db_client = events_client = metrics_client = null; + db.isHalted = true; -function isConnected(){ - try { - return database.serverConfig.isConnected(); - } catch (e){ - return false; + return callback(null); } -} + db.isHalted = false; -// // Much like db.collection, but caches the result for both events and metrics. -// // Also, this is synchronous, since we are opening a collection unsafely. -// function metrics(database) { -// var m_collections = {}; -// -// return function(name, callback){ -// if(m_collections[name]) return callback(null, m_collections[name]); -// -// var collection_name = name + "_metrics", -// _this = this; -// database.createCollection(collection_name, metric_options||{safe: false}, function(error, metrics) { -// -// if (error && error.errmsg == "collection already exists") return _this.metrics(name, callback); -// if (error) return callback(error); -// -// metrics = logging_clxn(metrics, 'metrics'); -// m_collections[name] = metrics; -// return callback(null, metrics); -// }); -// } -// } - -function metrics_collection_factory(database){ - return function metrics(name, on_collection, tr){ - return clxn_for(database, (name + "_metrics"), (metric_options||{safe: false}), on_collection, on_m_create, tr); - function on_m_create(clxn, oc_cb){ - clxn.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); - clxn.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, function(error){ oc_cb(error, clxn); }); + function isConnected(){ + try { + return db_client.serverConfig.isConnected(); + } catch (e){ + return false; } - }; -} + } -function events_collection_factory(database){ - return function events(name, on_collection, tr){ - return clxn_for(database, (name + "_events"), (event_options||{safe: false}), on_collection, on_e_create, tr); - function on_e_create(clxn, oc_cb){ - db.metrics(name, function(error){ - if (error) return oc_cb(error); - clxn.ensureIndex({"t": 1}, function(error){ oc_cb(error, clxn); }); - }); + function metrics_collection_factory(client){ + return function metrics(name, on_collection, tr){ + var clxnname = (collection_prefix + name + "_metrics"); + return clxn_for(client, clxnname, (metric_options||{safe: false}), on_collection, on_m_create, tr); + function on_m_create(clxn, oc_cb){ + clxn.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); + clxn.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, function(error){ oc_cb(error, clxn); }); + } + }; + } + + function events_collection_factory(client){ + return function events(name, on_collection, tr){ + var clxnname = (collection_prefix + name + "_events"); + return clxn_for(client, clxnname, (event_options||{safe: false}), on_collection, on_e_create, tr); + function on_e_create(clxn, oc_cb){ + db.metrics(name, function also_make_metrics_clxn(error){ + if (error) return oc_cb(error); + clxn.ensureIndex({"t": 1}, function(error){ oc_cb(error, clxn); }); + }); + } + }; + } + + function clxn_for(client, clxnname, clxnopts, on_collection, on_create, tr){ + on_collection = on_collection || function(){}; + + // If we've cached the collection, call back immediately + if (collections[clxnname]) return on_collection(null, collections[clxnname]); + + // If someone is already creating the collection for this new type, + // then append the callback to the queue for later save. + if (clxnname in pending_callbacks){ + metalog.trace('clxnQ', tr, {clxnname: clxnname, id: db_id}); + return pending_callbacks[clxnname].push(on_collection); } - }; -} -var pending_callbacks = {}; + // Otherwise, it's up to us to create the corresponding collection, then save + // any requests that have queued up in the interim! -function clxn_for(database, clxnname, clxnopts, on_collection, on_create, tr){ - on_collection = on_collection || function(){}; - // If we've cached the collection, call back immediately - if (collections[clxnname]) return on_collection(null, collections[clxnname]); + // First add the new event to the queue. + pending_callbacks[clxnname] = [on_collection]; + + // Create collection, and issue callback for index creation, etc + client.createCollection(clxnname, clxnopts, function(error, clxn){ + metalog.trace('clxnC', tr); + if (error && (error.errmsg == "collection already exists")){ + metalog.trace('clxnE', tr); + return client.collection(clxnname, adopt_collection); + } + if (!error) return on_create(clxn, adopt_collection); + if (error) return adopt_collection(error); + }); - // If someone is already creating the collection for this new type, - // then append the callback to the queue for later save. - if (clxnname in pending_callbacks){ - metalog.trace('clxnQ', tr); - return pending_callbacks[clxnname].push(on_collection); + function adopt_collection(error, clxn){ + var callbacks = pending_callbacks[clxnname]; + delete pending_callbacks[clxnname]; + + metalog.trace('clxnD', tr); + if (! error){ + clxn = logging_clxn(clxn, clxnname); + collections[clxnname] = clxn; + } + callbacks.forEach(function(cb){ + try{ cb(error, clxn); } + catch(error){ metalog.error('db_pending_callbacks', error); } + }); + } } - // Otherwise, it's up to us to create the corresponding collection, then save - // any requests that have queued up in the interim! - - // First add the new event to the queue. - pending_callbacks[clxnname] = [on_collection]; - - // Create collection, and issue callback for index creation, etc - database.createCollection(clxnname, clxnopts, function(error, clxn){ - metalog.minor('create collection', {name: clxnname}); - if (error && (error.errmsg == "collection already exists")) return db.collection(clxnname, adopt_collection); - if (!error) return on_create(clxn, adopt_collection); - if (error) return adopt_collection(error); - }); - - function adopt_collection(error, clxn){ - if (! error){ - clxn = logging_clxn(clxn, clxnname); - collections[clxnname] = clxn; + function collection(client){ + return function(name, callback){ + if (collections[name]) return callback(null, collections[name]); + client.collection.apply(client, arguments); } - pending_callbacks[clxnname].forEach(function(cb){ cb(error, clxn) }); - delete pending_callbacks[clxnname]; } -} -function collection(database){ - return function(name, callback){ - if (collections[name]) return callback(null, collections[name]); - database.collection.apply(database, arguments); + function logging_clxn(collection, type){ + var orig_find = collection.find; + collection.find = function(){ + //metalog.minor((type + '_find'), {query: arguments}); + return orig_find.apply(this, arguments); + }; + return collection; } -} -function logging_clxn(collection, type){ - var orig_find = collection.find; - collection.find = function(){ - //metalog.minor((type + '_find'), {query: arguments}); - return orig_find.apply(this, arguments); + var eventRe = /_events$/; + + function types(client) { + return function(request, callback) { + client.collectionNames(function(error, names) { + handle(error); + callback(names + .map(function(d) { return d.name.split(".")[1]; }) + .filter(function(d) { return eventRe.test(d); }) + .map(function(d) { return d.substring(0, d.length - 7); }) + .sort()); + }); + }; }; - return collection; -} -var eventRe = /_events$/; - -function types(database) { - return function(request, callback) { - database.collectionNames(function(error, names) { - handle(error); - callback(names - .map(function(d) { return d.name.split(".")[1]; }) - .filter(function(d) { return eventRe.test(d); }) - .map(function(d) { return d.substring(0, d.length - 7); }) - .sort()); - }); - }; -}; +} function handle(error){ if (!error) return; @@ -202,4 +207,4 @@ function handle(error){ function noop(){}; -module.exports = db; +module.exports = Db; diff --git a/lib/cube/evaluator.js b/lib/cube/evaluator.js index fa53675e..3bd364a4 100644 --- a/lib/cube/evaluator.js +++ b/lib/cube/evaluator.js @@ -44,7 +44,8 @@ exports.register = function(db, endpoints) { request = url.parse(request.url, true).query; if (! (request.start && request.stop && request.type)) { - metalog.info('cube_request', {is: 'error', error: 'specify type, start and stop'}); + resp = {error: 'specify type, start and stop'}; + metalog.info('evaluator_refresh', resp); response.writeHead(400, headers); response.end('{}'); } else { @@ -52,8 +53,7 @@ exports.register = function(db, endpoints) { start = new Date(request.start), stop = new Date(request.stop); - metalog.info('cube_request', {is: 'refresh', type: type, start: start, stop: stop}); - + metalog.info('evaluator_refresh', {type: type, start: start, stop: stop}); event_putter.invalidate_range(type, start, stop); response.writeHead(202, headers); diff --git a/lib/cube/event.js b/lib/cube/event.js index 158fd72a..059b2d44 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -22,10 +22,8 @@ var streamDelayDefault = 5000, // How frequently to invalidate metrics after receiving events. var invalidateInterval = 5000; -exports.stop = function(){ Invalidator.stop_flushers(); }; - -var invalidator = new Invalidator(); -exports.invalidator = function(){ return invalidator; }; +// serial id so we can track flushers +var putter_id = 0; // event.putter -- save the event, invalidate any cached metrics impacted by it. // @@ -37,12 +35,15 @@ exports.invalidator = function(){ return invalidator; }; // exports.putter = function(db){ + var invalidator = new Invalidator(); + function putter(request, callback){ var time = "time" in request ? new Date(request.time) : new Date(), type = request.type; callback = callback || function(){}; - // metalog.trace('putter', request); + metalog.trace('eput', request); + // // Drop events from before invalidation horizon // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; @@ -54,7 +55,7 @@ exports.putter = function(db){ var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } - // metalog.trace('ev0', event, { using: request }); + metalog.trace('ebeg', event, { using: request }); // Save the event, then queue invalidation of its associated cached metrics. // @@ -63,26 +64,35 @@ exports.putter = function(db){ // short delay between saving the event and invalidating the metrics reduces // the likelihood of a race condition between when the events are read by // the evaluator and when the newly-computed metrics are saved. - event.save(function after_save(error, event){ + event.save(db, function after_save(error, event){ + metalog.trace('esav', event); if (event) invalidator.add(event.type, event); callback(error, event); }); } + putter.id = ++putter_id; + // Process any deferred metric invalidations, flushing the queues. Note that // the queue (timesToInvalidateByTierByType) is copied-on-write, so while the // previous batch of events are being invalidated, new events can arrive. - Invalidator.start_flusher(function(){ + Invalidator.start_flusher(putter.id, function(){ + if (db.isHalted) return putter.stop(); invalidator.flush(db); invalidator = new Invalidator(); // copy-on-write }); - metalog.info('putter_start'); + putter.invalidator = function(){ return invalidator; }; + putter.stop = function(on_stop){ + metalog.info('putter_stopping', {id: putter.id}); + Invalidator.stop_flusher(putter.id, on_stop); + invalidator = null + }; + + metalog.info('putter_start', {id: putter.id, inv: invalidator}); return putter; }; - - // -------------------------------------------------------------------------- // Schedule deferred invalidation of metrics by type and tier. @@ -103,7 +113,7 @@ function Invalidator(){ _.each(type_tset, function(tset, tier){ var times = dateify(tset); - metalog.info("cube_compute", { is: "flush", type: type, tier: tier, times: times }); + metalog.info("event_flush", { type: type, tier: tier, times: times }); collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); }); }); @@ -111,17 +121,21 @@ function Invalidator(){ }; this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }); }; + this._empty = function(){ type_tsets = {}; } // for testing only function type_tset(type){ - if (! (type in type_tsets)) type_tsets[type] = empty_tsets(); + if (! (type in type_tsets)) type_tsets[type] = _.mapHash(tiers, function(){ return {}; });; return type_tsets[type]; } - function empty_tsets(){ return _.mapHash(tiers, function(){ return {}; }); } function dateify(tset){ return _.map(_.keys(tset), function(time){ return new Date(+time); }).sort(function(aa,bb){return aa-bb;}); } } -Invalidator.flushers = []; -Invalidator.start_flusher = function(cb){ Invalidator.flushers.push(setInterval(cb, invalidateInterval)); }; -Invalidator.stop_flushers = function(){ Invalidator.flushers.forEach(clearInterval); Invalidator.flushers = []; }; +Invalidator.flushers = {}; +Invalidator.start_flusher = function(id, cb){ Invalidator.flushers[id] = setInterval(cb, invalidateInterval); }; +Invalidator.stop_flusher = function(id, on_stop){ + clearInterval(Invalidator.flushers[id]); + delete Invalidator.flushers[id]; + if (on_stop) on_stop(); +}; // // event.getter - subscribe to event type @@ -153,8 +167,8 @@ exports.getter = function(db) { try { expression = parser.parse(request.expression); } catch (error) { - var resp = { is: "invalid expression", expression: request.expression, error: error }; - metalog.info('cube_getter', resp); + var resp = { error: "invalid expression", expression: request.expression, message: error }; + metalog.info('event_getter', resp); return callback(resp), -1; } diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 41df7358..73c20d67 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -9,14 +9,23 @@ var util = require("util"), _ = require("underscore"); +// 2012-08-10 04:25:55.736 +function timestamp() { + function pad(n){ return n < 10 ? '0' + n.toString(10) : n.toString(10); } + var d = new Date(); + var time = [pad(d.getUTCHours()), pad(d.getUTCMinutes()), pad(d.getUTCSeconds())].join(':'); + var date = [d.getUTCFullYear(), pad(d.getUTCMonth()+1), pad(d.getUTCDate()) ].join('-'); + return [date, ' ', time, '.', d.getUTCMilliseconds()].join(''); +} + var metalog = { putter: null, - log: util.log, + log: function(msg){ process.stderr.write(timestamp() + ' - ' + msg.toString() + "\n"); }, silent: function(){ } }; -var tracestack = {}, // container for traces - tracecap = 10000; // max traces to track + +var tracing = true; // false to mute tracing // adjust verboseness by reassigning `metalog.loggers.{level}` // @example: quiet all logs @@ -85,15 +94,13 @@ function dump(hsh){ var fields = _.map(dump_keys, function(len, key){ return ((hsh[key] === undefined ? '' : hsh[key])+' ').slice(0,len); }); var extras = {}; for (var key in hsh){ if (!(key in dump_keys)) extras[key] = hsh[key]; } if (_.isNumber(value)) fields.push((value*100)/100); - metalog.log(fields.join('|') + "\t" + (JSON.stringify(extras).slice(0,150))); + // metalog.log(fields.join('|') + "\t" + (JSON.stringify(extras).slice(0,150))); } function dump_header(){ - metalog.log(_.map(dump_keys, function(len, key){ return (key+' ').slice(0,len); }).join('|')); + // metalog.log(_.map(dump_keys, function(len, key){ return (key+' ').slice(0,len); }).join('|')); } -var tracing = false; - if (tracing){ metalog.trace = function(label, item, hsh){ @@ -128,6 +135,7 @@ if (tracing){ } catch(err){ metalog.log(err); metalog.log(err.stack); } return item; }; + } else { metalog.trace = function(){}; metalog.dump_trace = function(){}; @@ -137,12 +145,12 @@ if (tracing){ // Dump the 'util.inspect' view of each argument to the console. metalog.inspectify = function inspectify(args){ for (var idx in arguments) { - util.print(idx + ": "); + process.stderr.write(idx + ": "); var val = arguments[idx]; if (_.isFunction(val)) val = val.toString().slice(0, 80); - util.print(util.inspect(val, false, 2, true)+"\n"); // , null, true + process.stderr.write(util.inspect(val, false, 2, true)+"\n"); // , null, true } - util.print('----\n'); + process.stderr.write('----\n'); }; // wraps a callback, inspectifies its contents as it goes by @@ -154,7 +162,7 @@ metalog.spy = function spy(callback, label, ctxt){ if (! label) label = 'spyable'; if (! ctxt) ctxt = null; return function(){ - util.print(label+': ', callback, ' called with ', util.inspect(arguments), '\n'); + process.stderr.write(label+': ', callback, ' called with ', util.inspect(arguments), '\n'); // metalog.inspectify(callback, arguments); return callback.apply(ctxt, arguments); }; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 5e813b20..2c6d6397 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -3,10 +3,8 @@ // TODO use expression ids or hashes for more compact storage var _ = require("underscore"), - mongodb = require("mongodb"), util = require("util"), queuer = require("../queue-async/queue"), - db = require("./db"), parser = require("./metric-expression"), tiers = require("./tiers"), reduces = require("./reduces"), @@ -22,7 +20,7 @@ var metric_fields = {v: 1}, // Query for metrics. exports.getter = function(db){ - var Double = mongodb.Double, + var Double = require("mongodb").Double, queueByName = {}, request_queues = {}, qcount = 0; diff --git a/lib/cube/models.js b/lib/cube/models.js index 4f1ee43e..35f7a472 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -1,8 +1,7 @@ 'use strict'; var _ = require("underscore"), - metalog = require('./metalog'), - db = require('./db'); + metalog = require('./metalog'); var second = 1e3, second10 = 10e3, @@ -52,16 +51,16 @@ Event.prototype = { } }; -Event.prototype.save = function(callback){ - var self=this - db.events(self.type, function(error, collection){ +Event.prototype.save = function(db, callback){ + var self = this; + // metalog.trace('eSva', self); + db.events(self.type, function event_saver(error, collection){ if (error) return callback(error); - metalog.trace('eSvc', self); + // metalog.trace('eSvc', self); collection.save(self.to_wire, function saver(error){ - callback(error, self - // , metalog.trace('eSvd', self) - ); }); - }, metalog.trace('eSva', self)); + // metalog.trace('eSvd', self); + callback(error, self, self); + }); }, self); }; // Validate the date and type. diff --git a/lib/cube/server.js b/lib/cube/server.js index 0692cf1e..49021aff 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -23,7 +23,7 @@ var util = require("util"), authentication = require("./authentication"), event = require("./event"), metalog = require("./metalog"), - db = require("./db"); + Db = require("./db"); // Don't crash on errors. process.on("uncaughtException", function(error) { @@ -46,7 +46,7 @@ var wsOptions = { closeTimeout: 5000 }; -module.exports = function(options) { +module.exports = function(options, db) { var server = {}, primary = http.createServer(), secondary = websprocket.createServer(), @@ -56,6 +56,9 @@ module.exports = function(options) { id = 0, authenticator; + // allows dependency injection from test_helper + if (! db) db = new Db(); + secondary.server = primary; function is_sec_ws_initiation(request){ @@ -204,12 +207,11 @@ module.exports = function(options) { } catch(error){} } } server.stop = function(cb){ - event.stop(); // stop flushing try_close('http', primary); // stop serving try_close('ws', secondary); try_close('udp', udp); - setTimeout(function(){try_close('mongo', db);},100); // stop db'ing - if (cb) setTimeout(function(){ cb(); }, 200); + setTimeout(function(){try_close('mongo', db);}, 50); // stop db'ing + if (cb) cb(); }; return server; diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js index baa5f795..5475027d 100644 --- a/lib/cube/visualizer.js +++ b/lib/cube/visualizer.js @@ -17,7 +17,7 @@ function viewBoard(db) { callbacksByBoard = {}; function dispatch(request, callback) { - if (request.type != 'ping') metalog.info("cube_request", { is: request.type, bd: request.id, pc: callback.id }); + if (request.type != 'ping') metalog.info("viz_req", { is: request.type, bd: request.id, pc: callback.id }); request.id = require('mongodb').ObjectID(request.id); db.collection("boards", function(error, collection) { @@ -34,7 +34,7 @@ function viewBoard(db) { function check_authorization(request, action){ if (request.authorized.admin) return true; - metalog.info('cube_request', { is: 'denied', action: action, u: request.authorized }); + metalog.info('viz_auth_no', { action: action, u: request.authorized }); return false; } diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index f8c82c49..9fe35d14 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -2,11 +2,11 @@ var cluster = require('cluster'), metric = require('./metric'), - db = require('./db'), tiers = require('./tiers'), metalog = require('./metalog'); module.exports = function(options){ + throw(new Error('TODO: needs a db arg')); var calculate_metric, tier; function fetch_metrics(callback){ diff --git a/test/broker-test.js b/test/broker-test.js index dcc29fe6..8cbb8a63 100644 --- a/test/broker-test.js +++ b/test/broker-test.js @@ -16,99 +16,99 @@ var squarer = function(ii, cb){ cb(null, ii*ii, 'squarer'); }; function example_worker(){ var worker = new Worker('test', 50); worker.start(); return worker; } function example_job(){ return (new Job('smurf', squarer, [7])); } -suite.addBatch({ - 'Worker': { - '.new': { - topic: function(){ return new Worker('test', 50); }, - 'has an empty queue': function(worker){ assert.deepEqual(worker.size, 0); }, - 'is stopped': function(worker){ assert.equal(worker.state, 'stop'); } - }, - '.add': { - topic: example_worker, - 'pushes a job onto the queue': function(worker){ - worker.add('smurfette', function(){ setTimeout(_.identity, 50) }, [3], _.identity); - assert.deepEqual(worker.report(), { qname: 'test', size: 1, state: 'idle', queue: [ 'smurfette' ] }); - } - }, - '.invoke': { - topic: function(){ - var worker = this.worker = example_worker(); - var ctxt = this; - ctxt.checker = assert.isCalledTimes(ctxt, 3); - ctxt.performances = 0; - // shortly, worker will invoke perfom (once). 200 ms later, `perform` - // will call `worker`'s proxy callback, which invokes all 3 callbacks. - var perform = function(cb){ ctxt.performances++; setTimeout(function(){cb('hi');}, 20); }; - worker.add('thrice', perform, [], ctxt.checker); - worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); - worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); - }, - 'calls perform exactly once': function(){ assert.equal(this.performances, 1); }, - 'calls all registered callbacks': function(results){ assert.deepEqual(results, [['hi'], ['hi'], ['hi']]); }, - teardown: function(){ - this.worker.stop(); - } - } - }, - - 'Job': { - '.new': { - topic: example_job, - '': function(job){ - assert.deepEqual(job, {mbox: 'smurf', perform: squarer, args: [7], on_completes: []}); - } - }, - '.listen': { - topic: function(){ - var job = example_job(); - job.listen(squarer); - return job; - } - } - }, - - 'Broker': { - 'handles interleaved jobs': { - topic: function(){ - var ctxt = this, - broker = this.broker = new Broker('test', 10), - ignored = assert.isNotCalled('perform'); - ctxt.perfs = {a: 0, b: 0, c:0}; - ctxt.checker = assert.isCalledTimes(ctxt, 8); - var task_a = function(ii, a2, cb){ ctxt.perfs.a++; setTimeout(function(){cb('result_a', ii*ii, a2);}, 10); }; - var task_b = function(ii, cb){ ctxt.perfs.b++; setTimeout(function(){cb('result_b', ii*ii );}, 20); }; - var task_c = function(ii, cb){ ctxt.perfs.c++; setTimeout(function(){cb('result_c', ii*ii );}, 300); }; - // will go second: jobs are sorted - broker.deferProxy('task_b', task_b, 1, ctxt.checker); - broker.deferProxy('task_b', ignored, '<>', ctxt.checker); - // will go first - broker.deferProxy('task_a', task_a, 0, '?', ctxt.checker); - broker.deferProxy('task_a', ignored, '<>', ctxt.checker); - // will go third - broker.deferProxy('task_c', task_c, 2, ctxt.checker); - // a & b will be done; c (takes 300ms) will still be running. - setTimeout(function(){ - broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); - broker.deferProxy('task_c', ignored, '<>', ctxt.checker); - broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); - }, 200); - }, - 'calls perform exactly once': function(){ assert.deepEqual(this.perfs, {a: 2, b: 1, c: 1}); }, - 'calls all registered callbacks': function(results){ - assert.deepEqual(results, [ - ['result_a', 0, '?'], ['result_a', 0, '?'], - ['result_b', 1], ['result_b', 1], - ['result_c', 4], ['result_c', 4], - ['result_a', 9, '!'], ['result_a', 9, '!'] - ]); - }, - teardown: function(){ - this.broker.stop(); - } - } - } // broker - -}); +// suite.addBatch({ +// 'Worker': { +// '.new': { +// topic: function(){ return new Worker('test', 50); }, +// 'has an empty queue': function(worker){ assert.deepEqual(worker.size, 0); }, +// 'is stopped': function(worker){ assert.equal(worker.state, 'stop'); } +// }, +// '.add': { +// topic: example_worker, +// 'pushes a job onto the queue': function(worker){ +// worker.add('smurfette', function(){ setTimeout(_.identity, 50) }, [3], _.identity); +// assert.deepEqual(worker.report(), { qname: 'test', size: 1, state: 'idle', queue: [ 'smurfette' ] }); +// } +// }, +// '.invoke': { +// topic: function(){ +// var worker = this.worker = example_worker(); +// var ctxt = this; +// ctxt.checker = assert.isCalledTimes(ctxt, 3); +// ctxt.performances = 0; +// // shortly, worker will invoke perfom (once). 200 ms later, `perform` +// // will call `worker`'s proxy callback, which invokes all 3 callbacks. +// var perform = function(cb){ ctxt.performances++; setTimeout(function(){cb('hi');}, 20); }; +// worker.add('thrice', perform, [], ctxt.checker); +// worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); +// worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); +// }, +// 'calls perform exactly once': function(){ assert.equal(this.performances, 1); }, +// 'calls all registered callbacks': function(results){ assert.deepEqual(results, [['hi'], ['hi'], ['hi']]); }, +// teardown: function(){ +// this.worker.stop(); +// } +// } +// }, +// +// 'Job': { +// '.new': { +// topic: example_job, +// '': function(job){ +// assert.deepEqual(job, {mbox: 'smurf', perform: squarer, args: [7], on_completes: []}); +// } +// }, +// '.listen': { +// topic: function(){ +// var job = example_job(); +// job.listen(squarer); +// return job; +// } +// } +// }, +// +// 'Broker': { +// 'handles interleaved jobs': { +// topic: function(){ +// var ctxt = this, +// broker = this.broker = new Broker('test', 10), +// ignored = assert.isNotCalled('perform'); +// ctxt.perfs = {a: 0, b: 0, c:0}; +// ctxt.checker = assert.isCalledTimes(ctxt, 8); +// var task_a = function(ii, a2, cb){ ctxt.perfs.a++; setTimeout(function(){cb('result_a', ii*ii, a2);}, 10); }; +// var task_b = function(ii, cb){ ctxt.perfs.b++; setTimeout(function(){cb('result_b', ii*ii );}, 20); }; +// var task_c = function(ii, cb){ ctxt.perfs.c++; setTimeout(function(){cb('result_c', ii*ii );}, 300); }; +// // will go second: jobs are sorted +// broker.deferProxy('task_b', task_b, 1, ctxt.checker); +// broker.deferProxy('task_b', ignored, '<>', ctxt.checker); +// // will go first +// broker.deferProxy('task_a', task_a, 0, '?', ctxt.checker); +// broker.deferProxy('task_a', ignored, '<>', ctxt.checker); +// // will go third +// broker.deferProxy('task_c', task_c, 2, ctxt.checker); +// // a & b will be done; c (takes 300ms) will still be running. +// setTimeout(function(){ +// broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); +// broker.deferProxy('task_c', ignored, '<>', ctxt.checker); +// broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); +// }, 200); +// }, +// 'calls perform exactly once': function(){ assert.deepEqual(this.perfs, {a: 2, b: 1, c: 1}); }, +// 'calls all registered callbacks': function(results){ +// assert.deepEqual(results, [ +// ['result_a', 0, '?'], ['result_a', 0, '?'], +// ['result_b', 1], ['result_b', 1], +// ['result_c', 4], ['result_c', 4], +// ['result_a', 9, '!'], ['result_a', 9, '!'] +// ]); +// }, +// teardown: function(){ +// this.broker.stop(); +// } +// } +// } // broker +// +// }); diff --git a/test/evaluator-test.js b/test/evaluator-test.js index aa3f92ed..435319ed 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -13,16 +13,16 @@ function frontend_components() { cube.visualizer.register.apply(this, arguments); } -// suite.addBatch( -// test_helper.with_server(server_options, frontend_components, { -// -// "POST /event/put with invalid JSON": { -// topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), -// "responds with status 400": function(response) { -// assert.equal(response.statusCode, 400); -// assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); -// } -// } -// })); +suite.addBatch( + test_helper.with_server(server_options, frontend_components, { + + "POST /event/put with invalid JSON": { + topic: test_helper.request({method: "GET", path: "/1.0/event/get"}, "This ain't JSON.\n"), + "responds with status 400": function(response) { + assert.equal(response.statusCode, 400); + assert.deepEqual(JSON.parse(response.body), {error: "invalid expression", message: {}}); + } + } +})); suite['export'](module); diff --git a/test/event-test.js b/test/event-test.js index 3f0675db..e7076a7a 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -13,20 +13,19 @@ var ice_cubes_good_day = Date.UTC(1992, 1, 20, 1, 8, 7), suite.addBatch(test_helper.batch({ topic: function(test_db) { - test_helper.inspectify(test_db); - return event.putter(test_db); + return this.putter = event.putter(test_db); }, 'invalidates': { - topic: function(putter){ + topic: function(){ var ctxt = this; test_helper.inspectify('a'); - putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ + ctxt.putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ test_helper.inspectify('b'); - putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); + ctxt.putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); }, 'correct tiers': function(){ test_helper.inspectify('invalidates putter tiers', arguments); - var ts = event.invalidator().tsets(); + var ts = this.putter.invalidator().tsets(); assert.deepEqual(ts, { 'test': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], 60e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:00Z') ], @@ -35,7 +34,8 @@ suite.addBatch(test_helper.batch({ 86400e3: [new Date('1992-02-20T00:00:00Z'), new Date('1993-03-18T00:00:00Z') ] }}); } - } + }, + teardown: function(){ this.putter.stop(this.callback); } })); suite['export'](module); diff --git a/test/metalog-test.js b/test/metalog-test.js index 27b9639a..4f36b27d 100644 --- a/test/metalog-test.js +++ b/test/metalog-test.js @@ -84,23 +84,23 @@ suite.with_log({ } } } -}).with_log({ - '.event': { - 'last parameter overrides log target': { - topic: function(metalog) { - metalog.send_events = true; - metalog.event('reactor_level', { criticality: 3, hemiconducers: 'cromulent' }, 'minor'); - metalog.event('reactor_level', { criticality: 2, hemiconducers: 'whispery' }, 'silent'); - return metalog; - }, - '': function(metalog){ - assert.equal(this.logged.minored.pop(), 'reactor_level\t{"criticality":3,"hemiconducers":"cromulent"}'); - assert.equal(this.logged.putted.pop().data.criticality, 2); - assert.equal(this.logged.putted.pop().data.criticality, 3); - assert.deepEqual(this.logged, { infoed: [], minored: [], putted: [] }); - } - } - } +// }).with_log({ +// '.event': { +// 'last parameter overrides log target': { +// topic: function(metalog) { +// metalog.send_events = true; +// metalog.event('reactor_level', { criticality: 3, hemiconducers: 'cromulent' }, 'minor'); +// metalog.event('reactor_level', { criticality: 2, hemiconducers: 'whispery' }, 'silent'); +// return metalog; +// }, +// '': function(metalog){ +// assert.equal(this.logged.minored.pop(), 'reactor_level\t{"criticality":3,"hemiconducers":"cromulent"}'); +// assert.equal(this.logged.putted.pop().data.criticality, 2); +// assert.equal(this.logged.putted.pop().data.criticality, 3); +// assert.deepEqual(this.logged, { infoed: [], minored: [], putted: [] }); +// } +// } +// } }); function dummy_logger(arg){} diff --git a/test/metric-test.js b/test/metric-test.js index e085d6ba..9159b8ee 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -133,6 +133,7 @@ suite.addBatch(test_helper.batch({ getter = metric.getter(test_db), callback = this.callback, put_queue = queuer(10); + this.putter = putter; // Seed the events table with a simple event: a value going from 0 to 2499 for (var i = 0; i < 2500; i++){ @@ -159,6 +160,7 @@ suite.addBatch(test_helper.batch({ // // So the events can settle in, wait `batch_testing_delay` ms before continuing // setTimeout(function() { callback(null, getter); }, batch_testing_delay); }, + teardown: function(){ this.putter.stop(this.callback); }, // FIXME: ---- remove below ------------------------------------ diff --git a/test/test_helper.js b/test/test_helper.js index 324a9121..9b55b5d1 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -5,7 +5,7 @@ var _ = require("underscore"), assert = require("assert"), http = require("http"), dgram = require('dgram'), - test_db = require("../lib/cube/db"), + Db = require("../lib/cube/db"), metalog = require("../lib/cube/metalog"); // ========================================================================== @@ -30,7 +30,6 @@ test_helper.settings = { // Disable logging for tests. metalog.loggers.info = metalog.silent; // log metalog.loggers.minor = metalog.silent; // log -metalog.loggers.warn = metalog.silent; // log metalog.send_events = false; // ========================================================================== @@ -75,10 +74,10 @@ test_helper.udp_request = function (data){ return function(){ var udp_client = dgram.createSocket('udp4'); var buffer = new Buffer(JSON.stringify(data)); - var ctxt = this; - metalog.info('sending_udp', { data: data }); + var ctxt = this, cb = ctxt.callback; + metalog.info('test_sending_udp', { data: data }); udp_client.send(buffer, 0, buffer.length, ctxt.udp_port, 'localhost', - function(err, val){ delay(ctxt.callback, ctxt)(err, val); udp_client.close(); } ); + function(err, val){ delay(cb, ctxt)(err, val); udp_client.close(); } ); }; }; @@ -129,23 +128,32 @@ test_helper.delay = delay; // @param components -- passed to server.register() // @param batch -- the tests to run test_helper.with_server = function(options, components, batch){ - return { '': { - topic: function(){ start_server(options, components, this); }, + return test_helper.batch({ '': { + topic: function(test_db){ + var ctxt = this, cb = ctxt.callback; + start_server(options, components, ctxt, test_db); + }, '': batch, - teardown: function(svr){ this.server.stop(this.callback); } - } }; + teardown: function(j_, test_db){ + var callback = this.callback; + this.server.stop(function(){ + metalog.info('test_server_batch_closed'); + callback(); + }); + } + } }); }; // @see test_helper.with_server -function start_server(options, register, vow){ +function start_server(options, register, ctxt, test_db){ for (var key in test_helper.settings){ if (! options[key]){ options[key] = test_helper.settings[key]; } } - vow.http_port = options['http-port']; - vow.udp_port = options['udp-port']; - vow.server = require('../lib/cube/server')(options); - vow.server.register = register; - vow.server.start(vow.callback); + ctxt.http_port = options['http-port']; + ctxt.udp_port = options['udp-port']; + ctxt.server = require('../lib/cube/server')(options, test_db); + ctxt.server.register = register; + ctxt.server.start(ctxt.callback); } // ========================================================================== @@ -153,22 +161,32 @@ function start_server(options, register, vow){ // db helpers // +var suite_id = 0; + // test_helper.batch -- // * connect to db, drop relevant collections // * run tests once db is ready; // * close db when tests are done test_helper.batch = function(batch) { + metalog.info('batch', batch); return { "": { topic: function() { - var _this = this; - test_db.open(test_helper.settings, function(error){ - drop_collections(_this.callback); + metalog.warn('got here'); + var ctxt = this; + ctxt.db = new Db(); + var options = _.extend({}, test_helper.settings, {collection_prefix: ('ts'+(++suite_id)+'_')}); + ctxt.db.open(options, function(error){ + drop_and_reopen_collections(ctxt.db, ctxt.callback); }); }, "": batch, - teardown: function(test_db) { - test_db.close(this.callback); + teardown: function(){ + var callback = this.callback; + this.db.close(function(){ + metalog.info('test_db_batch_closed'); + callback(); + }); } } }; @@ -177,14 +195,15 @@ test_helper.batch = function(batch) { // test_db.using_objects -- scaffold fixtures into the database, run tests once loaded. // // Wrap your tests in test_helper.batch to get the test_db object. -test_db.using_objects = function (clxn_name, test_objects, context){ - metalog.minor('cube_testdb', {state: 'loading test objects', test_objects: test_objects }); +Db.prototype.using_objects = function (clxn_name, test_objects, ctxt){ + var test_db = this; + metalog.minor('test_db_loading_objects', test_objects); test_db.collection(clxn_name, function(err, clxn){ if (err) throw(err); - context[clxn_name] = clxn; + ctxt[clxn_name] = clxn; clxn.remove({ dummy: true }, function(){ clxn.insert(test_objects, function(){ - context.callback(null, test_db); + ctxt.callback(null, test_db); }); }); }); }; @@ -195,20 +214,20 @@ test_db.using_objects = function (clxn_name, test_objects, context){ // // @see test_helper.batch -function drop_collections(cb){ - metalog.minor('cube_testdb', { state: 'dropping test collections', collections: test_collections }); +function drop_and_reopen_collections(test_db, cb){ + metalog.minor('test_db_drop_collections', { collections: test_collections }); var collectionsRemaining = test_collections.length; - test_collections.forEach(function(collection_name){ - test_db.collection(collection_name, function(error, collection){ - collection.drop(collectionReady); - }) - }); - function collectionReady() { - if (!--collectionsRemaining) { - cb(null, test_db); - } - } + // test_collections.forEach(function(collection_name){ + // test_db.collection(collection_name, function(error, collection){ + // collection.drop(collectionReady); + // }) + // }); + // function collectionReady() { + // if (!--collectionsRemaining) { + test_db.events('test', function test_helper_go(){ cb(null, test_db); }); + // } + // } } // ========================================================================== From e2be0b1a342430b2ce8cf4807a84b739f2124f04 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 13 Sep 2012 10:34:48 -0500 Subject: [PATCH 41/87] Misc bug fixes --- examples/random-emitter/random-emitter.js | 17 ++++++++++------- lib/cube/db.js | 11 ++++++----- lib/cube/event.js | 6 +++--- lib/cube/models.js | 10 +++++----- test/event-test.js | 5 +---- test/test_helper.js | 1 - 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/random-emitter/random-emitter.js b/examples/random-emitter/random-emitter.js index 9b4cf45c..af3d405f 100644 --- a/examples/random-emitter/random-emitter.js +++ b/examples/random-emitter/random-emitter.js @@ -13,7 +13,14 @@ var start = Date.now() + options["offset"], value = 0, count = 0; -while (start < stop) { +function insert(start){ + if(start > stop){ + util.log("sent " + count + " events"); + util.log("stopping emitter"); + emitter.close(); + return; + } + count++; emitter.send({ type: "random", time: new Date(start), @@ -21,11 +28,7 @@ while (start < stop) { value: value += Math.random() - .5 } }); - start += step; - ++count; - if (count > 3) break; + process.nextTick(function(){ insert(start + step) }); } -util.log("sent " + count + " events"); -util.log("stopping emitter"); -emitter.close(); +insert(start); diff --git a/lib/cube/db.js b/lib/cube/db.js index 25e80a5e..e1ca06d4 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -58,7 +58,7 @@ function Db(){ db.collection = collection(db_client); if (! options["mongo-username"]) return callback(null, db); - + db_client.authenticate(options["mongo-username"], mongo_password, function(error, success) { if (error) return callback(error); if (!success) return callback(new Error("authentication failed")); @@ -122,10 +122,10 @@ function Db(){ function clxn_for(client, clxnname, clxnopts, on_collection, on_create, tr){ on_collection = on_collection || function(){}; - + // If we've cached the collection, call back immediately if (collections[clxnname]) return on_collection(null, collections[clxnname]); - + // If someone is already creating the collection for this new type, // then append the callback to the queue for later save. if (clxnname in pending_callbacks){ @@ -138,7 +138,7 @@ function Db(){ // First add the new event to the queue. pending_callbacks[clxnname] = [on_collection]; - + // Create collection, and issue callback for index creation, etc client.createCollection(clxnname, clxnopts, function(error, clxn){ metalog.trace('clxnC', tr); @@ -169,7 +169,8 @@ function Db(){ function collection(client){ return function(name, callback){ if (collections[name]) return callback(null, collections[name]); - client.collection.apply(client, arguments); + collections[name] = client.collection(name, callback); + return callback(null, collections[name]); } } diff --git a/lib/cube/event.js b/lib/cube/event.js index 059b2d44..c89e3509 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -43,7 +43,7 @@ exports.putter = function(db){ callback = callback || function(){}; metalog.trace('eput', request); - + // // Drop events from before invalidation horizon // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; @@ -56,7 +56,7 @@ exports.putter = function(db){ var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } metalog.trace('ebeg', event, { using: request }); - + // Save the event, then queue invalidation of its associated cached metrics. // // We don't invalidate the events immediately. This would cause redundant @@ -67,7 +67,7 @@ exports.putter = function(db){ event.save(db, function after_save(error, event){ metalog.trace('esav', event); if (event) invalidator.add(event.type, event); - callback(error, event); + callback(event); }); } diff --git a/lib/cube/models.js b/lib/cube/models.js index 35f7a472..ab759c37 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -28,12 +28,12 @@ function Event(type, time, data, id){ if (id) this._id = id; var self = this; - + if (this._id) Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, _id: _id }, enumerable: false, writable: false }); else Object.defineProperty(this, 'to_wire', { value: { t: time, d: data }, enumerable: false, writable: false }); - + Object.defineProperties(this, { - _trace: { value: null, enumerable: false, writable: true } + _trace: { value: null, enumerable: false, writable: true, configurable: true } }); } Event.prototype = { @@ -85,12 +85,12 @@ function Metric(time, value, id, measurement){ this.time = time; this.value = value; if (id) this.id = id; - + Object.defineProperties(this, { e: { value: measurement.expression.source, enumerable: false, writable: false }, l: { value: measurement.tier.key, enumerable: false, writable: false }, measurement: { value: measurement, enumerable: false, writable: false }, - _trace: { value: null, enumerable: false, writable: true }, + _trace: { value: null, enumerable: false, writable: true, configurable: true }, }); } Object.defineProperties(Metric.prototype, { diff --git a/test/event-test.js b/test/event-test.js index e7076a7a..3d1b39de 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -18,13 +18,10 @@ suite.addBatch(test_helper.batch({ 'invalidates': { topic: function(){ var ctxt = this; - test_helper.inspectify('a'); ctxt.putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ - test_helper.inspectify('b'); ctxt.putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); }, - 'correct tiers': function(){ - test_helper.inspectify('invalidates putter tiers', arguments); + 'correct tiers': function(a,b){ var ts = this.putter.invalidator().tsets(); assert.deepEqual(ts, { 'test': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], diff --git a/test/test_helper.js b/test/test_helper.js index 9b55b5d1..e3198ce2 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -172,7 +172,6 @@ test_helper.batch = function(batch) { return { "": { topic: function() { - metalog.warn('got here'); var ctxt = this; ctxt.db = new Db(); var options = _.extend({}, test_helper.settings, {collection_prefix: ('ts'+(++suite_id)+'_')}); From 1112467ba75865b1c9d2bce73198aed3e60254d5 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 13 Sep 2012 13:50:27 -0500 Subject: [PATCH 42/87] Fix stuff --- lib/cube/db.js | 7 +++++-- lib/cube/warmer.js | 13 +++++++++---- test/event-test.js | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/cube/db.js b/lib/cube/db.js index e1ca06d4..94b366a7 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -169,8 +169,11 @@ function Db(){ function collection(client){ return function(name, callback){ if (collections[name]) return callback(null, collections[name]); - collections[name] = client.collection(name, callback); - return callback(null, collections[name]); + + client.collection(name, function(error, collection){ + if(!error) collections[name] = collection; + callback(error, collection); + }); } } diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index 9fe35d14..96b34808 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -3,11 +3,11 @@ var cluster = require('cluster'), metric = require('./metric'), tiers = require('./tiers'), - metalog = require('./metalog'); + metalog = require('./metalog'), + db = new (require('./db'))(); module.exports = function(options){ - throw(new Error('TODO: needs a db arg')); - var calculate_metric, tier; + var calculate_metric, tier, timeout; function fetch_metrics(callback){ var expressions = []; @@ -40,7 +40,7 @@ module.exports = function(options){ // fake metrics request calculate_metric({ step: tier.key, expression: expression, start: start, stop: stop }, noop); }); - setTimeout(function(){ fetch_metrics(process_metrics); }, options['warmer-interval']); + timeout = setTimeout(function(){ fetch_metrics(process_metrics); }, options['warmer-interval']); } function noop(){}; @@ -58,6 +58,11 @@ module.exports = function(options){ calculate_metric = metric.getter(db); fetch_metrics(process_metrics); }); + }, + + stop: function(){ + metalog.event("cube_life", { is: 'stop_warmer' }); + clearTimeout(timeout); } }; }; diff --git a/test/event-test.js b/test/event-test.js index 3d1b39de..d1b5ecd7 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -16,10 +16,10 @@ suite.addBatch(test_helper.batch({ return this.putter = event.putter(test_db); }, 'invalidates': { - topic: function(){ - var ctxt = this; - ctxt.putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ - ctxt.putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), ctxt.callback);}); + topic: function(putter){ + var _this = this; + putter((new Event('test', ice_cubes_good_day, {value: 3})).to_request(), function(){ + putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), _this.callback);}); }, 'correct tiers': function(a,b){ var ts = this.putter.invalidator().tsets(); @@ -32,7 +32,16 @@ suite.addBatch(test_helper.batch({ }}); } }, - teardown: function(){ this.putter.stop(this.callback); } + 'callback': { + topic: function(putter){ + putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), this.callback); + }, + 'no error arg': function(arg1, arg2){ + assert.instanceOf(arg1, Event); + assert.typeOf(arg2, 'undefined'); + } + }, + teardown: function(putter){ putter.stop(this.callback); } })); suite['export'](module); From 5fe88441c85fc8acee1ed2c53dcba2ee9fd2185a Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 13 Sep 2012 16:22:46 -0500 Subject: [PATCH 43/87] Add warmer test --- lib/cube/db.js | 6 ++++- test/test_helper.js | 27 +++++++++++++---------- test/warmer-test.js | 53 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 test/warmer-test.js diff --git a/lib/cube/db.js b/lib/cube/db.js index 94b366a7..5cb4a26b 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -56,6 +56,7 @@ function Db(){ db.events = events_collection_factory(events_client); db.types = types(events_client); db.collection = collection(db_client); + db.clearCache = function(callback){ collections = {}; if(callback) callback(null, db); }; if (! options["mongo-username"]) return callback(null, db); @@ -75,11 +76,14 @@ function Db(){ function close(callback){ metalog.trace('mongo_closing', db.report()); if (! db_client) return metalog.minor('mongo_already_closed'); - collections = {}; + + db.clearCache(); delete db.metrics; delete db.events; delete db.types; delete db.collection; + delete db.clearCache; + if (isConnected()){ db_client.close(); } db_client = events_client = metrics_client = null; db.isHalted = true; diff --git a/test/test_helper.js b/test/test_helper.js index e3198ce2..2190a2a0 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -174,9 +174,12 @@ test_helper.batch = function(batch) { topic: function() { var ctxt = this; ctxt.db = new Db(); - var options = _.extend({}, test_helper.settings, {collection_prefix: ('ts'+(++suite_id)+'_')}); + var options = _.extend({}, test_helper.settings); ctxt.db.open(options, function(error){ - drop_and_reopen_collections(ctxt.db, ctxt.callback); + drop_and_reopen_collections(ctxt.db, function(error){ + ctxt.callback.apply(ctxt, arguments); + ctxt.db.clearCache(); + }); }); }, "": batch, @@ -217,16 +220,16 @@ function drop_and_reopen_collections(test_db, cb){ metalog.minor('test_db_drop_collections', { collections: test_collections }); var collectionsRemaining = test_collections.length; - // test_collections.forEach(function(collection_name){ - // test_db.collection(collection_name, function(error, collection){ - // collection.drop(collectionReady); - // }) - // }); - // function collectionReady() { - // if (!--collectionsRemaining) { - test_db.events('test', function test_helper_go(){ cb(null, test_db); }); - // } - // } + test_collections.forEach(function(collection_name){ + test_db.collection(collection_name, function(error, collection){ + collection.drop(collectionReady); + }) + }); + function collectionReady() { + if (!--collectionsRemaining) { + cb(null, test_db); + } + } } // ========================================================================== diff --git a/test/warmer-test.js b/test/warmer-test.js new file mode 100644 index 00000000..fd9a41ca --- /dev/null +++ b/test/warmer-test.js @@ -0,0 +1,53 @@ +'use strict'; + +var _ = require('underscore'), + vows = require("vows"), + assert = require("assert"), + test_helper = require("./test_helper"), + event = require("../lib/cube/event"), + settings = {"warmer-tier": 10000, "warmer-interval": 10000, horizons: { calculation: 30000 } }, + warmer = require("../lib/cube/warmer")(_.extend({}, settings, test_helper.settings)); + +var suite = vows.describe("warmer"); + +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + var board = { pieces: [{ query: "sum(test(value))" }] }, + nowish = this.nowish = (10e3 * Math.floor(new Date()/10e3)), + putter = this.putter = event.putter(test_db), + _this = this; + + putter({type: 'test', time: nowish + 500, data: {value: 10}}, function(){ + putter({type: 'test', time: nowish + 2000, data: {value: 5}}, function(){ + test_db.using_objects("boards", [board], {callback: function(error){ + if(error) return _this.callback(error); + _this.callback(null, test_db); + }}); + }); + }); + }, + 'calculates': { + topic: function(test_db){ + var _this = this; + + warmer.start() + setTimeout(function(){ + test_db.metrics('test', function(error, collection){ + collection.find().toArray(_this.callback) + }) + }, 1000); + }, + 'correct number of metrics': function(metrics){ assert.equal(metrics.length, 3); }, + 'correct values for metrics': function(metrics){ + assert.equal(metrics[0].v, 0); assert.equal(+metrics[0]._id.t, +this.nowish - 20000); + assert.equal(metrics[1].v, 0); assert.equal(+metrics[1]._id.t, +this.nowish - 10000); + assert.equal(metrics[2].v, 15); assert.equal(+metrics[2]._id.t, +this.nowish); + } + }, + teardown: function(){ + warmer.stop(); + this.putter.stop(this.callback); + } +})); + +suite['export'](module); From a1b71071f0dd4e91dae6f6479fbe0b10cd3a564e Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 13 Sep 2012 18:36:52 -0500 Subject: [PATCH 44/87] Update random streamer example to be real time --- examples/random-emitter/random-streamer.js | 40 ++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/examples/random-emitter/random-streamer.js b/examples/random-emitter/random-streamer.js index 18a4c0b7..a08776ed 100644 --- a/examples/random-emitter/random-streamer.js +++ b/examples/random-emitter/random-streamer.js @@ -13,31 +13,35 @@ var options = { // For example, if the offset is minus four hours, then the first event that // the random emitter sends will be four hours old. It will then generate more // recent events based on the step interval, all the way up to the duration. - "offset": -0.49 * 60 * 60 * 1000, - "duration": 0.49 * 60 * 60 * 1000, - - // The time between random events. - "step": 1000 * 2, + "event_frequency": 5000, // per second event_type: "doh" }; var emitter = cube.emitter(options["collector"]); -cromulator.start = Date.now() + options.offset; -cromulator.stop = cromulator.start + options.duration; -cromulator.step = options.step; - metalog.info('emitter', cromulator.report('starting')); -var time = cromulator.start; -while (time < cromulator.stop) { - var event = new Event(options.event_type, cromulator.spread_time(time), cromulator.data_at(time)); - if (cromulator.count % 1000 == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); - event.force = true; - emitter.send(event.to_request()); - time += cromulator.step; +function send(){ + cromulator.start = new Date(Math.floor(new Date() / (1000 * 60 * 60 * 24)) * 1000 * 60 * 60 * 24); + cromulator.stop = new Date(+cromulator.start + (1000 * 60 * 60 * 24)); + cromulator.step = 1000 / options.event_frequency; + + var base_time = new Date(Math.floor(new Date() / 1000) * 1000), + step = 1000 / options.event_frequency, + i = 0; + + while (i < options.event_frequency) { + var time = new Date(+base_time + (step * i)), + event = new Event(options.event_type, time, cromulator.data_at(time)); + if (i % 1000 == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); + event.force = true; + emitter.send(event.to_request()); + i = i + 1; + } } -metalog.info('emitter', cromulator.report('stopping', time)); -emitter.close(); +setInterval(send, 1000); + +//metalog.info('emitter', cromulator.report('stopping', new Date())); +//emitter.close(); From f45342627a81b62a8f91f67a12dc8c8888a18e9e Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Wed, 19 Sep 2012 10:57:31 -0500 Subject: [PATCH 45/87] Stop random streamer example from batching --- examples/random-emitter/random-streamer.js | 35 +++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/examples/random-emitter/random-streamer.js b/examples/random-emitter/random-streamer.js index a08776ed..33419b32 100644 --- a/examples/random-emitter/random-streamer.js +++ b/examples/random-emitter/random-streamer.js @@ -18,30 +18,29 @@ var options = { event_type: "doh" }; -var emitter = cube.emitter(options["collector"]); +var emitter = cube.emitter(options["collector"]), + step = 1 / options.event_frequency; -metalog.info('emitter', cromulator.report('starting')); -function send(){ +function setup_cromulator(){ cromulator.start = new Date(Math.floor(new Date() / (1000 * 60 * 60 * 24)) * 1000 * 60 * 60 * 24); cromulator.stop = new Date(+cromulator.start + (1000 * 60 * 60 * 24)); - cromulator.step = 1000 / options.event_frequency; - - var base_time = new Date(Math.floor(new Date() / 1000) * 1000), - step = 1000 / options.event_frequency, - i = 0; - - while (i < options.event_frequency) { - var time = new Date(+base_time + (step * i)), - event = new Event(options.event_type, time, cromulator.data_at(time)); - if (i % 1000 == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); - event.force = true; - emitter.send(event.to_request()); - i = i + 1; - } + cromulator.step = 1 / options.event_frequency; +} + +metalog.info('emitter', cromulator.report('starting')); + +function send(){ + if(+cromulator.stop <= +(new Date())) setup_cromulator(); + + var time = new Date(), + event = new Event(options.event_type, time, cromulator.data_at(time)); + if (+time % 1000 == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); + event.force = true; + emitter.send(event.to_request()); } -setInterval(send, 1000); +setInterval(send, step); //metalog.info('emitter', cromulator.report('stopping', new Date())); //emitter.close(); From f6cfbc859e971f13dba7aa35e5a11de5acc78d5b Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Wed, 19 Sep 2012 14:26:27 -0500 Subject: [PATCH 46/87] Update streamer, remove tracing from events --- examples/random-emitter/random-streamer.js | 38 ++++++++++++---------- lib/cube/event.js | 6 ++-- lib/cube/metalog.js | 18 +++++----- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/examples/random-emitter/random-streamer.js b/examples/random-emitter/random-streamer.js index 33419b32..5587b60c 100644 --- a/examples/random-emitter/random-streamer.js +++ b/examples/random-emitter/random-streamer.js @@ -7,7 +7,7 @@ var cube = require("../../"), // replace with require("cube") models = require("../../lib/cube/models"), Event = models.Event; var options = { - "collector": "ws://127.0.0.1:1080", + "collector": "ws://127.0.0.1:6000", // The offset and duration to backfill, in milliseconds. // For example, if the offset is minus four hours, then the first event that @@ -15,32 +15,36 @@ var options = { // recent events based on the step interval, all the way up to the duration. "event_frequency": 5000, // per second + event_batch: 500, event_type: "doh" }; var emitter = cube.emitter(options["collector"]), - step = 1 / options.event_frequency; + step = 1000 / options.event_frequency, + batch = options.event_batch || (step < 1 ? (1 / step) : 1); function setup_cromulator(){ - cromulator.start = new Date(Math.floor(new Date() / (1000 * 60 * 60 * 24)) * 1000 * 60 * 60 * 24); - cromulator.stop = new Date(+cromulator.start + (1000 * 60 * 60 * 24)); - cromulator.step = 1 / options.event_frequency; -} + var day = 1000 * 60 * 60 * 24; + cromulator.start = new Date(Math.floor(new Date() / day) * day); + cromulator.stop = new Date(+cromulator.start + day); + cromulator.step = step; + } metalog.info('emitter', cromulator.report('starting')); function send(){ - if(+cromulator.stop <= +(new Date())) setup_cromulator(); - - var time = new Date(), - event = new Event(options.event_type, time, cromulator.data_at(time)); - if (+time % 1000 == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); - event.force = true; - emitter.send(event.to_request()); + if (!cromulator.stop || +cromulator.stop <= +(new Date())) setup_cromulator(); + var i = -1; + + while (++i < batch) { + var time = new Date(), + event = new Event(options.event_type, time, cromulator.data_at(time)); + if (cromulator.count % options.event_frequency == 0) metalog.info('emitter', {em: emitter.report(), cr: cromulator.report('progress', time)}); + event.force = true; + emitter.send(event.to_request()); + } } -setInterval(send, step); - -//metalog.info('emitter', cromulator.report('stopping', new Date())); -//emitter.close(); +var interval = 1000 / (options.event_frequency / batch); +setInterval(send, interval); \ No newline at end of file diff --git a/lib/cube/event.js b/lib/cube/event.js index c89e3509..0a662b17 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -41,7 +41,7 @@ exports.putter = function(db){ var time = "time" in request ? new Date(request.time) : new Date(), type = request.type; callback = callback || function(){}; - metalog.trace('eput', request); + //metalog.trace('eput', request); // // Drop events from before invalidation horizon @@ -55,7 +55,7 @@ exports.putter = function(db){ var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } - metalog.trace('ebeg', event, { using: request }); + //metalog.trace('ebeg', event, { using: request }); // Save the event, then queue invalidation of its associated cached metrics. // @@ -65,7 +65,7 @@ exports.putter = function(db){ // the likelihood of a race condition between when the events are read by // the evaluator and when the newly-computed metrics are saved. event.save(db, function after_save(error, event){ - metalog.trace('esav', event); + //metalog.trace('esav', event); if (event) invalidator.add(event.type, event); callback(event); }); diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 73c20d67..4b25f610 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -4,12 +4,12 @@ // // * log progress to disk // * trace a request through the process stack -// * +// * var util = require("util"), _ = require("underscore"); -// 2012-08-10 04:25:55.736 +// 2012-08-10 04:25:55.736 function timestamp() { function pad(n){ return n < 10 ? '0' + n.toString(10) : n.toString(10); } var d = new Date(); @@ -25,7 +25,7 @@ var metalog = { }; -var tracing = true; // false to mute tracing +var tracing = false; // false to mute tracing // adjust verboseness by reassigning `metalog.loggers.{level}` // @example: quiet all logs @@ -83,7 +83,7 @@ metalog.error = function(at, error, info){ var trace_id = 1, boot = +(new Date()); -var dump_keys = {tid: 4, beg: 15, boot: 4, +var dump_keys = {tid: 4, beg: 15, boot: 4, tier: 8, start: 9, stop: 9, bin: 9, mget: 4, find: 4, mf0: 4, mf1: 4, msb: 4, mflt: 4, mfl0: 4, mfly: 4, msav: 4, mres: 4, resp: 4, @@ -95,18 +95,18 @@ function dump(hsh){ var extras = {}; for (var key in hsh){ if (!(key in dump_keys)) extras[key] = hsh[key]; } if (_.isNumber(value)) fields.push((value*100)/100); // metalog.log(fields.join('|') + "\t" + (JSON.stringify(extras).slice(0,150))); -} +} function dump_header(){ // metalog.log(_.map(dump_keys, function(len, key){ return (key+' ').slice(0,len); }).join('|')); -} +} if (tracing){ metalog.trace = function(label, item, hsh){ - + // try{ throw new Error("Trace"); } catch(e){ console.log(e.stack); } - + item = item || {}; hsh = hsh || {}; var using = (hsh.using||{}); delete hsh.using; if (! item._trace) item._trace = { beg: +(new Date()), boot: ((new Date()) - boot) }; @@ -135,7 +135,7 @@ if (tracing){ } catch(err){ metalog.log(err); metalog.log(err.stack); } return item; }; - + } else { metalog.trace = function(){}; metalog.dump_trace = function(){}; From a12ec7fd684008ff90600f559c117bd045382d27 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Mon, 24 Sep 2012 11:42:41 -0500 Subject: [PATCH 47/87] Events include their bins when they are saved. Tiers cascade down to 10 seconds --- lib/cube/models.js | 8 +++---- lib/cube/tiers.js | 55 +++++++++++++++++++++++++++------------------- test/tiers-test.js | 17 +++++++------- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/lib/cube/models.js b/lib/cube/models.js index ab759c37..bf1ca1f3 100644 --- a/lib/cube/models.js +++ b/lib/cube/models.js @@ -29,8 +29,8 @@ function Event(type, time, data, id){ var self = this; - if (this._id) Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, _id: _id }, enumerable: false, writable: false }); - else Object.defineProperty(this, 'to_wire', { value: { t: time, d: data }, enumerable: false, writable: false }); + if (this._id) Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, _id: _id, b: this.bins() }, enumerable: false, writable: false }); + else Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, b: this.bins() }, enumerable: false, writable: false }); Object.defineProperties(this, { _trace: { value: null, enumerable: false, writable: true, configurable: true } @@ -44,8 +44,8 @@ Event.prototype = { day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, - bins: function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }, - agos: function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, + bins: function bins(){ return tiers.bins(this.t); }, + agos: function agos(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, report: function report(){ return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; } diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index 13d38b0f..dc843229 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -12,51 +12,60 @@ var second = 1000, tiers[second10] = { key: second10, floor: function(d) { return new Date(Math.floor(d / second10) * second10); }, - bin: function(d) { return Math.floor(d / second10); }, - ceil: tier_ceil, - step: function(d) { return new Date(+d + second10); } + bin: function(d) { return new Date(Math.floor(d / second10) * second10); }, + ceil: tier_ceil, + step: function(d) { return new Date(+d + second10); } }; tiers[minute] = { key: minute, floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, - bin: function(d) { return Math.floor(d / minute); }, - ceil: tier_ceil, - step: function(d) { return new Date(+d + minute); }, - // next: tiers[second10], - // size: function() { return 6; } + bin: function(d) { return new Date(Math.floor(d / minute) * minute); }, + ceil: tier_ceil, + step: function(d) { return new Date(+d + minute); }, + next: tiers[second10], + size: function() { return 6; } }; tiers[minute5] = { key: minute5, floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, - bin: function(d) { return Math.floor(d / minute5); }, - ceil: tier_ceil, - step: function(d) { return new Date(+d + minute5); }, - // next: tiers[minute], - // size: function() { return 5; } + bin: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, + ceil: tier_ceil, + step: function(d) { return new Date(+d + minute5); }, + next: tiers[minute], + size: function() { return 5; } }; tiers[hour] = { key: hour, floor: function(d) { return new Date(Math.floor(d / hour) * hour); }, - bin: function(d) { return Math.floor(d / hour); }, - ceil: tier_ceil, - step: function(d) { return new Date(+d + hour); }, - next: tiers[minute5], - size: function() { return 12; } + bin: function(d) { return new Date(Math.floor(d / hour) * hour); }, + ceil: tier_ceil, + step: function(d) { return new Date(+d + hour); }, + next: tiers[minute5], + size: function() { return 12; } }; tiers[day] = { key: day, floor: function(d) { return new Date(Math.floor(d / day) * day); }, - bin: function(d) { return Math.floor(d / day); }, - ceil: tier_ceil, - step: function(d) { return new Date(+d + day); }, - next: tiers[hour], - size: function() { return 24; } + bin: function(d) { return new Date(Math.floor(d / day) * day); }, + ceil: tier_ceil, + step: function(d) { return new Date(+d + day); }, + next: tiers[hour], + size: function() { return 24; } }; +Object.defineProperty(tiers, "bins", { + enumberable: false, + value: function(d){ + var bins = {}; + for(var bin in tiers) bins[tiers[bin].key] = tiers[bin].bin(d); + return bins; + } +}) + function tier_ceil(date) { return this.step(this.floor(new Date(date - 1))); } diff --git a/test/tiers-test.js b/test/tiers-test.js index 0ebea835..38c6b10f 100644 --- a/test/tiers-test.js +++ b/test/tiers-test.js @@ -87,13 +87,12 @@ suite.addBatch({ "has the key 6e4": function(tier) { assert.strictEqual(tier.key, 6e4); }, - "next is undefined": function(tier) { - assert.isUndefined(tier.next); + "next is the 10-second tier": function(tier) { + assert.equal(tier.next, tiers[1e4]); }, - "size is undefined": function(tier) { - assert.isUndefined(tier.size); + "size is 6": function(tier) { + assert.strictEqual(tier.size(), 6); }, - "floor": { "rounds down to minutes": function(tier) { assert.deepEqual(tier.floor(utc(2011, 8, 2, 12, 20, 0)), utc(2011, 8, 2, 12, 20)); @@ -151,11 +150,11 @@ suite.addBatch({ "has the key 3e5": function(tier) { assert.strictEqual(tier.key, 3e5); }, - "next is undefined": function(tier) { - assert.isUndefined(tier.next); + "next is the minute tier": function(tier) { + assert.equal(tier.next, tiers[6e4]); }, - "size is undefined": function(tier) { - assert.isUndefined(tier.size); + "size is 5": function(tier) { + assert.strictEqual(tier.size(), 5); }, "floor": { From 3e6bb527d5e6469111c61e5334a6270f0316d71c Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Mon, 24 Sep 2012 11:56:42 -0500 Subject: [PATCH 48/87] Add array and object types to random (cromulator) example --- examples/random-emitter/cromulator.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/random-emitter/cromulator.js b/examples/random-emitter/cromulator.js index 4082a70c..23263684 100644 --- a/examples/random-emitter/cromulator.js +++ b/examples/random-emitter/cromulator.js @@ -19,7 +19,7 @@ var ev = { homer: 324, bart: 263, lisa: 203, marge: 142, scratchy: 79, itchy: 79, maggie: 51, mr_burns: 49, ned_flanders: 39, milhouse: 38, skinner: 37, sideshow_mel: 31, willie: 30, quimby: 25, moe: 25, krusty: 24, nelson: 23, wiggum: 22, grampa: 22, frink: 19, apu: 16, sideshow_bob: 14, - selma: 14, patty: 14, barney: 13, mrs_krabappel: 12, comic_book_guy: 12, martin: 10, + selma: 14, patty: 14, barney: 13, mrs_krabappel: 12, comic_book_guy: 12, martin: 10, dr_hibbert: 10, smithers: 9, ralph: 9, rev_lovejoy: 8, lionel_hutz: 8, fat_tony: 8, chalmers: 8, snake: 7, otto: 6, dr_nick: 6, cletus: 5, troy_mcclure: 4, todd: 3, rodd: 3, kent_brockman: 3 }, characters_pool: [] @@ -59,7 +59,9 @@ cromulator.data_at = function(time){ walk: ev.walk += (Math.random() - 0.5), ramp: ev.ramp += (Math.random() * 20 * cromulator.step / (stop - cromulator.start)), visits: exprand( un.sec * cromulator.visit_rate / cromulator.step ), - who: rand_element(ev.characters_pool) + who: rand_element(ev.characters_pool), + people: [rand_element(ev.characters_pool), rand_element(ev.characters_pool), rand_element(ev.characters_pool)], + _meta: { time: time, stop: stop, start: cromulator.start, step: cromulator.step } }; ++cromulator.count; return data; From c629714766192c0bae465339ea46559c1f1c72ad Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 2 Oct 2012 17:56:29 -0500 Subject: [PATCH 49/87] Modelize event, metric, and measurement objects --- lib/cube/core_ext/model.js | 55 ++++++ lib/cube/event.js | 8 +- lib/cube/index.js | 3 - lib/cube/metalog.js | 6 - lib/cube/metric.js | 307 +-------------------------------- lib/cube/models.js | 122 ------------- lib/cube/models/event.js | 116 +++++++++++++ lib/cube/models/measurement.js | 232 +++++++++++++++++++++++++ lib/cube/models/metric.js | 104 +++++++++++ lib/cube/reduces.js | 5 +- lib/cube/tiers.js | 9 +- test/event-test.js | 3 +- test/metric-test.js | 80 +-------- test/test_helper.js | 2 +- 14 files changed, 539 insertions(+), 513 deletions(-) create mode 100644 lib/cube/core_ext/model.js delete mode 100644 lib/cube/models.js create mode 100644 lib/cube/models/event.js create mode 100644 lib/cube/models/measurement.js create mode 100644 lib/cube/models/metric.js diff --git a/lib/cube/core_ext/model.js b/lib/cube/core_ext/model.js new file mode 100644 index 00000000..825d5f86 --- /dev/null +++ b/lib/cube/core_ext/model.js @@ -0,0 +1,55 @@ +'use strict'; + +var util = require('util'), + _ = require('underscore'); + +function Model(){}; + +function modelize(ctor, super_) { + super_ = super_ || Model; + if(super_ !== ctor) util.inherits(ctor, super_); + + var properties = {}; + + properties['eventize'] = { value: eventize }; + properties['_trace'] = { value: null, enumerable: false, writable: true, configurable: true }; + properties['setProperty'] = { value: function(name, desc){ Object.defineProperty(this, name, desc) }}; + properties['setProperties'] = { + value: function(descs){ + var _this = this; + _.keys(descs).forEach(function(key){ + Object.defineProperty(_this, key, descs[key]); + }) + } + }; + + Object.defineProperties(ctor.prototype, properties); + Object.defineProperty(ctor, 'setProperty', { value: function(name, desc){ Object.defineProperty(ctor.prototype, name, desc); }}); + Object.defineProperty(ctor, 'setProperties', { + value: function(descs){ + var _this = this; + _.keys(descs).forEach(function(key){ + Object.defineProperty(ctor.prototype, key, descs[key]); + }) + } + }); + + return ctor; +}; + +function eventize() { + var emitter = new (require('events').EventEmitter)(), + _this = this; + + Object.keys(Object.getPrototypeOf(emitter)).forEach(function(prop){ + Object.defineProperty(_this, prop, { value: function(args){ + emitter[prop].apply(emitter, arguments); + }}); + }); +} + +modelize(Model); + +Model.modelize = modelize; + +module.exports = Model; \ No newline at end of file diff --git a/lib/cube/event.js b/lib/cube/event.js index 0a662b17..0ebb9ab9 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -8,7 +8,8 @@ var _ = require("underscore"), ObjectID = mongodb.ObjectID, util = require("util"), tiers = require("./tiers"), - models = require("./models"), Event = models.Event, Metric = models.Metric, + Event = require("./models/event"), + Metric = require("./models/metric"), parser = require("./event-expression"), bisect = require("./bisect"), metalog = require("./metalog"); @@ -41,7 +42,6 @@ exports.putter = function(db){ var time = "time" in request ? new Date(request.time) : new Date(), type = request.type; callback = callback || function(){}; - //metalog.trace('eput', request); // // Drop events from before invalidation horizon @@ -55,7 +55,6 @@ exports.putter = function(db){ var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } - //metalog.trace('ebeg', event, { using: request }); // Save the event, then queue invalidation of its associated cached metrics. // @@ -65,7 +64,6 @@ exports.putter = function(db){ // the likelihood of a race condition between when the events are read by // the evaluator and when the newly-computed metrics are saved. event.save(db, function after_save(error, event){ - //metalog.trace('esav', event); if (event) invalidator.add(event.type, event); callback(event); }); @@ -103,7 +101,7 @@ function Invalidator(){ this.add = function(type, ev){ var tt = type_tset(type); - for (var tier in tiers){ tt[tier][tier*Math.floor(ev.t/tier)] = true; } + for (var tier in tiers){ tt[tier][tier*Math.floor(ev.time/tier)] = true; } }; this.flush = function(db){ diff --git a/lib/cube/index.js b/lib/cube/index.js index 78313693..ef9c31b8 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -1,9 +1,6 @@ 'use strict'; process.env.TZ = 'UTC'; -exports.models = require("./models"); -exports.Event = exports.models.Event; -// exports.Metric = require("./metric").Metric; exports.authentication = require("./authentication"); exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); diff --git a/lib/cube/metalog.js b/lib/cube/metalog.js index 4b25f610..7f5fef24 100644 --- a/lib/cube/metalog.js +++ b/lib/cube/metalog.js @@ -115,11 +115,6 @@ if (tracing){ // item._trace[label] = ((new Date()) - item._trace['beg']); if (! item._trace.tid) item._trace.tid = trace_id++; - // if (value) item._trace['val'] = (Math.round(100*value)/100.0); - // } catch(err){ metalog.log(err); } - - // metalog.warn('tr_'+label, item); - // if (label == 'msbl') metalog.inspectify(label, item, using, hsh) return item; }; @@ -163,7 +158,6 @@ metalog.spy = function spy(callback, label, ctxt){ if (! ctxt) ctxt = null; return function(){ process.stderr.write(label+': ', callback, ' called with ', util.inspect(arguments), '\n'); - // metalog.inspectify(callback, arguments); return callback.apply(ctxt, arguments); }; }; diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 2c6d6397..4bdee361 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -8,23 +8,14 @@ var _ = require("underscore"), parser = require("./metric-expression"), tiers = require("./tiers"), reduces = require("./reduces"), - models = require("./models"), Metric = models.Metric, + Metric = require("./models/metric"), + Measurement = require("./models/measurement"), event = require("./event"), metalog = require('./metalog'), options = require('../../config/cube'); -var metric_fields = {v: 1}, - metric_options = {sort: {"_id.t": 1}, batchSize: 1000}, - event_options = {sort: {t: 1}, batchSize: 1000}, - queue_parallelism = 1; - // Query for metrics. exports.getter = function(db){ - var Double = require("mongodb").Double, - queueByName = {}, - request_queues = {}, - qcount = 0; - function getter(request, callback) { var measurement, expression, tier = tiers[+request.step], @@ -41,301 +32,19 @@ exports.getter = function(db){ start = tier.floor(start); stop = tier.ceil(stop); expression = parser.parse(request.expression); - measurement = new Measurement(expression, start, stop, tier, send_response); + measurement = new Measurement(expression, start, stop, tier); + measurement.on('complete', function(){ handle_response(new Metric(stop, null, measurement)); }); } catch(error) { metalog.error('mget', error, { info: util.inspect([start, stop, tier, expression] )}); return callback({error: error, _trace: request._trace}), -1; } - function send_response(time, value, tr){ - var resp = new Metric(time, value, request.id, measurement); - metalog.dump_trace('resp', resp, {using: tr, bin: resp.bin }); - callback(resp); + function handle_response(metric){ + callback(metric); + if (metric.value || metric.value === 0) metric.save(db, handle); } - // Compute the request metric! - metalog.trace('mget', measurement, { using: request }); - measurement.measure(); - } - - function Measurement(expression, start, stop, tier, sender){ - // Round the start/stop to the tier edges - this.expression = expression; - this.start = start; - this.stop = stop; - this.tier = tier; - this.sender = sender; - this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); - - // Object.defineProperties(this, { - // }); - } - Measurement.prototype.report = function(){ - return { flavor: this.flavor, tier: this.tier.key, start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), expr: (this.expression.op||this.expression.source||this.expression.value()) }; - }; - - Measurement.prototype.send_result = function(time, value, tr){ - var ret_tr = metalog.dump_trace('mres', tr, { using: this, bin: this.tier.bin(time), val: value }); - this.sender(time, value, ret_tr); - }; - Measurement.prototype.complete = function(tr){ this.send_result(this.stop, null, tr); }; - - // Computes the metric for the given expression for the time interval from - // start (inclusive) to stop (exclusive). The time granularity is determined - // by the specified tier, such as daily or hourly. The callback is invoked - // repeatedly for each metric value, being passed two arguments: the time and - // the value. The values may be out of order due to partial cache hits. - Measurement.prototype.measure = function measure() { - metalog.trace('meas', this, this.report()); - this[this.flavor](this.sender); - }; - - // Computes a constant expression like the "7" in "x * 7" - Measurement.prototype.constant = function constant() { - var self = this, value = this.expression.value(); - walk(this.start, this.stop, this.tier, function(time){ self.send_result(time, value); }); - self.complete(); - }; - - // // Serializes a unary expression for computation. - // Measurement.prototype.unary = function unary() { - // var self = this, - // remaining = 0, - // queue = get_queue(this.expression.source); - // - // // Compute the expected number of values; if no results were requested, return immediately. - // walk(self.start, self.stop, self.tier, function(time){ remaining++; }); - // if (! remaining) return this.complete(); - // - // // Add this task to the appropriate queue. - // queue.defer(function task(q_callback){ - // self.run_unary(function(time, value, tr){ - // self.send_result(time, value); - // if (!--remaining) { - // process.nextTick(function(){ q_callback(null, [time, value]); }); - // self.complete(); - // } - // }); - // }).await(_.identity); - // } - - // Serializes a unary expression for computation. - Measurement.prototype.unary = function unary(callback) { - var self = this, expression = this.expression, start = this.start, stop = this.stop, tier = this.tier; - var remaining = 0, - time0 = Date.now(), - name = expression.source, - queue = queueByName[name], - step = tier.key; - - // Compute the expected number of values. - walk(start, stop, tier, function(time){ ++remaining; }); - - // If no results were requested, return immediately. - if (!remaining) return callback(stop); - - // Add this task to the appropriate queue. - if (queue) queue.next = task; - else process.nextTick(task); - queueByName[name] = task; - - function task() { - findOrComputeUnary(expression, start, stop, tier, function(time, value, tr) { - self.send_result(time, value, tr); - if (!--remaining) { - self.complete(tr); - if (task.next) process.nextTick(task.next); - else delete queueByName[name]; - - // Record how long it took us to compute as an event! - var time1 = Date.now(); - metalog.event("cube_compute", { - expression: expression.source, - ms: time1 - time0 - }); - } - }, self); - } - } - - // Finds or computes a unary (primary) expression. - function findOrComputeUnary(expression, start, stop, tier, callback, tr) { - var name = expression.type, - map = expression.value, - reduce = reduces[expression.reduce], - filter = {t: {}}, - fields = {t: 1}, - metrics, events; - - metalog.trace('find', this); - - // Copy any expression filters into the query object. - expression.filter(filter); - // Request any needed fields. - expression.fields(fields); - - db.metrics(name, function(error, collection){ - handle(error); - metrics = collection; - find(start, stop, tier, tr, callback); - }); - - // The metric is computed recursively, reusing the above variables. - function find(start, stop, tier, tr, callback) { - var compute = ((tier.next && reduce.pyramidal) ? computePyramidal : computeFlat), - step = tier.key; - - metalog.trace('mf0', tr); - - // Query for the desired metric in the cache. - metrics.find({ - i: false, - "_id.e": expression.source, - "_id.l": tier.key, - "_id.t": { - $gte: start, - $lt: stop - } - }, metric_fields, metric_options, foundMetrics); - - // Immediately report back whatever we have. If any values are missing, - // merge them into contiguous intervals and asynchronously compute them. - function foundMetrics(error, cursor) { - handle(error); - var time = start; - cursor.each(function(error, row) { - var mftr = metalog.trace('mf1', {}, { using: tr }); - - handle(error); - if (row) { - callback(row._id.t, row.v, metalog.trace('mfy', mftr)); // send back value for this timeslot - if (time < row._id.t) compute(time, row._id.t, mftr); // recurse from last value seen up to this timeslot - time = tier.step(row._id.t); // update the last-observed timeslot - } else { - if (time < stop) compute(time, stop, mftr); // once last row is seen, compute rest of range - } - }); - } - - // Group metrics from the next tier. - function computePyramidal(start, stop, tr) { - var bins = {}; - // - // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); - metalog.trace('mfl0', tr, { start: start, stop: stop }); - find(start, stop, tier.next, tr, function(time, value, tr) { - var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); - if (bin.values.push(value) === bin.size) { - save(time, reduce(bin.values)); - delete bins[time]; - } - }); - } - - // Group raw events. Unlike the pyramidal computation, here we can control - // the order in which rows are returned from the database. Thus, we know - // when we've seen all of the events for a given time interval. - function computeFlat(start, stop, tr) { - // metalog.warn('computeFlat', { expr: expression.source, start: start, stop: stop, tier: tier.key }); - metalog.trace('mfl0', tr, { start: start, stop: stop }); - - // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ - // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); - // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) - // } - filter.t.$gte = start; - filter.t.$lt = stop; - - db.events(name, function (error, collection) { - handle(error); - - collection.find(filter, fields, event_options, function(error, cursor) { - handle(error); - var time = start, values = []; - cursor.each(function(error, row) { - var res_tr = metalog.trace('mflt', {}, { using: tr }); - handle(error); - if (row) { - var then = tier.floor(row.t); - if (time < then) { - save(time, values.length ? reduce(values) : reduce.empty, res_tr); - while ((time = tier.step(time)) < then) save(time, reduce.empty, res_tr); - values = [map(row)]; - } else { - values.push(map(row)); - } - } else { - save(time, values.length ? reduce(values) : reduce.empty, res_tr); - while ((time = tier.step(time)) < stop) save(time, reduce.empty, res_tr); - } - }); - }); - }) - } - - function save(time, value, tr) { - callback(time, value, metalog.trace('msav', tr)); - if ((! value) && (value !== 0)) return; - var metric = { - _id: { - e: expression.source, - l: tier.key, - t: time - }, - i: false, - v: new Double(value) - }; - // metalog.trace('cube_compute', {is: 'metric_save', metric: metric }); - metrics.save(metric, handle); - } - } - } - - // Computes a binary expression by merging two subexpressions - // - // "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and - // unary "sum(resp)". We don't know what order they'll show up in, so if say - // the value for left appears first, it parks that value as left[time], where - // the result for right will eventually find it. - Measurement.prototype.binary = function binary() { - var self = this, expression = this.expression, value; - var left = new Measurement(expression.left, this.start, this.stop, this.tier, this.id), - right = new Measurement(expression.right, this.start, this.stop, this.tier, this.id); - metalog.trace('msb0', left, {using: self}); metalog.trace('msb0', right, {using: self}); - - left.sender = function(time, vall, tr) { - if (time in right) { // right val already appeared; get a result - self.send_result(time, (time < self.stop ? expression.op(vall, right[time]) : vall), metalog.trace('msb', tr)); - delete right[time]; - } else { // right val still on the way; stash the value - left[time] = vall; - } - }; - - right.sender = function(time, valr, tr) { - if (time in left) { - self.send_result(time, (time < self.stop ? expression.op(left[time], valr) : valr), metalog.trace('msb', tr)); - delete left[time]; - } else { - right[time] = valr; - } - }; - - left.measure(); - right.measure(); - }; - - function get_queue(name){ - if (! request_queues[name]) request_queues[name] = queuer(queue_parallelism); - return request_queues[name]; - } - - // execute cb on each interval from t1 to t2 - function walk(t1, t2, tier, cb){ - while (t1 < t2) { - cb(t1, t2); - t1 = tier.step(t1); - } + measurement.measure(db, handle_response); } return getter; diff --git a/lib/cube/models.js b/lib/cube/models.js deleted file mode 100644 index bf1ca1f3..00000000 --- a/lib/cube/models.js +++ /dev/null @@ -1,122 +0,0 @@ -'use strict'; - -var _ = require("underscore"), - metalog = require('./metalog'); - -var second = 1e3, - second10 = 10e3, - minute = 60e3, - minute5 = 300e3, - hour = 3600e3, - day = 86400e3; -exports.units = { second: second, second10: second10, minute: minute, minute5: minute5, hour: hour, day: day }; - -var tiers = require("./tiers"), - tensec = tiers[second10], - type_re = /^[a-z][a-zA-Z0-9_]+$/; - -_.mapHash = function(obj, func){ - var res = {}; - _.each(obj, function(val, key){ res[key] = func(val, key, res); }); - return res; -}; - -function Event(type, time, data, id){ - this.t = time; - this.d = data; - this.type = type; - if (id) this._id = id; - - var self = this; - - if (this._id) Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, _id: _id, b: this.bins() }, enumerable: false, writable: false }); - else Object.defineProperty(this, 'to_wire', { value: { t: time, d: data, b: this.bins() }, enumerable: false, writable: false }); - - Object.defineProperties(this, { - _trace: { value: null, enumerable: false, writable: true, configurable: true } - }); -} -Event.prototype = { - bin: function(tr){ return tiers[tr].bin(this.t); }, - day_bin: function(){ return tiers[day ].bin(this.t); }, - m05_bin: function(){ return tiers[minute5 ].bin(this.t); }, - s10_bin: function(){ return tiers[second10].bin(this.t); }, - day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, - m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, - s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, - bins: function bins(){ return tiers.bins(this.t); }, - agos: function agos(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, - report: function report(){ - return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; - } -}; - -Event.prototype.save = function(db, callback){ - var self = this; - // metalog.trace('eSva', self); - db.events(self.type, function event_saver(error, collection){ - if (error) return callback(error); - // metalog.trace('eSvc', self); - collection.save(self.to_wire, function saver(error){ - // metalog.trace('eSvd', self); - callback(error, self, self); - }); }, self); -}; - -// Validate the date and type. -Event.prototype.validate = function(){ - if (!type_re.test(this.type)) throw("invalid type"); - if (isNaN(this.t)) throw("invalid time"); -}; - -Event.prototype.to_request = function(attrs){ - var ev = { time: this.t, data: this.d, type: this.type }; - if (this._id) ev.id = this._id; - for (var key in attrs){ ev[key] = attrs[key]; } - metalog.trace('new_event', ev); - return ev; -}; - -exports.Event = Event; - -// -------------------------------------------------------------------------- - -function Metric(time, value, id, measurement){ - this.time = time; - this.value = value; - if (id) this.id = id; - - Object.defineProperties(this, { - e: { value: measurement.expression.source, enumerable: false, writable: false }, - l: { value: measurement.tier.key, enumerable: false, writable: false }, - measurement: { value: measurement, enumerable: false, writable: false }, - _trace: { value: null, enumerable: false, writable: true, configurable: true }, - }); -} -Object.defineProperties(Metric.prototype, { - tier: { get: function(){ return this.measurement.tier } }, - bin: { get: function(){ return this.tier.bin(this.time); } }, - // day_bin: function(){ return tiers[day ].bin(this.t); }, - // m05_bin: function(){ return tiers[minute5 ].bin(this.t); }, - // s10_bin: function(){ return tiers[second10].bin(this.t); }, - // day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, - // m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, - // s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, - // bins: function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }, - // agos: function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, - // report: function report(){ - // return { time: this.t, type: this.type, bin: this.bins(), ago: this.agos() }; - // } -}); - -Metric.prototype.to_wire = function to_wire(){ - return { i: false, v: this.value, _id: { e: this.e, l: this.l, t: time } }; -}; - -Metric.prototype.report = function report(){ - var hsh = { time: this.time, value: this.value }; - if (this.id) hsh.id = this.id; - return hsh; -}; - -exports.Metric = Metric; diff --git a/lib/cube/models/event.js b/lib/cube/models/event.js new file mode 100644 index 00000000..73d3a14c --- /dev/null +++ b/lib/cube/models/event.js @@ -0,0 +1,116 @@ +'use strict'; + +var _ = require("underscore"), + metalog = require('../metalog'); + +var tiers = require("../tiers"), + Model = require("../core_ext/model"), + tensec = tiers[tiers.units['second10']], + type_re = /^[a-z][a-zA-Z0-9_]+$/, + event_options = {sort: {t: 1}, batchSize: 1000}; + +_.mapHash = function(obj, func){ + var res = {}; + _.each(obj, function(val, key){ res[key] = func(val, key, res); }); + return res; +}; + +function Event(type, time, data, id){ + this.time = time; + this.data = data; + if (id) this.id = id; + + this.setProperty("type", { value: type }); +} + +Model.modelize(Event); + +Event.setProperties({ + bin: { value: function(tr){ return tiers[tr].bin(this.time); }}, + day_bin: { value: function(){ return tiers[day ].bin(this.time); }}, + m05_bin: { value: function(){ return tiers[minute5 ].bin(this.time); }}, + s10_bin: { value: function(){ return tiers[second10].bin(this.time); }}, + day_ago: { value: function day_ago(){ return Math.floor((Date.now() - this.time) / day); }}, + m05_ago: { value: function m05_ago(){ return Math.floor((Date.now() - this.time) / minute5); }}, + s10_ago: { value: function s10_ago(){ return Math.floor((Date.now() - this.time) / second10); }}, + bins: { value: function bins(){ return tiers.bins(this.time); }}, + agos: { value: function agos(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }}, + report: { value: function report(){ + return { time: this.time, type: this.type, bin: this.bins(), ago: this.agos() }; + }}, + + to_wire: { value: function(){ var event = { t: this.time, d: this.data, b: this.bins() }; if (this.id) event._id = this.id; return event; }}, + + save: { value: save }, + validate: { value: validate }, + to_request: {value: to_request } +}); + +function find(db, measurement, callback){ + var expression = measurement.expression, + type = expression.type, + start = measurement.start, + stop = measurement.stop, + filter = {t: {}}, + fields = {t: 1}; + + // Copy any expression filters into the query object. + expression.filter(filter); + // Request any needed fields. + expression.fields(fields); + + // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); + // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + // } + filter.t.$gte = start; + filter.t.$lt = stop; + + db.events(type, function (error, collection) { + if(error) return callback(error); + collection.find(filter, fields, event_options, handleResponse); + }); + + function handleResponse(error, cursor){ + if (error) return callback(error); + cursor.each(function(error, row) { + if (error) return callback(error); + if (row) callback(error, new Event(type, row.t, row.d, row._id)); + else callback(); + }) + } +} +Object.defineProperty(Event, "find", { value: find }); + +function save(db, callback){ + var self = this; + if (this.validate) this.validate(); + + db.events(self.type, function event_saver(error, collection){ + if (error) return callback(error); + collection.save(self.to_wire(), function saver(error){ + callback(error, self); + }); + }); +}; + +// Validate the date and type. +function validate(){ + if (!type_re.test(this.type)) throw("invalid type"); + if (isNaN(this.time)) throw("invalid time"); +}; + +function to_request(attrs){ + var ev = { time: this.time, data: this.data, type: this.type }; + if (this.id) ev.id = this.id; + for (var key in attrs){ ev[key] = attrs[key]; } + return ev; +}; + +function handle(error) { + if (!error) return; + metalog.error('event', error); + throw error; +} + +module.exports = Event; diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js new file mode 100644 index 00000000..28d3fbd1 --- /dev/null +++ b/lib/cube/models/measurement.js @@ -0,0 +1,232 @@ +'use strict'; + +var metalog = require('../metalog'), + Model = require('../core_ext/model'), + reduces = require('../reduces'), + Metric = require('./metric'), + Event = require('./event'), + compute = {constant: constant, binary: binary, unary: unary}, + queueByName = {}; + +function Measurement(expression, start, stop, tier){ + // Round the start/stop to the tier edges + this.expression = expression; + this.start = start; + this.stop = stop; + this.tier = tier; + this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); + + this.eventize(); +} + +Model.modelize(Measurement); + +Measurement.prototype.report = function report(){ + return { flavor: this.flavor, tier: this.tier.key, start: this.tier.bin(this.start), stop: this.tier.bin(this.stop), expr: (this.expression.op||this.expression.source||this.expression.value()) }; +}; + +// Computes the metric for the given expression for the time interval from +// start (inclusive) to stop (exclusive). The time granularity is determined +// by the specified tier, such as daily or hourly. The callback is invoked +// repeatedly for each metric value, being passed two arguments: the time and +// the value. The values may be out of order due to partial cache hits. +Measurement.prototype.measure = function measure(db, callback) { + var _this = this; + + compute[this.flavor].call(this, db, function(time, value){ callback(new Metric(time, value, _this)); }); +}; + +// Computes a constant expression like the "7" in "x * 7" +function constant(db, callback) { + var self = this, value = this.expression.value(); + walk(this.start, this.stop, this.tier, function(time){ callback(time, value); }); + this.emit('complete'); +}; + +// Serializes a unary expression for computation. +function unary(db, callback) { + var self = this, + remaining = 0, + time0 = Date.now(), + name = this.expression.source, + queue = queueByName[name]; + + // Compute the expected number of values. + walk(this.start, this.stop, this.tier, function(time){ ++remaining; }); + + // If no results were requested, return immediately. + if (!remaining) return callback(stop); + + // Add this task to the appropriate queue. + if (queue) queue.next = task; + else process.nextTick(task); + queueByName[name] = task; + + function task() { + findOrComputeUnary.call(self, db, function(time, value) { + callback(time, value); + if (!--remaining) { + self.emit('complete'); + if (task.next) process.nextTick(task.next); + else delete queueByName[name]; + + // Record how long it took us to compute as an event! + var time1 = Date.now(); + metalog.event("cube_compute", { + expression: self.expression.source, + ms: time1 - time0 + }); + } + }, self); + } +} + +// Finds or computes a unary (primary) expression. +function findOrComputeUnary(db, callback) { + var measurement = this, + expression = this.expression, + name = expression.type, + map = expression.value, + reduce = reduces[expression.reduce]; + + find(measurement, callback); + + // The metric is computed recursively, reusing the above variables. + function find(measurement, callback) { + var start = measurement.start, + stop = measurement.stop, + tier = measurement.tier, + compute = ((tier.next && reduce.pyramidal) ? computePyramidal : computeFlat), + time = start; + + // Query for the desired metric in the cache. + Metric.find(db, measurement, foundMetrics); + + // Immediately report back whatever we have. If any values are missing, + // merge them into contiguous intervals and asynchronously compute them. + function foundMetrics(error, metric) { + handle(error); + if (metric) { + callback(metric.time, metric.value); // send back value for this timeslot + if (time < metric.time) compute(time, metric.time); // recurse from last value seen up to this timeslot + time = tier.step(metric.time); // update the last-observed timeslot + } else { + if (time < stop) compute(time, stop); // once last row is seen, compute rest of range + } + } + + // Group metrics from the next tier. + function computePyramidal(start, stop) { + var bins = {}, + measurement = new Measurement(expression, start, stop, tier.next); + // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); + + find(measurement, function(time, value) { + var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); + if (bin.values.push(value) === bin.size) { + callback(time, reduce(bin.values)); + delete bins[time]; + } + }); + } + + // Group raw events. Unlike the pyramidal computation, here we can control + // the order in which rows are returned from the database. Thus, we know + // when we've seen all of the events for a given time interval. + function computeFlat(start, stop) { + // metalog.warn('computeFlat', { expr: expression.source, start: start, stop: stop, tier: tier.key }); + + // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); + // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + // } + + var measurement = new Measurement(expression, start, stop, tier); + var time = start, values = []; + + + Event.find(db, measurement, function(error, event) { + handle(error); + if (event) { + var then = tier.floor(event.time); + if (time < then) { + callback(time, values.length ? reduce(values) : reduce.empty); + while ((time = tier.step(time)) < then) callback(time, reduce.empty); + values = [map(event.to_wire())]; + } else { + values.push(map(event.to_wire())); + } + } else { + callback(time, values.length ? reduce(values) : reduce.empty); + while ((time = tier.step(time)) < stop) callback(time, reduce.empty); + } + }); + } + } +} + +// Computes a binary expression by merging two subexpressions +// +// "sum(req) - sum(resp)" will op ('-') the result of unary "sum(req)" and +// unary "sum(resp)". We don't know what order they'll show up in, so if say +// the value for left appears first, it parks that value as left[time], where +// the result for right will eventually find it. +function binary(db, callback) { + var self = this, expression = this.expression, value; + var left = new Measurement(expression.left, this.start, this.stop, this.tier), + right = new Measurement(expression.right, this.start, this.stop, this.tier); + + left.on("complete", function(){ + var time = self.stop; + if (time in right){ + self.emit("complete"); + } else { + left[time] = undefined; + } + }); + + right.on("complete", function(){ + var time = self.stop; + if (time in left){ + self.emit("complete"); + } else { + right[time] = undefined; + } + }); + + left.measure(db, function(metric) { + var time = metric.time, value = metric.value; + if (time in right) { // right val already appeared; get a result + callback(time, expression.op(value, right[time])); + delete right[time]; + } else { // right val still on the way; stash the value + left[time] = value; + } + }); + + right.measure(db, function(metric) { + var time = metric.time, value = metric.value; + if (time in left) { + callback(time, expression.op(left[time], value)); + delete left[time]; + } else { + right[time] = value; + } + }); +}; + +// execute cb on each interval from t1 to t2 +function walk(t1, t2, tier, cb){ + while (t1 < t2) { + cb(t1, t2); + t1 = tier.step(t1); + } +} + +function handle(error) { + if (!error) return; + metalog.error('measurement', error); + throw error; +} + +module.exports = Measurement; \ No newline at end of file diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js new file mode 100644 index 00000000..d3299706 --- /dev/null +++ b/lib/cube/models/metric.js @@ -0,0 +1,104 @@ +'use strict'; + +var _ = require("underscore"), + metalog = require('../metalog'), + mongo = require('mongodb'), + Model = require("../core_ext/model"); + +var second = 1e3, + second10 = 10e3, + minute = 60e3, + minute5 = 300e3, + hour = 3600e3, + day = 86400e3; + +var tiers = require("../tiers"), + tensec = tiers[second10], + type_re = /^[a-z][a-zA-Z0-9_]+$/, + metric_fields = {v: 1}, + metric_options = {sort: {"_id.t": 1}, batchSize: 1000}; + +function Metric(time, value, measurement){ + this.time = time; + this.value = value; + + this.setProperty("measurement", { value: measurement }); +} + +Model.modelize(Metric); + +Metric.setProperties({ + tier: { get: function(){ return this.measurement.tier } }, + bin: { get: function(){ return this.tier.bin(this.time); } }, + e: { get: function(){ return this.measurement.expression.source }}, + l: { get: function(){ return this.measurement.tier.key }}, + type: { get: function(){ return this.measurement.expression.type }}, + + to_wire: { value: to_wire }, + report: { value: report }, + save: { value: save } + + // day_bin: function(){ return tiers[day ].bin(this.t); }, + // m05_bin: function(){ return tiers[minute5 ].bin(this.t); }, + // s10_bin: function(){ return tiers[second10].bin(this.t); }, + // day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, + // m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, + // s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, + // bins: function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }, + // agos: function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, +}); + +function find(db, measurement, callback){ + var expression = measurement.expression, + start = expression.start, + stop = expression.stop, + type = expression.type, + tier = measurement.tier; + + db.metrics(type, function(error, collection){ + if (error) return callback(error); + + collection.find({ + i: false, + "_id.e": expression.source, + "_id.l": tier.key, + "_id.t": { + $gte: start, + $lt: stop + } + }, metric_fields, metric_options, handleResponse); + }); + + function handleResponse(error, cursor){ + if (error) return callback(error); + cursor.each(function(error, row) { + if (error) return callback(error); + if (row) callback(error, new Metric(row._id.t, row.v, measurement)); + else callback(); + }) + } +} +Object.defineProperty(Metric, "find", { value: find }); + +function to_wire(){ + return { i: false, v: mongo.Double(this.value), _id: { e: this.e, l: this.l, t: this.time } }; +}; + +function report(){ + var hsh = { time: this.time, value: this.value }; + return hsh; +}; + +function save(db, callback){ + var self = this; + if (this.validate) this.validate(); + + db.metrics(self.type, function(error, collection){ + if (error) return callback(error); + collection.save(self.to_wire(), function(error){ + callback(error, self); + }); + }); +}; + +module.exports = Metric; diff --git a/lib/cube/reduces.js b/lib/cube/reduces.js index bacfdd0e..de76ba00 100644 --- a/lib/cube/reduces.js +++ b/lib/cube/reduces.js @@ -11,13 +11,13 @@ var reduces = module.exports = { min: function(values) { var i = -1, n = values.length, min = Infinity, value; while (++i < n){ if ((value = values[i]) < min){ min = value; } } - return min; + return isFinite(min) ? min : undefined; }, max: function(values) { var i = -1, n = values.length, max = -Infinity, value; while (++i < n){ if ((value = values[i]) > max){ max = value; } } - return max; + return isFinite(max) ? max : undefined; }, distinct: function(values) { @@ -40,6 +40,7 @@ reduces.distinct.empty = 0; reduces.sum.pyramidal = true; reduces.min.pyramidal = true; reduces.max.pyramidal = true; +//reduces.distinct.pyramidal = true; function ascending(a, b) { return a - b; diff --git a/lib/cube/tiers.js b/lib/cube/tiers.js index dc843229..a78eae37 100644 --- a/lib/cube/tiers.js +++ b/lib/cube/tiers.js @@ -64,7 +64,14 @@ Object.defineProperty(tiers, "bins", { for(var bin in tiers) bins[tiers[bin].key] = tiers[bin].bin(d); return bins; } -}) +}); + +Object.defineProperty(tiers, "units", { + enumberable: false, + value: { second: second, second10: second10, minute: minute, minute5: minute5, hour: hour, day: day } +}); + + function tier_ceil(date) { return this.step(this.floor(new Date(date - 1))); diff --git a/test/event-test.js b/test/event-test.js index d1b5ecd7..0020b928 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -3,7 +3,8 @@ var vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), - models = require("../lib/cube/models"), units = models.units, Event = models.Event, + Event = require("../lib/cube/models/event"), + Metric = require("../lib/cube/models/metric"), event = require("../lib/cube/event"); var suite = vows.describe("event"); diff --git a/test/metric-test.js b/test/metric-test.js index 9159b8ee..ab411ef1 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -7,7 +7,7 @@ var _ = require("underscore"), assert = require("assert"), test_helper = require("./test_helper"), queuer = require("../lib/queue-async/queue"), - models = require("../lib/cube/models"), units = models.units, + units = require("../lib/cube/tiers").units, event = require("../lib/cube/event"), metric = require("../lib/cube/metric"); @@ -25,7 +25,7 @@ var nowish = Date.now(), var invalid_expression_error = { error: { message: 'Expected "(", "-", "distinct", "max", "median", "min", "sum" or number but "D" found.', column: 1, line: 1, name: 'SyntaxError' }}; function gen_date(sec){ - return new Date(thenish + sec*units.second); + return new Date(thenish + sec * units.second); } var t1 = gen_date(3), t1_10s = new Date(10e3 * Math.floor(t1/10e3)), @@ -45,72 +45,6 @@ function assert_invalid_request(req, expected_err) { }; } -// suite.addBatch(test_helper.batch({ -// topic: function(test_db){ -// var putter = event.putter(test_db), -// getter = metric.getter(test_db), -// callback = this.callback, -// put_queue = queuer(10); -// // Seed the events table with a simple event: a value going from 0 to 2499 -// for (var i = 0; i < 250; i++){ -// put_queue.defer(function(num, dt, cb){ -// putter({ type: "test", time: dt, data: {i: num}}, function(){ cb(null, null); }); -// }, i, gen_date(i*10).toISOString()); -// } -// put_queue.await(function(){ callback(null, getter) }); -// }, -// // 'invalid start': assert_invalid_request({start: 'THEN'}, {error: "invalid start"}), -// // 'invalid stop': assert_invalid_request({stop: 'NOW'}, {error: "invalid stop"}), -// // 'invalid step': assert_invalid_request({step: 'LEFT'}, {error: "invalid step"}), -// // 'invalid expression': assert_invalid_request({expression: 'DANCE'}, invalid_expression_error), -// -// 'simple constant' : { -// topic: function(getter){ -// var checker = assert.isCalledTimes(this, 5); -// getter(gen_request({expression: '1'}), checker); -// }, -// 'gets a metric for each time slot': function(results){ -// _.each([0,10,20,30], function(step, idx){ -// assert.deepEqual(results[idx][0].report(), {time: gen_date(step), value: 1}); -// }); -// }, -// 'sends a null metric for the end slot': function(results){ assert.deepEqual(results[4][0].report(), {time: gen_date(40), value: null}); } -// }, -// -// 'no request id' : { -// topic: function(getter){ -// var checker = assert.isCalledTimes(this, 5); -// this.ret = getter(gen_request({}), checker); -// }, -// 'does not have id in result': function(results){ -// test_helper.inspectify(results) -// _.each([0,10,20,30], function(step, idx){ -// assert.isFalse("id" in results[idx][0]); -// assert.deepEqual(results[idx][0].report(), { time: gen_date(step), value: idx }); -// }); -// }, -// 'sends a null metric for the end slot': function(results){ -// assert.deepEqual(results[4][0].report(), {time: gen_date(40), value: null}); -// } -// }, -// -// 'with request id' : { -// topic: function(getter){ -// var checker = assert.isCalledTimes(this, 5); -// this.ret = getter(gen_request({id: 'joe', expression: 'sum(test(i))'}), checker); -// }, -// 'includes id in result': function(results){ -// metalog.inspectify(results); -// _.each([0,10,20,30], function(step, idx){ -// assert.deepEqual(results[idx][0].report(), { id: 'joe', time: gen_date(step), value: idx }); -// }); -// }, -// 'sends a null metric for the end slot': function(results){ -// assert.deepEqual(results[4][0].report(), {id: 'joe', time: gen_date(40), value: null}); -// } -// } -// -// })); function skip(){ // FIXME: remove ------------------------------------------------------------ @@ -165,7 +99,7 @@ suite.addBatch(test_helper.batch({ // FIXME: ---- remove below ------------------------------------ "constant expression": metricTest({ expression: "1", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] }), - + "unary expression a": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", @@ -173,7 +107,7 @@ suite.addBatch(test_helper.batch({ }, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), - + "unary expression b": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", stop: "2011-07-18T00:00:00.000Z"}, { 60e3: [ 0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17 ] }), "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), @@ -191,9 +125,9 @@ suite.addBatch(test_helper.batch({ 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }), - + // FIXME: ---- remove above ------------------------------------ - + "unary expression": metricTest({ expression: "sum(test)", start: "2011-07-17T23:47:00.000Z", @@ -326,7 +260,7 @@ function metricTest(request, expected) { function get_metrics_with_delay(depth){ return function(){ var actual = [], - timeout = setTimeout(function() { cb(new Error("Time's up!")); }, 20000), + timeout = setTimeout(function() { console.log(" TIMING OUT NOW", request ); cb(new Error("Time's up!")); }, 20000), cb = this.callback, req = Object.create(request), getter = arguments[depth]; diff --git a/test/test_helper.js b/test/test_helper.js index 2190a2a0..20a23674 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -14,7 +14,7 @@ var _ = require("underscore"), // var test_helper = {}; -var test_collections = ["test_users", "test_events", "test_metrics"]; +var test_collections = ["test_users", "test_events", "test_metrics", "test_boards"]; test_helper.inspectify = metalog.inspectify; test_helper.settings = { From 8e8971eb9025d5e32f6d86c05f67744899a0c526 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 4 Oct 2012 17:06:48 -0500 Subject: [PATCH 50/87] Add semi pyramidal calculations for non-pyramidal reduces --- lib/cube/models/measurement.js | 37 +++++++++++++++++----------------- lib/cube/models/metric.js | 33 ++++++++++++++++++++++-------- lib/cube/reduces.js | 1 - 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 28d3fbd1..a84325bb 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -33,7 +33,7 @@ Measurement.prototype.report = function report(){ Measurement.prototype.measure = function measure(db, callback) { var _this = this; - compute[this.flavor].call(this, db, function(time, value){ callback(new Metric(time, value, _this)); }); + compute[this.flavor].call(this, db, function(time, value, values){ callback(new Metric(time, value, _this, values)); }); }; // Computes a constant expression like the "7" in "x * 7" @@ -63,8 +63,8 @@ function unary(db, callback) { queueByName[name] = task; function task() { - findOrComputeUnary.call(self, db, function(time, value) { - callback(time, value); + findOrComputeUnary.call(self, db, function(time, value, values) { + callback(time, value, values); if (!--remaining) { self.emit('complete'); if (task.next) process.nextTick(task.next); @@ -77,26 +77,24 @@ function unary(db, callback) { ms: time1 - time0 }); } - }, self); + }); } } // Finds or computes a unary (primary) expression. function findOrComputeUnary(db, callback) { - var measurement = this, - expression = this.expression, - name = expression.type, - map = expression.value, - reduce = reduces[expression.reduce]; + var expression = this.expression, + map = expression.value, + reduce = reduces[expression.reduce]; - find(measurement, callback); + find(this, callback); // The metric is computed recursively, reusing the above variables. function find(measurement, callback) { var start = measurement.start, stop = measurement.stop, tier = measurement.tier, - compute = ((tier.next && reduce.pyramidal) ? computePyramidal : computeFlat), + compute = ((tier.next) ? computePyramidal : computeFlat), time = start; // Query for the desired metric in the cache. @@ -107,7 +105,7 @@ function findOrComputeUnary(db, callback) { function foundMetrics(error, metric) { handle(error); if (metric) { - callback(metric.time, metric.value); // send back value for this timeslot + callback(metric.time, metric.value, metric.values); // send back value for this timeslot if (time < metric.time) compute(time, metric.time); // recurse from last value seen up to this timeslot time = tier.step(metric.time); // update the last-observed timeslot } else { @@ -121,10 +119,14 @@ function findOrComputeUnary(db, callback) { measurement = new Measurement(expression, start, stop, tier.next); // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); - find(measurement, function(time, value) { + find(measurement, function(time, value, values) { var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); - if (bin.values.push(value) === bin.size) { - callback(time, reduce(bin.values)); + + if (reduce.pyramidal) bin.values.push(value); + else bin.values = bin.values.concat(values||[]); + + if (!--bin.size) { + callback(time, reduce(bin.values), bin.values); delete bins[time]; } }); @@ -144,20 +146,19 @@ function findOrComputeUnary(db, callback) { var measurement = new Measurement(expression, start, stop, tier); var time = start, values = []; - Event.find(db, measurement, function(error, event) { handle(error); if (event) { var then = tier.floor(event.time); if (time < then) { - callback(time, values.length ? reduce(values) : reduce.empty); + callback(time, (values.length ? reduce(values) : reduce.empty), values); while ((time = tier.step(time)) < then) callback(time, reduce.empty); values = [map(event.to_wire())]; } else { values.push(map(event.to_wire())); } } else { - callback(time, values.length ? reduce(values) : reduce.empty); + callback(time, (values.length ? reduce(values) : reduce.empty), values); while ((time = tier.step(time)) < stop) callback(time, reduce.empty); } }); diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index d3299706..9f503a98 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -15,13 +15,14 @@ var second = 1e3, var tiers = require("../tiers"), tensec = tiers[second10], type_re = /^[a-z][a-zA-Z0-9_]+$/, - metric_fields = {v: 1}, + metric_fields = {v: 1, vs: 1}, metric_options = {sort: {"_id.t": 1}, batchSize: 1000}; -function Metric(time, value, measurement){ +function Metric(time, value, measurement, values){ this.time = time; this.value = value; + this.setProperty("values", { value: values||[] }); this.setProperty("measurement", { value: measurement }); } @@ -50,15 +51,15 @@ Metric.setProperties({ function find(db, measurement, callback){ var expression = measurement.expression, - start = expression.start, - stop = expression.stop, + start = measurement.start, + stop = measurement.stop, type = expression.type, tier = measurement.tier; db.metrics(type, function(error, collection){ if (error) return callback(error); - collection.find({ + var query = { i: false, "_id.e": expression.source, "_id.l": tier.key, @@ -66,22 +67,38 @@ function find(db, measurement, callback){ $gte: start, $lt: stop } - }, metric_fields, metric_options, handleResponse); + }; + collection.find(query, metric_fields, metric_options, handleResponse); }); function handleResponse(error, cursor){ if (error) return callback(error); cursor.each(function(error, row) { if (error) return callback(error); - if (row) callback(error, new Metric(row._id.t, row.v, measurement)); + if (row) callback(error, Metric.from_wire(row, measurement)); else callback(); }) } } Object.defineProperty(Metric, "find", { value: find }); +Object.defineProperty(Metric, "from_wire", { value: from_wire }); + +function from_wire(row, measurement){ + var values = row.vs.reduce(function(expanded, value){ + _.times(value.c, function(){ expanded.push(value.v); }); + return expanded; + }, []); + return new Metric(row._id.t, row.v, measurement, values); +} function to_wire(){ - return { i: false, v: mongo.Double(this.value), _id: { e: this.e, l: this.l, t: this.time } }; + var values = this.values.reduce(function(values, value){ + var pair = _.find(values, function(pair){ return pair.value == value; }); + if (!pair) values.push(pair = { v: value, c: 0 }); + pair.c++; + return values; + }, []); + return { i: false, v: mongo.Double(this.value), vs: values, _id: { e: this.e, l: this.l, t: this.time } }; }; function report(){ diff --git a/lib/cube/reduces.js b/lib/cube/reduces.js index de76ba00..d73faac7 100644 --- a/lib/cube/reduces.js +++ b/lib/cube/reduces.js @@ -40,7 +40,6 @@ reduces.distinct.empty = 0; reduces.sum.pyramidal = true; reduces.min.pyramidal = true; reduces.max.pyramidal = true; -//reduces.distinct.pyramidal = true; function ascending(a, b) { return a - b; From bef5e48e35d5bf40afd61c758e73f1281e014de7 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Thu, 4 Oct 2012 17:11:25 -0500 Subject: [PATCH 51/87] Prepend numeric event data keys with 'k' Conflicts: lib/cube/models.js --- lib/cube/models/event.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/cube/models/event.js b/lib/cube/models/event.js index 73d3a14c..4d36e1a8 100644 --- a/lib/cube/models/event.js +++ b/lib/cube/models/event.js @@ -15,9 +15,23 @@ _.mapHash = function(obj, func){ return res; }; +function formatData(data){ + if (!_.isObject(data)) return data; + if (Array.isArray(data)) return data.map(formatData); + _.keys(data).forEach(function(key){ + data[key] = formatData(data[key]); + + if (_.isNumber(key) || /^[0-9]$/.test(key)) { + var val = data[key]; delete data[key]; + data['k' + key] = val; + } + }); + return data; +} + function Event(type, time, data, id){ this.time = time; - this.data = data; + this.data = formatData(data); if (id) this.id = id; this.setProperty("type", { value: type }); From 36c78fd414f1e8cc6f8d3fcbe95edc794a7fb859 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Mon, 8 Oct 2012 13:32:13 -0500 Subject: [PATCH 52/87] Add horizons back in, complete with tests Conflicts: lib/cube/db.js lib/cube/event.js lib/cube/metric.js --- lib/cube/event.js | 17 ++++++++------- lib/cube/metric.js | 8 +++++-- lib/cube/models/measurement.js | 38 ++++++++++++++++++++-------------- lib/cube/warmer.js | 2 +- test/event-test.js | 21 +++++++++++++++++++ test/metric-test.js | 33 +++++++++-------------------- test/test_helper.js | 12 ++++++++--- test/warmer-test.js | 2 +- 8 files changed, 81 insertions(+), 52 deletions(-) diff --git a/lib/cube/event.js b/lib/cube/event.js index 0ebb9ab9..9727fec3 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -34,7 +34,9 @@ var putter_id = 0; // - type, namespace for the events. A corresponding `foo_events` collection must exist in the DB -- /schema/schema-*.js illustrate how to set up a new event type. // - data, the event's payload // -exports.putter = function(db){ + +exports.putter = function(db, config){ + var options = (config || options || {}); var invalidator = new Invalidator(); @@ -48,10 +50,10 @@ exports.putter = function(db){ // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; // // Drop events from before invalidation horizon - // if ((! request.force) && (time < new Date(new Date() - options.horizons.invalidation))) { - // metric.info('cube_compute', {error: "event before invalidation horizon"}); - // return callback({error: "event before invalidation horizon"}), -1; - // } + if ((! request.force) && options.horizons && (time < new Date(new Date() - options.horizons.invalidation))) { + metalog.info('cube_compute', {error: "event before invalidation horizon"}); + return callback({error: "event before invalidation horizon"}), -1; + } var event = new Event(type, time, request.data, request.id); try{ event.validate(); } catch(err) { return callback({error: err}), -1; } @@ -147,8 +149,9 @@ Invalidator.stop_flusher = function(id, on_stop){ // * Issue the query; // * if streaming, register the query to be run at a regular interval // -exports.getter = function(db) { - var streamsBySource = {}; +exports.getter = function(db, config) { + var options = (config || options || {}), + streamsBySource = {}; function getter(request, callback) { var stream = !("stop" in request), diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 4bdee361..2a21c79e 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -15,7 +15,10 @@ var _ = require("underscore"), options = require('../../config/cube'); // Query for metrics. -exports.getter = function(db){ + +exports.getter = function(db, config) { + var options = (config || options || {}); + function getter(request, callback) { var measurement, expression, tier = tiers[+request.step], @@ -33,6 +36,7 @@ exports.getter = function(db){ stop = tier.ceil(stop); expression = parser.parse(request.expression); measurement = new Measurement(expression, start, stop, tier); + measurement.on('complete', function(){ handle_response(new Metric(stop, null, measurement)); }); } catch(error) { metalog.error('mget', error, { info: util.inspect([start, stop, tier, expression] )}); @@ -44,7 +48,7 @@ exports.getter = function(db){ if (metric.value || metric.value === 0) metric.save(db, handle); } - measurement.measure(db, handle_response); + measurement.measure(db, options, handle_response); } return getter; diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index a84325bb..58218aa2 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -30,21 +30,20 @@ Measurement.prototype.report = function report(){ // by the specified tier, such as daily or hourly. The callback is invoked // repeatedly for each metric value, being passed two arguments: the time and // the value. The values may be out of order due to partial cache hits. -Measurement.prototype.measure = function measure(db, callback) { +Measurement.prototype.measure = function measure(db, options, callback) { var _this = this; - - compute[this.flavor].call(this, db, function(time, value, values){ callback(new Metric(time, value, _this, values)); }); + compute[this.flavor].call(this, db, options, function(time, value, values){ callback(new Metric(time, value, _this, values)); }); }; // Computes a constant expression like the "7" in "x * 7" -function constant(db, callback) { +function constant(db, options, callback) { var self = this, value = this.expression.value(); walk(this.start, this.stop, this.tier, function(time){ callback(time, value); }); this.emit('complete'); }; // Serializes a unary expression for computation. -function unary(db, callback) { +function unary(db, options, callback) { var self = this, remaining = 0, time0 = Date.now(), @@ -55,7 +54,7 @@ function unary(db, callback) { walk(this.start, this.stop, this.tier, function(time){ ++remaining; }); // If no results were requested, return immediately. - if (!remaining) return callback(stop); + if (!remaining) return this.emit('complete'); // Add this task to the appropriate queue. if (queue) queue.next = task; @@ -63,7 +62,7 @@ function unary(db, callback) { queueByName[name] = task; function task() { - findOrComputeUnary.call(self, db, function(time, value, values) { + findOrComputeUnary.call(self, db, options, function(time, value, values) { callback(time, value, values); if (!--remaining) { self.emit('complete'); @@ -82,7 +81,7 @@ function unary(db, callback) { } // Finds or computes a unary (primary) expression. -function findOrComputeUnary(db, callback) { +function findOrComputeUnary(db, options, callback) { var expression = this.expression, map = expression.value, reduce = reduces[expression.reduce]; @@ -138,10 +137,19 @@ function findOrComputeUnary(db, callback) { function computeFlat(start, stop) { // metalog.warn('computeFlat', { expr: expression.source, start: start, stop: stop, tier: tier.key }); - // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ - // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); - // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) - // } + // Reset start time to calculation horizon if requested time span goes past it + if (options.horizons && tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + var old_start = start, + start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + metalog.info('cube_compute', {is: 'past_horizon', start: {was: old_start, updated_to: start}, stop: stop, tier: tier, expression: expression.source }); + } + + // Reset stop time to calculation horizon if requested time span goes past it + if (options.horizons && tier.floor(stop) < new Date(new Date() - options.horizons.calculation)){ + var old_stop = stop, + stop = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + metalog.info('cube_compute', {is: 'past_horizon', metric: {start: start, stop: {was: old_stop, updated_to: stop}, tier: tier, expression: expression.source } }); + } var measurement = new Measurement(expression, start, stop, tier); var time = start, values = []; @@ -172,7 +180,7 @@ function findOrComputeUnary(db, callback) { // unary "sum(resp)". We don't know what order they'll show up in, so if say // the value for left appears first, it parks that value as left[time], where // the result for right will eventually find it. -function binary(db, callback) { +function binary(db, options, callback) { var self = this, expression = this.expression, value; var left = new Measurement(expression.left, this.start, this.stop, this.tier), right = new Measurement(expression.right, this.start, this.stop, this.tier); @@ -195,7 +203,7 @@ function binary(db, callback) { } }); - left.measure(db, function(metric) { + left.measure(db, options, function(metric) { var time = metric.time, value = metric.value; if (time in right) { // right val already appeared; get a result callback(time, expression.op(value, right[time])); @@ -205,7 +213,7 @@ function binary(db, callback) { } }); - right.measure(db, function(metric) { + right.measure(db, options, function(metric) { var time = metric.time, value = metric.value; if (time in left) { callback(time, expression.op(left[time], value)); diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index 96b34808..466a01ca 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -55,7 +55,7 @@ module.exports = function(options){ db.open(options, function(error) { if (error) throw error; - calculate_metric = metric.getter(db); + calculate_metric = metric.getter(db, options); fetch_metrics(process_metrics); }); }, diff --git a/test/event-test.js b/test/event-test.js index 0020b928..3ef9cffc 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -45,4 +45,25 @@ suite.addBatch(test_helper.batch({ teardown: function(putter){ putter.stop(this.callback); } })); +suite.addBatch(test_helper.batch({ + topic: function(test_db) { + var horizon = new Date() - fuck_wit_dre_day + (1000 * 60), + options = this.settings = test_helper._.extend({}, test_helper.settings, {horizons: { invalidation: horizon }}); + return event.putter(test_db.db, options); + }, + 'events past invalidation horizon': { + topic: function(putter){ + var _this = this, + event = new Event('test', ice_cubes_good_day, {value: 3}); + this.ret = putter(event.to_request(), this.callback); + }, + 'should error': function(error, response){ + assert.deepEqual(error, {error: "event before invalidation horizon"}); + }, + 'should return -1': function(error, response){ + assert.equal(this.ret, -1); + } + } +})); + suite['export'](module); diff --git a/test/metric-test.js b/test/metric-test.js index ab411ef1..e1d47388 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -7,7 +7,8 @@ var _ = require("underscore"), assert = require("assert"), test_helper = require("./test_helper"), queuer = require("../lib/queue-async/queue"), - units = require("../lib/cube/tiers").units, + tiers = require("../lib/cube/tiers"), + units = tiers.units, event = require("../lib/cube/event"), metric = require("../lib/cube/metric"); @@ -45,7 +46,6 @@ function assert_invalid_request(req, expected_err) { }; } - function skip(){ // FIXME: remove ------------------------------------------------------------ var steps = { @@ -66,7 +66,7 @@ suite.addBatch(test_helper.batch({ var putter = event.putter(test_db), getter = metric.getter(test_db), callback = this.callback, - put_queue = queuer(10); + put_queue = queuer(10); this.putter = putter; // Seed the events table with a simple event: a value going from 0 to 2499 @@ -81,18 +81,6 @@ suite.addBatch(test_helper.batch({ } // continue when queue clears put_queue.await(function(){ callback(null, getter) }); - - // // Seed the events table with a simple event: a value going from 0 to 2499 - // for (var i = 0; i < 2500; i++) { - // putter({ - // type: "test", - // time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(i) - 10)).toISOString(), - // data: {i: i} - // }); - // } - - // // So the events can settle in, wait `batch_testing_delay` ms before continuing - // setTimeout(function() { callback(null, getter); }, batch_testing_delay); }, teardown: function(){ this.putter.stop(this.callback); }, @@ -112,7 +100,7 @@ suite.addBatch(test_helper.batch({ "unary expression c": metricTest({ expression: "sum(test)", start: "2011-07-17T23:48:00.000Z", stop: "2011-07-18T00:01:00.000Z"}, { 60e3: [ 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39 ] }), "unary expression d": metricTest({ expression: "sum(test)", start: "2011-07-17T23:49:00.000Z", stop: "2011-07-18T00:02:00.000Z"}, { 60e3: [ 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23 ] }), "unary expression e": metricTest({ expression: "sum(test)", start: "2011-07-17T23:50:00.000Z", stop: "2011-07-18T00:03:00.000Z"}, { 60e3: [ 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25 ] }), - + "unary expression f": metricTest({ expression: "sum(test)", start: "2011-07-17T23:57:00.000Z", @@ -138,7 +126,7 @@ suite.addBatch(test_helper.batch({ 3600e3: [82, 2418], 86400e3: [82, 2418] }), - + "unary expression with data accessor": metricTest({ expression: "sum(test(i))", start: "2011-07-17T23:47:00.000Z", @@ -148,7 +136,7 @@ suite.addBatch(test_helper.batch({ 3600e3: [3321, 3120429], 86400e3: [3321, 3120429] }), - + "unary expression with compound data accessor": metricTest({ expression: "sum(test(i / 100))", start: "2011-07-17T23:47:00.000Z", @@ -158,7 +146,7 @@ suite.addBatch(test_helper.batch({ 3600e3: [33.21, 31204.29], 86400e3: [33.21, 31204.29] }), - + "compound expression (sometimes fails due to race condition?)": metricTest({ expression: "max(test(i)) - min(test(i))", start: "2011-07-17T23:47:00.000Z", @@ -168,7 +156,7 @@ suite.addBatch(test_helper.batch({ 3600e3: [81, 2417], 86400e3: [81, 2417] }), - + "non-pyramidal expression": metricTest({ expression: "distinct(test(i))", start: "2011-07-17T23:47:00.000Z", @@ -178,7 +166,7 @@ suite.addBatch(test_helper.batch({ 3600e3: [82, 2418], 86400e3: [82, 2418] }), - + "compound pyramidal and non-pyramidal expression": metricTest({ expression: "sum(test(i)) - median(test(i))", start: "2011-07-17T23:47:00.000Z", @@ -188,7 +176,7 @@ suite.addBatch(test_helper.batch({ 3600e3: [3280.5, 3119138.5], 86400e3: [3280.5, 3119138.5] }), - + "compound with constant expression": metricTest({ expression: "-1 + sum(test)", start: "2011-07-17T23:47:00.000Z", @@ -198,7 +186,6 @@ suite.addBatch(test_helper.batch({ 3600e3: [81, 2417], 86400e3: [81, 2417] }) - })); // metricTest -- generates test tree for metrics. diff --git a/test/test_helper.js b/test/test_helper.js index 20a23674..f22a1da3 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -16,6 +16,8 @@ var _ = require("underscore"), var test_helper = {}; var test_collections = ["test_users", "test_events", "test_metrics", "test_boards"]; test_helper.inspectify = metalog.inspectify; +test_helper._ = require('underscore'); + test_helper.settings = { "mongo-host": "localhost", @@ -24,7 +26,13 @@ test_helper.settings = { "mongo-password": null, "mongo-database": "cube_test", "host": "localhost", - "authenticator": "allow_all" + "authenticator": "allow_all", + + + "horizons": { + "calculation": +(new Date()), + "invalidation": +(new Date()), + } }; // Disable logging for tests. @@ -161,8 +169,6 @@ function start_server(options, register, ctxt, test_db){ // db helpers // -var suite_id = 0; - // test_helper.batch -- // * connect to db, drop relevant collections // * run tests once db is ready; diff --git a/test/warmer-test.js b/test/warmer-test.js index fd9a41ca..da7d3fdf 100644 --- a/test/warmer-test.js +++ b/test/warmer-test.js @@ -6,7 +6,7 @@ var _ = require('underscore'), test_helper = require("./test_helper"), event = require("../lib/cube/event"), settings = {"warmer-tier": 10000, "warmer-interval": 10000, horizons: { calculation: 30000 } }, - warmer = require("../lib/cube/warmer")(_.extend({}, settings, test_helper.settings)); + warmer = require("../lib/cube/warmer")(_.extend({}, test_helper.settings, settings)); var suite = vows.describe("warmer"); From 48b0060d5858c42f6a38d4bd25123c1a97f74301 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 16 Oct 2012 10:17:07 -0500 Subject: [PATCH 53/87] Fix metrics bugs. --- lib/cube/metric.js | 1 - lib/cube/models/metric.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 2a21c79e..e61b9a3c 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -29,7 +29,6 @@ exports.getter = function(db, config) { if (!tier) throw "invalid step"; if (isNaN(start)) throw "invalid start"; if (isNaN(stop)) throw "invalid stop"; - // if (request.expression && (request.expression.match(/\(/mg).length > 2)){ throw("rejected complex expression"); } // Round start and stop to the appropriate time step. start = tier.floor(start); diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index 9f503a98..5b365a8e 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -93,7 +93,7 @@ function from_wire(row, measurement){ function to_wire(){ var values = this.values.reduce(function(values, value){ - var pair = _.find(values, function(pair){ return pair.value == value; }); + var pair = _.find(values, function(pair){ return pair.v == value; }); if (!pair) values.push(pair = { v: value, c: 0 }); pair.c++; return values; From 9e77305eb37f86424b6f843f09e0963495ddbf2f Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 16 Oct 2012 10:17:19 -0500 Subject: [PATCH 54/87] Send parse errors back to user for malformed queries --- lib/cube/visualizer.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js index 5475027d..3d6122c8 100644 --- a/lib/cube/visualizer.js +++ b/lib/cube/visualizer.js @@ -3,7 +3,8 @@ var url = require("url"), path = require("path"), endpoint = require("./endpoint"), - metalog = require('./metalog'); + metalog = require('./metalog'), + parser = require("./metric-expression"); exports.register = function(db, endpoints) { endpoints.ws.push( @@ -51,9 +52,16 @@ function viewBoard(db) { if (! check_authorization(request, 'move')) return; var boardId = request.id, - callbacks = callbacksByBoard[boardId].filter(function(c) { return c.id != callback.id; }); + callbacks = callbacksByBoard[boardId].filter(function(c) { return c.id != callback.id; }), + response = callbacksByBoard[boardId].filter(function(c) { return c.id == callback.id; }), + error; boards.update({_id: boardId, "pieces.id": request.piece.id}, {$set: {"pieces.$": request.piece}}); - if (callbacks.length) emit(callbacks, {type: request.type, piece: request.piece}); + if (request.piece.query) { + try { parser.parse(request.piece.query); } + catch (e) { error = e.message + ' Line: ' + e.line + ', Column: ' + e.column; } + } + if (error) emit(response, { type: 'error', piece: request.piece, error: error }); + if (callbacks.length && !error) emit(callbacks, {type: request.type, piece: request.piece}); } function remove(request, callback) { From 527d7e056d580d8347f1ace9c705ae27a334835a Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 2 Nov 2012 15:49:48 -0500 Subject: [PATCH 55/87] Fix random-streamer example model references --- examples/random-emitter/cromulator.js | 3 +-- examples/random-emitter/random-streamer.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/random-emitter/cromulator.js b/examples/random-emitter/cromulator.js index 23263684..290c5720 100644 --- a/examples/random-emitter/cromulator.js +++ b/examples/random-emitter/cromulator.js @@ -1,5 +1,4 @@ -var metalog = require("../../lib/cube/metalog"), - models = require("../../lib/cube/models"); +var metalog = require("../../lib/cube/metalog"); var un = {}; un.sec = 1000; un.min = 60 * un.sec; un.hr = 60 * un.min; diff --git a/examples/random-emitter/random-streamer.js b/examples/random-emitter/random-streamer.js index 5587b60c..841e8f25 100644 --- a/examples/random-emitter/random-streamer.js +++ b/examples/random-emitter/random-streamer.js @@ -4,7 +4,7 @@ var cube = require("../../"), // replace with require("cube") metalog = cube.metalog, options = require("./random-config"), cromulator = require("./cromulator"), - models = require("../../lib/cube/models"), Event = models.Event; + Event = require("../../lib/cube/models/event"); var options = { "collector": "ws://127.0.0.1:6000", From 1f6f78d8bf831a0f59249467d097c947364bc030 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 2 Nov 2012 15:50:30 -0500 Subject: [PATCH 56/87] Remove commented out code sections --- lib/cube/models/event.js | 4 ---- lib/cube/models/metric.js | 9 --------- 2 files changed, 13 deletions(-) diff --git a/lib/cube/models/event.js b/lib/cube/models/event.js index 4d36e1a8..a6c5aa8c 100644 --- a/lib/cube/models/event.js +++ b/lib/cube/models/event.js @@ -73,10 +73,6 @@ function find(db, measurement, callback){ // Request any needed fields. expression.fields(fields); - // if (tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ - // metalog.info('cube_compute', {is: 'past_horizon', metric: metric }); - // start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) - // } filter.t.$gte = start; filter.t.$lt = stop; diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index 5b365a8e..b717bf84 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -38,15 +38,6 @@ Metric.setProperties({ to_wire: { value: to_wire }, report: { value: report }, save: { value: save } - - // day_bin: function(){ return tiers[day ].bin(this.t); }, - // m05_bin: function(){ return tiers[minute5 ].bin(this.t); }, - // s10_bin: function(){ return tiers[second10].bin(this.t); }, - // day_ago: function day_ago(){ return Math.floor((Date.now() - this.t) / day); }, - // m05_ago: function m05_ago(){ return Math.floor((Date.now() - this.t) / minute5); }, - // s10_ago: function s10_ago(){ return Math.floor((Date.now() - this.t) / second10); }, - // bins: function bin(){ return [this.day_bin(), this.m05_bin(), this.s10_bin() ]; }, - // agos: function ago(){ return [this.day_ago(), this.m05_ago(), this.s10_ago() ]; }, }); function find(db, measurement, callback){ From 438efa3148810f649deef4808f89b1d9d9d4493d Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 2 Nov 2012 15:50:48 -0500 Subject: [PATCH 57/87] Modelize invalidator --- lib/cube/event.js | 71 ++++++---------------------------- lib/cube/models/invalidator.js | 68 ++++++++++++++++++++++++++++++++ test/event-test.js | 2 +- 3 files changed, 80 insertions(+), 61 deletions(-) create mode 100644 lib/cube/models/invalidator.js diff --git a/lib/cube/event.js b/lib/cube/event.js index 9727fec3..d33c222c 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -4,24 +4,21 @@ // TODO allow the event time to change when updating (fix invalidation) var _ = require("underscore"), - mongodb = require("mongodb"), - ObjectID = mongodb.ObjectID, - util = require("util"), - tiers = require("./tiers"), - Event = require("./models/event"), - Metric = require("./models/metric"), - parser = require("./event-expression"), - bisect = require("./bisect"), - metalog = require("./metalog"); + mongodb = require("mongodb"), + ObjectID = mongodb.ObjectID, + util = require("util"), + Event = require("./models/event"), + Metric = require("./models/metric"), + Invalidator = Metric = require("./models/invalidator"), + parser = require("./event-expression"), + bisect = require("./bisect"), + metalog = require("./metalog"); // When streaming events, we should allow a delay for events to arrive, or else // we risk skipping events that arrive after their event.time. This delay can be // customized by specifying a `delay` property as part of the request. var streamDelayDefault = 5000, - streamInterval = 1000; - -// How frequently to invalidate metrics after receiving events. -var invalidateInterval = 5000; + streamInterval = 1000; // serial id so we can track flushers var putter_id = 0; @@ -45,10 +42,6 @@ exports.putter = function(db, config){ type = request.type; callback = callback || function(){}; - - // // Drop events from before invalidation horizon - // if (time < new Date(new Date() - options.horizons.invalidation)) return callback({error: "event before invalidation horizon"}), -1; - // // Drop events from before invalidation horizon if ((! request.force) && options.horizons && (time < new Date(new Date() - options.horizons.invalidation))) { metalog.info('cube_compute', {error: "event before invalidation horizon"}); @@ -78,7 +71,7 @@ exports.putter = function(db, config){ // previous batch of events are being invalidated, new events can arrive. Invalidator.start_flusher(putter.id, function(){ if (db.isHalted) return putter.stop(); - invalidator.flush(db); + invalidator.flush(db, handle); invalidator = new Invalidator(); // copy-on-write }); @@ -95,48 +88,6 @@ exports.putter = function(db, config){ // -------------------------------------------------------------------------- -// Schedule deferred invalidation of metrics by type and tier. -function Invalidator(){ - var type_tsets = {}, - invalidate = { $set: {i: true} }, - multi = { multi: true }; - - this.add = function(type, ev){ - var tt = type_tset(type); - for (var tier in tiers){ tt[tier][tier*Math.floor(ev.time/tier)] = true; } - }; - - this.flush = function(db){ - _.each(type_tsets, function(type_tset, type){ - db.metrics(type, function(error, collection){ - handle(error); - - _.each(type_tset, function(tset, tier){ - var times = dateify(tset); - metalog.info("event_flush", { type: type, tier: tier, times: times }); - collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, invalidate, multi); - }); - }); - }); - }; - - this.tsets = function(){ return _.mapHash(type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }); }; - this._empty = function(){ type_tsets = {}; } // for testing only - - function type_tset(type){ - if (! (type in type_tsets)) type_tsets[type] = _.mapHash(tiers, function(){ return {}; });; - return type_tsets[type]; - } - function dateify(tset){ return _.map(_.keys(tset), function(time){ return new Date(+time); }).sort(function(aa,bb){return aa-bb;}); } -} -Invalidator.flushers = {}; -Invalidator.start_flusher = function(id, cb){ Invalidator.flushers[id] = setInterval(cb, invalidateInterval); }; -Invalidator.stop_flusher = function(id, on_stop){ - clearInterval(Invalidator.flushers[id]); - delete Invalidator.flushers[id]; - if (on_stop) on_stop(); -}; - // // event.getter - subscribe to event type // diff --git a/lib/cube/models/invalidator.js b/lib/cube/models/invalidator.js new file mode 100644 index 00000000..a9d7a656 --- /dev/null +++ b/lib/cube/models/invalidator.js @@ -0,0 +1,68 @@ +'use strict'; + +var _ = require("underscore"), + metalog = require('../metalog'), + tiers = require("../tiers"), + Model = require("../core_ext/model"); + + +// How frequently to invalidate metrics after receiving events. +var invalidateInterval = 5000; + +// Schedule deferred invalidation of metrics by type and tier. +function Invalidator(){ + this.type_tsets = {}, + this.invalidate = { $set: {i: true} }, + this.multi = { multi: true }; +} + +Model.modelize(Invalidator); + +function add(type, ev){ + var tt = this.type_tset(type); + for (var tier in tiers){ tt[tier][tier*Math.floor(ev.time/tier)] = true; } +}; + +function flush(db, callback){ + var _this = this; + _.each(_this.type_tsets, function(type_tset, type){ + db.metrics(type, function(error, collection){ + callback(error); + + _.each(type_tset, function(tset, tier){ + var times = dateify(tset); + metalog.info("event_flush", { type: type, tier: tier, times: times }); + collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, _this.invalidate, _this.multi); + }); + }); + }); +}; + +function tsets(){ return _.mapHash(this.type_tsets, function(tt, type){ return _.mapHash(tt, dateify); }); }; + +Invalidator.setProperties({ + add: { value: add }, + flush: { value: flush }, + tsets: { get: tsets }, + type_tset: { value: type_tset } +}); + +function type_tset(type){ + if (! (type in this.type_tsets)) this.type_tsets[type] = _.mapHash(tiers, function(){ return {}; });; + return this.type_tsets[type]; +}; +function dateify(tset){ + return _.map(_.keys(tset), function(time){ + return new Date(+time); + }).sort(function(aa,bb){return aa-bb;}); +} + +Invalidator.flushers = {}; +Invalidator.start_flusher = function(id, cb){ Invalidator.flushers[id] = setInterval(cb, invalidateInterval); }; +Invalidator.stop_flusher = function(id, on_stop){ + clearInterval(Invalidator.flushers[id]); + delete Invalidator.flushers[id]; + if (on_stop) on_stop(); +}; + +module.exports = Invalidator; \ No newline at end of file diff --git a/test/event-test.js b/test/event-test.js index 3ef9cffc..f877e1ee 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -23,7 +23,7 @@ suite.addBatch(test_helper.batch({ putter((new Event('test', fuck_wit_dre_day, {value: 3})).to_request(), _this.callback);}); }, 'correct tiers': function(a,b){ - var ts = this.putter.invalidator().tsets(); + var ts = this.putter.invalidator().tsets; assert.deepEqual(ts, { 'test': { 10e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:50Z') ], 60e3: [new Date('1992-02-20T01:08:00Z'), new Date('1993-03-18T08:44:00Z') ], From b6d3524bfd7243d01342c9238475a5f37b80b04b Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 6 Nov 2012 10:53:14 -0600 Subject: [PATCH 58/87] Fix error when running from_wire on row with no vs value --- lib/cube/models/metric.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index b717bf84..6810c865 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -75,7 +75,7 @@ Object.defineProperty(Metric, "find", { value: find }); Object.defineProperty(Metric, "from_wire", { value: from_wire }); function from_wire(row, measurement){ - var values = row.vs.reduce(function(expanded, value){ + var values = (row.vs || []).reduce(function(expanded, value){ _.times(value.c, function(){ expanded.push(value.v); }); return expanded; }, []); From 8208012975a8f195fbdd6173acb9a02aa9a7c316 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 6 Nov 2012 13:52:45 -0600 Subject: [PATCH 59/87] Add "jake" task to remove metrics older than the expiration horizon --- Jakefile | 42 ++++++++++++++++++++++++++++++++++++++++++ config/cube.js | 7 +++++-- lib/cube/db.js | 12 +++++++++++- package.json | 6 ++++-- 4 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 Jakefile diff --git a/Jakefile b/Jakefile new file mode 100644 index 00000000..bb393c69 --- /dev/null +++ b/Jakefile @@ -0,0 +1,42 @@ +var config = require('./config/cube'), + Db = require('./lib/cube/db'); + +namespace("db", function(){ + namespace("metrics", function(){ + desc("Remove metrics with times past the forced metric expiration horizon") + task("remove_expired", [], function(){ + if(!config.horizons.forced_metric_expiration){ + throw new Error("horizons.forced_metric_expiration MUST be set in: config/cube.js"); + } + + var db = new Db(), + expiration_date = new Date(new Date() - config.horizons.forced_metric_expiration); + + function handle(err) { + if(err) throw err; + } + + db.open(config, function(err, db){ + handle(err); + var metrics_db = db._dbs.metrics; + metrics_db.collectionNames({namesOnly: true}, function(err, names){ + handle(err); + var metric_names = names.filter(function(name){ return /_metrics$/.test(name); }), + remaining = metric_names.length; + metric_names.forEach(function(name){ + var segments = name.split('.'), + collection_name = (segments.shift(), segments.join('.')); + metrics_db.collection(collection_name, function(err, collection){ + handle(err); + collection.remove({'_id.t': {$lt: expiration_date }}, function(err){ + handle(err); + console.log('Removing ' + collection_name.split('_').join(' ') + ' older than ' + expiration_date); + if(!--remaining) db.close(); + }); + }); + }); + }) + }) + }); + }); +}); \ No newline at end of file diff --git a/config/cube.js b/config/cube.js index 5948b01a..489b15d4 100644 --- a/config/cube.js +++ b/config/cube.js @@ -1,3 +1,5 @@ +'use strict'; + var configs = {}, metalog = require('../lib/cube/metalog'); @@ -20,8 +22,9 @@ configs.common = { "separate-events-database": true, "horizons": { - "calculation": 1000 * 60 * 60 * 2, // 2 hours - "invalidation": 1000 * 60 * 60 * 1, // 1 hour + "calculation": 1000 * 60 * 60 * 2, // 2 hours + "invalidation": 1000 * 60 * 60 * 1, // 1 hour + "forced_metric_expiration": 1000 * 60 * 60 * 24 * 7, // 7 days } }; diff --git a/lib/cube/db.js b/lib/cube/db.js index 5cb4a26b..e99dc19f 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -52,6 +52,15 @@ function Db(){ if (options["separate-metrics-database"]) metrics_client = db_client.db(database_name + '-metrics'); else metrics_client = db_client; + Object.defineProperty(db, "_dbs", { + configurable: true, + value: { + main: db_client, + metrics: metrics_client, + events: events_client + } + }); + db.metrics = metrics_collection_factory(metrics_client); db.events = events_collection_factory(events_client); db.types = types(events_client); @@ -83,12 +92,13 @@ function Db(){ delete db.types; delete db.collection; delete db.clearCache; + delete db._clients; if (isConnected()){ db_client.close(); } db_client = events_client = metrics_client = null; db.isHalted = true; - return callback(null); + if(callback) return callback(null); } db.isHalted = false; diff --git a/package.json b/package.json index 69f39dd5..59452d34 100644 --- a/package.json +++ b/package.json @@ -17,10 +17,12 @@ "websocket-server": "1.4.04", "cookies": "0.3.1", "bcrypt": "0.7.1", - "underscore": "1.3.3" + "underscore": "1.3.3", + "jake": "0.5.6" }, "scripts": { "preinstall": "npm install mongodb --mongodb:native", - "test": "vows --isolate --spec" + "test": "vows --isolate --spec", + "remove_expired_metrics": "jake db:metrics:remove_expired" } } From 057e3921ec41c465f102d682294ad120cd9890d1 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 6 Nov 2012 14:15:26 -0600 Subject: [PATCH 60/87] Only store intermediate metric values for nonpyramidal reduces --- lib/cube/models/measurement.js | 9 +++++---- lib/cube/models/metric.js | 26 ++++++++++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 58218aa2..0c9db2c3 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -11,10 +11,11 @@ var metalog = require('../metalog'), function Measurement(expression, start, stop, tier){ // Round the start/stop to the tier edges this.expression = expression; - this.start = start; - this.stop = stop; - this.tier = tier; - this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); + this.start = start; + this.stop = stop; + this.tier = tier; + this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); + this.isPyramidal = reduces[expression.reduce].pyramidal; this.eventize(); } diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index 6810c865..2a914c9f 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -75,20 +75,26 @@ Object.defineProperty(Metric, "find", { value: find }); Object.defineProperty(Metric, "from_wire", { value: from_wire }); function from_wire(row, measurement){ - var values = (row.vs || []).reduce(function(expanded, value){ - _.times(value.c, function(){ expanded.push(value.v); }); - return expanded; - }, []); + var values = null; + if(!measurement.isPyramidal) { + values = (row.vs || []).reduce(function(expanded, value){ + _.times(value.c, function(){ expanded.push(value.v); }); + return expanded; + }, []); + } return new Metric(row._id.t, row.v, measurement, values); } function to_wire(){ - var values = this.values.reduce(function(values, value){ - var pair = _.find(values, function(pair){ return pair.v == value; }); - if (!pair) values.push(pair = { v: value, c: 0 }); - pair.c++; - return values; - }, []); + var values = null; + if(!this.measurement.isPyramidal){ + values = this.values.reduce(function(values, value){ + var pair = _.find(values, function(pair){ return pair.v == value; }); + if (!pair) values.push(pair = { v: value, c: 0 }); + pair.c++; + return values; + }, []); + } return { i: false, v: mongo.Double(this.value), vs: values, _id: { e: this.e, l: this.l, t: this.time } }; }; From 76873e15735d96cfc4cad8e4b4fc25b842916eab Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Tue, 6 Nov 2012 16:16:59 -0600 Subject: [PATCH 61/87] Oops, metrics weren't saving/cascading correctly --- lib/cube/metric.js | 9 ++---- lib/cube/models/measurement.js | 57 +++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/lib/cube/metric.js b/lib/cube/metric.js index e61b9a3c..058a2d30 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -36,18 +36,13 @@ exports.getter = function(db, config) { expression = parser.parse(request.expression); measurement = new Measurement(expression, start, stop, tier); - measurement.on('complete', function(){ handle_response(new Metric(stop, null, measurement)); }); + measurement.on('complete', function(){ callback(new Metric(stop, null, measurement)); }); } catch(error) { metalog.error('mget', error, { info: util.inspect([start, stop, tier, expression] )}); return callback({error: error, _trace: request._trace}), -1; } - function handle_response(metric){ - callback(metric); - if (metric.value || metric.value === 0) metric.save(db, handle); - } - - measurement.measure(db, options, handle_response); + measurement.measure(db, options, callback); } return getter; diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 0c9db2c3..28547aab 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -15,7 +15,7 @@ function Measurement(expression, start, stop, tier){ this.stop = stop; this.tier = tier; this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); - this.isPyramidal = reduces[expression.reduce].pyramidal; + this.isPyramidal = expression.type && reduces[expression.reduce].pyramidal; this.eventize(); } @@ -33,13 +33,13 @@ Measurement.prototype.report = function report(){ // the value. The values may be out of order due to partial cache hits. Measurement.prototype.measure = function measure(db, options, callback) { var _this = this; - compute[this.flavor].call(this, db, options, function(time, value, values){ callback(new Metric(time, value, _this, values)); }); + compute[this.flavor].call(this, db, options, callback); }; // Computes a constant expression like the "7" in "x * 7" function constant(db, options, callback) { - var self = this, value = this.expression.value(); - walk(this.start, this.stop, this.tier, function(time){ callback(time, value); }); + var _this = this, value = this.expression.value(); + walk(this.start, this.stop, this.tier, function(time){ callback(new Metric(time, value, _this)); }); this.emit('complete'); }; @@ -63,8 +63,8 @@ function unary(db, options, callback) { queueByName[name] = task; function task() { - findOrComputeUnary.call(self, db, options, function(time, value, values) { - callback(time, value, values); + findOrComputeUnary.call(self, db, options, function(metric) { + callback(metric); if (!--remaining) { self.emit('complete'); if (task.next) process.nextTick(task.next); @@ -83,11 +83,12 @@ function unary(db, options, callback) { // Finds or computes a unary (primary) expression. function findOrComputeUnary(db, options, callback) { - var expression = this.expression, - map = expression.value, - reduce = reduces[expression.reduce]; + var expression = this.expression, + map = expression.value, + reduce = reduces[expression.reduce], + measurement = this; - find(this, callback); + find(measurement, callback); // The metric is computed recursively, reusing the above variables. function find(measurement, callback) { @@ -105,7 +106,7 @@ function findOrComputeUnary(db, options, callback) { function foundMetrics(error, metric) { handle(error); if (metric) { - callback(metric.time, metric.value, metric.values); // send back value for this timeslot + callback(metric); // send back value for this timeslot if (time < metric.time) compute(time, metric.time); // recurse from last value seen up to this timeslot time = tier.step(metric.time); // update the last-observed timeslot } else { @@ -116,17 +117,19 @@ function findOrComputeUnary(db, options, callback) { // Group metrics from the next tier. function computePyramidal(start, stop) { var bins = {}, - measurement = new Measurement(expression, start, stop, tier.next); - // metalog.warn('computePyramidal', { expr: expression.source, start: start, stop: stop, tier: tier.key, tr: tr }); + query_measurement = new Measurement(expression, start, stop, tier.next); - find(measurement, function(time, value, values) { + find(query_measurement, function(metric) { + var value = metric.value, time = metric.time, values = metric.values; var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); if (reduce.pyramidal) bin.values.push(value); else bin.values = bin.values.concat(values||[]); if (!--bin.size) { - callback(time, reduce(bin.values), bin.values); + var metric = new Metric(time, reduce(bin.values), measurement, bin.values); + if (metric.value || metric.value === 0) metric.save(db, handle); + callback(metric); delete bins[time]; } }); @@ -136,8 +139,6 @@ function findOrComputeUnary(db, options, callback) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { - // metalog.warn('computeFlat', { expr: expression.source, start: start, stop: stop, tier: tier.key }); - // Reset start time to calculation horizon if requested time span goes past it if (options.horizons && tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ var old_start = start, @@ -152,23 +153,29 @@ function findOrComputeUnary(db, options, callback) { metalog.info('cube_compute', {is: 'past_horizon', metric: {start: start, stop: {was: old_stop, updated_to: stop}, tier: tier, expression: expression.source } }); } - var measurement = new Measurement(expression, start, stop, tier); + var query_measurement = new Measurement(expression, start, stop, tier); var time = start, values = []; - Event.find(db, measurement, function(error, event) { + function flat_callback(time, value, values){ + var metric = new Metric(time, value, measurement, values); + callback(metric); + if (metric.value || metric.value === 0) metric.save(db, handle); + } + + Event.find(db, query_measurement, function(error, event) { handle(error); if (event) { var then = tier.floor(event.time); if (time < then) { - callback(time, (values.length ? reduce(values) : reduce.empty), values); - while ((time = tier.step(time)) < then) callback(time, reduce.empty); + flat_callback(time, (values.length ? reduce(values) : reduce.empty), values); + while ((time = tier.step(time)) < then) flat_callback(time, reduce.empty); values = [map(event.to_wire())]; } else { values.push(map(event.to_wire())); } } else { - callback(time, (values.length ? reduce(values) : reduce.empty), values); - while ((time = tier.step(time)) < stop) callback(time, reduce.empty); + flat_callback(time, (values.length ? reduce(values) : reduce.empty), values); + while ((time = tier.step(time)) < stop) flat_callback(time, reduce.empty); } }); } @@ -207,7 +214,7 @@ function binary(db, options, callback) { left.measure(db, options, function(metric) { var time = metric.time, value = metric.value; if (time in right) { // right val already appeared; get a result - callback(time, expression.op(value, right[time])); + callback(new Metric(time, expression.op(value, right[time]))); delete right[time]; } else { // right val still on the way; stash the value left[time] = value; @@ -217,7 +224,7 @@ function binary(db, options, callback) { right.measure(db, options, function(metric) { var time = metric.time, value = metric.value; if (time in left) { - callback(time, expression.op(left[time], value)); + callback(new Metric(time, expression.op(left[time], value))); delete left[time]; } else { right[time] = value; From 59ec91d9ffd192359b5520455310048566a7a1fc Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 9 Nov 2012 15:31:13 -0600 Subject: [PATCH 62/87] Events only need validated once. Event putter and event.save were both validating. Now only validate on save. --- lib/cube/event.js | 2 +- lib/cube/models/event.js | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cube/event.js b/lib/cube/event.js index d33c222c..98e3c492 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -49,7 +49,6 @@ exports.putter = function(db, config){ } var event = new Event(type, time, request.data, request.id); - try{ event.validate(); } catch(err) { return callback({error: err}), -1; } // Save the event, then queue invalidation of its associated cached metrics. // @@ -59,6 +58,7 @@ exports.putter = function(db, config){ // the likelihood of a race condition between when the events are read by // the evaluator and when the newly-computed metrics are saved. event.save(db, function after_save(error, event){ + if (error) return callback({error: error}); if (event) invalidator.add(event.type, event); callback(event); }); diff --git a/lib/cube/models/event.js b/lib/cube/models/event.js index a6c5aa8c..1d02bf4d 100644 --- a/lib/cube/models/event.js +++ b/lib/cube/models/event.js @@ -94,7 +94,10 @@ Object.defineProperty(Event, "find", { value: find }); function save(db, callback){ var self = this; - if (this.validate) this.validate(); + if (this.validate) { + try{ this.validate(); } + catch(error) { return callback(error); } + } db.events(self.type, function event_saver(error, collection){ if (error) return callback(error); From b03010248e9ba748436f5c931f45c324760c7b05 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Fri, 30 Nov 2012 11:03:03 -0600 Subject: [PATCH 63/87] Add grouping to metric queries. --- lib/cube/metric-expression.js | 5140 +++++++++++++------------------- lib/cube/metric-expression.peg | 5 +- lib/cube/metric.js | 4 +- lib/cube/models/event.js | 51 +- lib/cube/models/measurement.js | 195 +- lib/cube/models/metric.js | 26 +- 6 files changed, 2342 insertions(+), 3079 deletions(-) diff --git a/lib/cube/metric-expression.js b/lib/cube/metric-expression.js index c0e15b8d..ebd8a1c5 100644 --- a/lib/cube/metric-expression.js +++ b/lib/cube/metric-expression.js @@ -1,5 +1,32 @@ module.exports = (function(){ - /* Generated by PEG.js 0.6.2 (http://pegjs.majda.cz/). */ + /* + * Generated by PEG.js 0.7.0. + * + * http://pegjs.majda.cz/ + */ + + function quote(s) { + /* + * ECMA-262, 5th ed., 7.8.4: All characters may appear literally in a + * string literal except for the closing quote character, backslash, + * carriage return, line separator, paragraph separator, and line feed. + * Any character may appear in the form of an escape sequence. + * + * For portability, we also escape escape all control and non-ASCII + * characters. Note that "\0" and "\v" escape sequences are not used + * because JSHint does not like the first and IE the second. + */ + return '"' + s + .replace(/\\/g, '\\\\') // backslash + .replace(/"/g, '\\"') // closing quote character + .replace(/\x08/g, '\\b') // backspace + .replace(/\t/g, '\\t') // horizontal tab + .replace(/\n/g, '\\n') // line feed + .replace(/\f/g, '\\f') // form feed + .replace(/\r/g, '\\r') // carriage return + .replace(/[\x00-\x07\x0B\x0E-\x1F\x80-\uFFFF]/g, escape) + + '"'; + } var result = { /* @@ -10,47 +37,48 @@ module.exports = (function(){ */ parse: function(input, startRule) { var parseFunctions = { - "_": parse__, - "additive_operator": parse_additive_operator, - "array_literal": parse_array_literal, - "character_escape_sequence": parse_character_escape_sequence, - "digit": parse_digit, - "digit19": parse_digit19, - "digits": parse_digits, - "double_string_char": parse_double_string_char, - "e": parse_e, - "escape_character": parse_escape_character, - "escape_sequence": parse_escape_sequence, - "event_additive_expression": parse_event_additive_expression, + "start": parse_start, + "metric_additive_expression": parse_metric_additive_expression, + "metric_multiplicative_expression": parse_metric_multiplicative_expression, + "metric_unary_expression": parse_metric_unary_expression, + "metric_primary_expression": parse_metric_primary_expression, + "metric_group_expression": parse_metric_group_expression, "event_expression": parse_event_expression, "event_filter_expression": parse_event_filter_expression, - "event_member_expression": parse_event_member_expression, + "event_value_expression": parse_event_value_expression, + "event_additive_expression": parse_event_additive_expression, "event_multiplicative_expression": parse_event_multiplicative_expression, - "event_primary_expression": parse_event_primary_expression, "event_unary_expression": parse_event_unary_expression, - "event_value_expression": parse_event_value_expression, - "exp": parse_exp, + "event_primary_expression": parse_event_primary_expression, + "event_member_expression": parse_event_member_expression, + "additive_operator": parse_additive_operator, + "multiplicative_operator": parse_multiplicative_operator, "filter_operator": parse_filter_operator, - "frac": parse_frac, - "hex_digit": parse_hex_digit, - "hex_escape_sequence": parse_hex_escape_sequence, + "reduce": parse_reduce, + "type": parse_type, "identifier": parse_identifier, - "int": parse_int, "literal": parse_literal, - "metric_additive_expression": parse_metric_additive_expression, - "metric_multiplicative_expression": parse_metric_multiplicative_expression, - "metric_primary_expression": parse_metric_primary_expression, - "metric_unary_expression": parse_metric_unary_expression, - "multiplicative_operator": parse_multiplicative_operator, - "non_escape_character": parse_non_escape_character, - "number": parse_number, - "reduce": parse_reduce, - "single_escape_character": parse_single_escape_character, - "single_string_char": parse_single_string_char, - "start": parse_start, + "array_literal": parse_array_literal, "string": parse_string, - "type": parse_type, + "double_string_char": parse_double_string_char, + "single_string_char": parse_single_string_char, + "escape_sequence": parse_escape_sequence, + "character_escape_sequence": parse_character_escape_sequence, + "single_escape_character": parse_single_escape_character, + "non_escape_character": parse_non_escape_character, + "escape_character": parse_escape_character, + "hex_escape_sequence": parse_hex_escape_sequence, "unicode_escape_sequence": parse_unicode_escape_sequence, + "number": parse_number, + "int": parse_int, + "frac": parse_frac, + "exp": parse_exp, + "digits": parse_digits, + "e": parse_e, + "digit": parse_digit, + "digit19": parse_digit19, + "hex_digit": parse_hex_digit, + "_": parse__, "whitespace": parse_whitespace }; @@ -63,10 +91,9 @@ module.exports = (function(){ } var pos = 0; - var reportMatchFailures = true; - var rightmostMatchFailuresPos = 0; - var rightmostMatchFailuresExpected = []; - var cache = {}; + var reportFailures = 0; + var rightmostFailuresPos = 0; + var rightmostFailuresExpected = []; function padLeft(input, padding, length) { var result = input; @@ -81,3769 +108,2960 @@ module.exports = (function(){ function escape(ch) { var charCode = ch.charCodeAt(0); + var escapeChar; + var length; if (charCode <= 0xFF) { - var escapeChar = 'x'; - var length = 2; + escapeChar = 'x'; + length = 2; } else { - var escapeChar = 'u'; - var length = 4; + escapeChar = 'u'; + length = 4; } return '\\' + escapeChar + padLeft(charCode.toString(16).toUpperCase(), '0', length); } - function quote(s) { - /* - * ECMA-262, 5th ed., 7.8.4: All characters may appear literally in a - * string literal except for the closing quote character, backslash, - * carriage return, line separator, paragraph separator, and line feed. - * Any character may appear in the form of an escape sequence. - */ - return '"' + s - .replace(/\\/g, '\\\\') // backslash - .replace(/"/g, '\\"') // closing quote character - .replace(/\r/g, '\\r') // carriage return - .replace(/\n/g, '\\n') // line feed - .replace(/[\x80-\uFFFF]/g, escape) // non-ASCII characters - + '"'; - } - function matchFailed(failure) { - if (pos < rightmostMatchFailuresPos) { + if (pos < rightmostFailuresPos) { return; } - if (pos > rightmostMatchFailuresPos) { - rightmostMatchFailuresPos = pos; - rightmostMatchFailuresExpected = []; + if (pos > rightmostFailuresPos) { + rightmostFailuresPos = pos; + rightmostFailuresExpected = []; } - rightmostMatchFailuresExpected.push(failure); + rightmostFailuresExpected.push(failure); } function parse_start() { - var cacheKey = 'start@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse__(); - if (result3 !== null) { - var result4 = parse_metric_additive_expression(); - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - var result1 = [result3, result4, result5]; + var result0, result1, result2; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse__(); + if (result0 !== null) { + result1 = parse_metric_additive_expression(); + if (result1 !== null) { + result2 = parse__(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(expression) { return expression; })(result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, expression) { return expression; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_metric_additive_expression() { - var cacheKey = 'metric_additive_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_metric_multiplicative_expression(); - if (result3 !== null) { - var result4 = []; - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_additive_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_metric_additive_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + var result0, result1, result2, result3, result4, result5; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_metric_multiplicative_expression(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_additive_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_metric_additive_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; - } - while (result5 !== null) { - result4.push(result5); - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_additive_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_metric_additive_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + result2 = null; + pos = pos2; + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_additive_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_metric_additive_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(head, tail) { return compoundMetric(head, tail); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, head, tail) { return compoundMetric(head, tail); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_metric_multiplicative_expression() { - var cacheKey = 'metric_multiplicative_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_metric_unary_expression(); - if (result3 !== null) { - var result4 = []; - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_multiplicative_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_metric_multiplicative_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + var result0, result1, result2, result3, result4, result5; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_metric_unary_expression(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_multiplicative_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_metric_multiplicative_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; - } - while (result5 !== null) { - result4.push(result5); - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_multiplicative_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_metric_multiplicative_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + result2 = null; + pos = pos2; + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_multiplicative_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_metric_multiplicative_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(head, tail) { return compoundMetric(head, tail); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, head, tail) { return compoundMetric(head, tail); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_metric_unary_expression() { - var cacheKey = 'metric_unary_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "-") { - var result5 = "-"; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 45) { + result0 = "-"; + pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"-\""); } } - if (result5 !== null) { - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_metric_unary_expression(); - if (result7 !== null) { - var result3 = [result5, result6, result7]; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_metric_unary_expression(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result3 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result3 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result3 = null; - pos = savedPos1; - } - var result4 = result3 !== null - ? (function(expression) { var value = expression.value; expression.value = function(o) { return -value(o); }; if (expression.source) expression.source = "-" + expression.source; return expression; })(result3[2]) - : null; - if (result4 !== null) { - var result2 = result4; - } else { - var result2 = null; - pos = savedPos0; + result0 = null; + pos = pos1; } - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_metric_primary_expression(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; + if (result0 !== null) { + result0 = (function(offset, expression) { var value = expression.value; expression.value = function(o) { return -value(o); }; if (expression.source) expression.source = "-" + expression.source; return expression; })(pos0, result0[2]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + result0 = parse_metric_primary_expression(); } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_metric_primary_expression() { - var cacheKey = 'metric_primary_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos3 = pos; - var savedPos4 = pos; - var result15 = parse_reduce(); - if (result15 !== null) { - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === "(") { - var result17 = "("; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6, result7, result8, result9, result10; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_reduce(); + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 40) { + result2 = "("; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_event_expression(); - if (result19 !== null) { - var result20 = parse__(); - if (result20 !== null) { - if (input.substr(pos, 1) === ")") { - var result21 = ")"; - pos += 1; + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + result4 = parse_event_expression(); + if (result4 !== null) { + result5 = parse__(); + if (result5 !== null) { + if (input.charCodeAt(pos) === 41) { + result6 = ")"; + pos++; } else { - var result21 = null; - if (reportMatchFailures) { + result6 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result21 !== null) { - var result13 = [result15, result16, result17, result18, result19, result20, result21]; + if (result6 !== null) { + pos2 = pos; + result7 = parse__(); + if (result7 !== null) { + if (input.charCodeAt(pos) === 46) { + result8 = "."; + pos++; + } else { + result8 = null; + if (reportFailures === 0) { + matchFailed("\".\""); + } + } + if (result8 !== null) { + result9 = parse__(); + if (result9 !== null) { + result10 = parse_metric_group_expression(); + if (result10 !== null) { + result7 = [result7, result8, result9, result10]; + } else { + result7 = null; + pos = pos2; + } + } else { + result7 = null; + pos = pos2; + } + } else { + result7 = null; + pos = pos2; + } + } else { + result7 = null; + pos = pos2; + } + result7 = result7 !== null ? result7 : ""; + if (result7 !== null) { + result0 = [result0, result1, result2, result3, result4, result5, result6, result7]; + } else { + result0 = null; + pos = pos1; + } } else { - var result13 = null; - pos = savedPos4; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos4; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos4; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos4; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos4; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos4; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos4; - } - var result14 = result13 !== null - ? (function(reduce, event) { event.reduce = reduce; event.source = input.substring(savedPos3, pos); return event; })(result13[0], result13[4]) - : null; - if (result14 !== null) { - var result12 = result14; - } else { - var result12 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } - if (result12 !== null) { - var result0 = result12; - } else { - var savedPos2 = pos; - var result10 = parse_number(); - var result11 = result10 !== null - ? (function(value) { return {value: function() { return value; }}; })(result10) - : null; - if (result11 !== null) { - var result9 = result11; - } else { - var result9 = null; - pos = savedPos2; + if (result0 !== null) { + result0 = (function(offset, reduce, event, group) { event.reduce = reduce; event.source = input.substring(pos0, pos); if(group) event.group = group[3]; return event; })(pos0, result0[0], result0[4], result0[7]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_number(); + if (result0 !== null) { + result0 = (function(offset, value) { return {value: function() { return value; }}; })(pos0, result0); } - if (result9 !== null) { - var result0 = result9; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "(") { - var result4 = "("; - pos += 1; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 40) { + result0 = "("; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - var result6 = parse_metric_additive_expression(); - if (result6 !== null) { - var result7 = parse__(); - if (result7 !== null) { - if (input.substr(pos, 1) === ")") { - var result8 = ")"; - pos += 1; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_metric_additive_expression(); + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + if (input.charCodeAt(pos) === 41) { + result4 = ")"; + pos++; } else { - var result8 = null; - if (reportMatchFailures) { + result4 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result8 !== null) { - var result2 = [result4, result5, result6, result7, result8]; + if (result4 !== null) { + result0 = [result0, result1, result2, result3, result4]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } - var result3 = result2 !== null - ? (function(expression) { return expression; })(result2[2]) - : null; - if (result3 !== null) { - var result1 = result3; + if (result0 !== null) { + result0 = (function(offset, expression) { return expression; })(pos0, result0[2]); + } + if (result0 === null) { + pos = pos0; + } + } + } + return result0; + } + + function parse_metric_group_expression() { + var result0, result1, result2, result3, result4, result5, result6; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + if (input.substr(pos, 5) === "group") { + result0 = "group"; + pos += 5; + } else { + result0 = null; + if (reportFailures === 0) { + matchFailed("\"group\""); + } + } + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 40) { + result2 = "("; + pos++; } else { - var result1 = null; - pos = savedPos0; + result2 = null; + if (reportFailures === 0) { + matchFailed("\"(\""); + } } - if (result1 !== null) { - var result0 = result1; + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + result4 = parse_event_member_expression(); + if (result4 !== null) { + result5 = parse__(); + if (result5 !== null) { + if (input.charCodeAt(pos) === 41) { + result6 = ")"; + pos++; + } else { + result6 = null; + if (reportFailures === 0) { + matchFailed("\")\""); + } + } + if (result6 !== null) { + result0 = [result0, result1, result2, result3, result4, result5, result6]; + } else { + result0 = null; + pos = pos1; + } + } else { + result0 = null; + pos = pos1; + } + } else { + result0 = null; + pos = pos1; + } + } else { + result0 = null; + pos = pos1; + } } else { - var result0 = null;; - }; - }; + result0 = null; + pos = pos1; + } + } else { + result0 = null; + pos = pos1; + } + } else { + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, member) { return member; })(pos0, result0[4]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_expression() { - var cacheKey = 'event_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_event_value_expression(); - if (result3 !== null) { - var result4 = []; - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - if (input.substr(pos, 1) === ".") { - var result7 = "."; - pos += 1; + var result0, result1, result2, result3, result4, result5; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_event_value_expression(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result7 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_filter_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_filter_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; - } - while (result5 !== null) { - result4.push(result5); - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - if (input.substr(pos, 1) === ".") { - var result7 = "."; - pos += 1; + result2 = null; + pos = pos2; + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result7 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_filter_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_filter_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(value, filters) { - value.filter = function(filter) { - var i = -1, n = filters.length; - while (++i < n) filters[i][3](filter); - value.exists(filter); - }; - return value; - })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, value, filters) { + value.filter = function(filter) { + var i = -1, n = filters.length; + while (++i < n) filters[i][3](filter); + value.exists(filter); + }; + return value; + })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_filter_expression() { - var cacheKey = 'event_filter_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_filter_operator(); - if (result3 !== null) { - var result4 = parse__(); - if (result4 !== null) { - if (input.substr(pos, 1) === "(") { - var result5 = "("; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6, result7, result8, result9, result10; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_filter_operator(); + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 40) { + result2 = "("; + pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result5 !== null) { - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_event_member_expression(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - if (input.substr(pos, 1) === ",") { - var result9 = ","; - pos += 1; + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + result4 = parse_event_member_expression(); + if (result4 !== null) { + result5 = parse__(); + if (result5 !== null) { + if (input.charCodeAt(pos) === 44) { + result6 = ","; + pos++; } else { - var result9 = null; - if (reportMatchFailures) { + result6 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result9 !== null) { - var result10 = parse__(); - if (result10 !== null) { - var result11 = parse_literal(); - if (result11 !== null) { - var result12 = parse__(); - if (result12 !== null) { - if (input.substr(pos, 1) === ")") { - var result13 = ")"; - pos += 1; + if (result6 !== null) { + result7 = parse__(); + if (result7 !== null) { + result8 = parse_literal(); + if (result8 !== null) { + result9 = parse__(); + if (result9 !== null) { + if (input.charCodeAt(pos) === 41) { + result10 = ")"; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result10 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result13 !== null) { - var result1 = [result3, result4, result5, result6, result7, result8, result9, result10, result11, result12, result13]; + if (result10 !== null) { + result0 = [result0, result1, result2, result3, result4, result5, result6, result7, result8, result9, result10]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(op, member, value) { return function(o) { op(o, member.field, value); }; })(result1[0], result1[4], result1[8]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, op, member, value) { return function(o) { op(o, member.field, value); }; })(pos0, result0[0], result0[4], result0[8]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_value_expression() { - var cacheKey = 'event_value_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos1 = pos; - var savedPos2 = pos; - var result7 = parse_type(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - if (input.substr(pos, 1) === "(") { - var result9 = "("; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_type(); + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 40) { + result2 = "("; + pos++; } else { - var result9 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result9 !== null) { - var result10 = parse__(); - if (result10 !== null) { - var result11 = parse_event_additive_expression(); - if (result11 !== null) { - var result12 = parse__(); - if (result12 !== null) { - if (input.substr(pos, 1) === ")") { - var result13 = ")"; - pos += 1; + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + result4 = parse_event_additive_expression(); + if (result4 !== null) { + result5 = parse__(); + if (result5 !== null) { + if (input.charCodeAt(pos) === 41) { + result6 = ")"; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result6 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result13 !== null) { - var result5 = [result7, result8, result9, result10, result11, result12, result13]; + if (result6 !== null) { + result0 = [result0, result1, result2, result3, result4, result5, result6]; } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; - } - var result6 = result5 !== null - ? (function(type, value) { value.type = type; return value; })(result5[0], result5[4]) - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; - var result2 = parse_type(); - var result3 = result2 !== null - ? (function(type) { return {type: type, value: one, exists: noop, fields: noop}; })(result2) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, type, value) { value.type = type; return value; })(pos0, result0[0], result0[4]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_type(); + if (result0 !== null) { + result0 = (function(offset, type) { return {type: type, value: one, exists: noop, fields: noop}; })(pos0, result0); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_additive_expression() { - var cacheKey = 'event_additive_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_event_multiplicative_expression(); - if (result3 !== null) { - var result4 = []; - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_additive_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_additive_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + var result0, result1, result2, result3, result4, result5; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_event_multiplicative_expression(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_additive_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_additive_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; - } - while (result5 !== null) { - result4.push(result5); - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_additive_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_additive_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + result2 = null; + pos = pos2; + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_additive_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_additive_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(head, tail) { return compoundValue(head, tail); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, head, tail) { return compoundValue(head, tail); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_multiplicative_expression() { - var cacheKey = 'event_multiplicative_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_event_unary_expression(); - if (result3 !== null) { - var result4 = []; - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_multiplicative_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_multiplicative_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + var result0, result1, result2, result3, result4, result5; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_event_unary_expression(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_multiplicative_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_multiplicative_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; - } - while (result5 !== null) { - result4.push(result5); - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_multiplicative_operator(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_multiplicative_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + result2 = null; + pos = pos2; + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + result3 = parse_multiplicative_operator(); + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_multiplicative_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(head, tail) { return compoundValue(head, tail); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, head, tail) { return compoundValue(head, tail); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_unary_expression() { - var cacheKey = 'event_unary_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result7 = parse_event_primary_expression(); - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "-") { - var result4 = "-"; - pos += 1; + var result0, result1, result2; + var pos0, pos1; + + result0 = parse_event_primary_expression(); + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 45) { + result0 = "-"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"-\""); } } - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - var result6 = parse_event_unary_expression(); - if (result6 !== null) { - var result2 = [result4, result5, result6]; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_event_unary_expression(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(unary) { return {value: function(o) { return -unary.value(o); }, exists: unary.exists, fields: unary.fields}; })(result2[2]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, unary) { return {value: function(o) { return -unary.value(o); }, exists: unary.exists, fields: unary.fields}; })(pos0, result0[2]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_primary_expression() { - var cacheKey = 'event_primary_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result12 = parse_event_member_expression(); - if (result12 !== null) { - var result0 = result12; - } else { - var savedPos2 = pos; - var result10 = parse_number(); - var result11 = result10 !== null - ? (function(number) { return {value: function() { return number; }, exists: noop, fields: noop}; })(result10) - : null; - if (result11 !== null) { - var result9 = result11; - } else { - var result9 = null; - pos = savedPos2; - } - if (result9 !== null) { - var result0 = result9; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "(") { - var result4 = "("; - pos += 1; + var result0, result1, result2, result3, result4; + var pos0, pos1; + + result0 = parse_event_member_expression(); + if (result0 === null) { + pos0 = pos; + result0 = parse_number(); + if (result0 !== null) { + result0 = (function(offset, number) { return {value: function() { return number; }, exists: noop, fields: noop}; })(pos0, result0); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 40) { + result0 = "("; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - var result6 = parse_event_additive_expression(); - if (result6 !== null) { - var result7 = parse__(); - if (result7 !== null) { - if (input.substr(pos, 1) === ")") { - var result8 = ")"; - pos += 1; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_event_additive_expression(); + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + if (input.charCodeAt(pos) === 41) { + result4 = ")"; + pos++; } else { - var result8 = null; - if (reportMatchFailures) { + result4 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result8 !== null) { - var result2 = [result4, result5, result6, result7, result8]; + if (result4 !== null) { + result0 = [result0, result1, result2, result3, result4]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } - var result3 = result2 !== null - ? (function(expression) { return expression; })(result2[2]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, expression) { return expression; })(pos0, result0[2]); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; + if (result0 === null) { + pos = pos0; + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_member_expression() { - var cacheKey = 'event_member_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_identifier(); - if (result3 !== null) { - var result4 = []; - var savedPos4 = pos; - var savedPos5 = pos; - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === "[") { - var result17 = "["; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6, result7; + var pos0, pos1, pos2, pos3; + + pos0 = pos; + pos1 = pos; + result0 = parse_identifier(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 91) { + result3 = "["; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_number(); - if (result19 !== null) { - var result20 = parse__(); - if (result20 !== null) { - if (input.substr(pos, 1) === "]") { - var result21 = "]"; - pos += 1; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_number(); + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 93) { + result7 = "]"; + pos++; } else { - var result21 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result21 !== null) { - var result14 = [result16, result17, result18, result19, result20, result21]; + if (result7 !== null) { + result2 = [result2, result3, result4, result5, result6, result7]; } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; - } - var result15 = result14 !== null - ? (function(name) { return arrayAccessor(name); })(result14[3]) - : null; - if (result15 !== null) { - var result13 = result15; - } else { - var result13 = null; - pos = savedPos4; + result2 = null; + pos = pos3; } - if (result13 !== null) { - var result5 = result13; - } else { - var savedPos2 = pos; - var savedPos3 = pos; - var result9 = parse__(); - if (result9 !== null) { - if (input.substr(pos, 1) === ".") { - var result10 = "."; - pos += 1; + if (result2 !== null) { + result2 = (function(offset, name) { return arrayAccessor(name); })(pos2, result2[3]); + } + if (result2 === null) { + pos = pos2; + } + if (result2 === null) { + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result10 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result10 !== null) { - var result11 = parse__(); - if (result11 !== null) { - var result12 = parse_identifier(); - if (result12 !== null) { - var result7 = [result9, result10, result11, result12]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_identifier(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } - var result8 = result7 !== null - ? (function(name) { return objectAccessor(name); })(result7[3]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + if (result2 !== null) { + result2 = (function(offset, name) { return objectAccessor(name); })(pos2, result2[3]); } - if (result6 !== null) { - var result5 = result6; - } else { - var result5 = null;; - }; - } - while (result5 !== null) { - result4.push(result5); - var savedPos4 = pos; - var savedPos5 = pos; - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === "[") { - var result17 = "["; - pos += 1; + if (result2 === null) { + pos = pos2; + } + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 91) { + result3 = "["; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_number(); - if (result19 !== null) { - var result20 = parse__(); - if (result20 !== null) { - if (input.substr(pos, 1) === "]") { - var result21 = "]"; - pos += 1; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_number(); + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 93) { + result7 = "]"; + pos++; } else { - var result21 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result21 !== null) { - var result14 = [result16, result17, result18, result19, result20, result21]; + if (result7 !== null) { + result2 = [result2, result3, result4, result5, result6, result7]; } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } - var result15 = result14 !== null - ? (function(name) { return arrayAccessor(name); })(result14[3]) - : null; - if (result15 !== null) { - var result13 = result15; - } else { - var result13 = null; - pos = savedPos4; + if (result2 !== null) { + result2 = (function(offset, name) { return arrayAccessor(name); })(pos2, result2[3]); } - if (result13 !== null) { - var result5 = result13; - } else { - var savedPos2 = pos; - var savedPos3 = pos; - var result9 = parse__(); - if (result9 !== null) { - if (input.substr(pos, 1) === ".") { - var result10 = "."; - pos += 1; + if (result2 === null) { + pos = pos2; + } + if (result2 === null) { + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result10 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result10 !== null) { - var result11 = parse__(); - if (result11 !== null) { - var result12 = parse_identifier(); - if (result12 !== null) { - var result7 = [result9, result10, result11, result12]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_identifier(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } - var result8 = result7 !== null - ? (function(name) { return objectAccessor(name); })(result7[3]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + if (result2 !== null) { + result2 = (function(offset, name) { return objectAccessor(name); })(pos2, result2[3]); + } + if (result2 === null) { + pos = pos2; } - if (result6 !== null) { - var result5 = result6; - } else { - var result5 = null;; - }; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(head, tail) { return member(head, tail); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, head, tail) { return member(head, tail); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_additive_operator() { - var cacheKey = 'additive_operator@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0; + var pos0; - - var savedPos1 = pos; - if (input.substr(pos, 1) === "+") { - var result5 = "+"; - pos += 1; + pos0 = pos; + if (input.charCodeAt(pos) === 43) { + result0 = "+"; + pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"+\""); } } - var result6 = result5 !== null - ? (function() { return add; })() - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 !== null) { + result0 = (function(offset) { return add; })(pos0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; - if (input.substr(pos, 1) === "-") { - var result2 = "-"; - pos += 1; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + if (input.charCodeAt(pos) === 45) { + result0 = "-"; + pos++; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"-\""); } } - var result3 = result2 !== null - ? (function() { return subtract; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset) { return subtract; })(pos0); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_multiplicative_operator() { - var cacheKey = 'multiplicative_operator@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; + var pos0; - var savedPos1 = pos; - if (input.substr(pos, 1) === "*") { - var result5 = "*"; - pos += 1; + pos0 = pos; + if (input.charCodeAt(pos) === 42) { + result0 = "*"; + pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"*\""); } } - var result6 = result5 !== null - ? (function() { return multiply; })() - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 !== null) { + result0 = (function(offset) { return multiply; })(pos0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; - if (input.substr(pos, 1) === "/") { - var result2 = "/"; - pos += 1; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + if (input.charCodeAt(pos) === 47) { + result0 = "/"; + pos++; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"/\""); } } - var result3 = result2 !== null - ? (function() { return divide; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset) { return divide; })(pos0); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_filter_operator() { - var cacheKey = 'filter_operator@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0; + var pos0; - - var savedPos7 = pos; + pos0 = pos; if (input.substr(pos, 2) === "eq") { - var result23 = "eq"; + result0 = "eq"; pos += 2; } else { - var result23 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"eq\""); } } - var result24 = result23 !== null - ? (function() { return filterEqual; })() - : null; - if (result24 !== null) { - var result22 = result24; - } else { - var result22 = null; - pos = savedPos7; + if (result0 !== null) { + result0 = (function(offset) { return filterEqual; })(pos0); } - if (result22 !== null) { - var result0 = result22; - } else { - var savedPos6 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "gt") { - var result20 = "gt"; + result0 = "gt"; pos += 2; } else { - var result20 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"gt\""); } } - var result21 = result20 !== null - ? (function() { return filterGreater; })() - : null; - if (result21 !== null) { - var result19 = result21; - } else { - var result19 = null; - pos = savedPos6; + if (result0 !== null) { + result0 = (function(offset) { return filterGreater; })(pos0); } - if (result19 !== null) { - var result0 = result19; - } else { - var savedPos5 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "ge") { - var result17 = "ge"; + result0 = "ge"; pos += 2; } else { - var result17 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"ge\""); } } - var result18 = result17 !== null - ? (function() { return filterGreaterOrEqual; })() - : null; - if (result18 !== null) { - var result16 = result18; - } else { - var result16 = null; - pos = savedPos5; + if (result0 !== null) { + result0 = (function(offset) { return filterGreaterOrEqual; })(pos0); } - if (result16 !== null) { - var result0 = result16; - } else { - var savedPos4 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "lt") { - var result14 = "lt"; + result0 = "lt"; pos += 2; } else { - var result14 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"lt\""); } } - var result15 = result14 !== null - ? (function() { return filterLess; })() - : null; - if (result15 !== null) { - var result13 = result15; - } else { - var result13 = null; - pos = savedPos4; + if (result0 !== null) { + result0 = (function(offset) { return filterLess; })(pos0); } - if (result13 !== null) { - var result0 = result13; - } else { - var savedPos3 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "le") { - var result11 = "le"; + result0 = "le"; pos += 2; } else { - var result11 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"le\""); } } - var result12 = result11 !== null - ? (function() { return filterLessOrEqual; })() - : null; - if (result12 !== null) { - var result10 = result12; - } else { - var result10 = null; - pos = savedPos3; + if (result0 !== null) { + result0 = (function(offset) { return filterLessOrEqual; })(pos0); } - if (result10 !== null) { - var result0 = result10; - } else { - var savedPos2 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "ne") { - var result8 = "ne"; + result0 = "ne"; pos += 2; } else { - var result8 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"ne\""); } } - var result9 = result8 !== null - ? (function() { return filterNotEqual; })() - : null; - if (result9 !== null) { - var result7 = result9; - } else { - var result7 = null; - pos = savedPos2; + if (result0 !== null) { + result0 = (function(offset) { return filterNotEqual; })(pos0); } - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos1 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "re") { - var result5 = "re"; + result0 = "re"; pos += 2; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"re\""); } } - var result6 = result5 !== null - ? (function() { return filterRegularExpression; })() - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 !== null) { + result0 = (function(offset) { return filterRegularExpression; })(pos0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "in") { - var result2 = "in"; + result0 = "in"; pos += 2; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"in\""); } } - var result3 = result2 !== null - ? (function() { return filterIn; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset) { return filterIn; })(pos0); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; - }; - }; - }; + if (result0 === null) { + pos = pos0; + } + } + } + } + } + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_reduce() { - var cacheKey = 'reduce@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; if (input.substr(pos, 3) === "sum") { - var result5 = "sum"; + result0 = "sum"; pos += 3; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"sum\""); } } - if (result5 !== null) { - var result0 = result5; - } else { + if (result0 === null) { if (input.substr(pos, 3) === "min") { - var result4 = "min"; + result0 = "min"; pos += 3; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"min\""); } } - if (result4 !== null) { - var result0 = result4; - } else { + if (result0 === null) { if (input.substr(pos, 3) === "max") { - var result3 = "max"; + result0 = "max"; pos += 3; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"max\""); } } - if (result3 !== null) { - var result0 = result3; - } else { + if (result0 === null) { if (input.substr(pos, 8) === "distinct") { - var result2 = "distinct"; + result0 = "distinct"; pos += 8; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"distinct\""); } } - if (result2 !== null) { - var result0 = result2; - } else { + if (result0 === null) { if (input.substr(pos, 6) === "median") { - var result1 = "median"; + result0 = "median"; pos += 6; } else { - var result1 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"median\""); } } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; + } + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_type() { - var cacheKey = 'type@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos).match(/^[a-z]/) !== null) { - var result3 = input.charAt(pos); + pos0 = pos; + pos1 = pos; + if (/^[a-z]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[a-z]"); } } - if (result3 !== null) { - if (input.substr(pos).match(/^[a-zA-Z0-9_]/) !== null) { - var result5 = input.charAt(pos); + if (result0 !== null) { + if (/^[a-zA-Z0-9_]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_]"); } } - if (result5 !== null) { - var result4 = []; - while (result5 !== null) { - result4.push(result5); - if (input.substr(pos).match(/^[a-zA-Z0-9_]/) !== null) { - var result5 = input.charAt(pos); + if (result2 !== null) { + result1 = []; + while (result2 !== null) { + result1.push(result2); + if (/^[a-zA-Z0-9_]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_]"); } } } } else { - var result4 = null; + result1 = null; } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(first, rest) { return first + rest.join(""); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, first, rest) { return first + rest.join(""); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_identifier() { - var cacheKey = 'identifier@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1, result2; + var pos0, pos1; - - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos).match(/^[a-zA-Z_]/) !== null) { - var result3 = input.charAt(pos); + pos0 = pos; + pos1 = pos; + if (/^[a-zA-Z_]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z_]"); } } - if (result3 !== null) { - var result4 = []; - if (input.substr(pos).match(/^[a-zA-Z0-9_$]/) !== null) { - var result5 = input.charAt(pos); + if (result0 !== null) { + result1 = []; + if (/^[a-zA-Z0-9_$]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_$]"); } } - while (result5 !== null) { - result4.push(result5); - if (input.substr(pos).match(/^[a-zA-Z0-9_$]/) !== null) { - var result5 = input.charAt(pos); + while (result2 !== null) { + result1.push(result2); + if (/^[a-zA-Z0-9_$]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_$]"); } } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(first, rest) { return first + rest.join(""); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, first, rest) { return first + rest.join(""); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_literal() { - var cacheKey = 'literal@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result9 = parse_array_literal(); - if (result9 !== null) { - var result0 = result9; - } else { - var result8 = parse_string(); - if (result8 !== null) { - var result0 = result8; - } else { - var result7 = parse_number(); - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos1 = pos; + var result0; + var pos0; + + result0 = parse_array_literal(); + if (result0 === null) { + result0 = parse_string(); + if (result0 === null) { + result0 = parse_number(); + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 4) === "true") { - var result5 = "true"; + result0 = "true"; pos += 4; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"true\""); } } - var result6 = result5 !== null - ? (function() { return true; })() - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 !== null) { + result0 = (function(offset) { return true; })(pos0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 5) === "false") { - var result2 = "false"; + result0 = "false"; pos += 5; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"false\""); } } - var result3 = result2 !== null - ? (function() { return false; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset) { return false; })(pos0); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; + if (result0 === null) { + pos = pos0; + } + } + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_array_literal() { - var cacheKey = 'array_literal@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2, result3, result4, result5, result6, result7; + var pos0, pos1, pos2; - var savedPos2 = pos; - var savedPos3 = pos; - if (input.substr(pos, 1) === "[") { - var result10 = "["; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 91) { + result0 = "["; + pos++; } else { - var result10 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result10 !== null) { - var result11 = parse__(); - if (result11 !== null) { - var result12 = parse_literal(); - if (result12 !== null) { - var result13 = []; - var savedPos4 = pos; - var result17 = parse__(); - if (result17 !== null) { - if (input.substr(pos, 1) === ",") { - var result18 = ","; - pos += 1; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_literal(); + if (result2 !== null) { + result3 = []; + pos2 = pos; + result4 = parse__(); + if (result4 !== null) { + if (input.charCodeAt(pos) === 44) { + result5 = ","; + pos++; } else { - var result18 = null; - if (reportMatchFailures) { + result5 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result18 !== null) { - var result19 = parse__(); - if (result19 !== null) { - var result20 = parse_literal(); - if (result20 !== null) { - var result16 = [result17, result18, result19, result20]; + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + result7 = parse_literal(); + if (result7 !== null) { + result4 = [result4, result5, result6, result7]; } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } - while (result16 !== null) { - result13.push(result16); - var savedPos4 = pos; - var result17 = parse__(); - if (result17 !== null) { - if (input.substr(pos, 1) === ",") { - var result18 = ","; - pos += 1; + while (result4 !== null) { + result3.push(result4); + pos2 = pos; + result4 = parse__(); + if (result4 !== null) { + if (input.charCodeAt(pos) === 44) { + result5 = ","; + pos++; } else { - var result18 = null; - if (reportMatchFailures) { + result5 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result18 !== null) { - var result19 = parse__(); - if (result19 !== null) { - var result20 = parse_literal(); - if (result20 !== null) { - var result16 = [result17, result18, result19, result20]; + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + result7 = parse_literal(); + if (result7 !== null) { + result4 = [result4, result5, result6, result7]; } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } - if (result13 !== null) { - var result14 = parse__(); - if (result14 !== null) { - if (input.substr(pos, 1) === "]") { - var result15 = "]"; - pos += 1; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + if (input.charCodeAt(pos) === 93) { + result5 = "]"; + pos++; } else { - var result15 = null; - if (reportMatchFailures) { + result5 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result15 !== null) { - var result8 = [result10, result11, result12, result13, result14, result15]; + if (result5 !== null) { + result0 = [result0, result1, result2, result3, result4, result5]; } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; - } - var result9 = result8 !== null - ? (function(first, rest) { return [first].concat(rest.map(function(d) { return d[3]; })); })(result8[2], result8[3]) - : null; - if (result9 !== null) { - var result7 = result9; - } else { - var result7 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "[") { - var result4 = "["; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, first, rest) { return [first].concat(rest.map(function(d) { return d[3]; })); })(pos0, result0[2], result0[3]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 91) { + result0 = "["; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - if (input.substr(pos, 1) === "]") { - var result6 = "]"; - pos += 1; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 93) { + result2 = "]"; + pos++; } else { - var result6 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result6 !== null) { - var result2 = [result4, result5, result6]; + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function() { return []; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset) { return []; })(pos0); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_string() { - var cacheKey = 'string@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - var savedReportMatchFailures = reportMatchFailures; - reportMatchFailures = false; - var savedPos2 = pos; - var savedPos3 = pos; - if (input.substr(pos, 1) === "\"") { - var result11 = "\""; - pos += 1; + var result0, result1, result2; + var pos0, pos1; + + reportFailures++; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 34) { + result0 = "\""; + pos++; } else { - var result11 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\"\""); } } - if (result11 !== null) { - var result12 = []; - var result14 = parse_double_string_char(); - while (result14 !== null) { - result12.push(result14); - var result14 = parse_double_string_char(); + if (result0 !== null) { + result1 = []; + result2 = parse_double_string_char(); + while (result2 !== null) { + result1.push(result2); + result2 = parse_double_string_char(); } - if (result12 !== null) { - if (input.substr(pos, 1) === "\"") { - var result13 = "\""; - pos += 1; + if (result1 !== null) { + if (input.charCodeAt(pos) === 34) { + result2 = "\""; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"\\\"\""); } } - if (result13 !== null) { - var result9 = [result11, result12, result13]; + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result9 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result9 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result9 = null; - pos = savedPos3; - } - var result10 = result9 !== null - ? (function(chars) { return chars.join(""); })(result9[1]) - : null; - if (result10 !== null) { - var result8 = result10; - } else { - var result8 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result8 !== null) { - var result0 = result8; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "'") { - var result4 = "'"; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, chars) { return chars.join(""); })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 39) { + result0 = "'"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"'\""); } } - if (result4 !== null) { - var result5 = []; - var result7 = parse_single_string_char(); - while (result7 !== null) { - result5.push(result7); - var result7 = parse_single_string_char(); + if (result0 !== null) { + result1 = []; + result2 = parse_single_string_char(); + while (result2 !== null) { + result1.push(result2); + result2 = parse_single_string_char(); } - if (result5 !== null) { - if (input.substr(pos, 1) === "'") { - var result6 = "'"; - pos += 1; + if (result1 !== null) { + if (input.charCodeAt(pos) === 39) { + result2 = "'"; + pos++; } else { - var result6 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"'\""); } } - if (result6 !== null) { - var result2 = [result4, result5, result6]; + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(chars) { return chars.join(""); })(result2[1]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, chars) { return chars.join(""); })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - reportMatchFailures = savedReportMatchFailures; - if (reportMatchFailures && result0 === null) { + reportFailures--; + if (reportFailures === 0 && result0 === null) { matchFailed("string"); } - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_double_string_char() { - var cacheKey = 'double_string_char@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos2 = pos; - var savedPos3 = pos; - var savedPos4 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - if (input.substr(pos, 1) === "\"") { - var result13 = "\""; - pos += 1; + var result0, result1; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + pos2 = pos; + reportFailures++; + if (input.charCodeAt(pos) === 34) { + result0 = "\""; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\"\""); } } - if (result13 !== null) { - var result11 = result13; - } else { - if (input.substr(pos, 1) === "\\") { - var result12 = "\\"; - pos += 1; + if (result0 === null) { + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result12 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result12 !== null) { - var result11 = result12; - } else { - var result11 = null;; - }; } - reportMatchFailures = savedReportMatchFailuresVar0; - if (result11 === null) { - var result9 = ''; + reportFailures--; + if (result0 === null) { + result0 = ""; } else { - var result9 = null; - pos = savedPos4; + result0 = null; + pos = pos2; } - if (result9 !== null) { + if (result0 !== null) { if (input.length > pos) { - var result10 = input.charAt(pos); + result1 = input.charAt(pos); pos++; } else { - var result10 = null; - if (reportMatchFailures) { - matchFailed('any character'); + result1 = null; + if (reportFailures === 0) { + matchFailed("any character"); } } - if (result10 !== null) { - var result7 = [result9, result10]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result7 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result7 = null; - pos = savedPos3; - } - var result8 = result7 !== null - ? (function(char_) { return char_; })(result7[1]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result6 !== null) { - var result0 = result6; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "\\") { - var result4 = "\\"; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, char_) { return char_; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result4 !== null) { - var result5 = parse_escape_sequence(); - if (result5 !== null) { - var result2 = [result4, result5]; + if (result0 !== null) { + result1 = parse_escape_sequence(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(sequence) { return sequence; })(result2[1]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, sequence) { return sequence; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_single_string_char() { - var cacheKey = 'single_string_char@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos2 = pos; - var savedPos3 = pos; - var savedPos4 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - if (input.substr(pos, 1) === "'") { - var result13 = "'"; - pos += 1; + var result0, result1; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + pos2 = pos; + reportFailures++; + if (input.charCodeAt(pos) === 39) { + result0 = "'"; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"'\""); } } - if (result13 !== null) { - var result11 = result13; - } else { - if (input.substr(pos, 1) === "\\") { - var result12 = "\\"; - pos += 1; + if (result0 === null) { + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result12 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result12 !== null) { - var result11 = result12; - } else { - var result11 = null;; - }; } - reportMatchFailures = savedReportMatchFailuresVar0; - if (result11 === null) { - var result9 = ''; + reportFailures--; + if (result0 === null) { + result0 = ""; } else { - var result9 = null; - pos = savedPos4; + result0 = null; + pos = pos2; } - if (result9 !== null) { + if (result0 !== null) { if (input.length > pos) { - var result10 = input.charAt(pos); + result1 = input.charAt(pos); pos++; } else { - var result10 = null; - if (reportMatchFailures) { - matchFailed('any character'); + result1 = null; + if (reportFailures === 0) { + matchFailed("any character"); } } - if (result10 !== null) { - var result7 = [result9, result10]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result7 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result7 = null; - pos = savedPos3; - } - var result8 = result7 !== null - ? (function(char_) { return char_; })(result7[1]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result6 !== null) { - var result0 = result6; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "\\") { - var result4 = "\\"; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, char_) { return char_; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result4 !== null) { - var result5 = parse_escape_sequence(); - if (result5 !== null) { - var result2 = [result4, result5]; + if (result0 !== null) { + result1 = parse_escape_sequence(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(sequence) { return sequence; })(result2[1]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, sequence) { return sequence; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_escape_sequence() { - var cacheKey = 'escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result9 = parse_character_escape_sequence(); - if (result9 !== null) { - var result0 = result9; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "0") { - var result6 = "0"; - pos += 1; + var result0, result1; + var pos0, pos1, pos2; + + result0 = parse_character_escape_sequence(); + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 48) { + result0 = "0"; + pos++; } else { - var result6 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"0\""); } } - if (result6 !== null) { - var savedPos2 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - var result8 = parse_digit(); - reportMatchFailures = savedReportMatchFailuresVar0; - if (result8 === null) { - var result7 = ''; + if (result0 !== null) { + pos2 = pos; + reportFailures++; + result1 = parse_digit(); + reportFailures--; + if (result1 === null) { + result1 = ""; } else { - var result7 = null; - pos = savedPos2; + result1 = null; + pos = pos2; } - if (result7 !== null) { - var result4 = [result6, result7]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result4 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result4 = null; - pos = savedPos1; - } - var result5 = result4 !== null - ? (function() { return "\0"; })() - : null; - if (result5 !== null) { - var result3 = result5; - } else { - var result3 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset) { return "\0"; })(pos0); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + result0 = parse_hex_escape_sequence(); + if (result0 === null) { + result0 = parse_unicode_escape_sequence(); + } } - if (result3 !== null) { - var result0 = result3; - } else { - var result2 = parse_hex_escape_sequence(); - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_unicode_escape_sequence(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_character_escape_sequence() { - var cacheKey = 'character_escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - var result2 = parse_single_escape_character(); - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_non_escape_character(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; + result0 = parse_single_escape_character(); + if (result0 === null) { + result0 = parse_non_escape_character(); } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_single_escape_character() { - var cacheKey = 'single_escape_character@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; + var pos0; - var savedPos0 = pos; - if (input.substr(pos).match(/^['"\\bfnrtv]/) !== null) { - var result1 = input.charAt(pos); + pos0 = pos; + if (/^['"\\bfnrtv]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result1 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("['\"\\\\bfnrtv]"); } } - var result2 = result1 !== null - ? (function(char_) { return char_.replace("b", "\b").replace("f", "\f").replace("n", "\n").replace("r", "\r").replace("t", "\t").replace("v", "\x0B"); })(result1) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, char_) { return char_.replace("b", "\b").replace("f", "\f").replace("n", "\n").replace("r", "\r").replace("t", "\t").replace("v", "\x0B"); })(pos0, result0); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_non_escape_character() { - var cacheKey = 'non_escape_character@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var savedPos2 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - var result5 = parse_escape_character(); - reportMatchFailures = savedReportMatchFailuresVar0; - if (result5 === null) { - var result3 = ''; - } else { - var result3 = null; - pos = savedPos2; - } - if (result3 !== null) { + var result0, result1; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + pos2 = pos; + reportFailures++; + result0 = parse_escape_character(); + reportFailures--; + if (result0 === null) { + result0 = ""; + } else { + result0 = null; + pos = pos2; + } + if (result0 !== null) { if (input.length > pos) { - var result4 = input.charAt(pos); + result1 = input.charAt(pos); pos++; } else { - var result4 = null; - if (reportMatchFailures) { - matchFailed('any character'); + result1 = null; + if (reportFailures === 0) { + matchFailed("any character"); } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(char_) { return char_; })(result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, char_) { return char_; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_escape_character() { - var cacheKey = 'escape_character@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result4 = parse_single_escape_character(); - if (result4 !== null) { - var result0 = result4; - } else { - var result3 = parse_digit(); - if (result3 !== null) { - var result0 = result3; - } else { - if (input.substr(pos, 1) === "x") { - var result2 = "x"; - pos += 1; + var result0; + + result0 = parse_single_escape_character(); + if (result0 === null) { + result0 = parse_digit(); + if (result0 === null) { + if (input.charCodeAt(pos) === 120) { + result0 = "x"; + pos++; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"x\""); } } - if (result2 !== null) { - var result0 = result2; - } else { - if (input.substr(pos, 1) === "u") { - var result1 = "u"; - pos += 1; + if (result0 === null) { + if (input.charCodeAt(pos) === 117) { + result0 = "u"; + pos++; } else { - var result1 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"u\""); } } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_hex_escape_sequence() { - var cacheKey = 'hex_escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "x") { - var result3 = "x"; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 120) { + result0 = "x"; + pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"x\""); } } - if (result3 !== null) { - var result4 = parse_hex_digit(); - if (result4 !== null) { - var result5 = parse_hex_digit(); - if (result5 !== null) { - var result1 = [result3, result4, result5]; + if (result0 !== null) { + result1 = parse_hex_digit(); + if (result1 !== null) { + result2 = parse_hex_digit(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(h1, h2) { return String.fromCharCode(+("0x" + h1 + h2)); })(result1[1], result1[2]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, h1, h2) { return String.fromCharCode(+("0x" + h1 + h2)); })(pos0, result0[1], result0[2]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_unicode_escape_sequence() { - var cacheKey = 'unicode_escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1, result2, result3, result4; + var pos0, pos1; - - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "u") { - var result3 = "u"; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 117) { + result0 = "u"; + pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"u\""); } } - if (result3 !== null) { - var result4 = parse_hex_digit(); - if (result4 !== null) { - var result5 = parse_hex_digit(); - if (result5 !== null) { - var result6 = parse_hex_digit(); - if (result6 !== null) { - var result7 = parse_hex_digit(); - if (result7 !== null) { - var result1 = [result3, result4, result5, result6, result7]; + if (result0 !== null) { + result1 = parse_hex_digit(); + if (result1 !== null) { + result2 = parse_hex_digit(); + if (result2 !== null) { + result3 = parse_hex_digit(); + if (result3 !== null) { + result4 = parse_hex_digit(); + if (result4 !== null) { + result0 = [result0, result1, result2, result3, result4]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(h1, h2, h3, h4) { return String.fromCharCode(+("0x" + h1 + h2 + h3 + h4)); })(result1[1], result1[2], result1[3], result1[4]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, h1, h2, h3, h4) { return String.fromCharCode(+("0x" + h1 + h2 + h3 + h4)); })(pos0, result0[1], result0[2], result0[3], result0[4]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_number() { - var cacheKey = 'number@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - var savedReportMatchFailures = reportMatchFailures; - reportMatchFailures = false; - var savedPos8 = pos; - var savedPos9 = pos; - if (input.substr(pos, 1) === "-") { - var result26 = "-"; - pos += 1; + var result0, result1, result2; + var pos0, pos1; + + reportFailures++; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 45) { + result0 = "-"; + pos++; } else { - var result26 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"-\""); } } - if (result26 !== null) { - var result27 = parse__(); - if (result27 !== null) { - var result28 = parse_number(); - if (result28 !== null) { - var result24 = [result26, result27, result28]; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_number(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result24 = null; - pos = savedPos9; + result0 = null; + pos = pos1; } } else { - var result24 = null; - pos = savedPos9; + result0 = null; + pos = pos1; } } else { - var result24 = null; - pos = savedPos9; - } - var result25 = result24 !== null - ? (function(number) { return -number; })(result24[2]) - : null; - if (result25 !== null) { - var result23 = result25; - } else { - var result23 = null; - pos = savedPos8; + result0 = null; + pos = pos1; } - if (result23 !== null) { - var result0 = result23; - } else { - var savedPos6 = pos; - var savedPos7 = pos; - var result20 = parse_int(); - if (result20 !== null) { - var result21 = parse_frac(); - if (result21 !== null) { - var result22 = parse_exp(); - if (result22 !== null) { - var result18 = [result20, result21, result22]; + if (result0 !== null) { + result0 = (function(offset, number) { return -number; })(pos0, result0[2]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + result0 = parse_int(); + if (result0 !== null) { + result1 = parse_frac(); + if (result1 !== null) { + result2 = parse_exp(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result18 = null; - pos = savedPos7; + result0 = null; + pos = pos1; } } else { - var result18 = null; - pos = savedPos7; + result0 = null; + pos = pos1; } } else { - var result18 = null; - pos = savedPos7; - } - var result19 = result18 !== null - ? (function(int_, frac, exp) { return +(int_ + frac + exp); })(result18[0], result18[1], result18[2]) - : null; - if (result19 !== null) { - var result17 = result19; - } else { - var result17 = null; - pos = savedPos6; + result0 = null; + pos = pos1; } - if (result17 !== null) { - var result0 = result17; - } else { - var savedPos4 = pos; - var savedPos5 = pos; - var result15 = parse_int(); - if (result15 !== null) { - var result16 = parse_frac(); - if (result16 !== null) { - var result13 = [result15, result16]; + if (result0 !== null) { + result0 = (function(offset, int_, frac, exp) { return +(int_ + frac + exp); })(pos0, result0[0], result0[1], result0[2]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + result0 = parse_int(); + if (result0 !== null) { + result1 = parse_frac(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result13 = null; - pos = savedPos5; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos5; + result0 = null; + pos = pos1; } - var result14 = result13 !== null - ? (function(int_, frac) { return +(int_ + frac); })(result13[0], result13[1]) - : null; - if (result14 !== null) { - var result12 = result14; - } else { - var result12 = null; - pos = savedPos4; + if (result0 !== null) { + result0 = (function(offset, int_, frac) { return +(int_ + frac); })(pos0, result0[0], result0[1]); } - if (result12 !== null) { - var result0 = result12; - } else { - var savedPos2 = pos; - var savedPos3 = pos; - var result10 = parse_int(); - if (result10 !== null) { - var result11 = parse_exp(); - if (result11 !== null) { - var result8 = [result10, result11]; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + result0 = parse_int(); + if (result0 !== null) { + result1 = parse_exp(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } - var result9 = result8 !== null - ? (function(int_, exp) { return +(int_ + exp); })(result8[0], result8[1]) - : null; - if (result9 !== null) { - var result7 = result9; - } else { - var result7 = null; - pos = savedPos2; + if (result0 !== null) { + result0 = (function(offset, int_, exp) { return +(int_ + exp); })(pos0, result0[0], result0[1]); } - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos1 = pos; - var result5 = parse_frac(); - var result6 = result5 !== null - ? (function(frac) { return +frac; })(result5) - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_frac(); + if (result0 !== null) { + result0 = (function(offset, frac) { return +frac; })(pos0, result0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; - var result2 = parse_int(); - var result3 = result2 !== null - ? (function(int_) { return +int_; })(result2) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_int(); + if (result0 !== null) { + result0 = (function(offset, int_) { return +int_; })(pos0, result0); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; - }; + if (result0 === null) { + pos = pos0; + } + } + } + } + } } - reportMatchFailures = savedReportMatchFailures; - if (reportMatchFailures && result0 === null) { + reportFailures--; + if (reportFailures === 0 && result0 === null) { matchFailed("number"); } - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_int() { - var cacheKey = 'int@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result5 = parse_digit19(); - if (result5 !== null) { - var result6 = parse_digits(); - if (result6 !== null) { - var result3 = [result5, result6]; + var result0, result1; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_digit19(); + if (result0 !== null) { + result1 = parse_digits(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result3 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result3 = null; - pos = savedPos1; - } - var result4 = result3 !== null - ? (function(digit19, digits) { return digit19 + digits; })(result3[0], result3[1]) - : null; - if (result4 !== null) { - var result2 = result4; - } else { - var result2 = null; - pos = savedPos0; + result0 = null; + pos = pos1; } - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_digit(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; + if (result0 !== null) { + result0 = (function(offset, digit19, digits) { return digit19 + digits; })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + result0 = parse_digit(); } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_frac() { - var cacheKey = 'frac@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1; + var pos0, pos1; - - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === ".") { - var result3 = "."; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 46) { + result0 = "."; + pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result3 !== null) { - var result4 = parse_digits(); - if (result4 !== null) { - var result1 = [result3, result4]; + if (result0 !== null) { + result1 = parse_digits(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(digits) { return "." + digits; })(result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, digits) { return "." + digits; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_exp() { - var cacheKey = 'exp@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_e(); - if (result3 !== null) { - var result4 = parse_digits(); - if (result4 !== null) { - var result1 = [result3, result4]; + var result0, result1; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_e(); + if (result0 !== null) { + result1 = parse_digits(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(e, digits) { return e + digits; })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, e, digits) { return e + digits; })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_digits() { - var cacheKey = 'digits@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1; + var pos0; - var savedPos0 = pos; - var result3 = parse_digit(); - if (result3 !== null) { - var result1 = []; - while (result3 !== null) { - result1.push(result3); - var result3 = parse_digit(); + pos0 = pos; + result1 = parse_digit(); + if (result1 !== null) { + result0 = []; + while (result1 !== null) { + result0.push(result1); + result1 = parse_digit(); } } else { - var result1 = null; + result0 = null; } - var result2 = result1 !== null - ? (function(digits) { return digits.join(""); })(result1) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, digits) { return digits.join(""); })(pos0, result0); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_e() { - var cacheKey = 'e@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos).match(/^[eE]/) !== null) { - var result3 = input.charAt(pos); + pos0 = pos; + pos1 = pos; + if (/^[eE]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[eE]"); } } - if (result3 !== null) { - if (input.substr(pos).match(/^[+\-]/) !== null) { - var result5 = input.charAt(pos); + if (result0 !== null) { + if (/^[+\-]/.test(input.charAt(pos))) { + result1 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result1 = null; + if (reportFailures === 0) { matchFailed("[+\\-]"); } } - var result4 = result5 !== null ? result5 : ''; - if (result4 !== null) { - var result1 = [result3, result4]; + result1 = result1 !== null ? result1 : ""; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(e, sign) { return e + sign; })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, e, sign) { return e + sign; })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_digit() { - var cacheKey = 'digit@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - if (input.substr(pos).match(/^[0-9]/) !== null) { - var result0 = input.charAt(pos); + if (/^[0-9]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[0-9]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_digit19() { - var cacheKey = 'digit19@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0; - - if (input.substr(pos).match(/^[1-9]/) !== null) { - var result0 = input.charAt(pos); + if (/^[1-9]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[1-9]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_hex_digit() { - var cacheKey = 'hex_digit@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - if (input.substr(pos).match(/^[0-9a-fA-F]/) !== null) { - var result0 = input.charAt(pos); + if (/^[0-9a-fA-F]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[0-9a-fA-F]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse__() { - var cacheKey = '_@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1; - var savedReportMatchFailures = reportMatchFailures; - reportMatchFailures = false; - var result0 = []; - var result1 = parse_whitespace(); + reportFailures++; + result0 = []; + result1 = parse_whitespace(); while (result1 !== null) { result0.push(result1); - var result1 = parse_whitespace(); + result1 = parse_whitespace(); } - reportMatchFailures = savedReportMatchFailures; - if (reportMatchFailures && result0 === null) { + reportFailures--; + if (reportFailures === 0 && result0 === null) { matchFailed("whitespace"); } - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_whitespace() { - var cacheKey = 'whitespace@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - if (input.substr(pos).match(/^[ \n\r]/) !== null) { - var result0 = input.charAt(pos); + if (/^[ \t\n\r]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { - matchFailed("[ \\n\\r]"); + result0 = null; + if (reportFailures === 0) { + matchFailed("[ \\t\\n\\r]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } - function buildErrorMessage() { - function buildExpected(failuresExpected) { - failuresExpected.sort(); - - var lastFailure = null; - var failuresExpectedUnique = []; - for (var i = 0; i < failuresExpected.length; i++) { - if (failuresExpected[i] !== lastFailure) { - failuresExpectedUnique.push(failuresExpected[i]); - lastFailure = failuresExpected[i]; - } - } - - switch (failuresExpectedUnique.length) { - case 0: - return 'end of input'; - case 1: - return failuresExpectedUnique[0]; - default: - return failuresExpectedUnique.slice(0, failuresExpectedUnique.length - 1).join(', ') - + ' or ' - + failuresExpectedUnique[failuresExpectedUnique.length - 1]; + + function cleanupExpected(expected) { + expected.sort(); + + var lastExpected = null; + var cleanExpected = []; + for (var i = 0; i < expected.length; i++) { + if (expected[i] !== lastExpected) { + cleanExpected.push(expected[i]); + lastExpected = expected[i]; } } - - var expected = buildExpected(rightmostMatchFailuresExpected); - var actualPos = Math.max(pos, rightmostMatchFailuresPos); - var actual = actualPos < input.length - ? quote(input.charAt(actualPos)) - : 'end of input'; - - return 'Expected ' + expected + ' but ' + actual + ' found.'; + return cleanExpected; } function computeErrorPosition() { @@ -3858,13 +3076,13 @@ module.exports = (function(){ var column = 1; var seenCR = false; - for (var i = 0; i < rightmostMatchFailuresPos; i++) { + for (var i = 0; i < Math.max(pos, rightmostFailuresPos); i++) { var ch = input.charAt(i); - if (ch === '\n') { + if (ch === "\n") { if (!seenCR) { line++; } column = 1; seenCR = false; - } else if (ch === '\r' | ch === '\u2028' || ch === '\u2029') { + } else if (ch === "\r" || ch === "\u2028" || ch === "\u2029") { line++; column = 1; seenCR = true; @@ -3878,208 +3096,107 @@ module.exports = (function(){ } - - var filterEqual = function(o, k, v) { o[k] = v; }, - - filterGreater = filter("$gt"), - - filterGreaterOrEqual = filter("$gte"), - - filterLess = filter("$lt"), - - filterLessOrEqual = filter("$lte"), - - filterNotEqual = filter("$ne"), - - filterRegularExpression = filter("$regex"), - - filterIn = filter("$in"), - - exists = {$exists: true}; - - - - function add(a, b) { return a + b; } - - function subtract(a, b) { return a - b; } - - function multiply(a, b) { return a * b; } - - function divide(a, b) { return a / b; } - - - - function one() { return 1; } - - function noop() {} - - - - function filter(op) { - - return function(o, k, v) { - - var f = o[k]; - - switch (typeof f) { - - case "undefined": o[k] = f = {}; // continue - - case "object": f[op] = v; break; - - // otherwise, observe the existing equals (literal) filter - + var filterEqual = function(o, k, v) { o[k] = v; }, + filterGreater = filter("$gt"), + filterGreaterOrEqual = filter("$gte"), + filterLess = filter("$lt"), + filterLessOrEqual = filter("$lte"), + filterNotEqual = filter("$ne"), + filterRegularExpression = filter("$regex"), + filterIn = filter("$in"), + exists = {$exists: true}; + + function add(a, b) { return a + b; } + function subtract(a, b) { return a - b; } + function multiply(a, b) { return a * b; } + function divide(a, b) { return a / b; } + + function one() { return 1; } + function noop() {} + + function filter(op) { + return function(o, k, v) { + var f = o[k]; + switch (typeof f) { + case "undefined": o[k] = f = {}; // continue + case "object": f[op] = v; break; + // otherwise, observe the existing equals (literal) filter + } + }; } - }; - - } - - - - function arrayAccessor(name) { - - name = new String(name); - - name.array = true; - - return name; - - } - - - - function objectAccessor(name) { - - return name; - - } - - - - function compoundMetric(head, tail) { - - var i = -1, - - n = tail.length, - - t, - - e = head; - - while (++i < n) { - - t = tail[i]; - - e = {left: e, op: t[1], right: t[3]}; - - if (!i) head = e; - - } - - return head; - - } - - - - function compoundValue(head, tail) { - - var n = tail.length; - - return { - - exists: function(o) { - - var i = -1; - - head.exists(o); - - while (++i < n) tail[i][3].exists(o); - - }, - - fields: function(o) { - - var i = -1; - - head.fields(o); - - while (++i < n) tail[i][3].fields(o); - - }, - - value: function(o) { - - var v = head.value(o), - - i = -1, - - t; - - while (++i < n) v = (t = tail[i])[1](v, t[3].value(o)); - - return v; - + function arrayAccessor(name) { + name = new String(name); + name.array = true; + return name; } - }; - - } - - - - function member(head, tail) { - - var fields = ["d", head].concat(tail), - - shortName = fields.filter(function(d) { return !d.array; }).join("."), - - longName = fields.join("."), - - i = -1, - - n = fields.length; - - return { - - field: longName, - - exists: function(o) { - - if (!(shortName in o)) { - - o[shortName] = exists; - - } - - }, - - fields: function(o) { - - o[shortName] = 1; - - }, - - value: function(o) { - - var i = -1; + function objectAccessor(name) { + return name; + } + function compoundMetric(head, tail) { + var i = -1, + n = tail.length, + t, + e = head; while (++i < n) { - - o = o[fields[i]]; - - } - - return o; - + t = tail[i]; + e = {left: e, op: t[1], right: t[3]}; + if (!i) head = e; + } + return head; + } + + function compoundValue(head, tail) { + var n = tail.length; + return { + exists: function(o) { + var i = -1; + head.exists(o); + while (++i < n) tail[i][3].exists(o); + }, + fields: function(o) { + var i = -1; + head.fields(o); + while (++i < n) tail[i][3].fields(o); + }, + value: function(o) { + var v = head.value(o), + i = -1, + t; + while (++i < n) v = (t = tail[i])[1](v, t[3].value(o)); + return v; + } + }; } - }; - - } + function member(head, tail) { + var fields = ["d", head].concat(tail), + shortName = fields.filter(function(d) { return !d.array; }).join("."), + longName = fields.join("."), + i = -1, + n = fields.length; + return { + field: longName, + exists: function(o) { + if (!(shortName in o)) { + o[shortName] = exists; + } + }, + fields: function(o) { + o[shortName] = 1; + }, + value: function(o) { + var i = -1; + while (++i < n) { + o = o[fields[i]]; + } + return o; + } + }; + } - var result = parseFunctions[startRule](); @@ -4090,27 +3207,32 @@ module.exports = (function(){ * * - |result !== null| * - |pos === input.length| - * - |rightmostMatchFailuresExpected| may or may not contain something + * - |rightmostFailuresExpected| may or may not contain something * * 2. The parser successfully parsed only a part of the input. * * - |result !== null| * - |pos < input.length| - * - |rightmostMatchFailuresExpected| may or may not contain something + * - |rightmostFailuresExpected| may or may not contain something * * 3. The parser did not successfully parse any part of the input. * * - |result === null| * - |pos === 0| - * - |rightmostMatchFailuresExpected| contains at least one failure + * - |rightmostFailuresExpected| contains at least one failure * * All code following this comment (including called functions) must * handle these states. */ if (result === null || pos !== input.length) { + var offset = Math.max(pos, rightmostFailuresPos); + var found = offset < input.length ? input.charAt(offset) : null; var errorPosition = computeErrorPosition(); + throw new this.SyntaxError( - buildErrorMessage(), + cleanupExpected(rightmostFailuresExpected), + found, + offset, errorPosition.line, errorPosition.column ); @@ -4125,9 +3247,33 @@ module.exports = (function(){ /* Thrown when a parser encounters a syntax error. */ - result.SyntaxError = function(message, line, column) { - this.name = 'SyntaxError'; - this.message = message; + result.SyntaxError = function(expected, found, offset, line, column) { + function buildMessage(expected, found) { + var expectedHumanized, foundHumanized; + + switch (expected.length) { + case 0: + expectedHumanized = "end of input"; + break; + case 1: + expectedHumanized = expected[0]; + break; + default: + expectedHumanized = expected.slice(0, expected.length - 1).join(", ") + + " or " + + expected[expected.length - 1]; + } + + foundHumanized = found ? quote(found) : "end of input"; + + return "Expected " + expectedHumanized + " but " + foundHumanized + " found."; + } + + this.name = "SyntaxError"; + this.expected = expected; + this.found = found; + this.message = buildMessage(expected, found); + this.offset = offset; this.line = line; this.column = column; }; diff --git a/lib/cube/metric-expression.peg b/lib/cube/metric-expression.peg index 80aa9bcf..666626bb 100644 --- a/lib/cube/metric-expression.peg +++ b/lib/cube/metric-expression.peg @@ -117,10 +117,13 @@ metric_unary_expression / metric_primary_expression metric_primary_expression - = reduce:reduce _ "(" _ event:event_expression _ ")" { event.reduce = reduce; event.source = input.substring(savedPos3, pos); return event; } + = reduce:reduce _ "(" _ event:event_expression _ ")" group:(_ "." _ metric_group_expression)? { event.reduce = reduce; event.source = input.substring(pos0, pos); if(group) event.group = group[3]; return event; } / value:number { return {value: function() { return value; }}; } / "(" _ expression:metric_additive_expression _ ")" { return expression; } +metric_group_expression + = "group" _ "(" _ member:event_member_expression _ ")" { return member; } + event_expression = value:event_value_expression filters:(_ "." _ event_filter_expression)* { diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 058a2d30..19405096 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -36,12 +36,14 @@ exports.getter = function(db, config) { expression = parser.parse(request.expression); measurement = new Measurement(expression, start, stop, tier); - measurement.on('complete', function(){ callback(new Metric(stop, null, measurement)); }); + measurement.on('complete', function(){ callback(new Metric({time: stop, value: null}, measurement)); }); } catch(error) { metalog.error('mget', error, { info: util.inspect([start, stop, tier, expression] )}); return callback({error: error, _trace: request._trace}), -1; } + + measurement.measure(db, options, callback); } diff --git a/lib/cube/models/event.js b/lib/cube/models/event.js index 1d02bf4d..d8f55968 100644 --- a/lib/cube/models/event.js +++ b/lib/cube/models/event.js @@ -62,6 +62,7 @@ Event.setProperties({ function find(db, measurement, callback){ var expression = measurement.expression, + group = expression.group, type = expression.type, start = measurement.start, stop = measurement.stop, @@ -70,28 +71,54 @@ function find(db, measurement, callback){ // Copy any expression filters into the query object. expression.filter(filter); - // Request any needed fields. - expression.fields(fields); - filter.t.$gte = start; filter.t.$lt = stop; + if(group) { + filter[group.field] = measurement.group; + fields[group.field] = 1; + } + + // Request any needed fields. + expression.fields(fields); + db.events(type, function (error, collection) { if(error) return callback(error); - collection.find(filter, fields, event_options, handleResponse); - }); - function handleResponse(error, cursor){ - if (error) return callback(error); - cursor.each(function(error, row) { + function handleResponse(error, cursor){ + var group_name; if (error) return callback(error); - if (row) callback(error, new Event(type, row.t, row.d, row._id)); - else callback(); - }) - } + cursor.each(function(error, row) { + if (error) return callback(error); + if (row) callback(error, new Event(type, row.t, row.d, row._id)); + else { + callback(); + } + }); + } + + collection.find(filter, fields, event_options, handleResponse); + }); } Object.defineProperty(Event, "find", { value: find }); +function groups(db, measurement, callback){ + var group = measurement.isGrouped, + type = group.type, + start = measurement.start, + stop = measurement.stop, + filter = {t: { $gte: start, $lt: stop }}; + + // Copy any expression filters into the query object. + group.filter(filter); + + db.events(type, function (error, collection) { + if(error) return callback(error); + collection.distinct(group.field, filter, callback); + }); +} +Object.defineProperty(Event, "groups", { value: groups }); + function save(db, callback){ var self = this; if (this.validate) { diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 28547aab..bc872932 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -5,9 +5,20 @@ var metalog = require('../metalog'), reduces = require('../reduces'), Metric = require('./metric'), Event = require('./event'), + _ = require('underscore'), compute = {constant: constant, binary: binary, unary: unary}, queueByName = {}; +function isGrouped(expression){ + if (expression.type && expression.group){ + expression.group.type = expression.type; + expression.group.filter = expression.filter; + return expression.group; + } + if (expression.op) return (isGrouped(expression.left) || isGrouped(expression.right)); + return; +} + function Measurement(expression, start, stop, tier){ // Round the start/stop to the tier edges this.expression = expression; @@ -16,6 +27,7 @@ function Measurement(expression, start, stop, tier){ this.tier = tier; this.flavor = (expression.op ? 'binary' : (expression.type ? 'unary' : 'constant')); this.isPyramidal = expression.type && reduces[expression.reduce].pyramidal; + this.isGrouped = isGrouped(expression); this.eventize(); } @@ -29,27 +41,38 @@ Measurement.prototype.report = function report(){ // Computes the metric for the given expression for the time interval from // start (inclusive) to stop (exclusive). The time granularity is determined // by the specified tier, such as daily or hourly. The callback is invoked -// repeatedly for each metric value, being passed two arguments: the time and -// the value. The values may be out of order due to partial cache hits. +// repeatedly for each metric value. The values may be out of order due +// to partial cache hits. Measurement.prototype.measure = function measure(db, options, callback) { var _this = this; - compute[this.flavor].call(this, db, options, callback); + if (this.isGrouped && !Array.isArray(this.isGrouped.groups)) { + Event.groups(db, this, function(error, group_names){ + handle(error); + _this.isGrouped.groups = group_names.sort(); + compute[_this.flavor].call(_this, db, options, callback); + }); + } else { + compute[this.flavor].call(this, db, options, callback); + } }; // Computes a constant expression like the "7" in "x * 7" function constant(db, options, callback) { var _this = this, value = this.expression.value(); - walk(this.start, this.stop, this.tier, function(time){ callback(new Metric(time, value, _this)); }); + walk(this.start, this.stop, this.tier, function(time){ + callback(new Metric({time: time, value: value}, _this)); + }); this.emit('complete'); }; // Serializes a unary expression for computation. function unary(db, options, callback) { - var self = this, - remaining = 0, - time0 = Date.now(), - name = this.expression.source, - queue = queueByName[name]; + var self = this, + total_remaining = 0, + remaining = 0, + time0 = Date.now(), + name = this.expression.source, + queue = queueByName[name]; // Compute the expected number of values. walk(this.start, this.stop, this.tier, function(time){ ++remaining; }); @@ -57,15 +80,18 @@ function unary(db, options, callback) { // If no results were requested, return immediately. if (!remaining) return this.emit('complete'); + total_remaining = remaining; + if(this.isGrouped) total_remaining *= this.isGrouped.groups.length; + // Add this task to the appropriate queue. if (queue) queue.next = task; else process.nextTick(task); queueByName[name] = task; function task() { - findOrComputeUnary.call(self, db, options, function(metric) { + function onMetric(metric){ callback(metric); - if (!--remaining) { + if (!--total_remaining) { self.emit('complete'); if (task.next) process.nextTick(task.next); else delete queueByName[name]; @@ -77,13 +103,36 @@ function unary(db, options, callback) { ms: time1 - time0 }); } - }); + } + + function nextGroup(prev_group){ + var query_measurement = new Measurement(_.clone(self.expression), self.start, self.stop, self.tier), + group_idx = self.isGrouped.groups.indexOf(prev_group) + 1, + group_name = self.isGrouped.groups[group_idx], + group_remaining = remaining; + + if (group_idx >= self.isGrouped.groups.length) return; + + query_measurement.group = group_name; + + findOrComputeUnary.call(query_measurement, db, options, function(metric){ + onMetric(metric); + if(!--group_remaining) nextGroup(group_name); + }); + } + + if (self.expression.group) { + nextGroup(); + } else { + findOrComputeUnary.call(self, db, options, onMetric); + } } } // Finds or computes a unary (primary) expression. function findOrComputeUnary(db, options, callback) { var expression = this.expression, + group_name = this.group, map = expression.value, reduce = reduces[expression.reduce], measurement = this; @@ -116,18 +165,21 @@ function findOrComputeUnary(db, options, callback) { // Group metrics from the next tier. function computePyramidal(start, stop) { - var bins = {}, - query_measurement = new Measurement(expression, start, stop, tier.next); + var query_measurement = new Measurement(expression, start, stop, tier.next), + bins = {}; + + if(group_name) query_measurement.group = group_name; find(query_measurement, function(metric) { var value = metric.value, time = metric.time, values = metric.values; - var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); + + var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); if (reduce.pyramidal) bin.values.push(value); else bin.values = bin.values.concat(values||[]); if (!--bin.size) { - var metric = new Metric(time, reduce(bin.values), measurement, bin.values); + var metric = new Metric({time: time, value: reduce(bin.values), group: group_name}, measurement, bin.values); if (metric.value || metric.value === 0) metric.save(db, handle); callback(metric); delete bins[time]; @@ -153,31 +205,39 @@ function findOrComputeUnary(db, options, callback) { metalog.info('cube_compute', {is: 'past_horizon', metric: {start: start, stop: {was: old_stop, updated_to: stop}, tier: tier, expression: expression.source } }); } - var query_measurement = new Measurement(expression, start, stop, tier); - var time = start, values = []; + var query_measurement = new Measurement(expression, start, stop, tier), + time = start, values = []; + + if (group_name) query_measurement.group = group_name; + + function flat_callback(time, values){ + var value = (values.length ? reduce(values) : reduce.empty), + metric = new Metric({ time: time, value: value, group: group_name }, measurement, values); - function flat_callback(time, value, values){ - var metric = new Metric(time, value, measurement, values); callback(metric); if (metric.value || metric.value === 0) metric.save(db, handle); } - Event.find(db, query_measurement, function(error, event) { + function process(error, event){ handle(error); + if (event) { var then = tier.floor(event.time); + if (time < then) { - flat_callback(time, (values.length ? reduce(values) : reduce.empty), values); - while ((time = tier.step(time)) < then) flat_callback(time, reduce.empty); + flat_callback(time, values, group_name); + while ((time = tier.step(time)) < then) flat_callback(time, [], group_name); values = [map(event.to_wire())]; } else { values.push(map(event.to_wire())); } } else { - flat_callback(time, (values.length ? reduce(values) : reduce.empty), values); - while ((time = tier.step(time)) < stop) flat_callback(time, reduce.empty); + flat_callback(time, values, group_name); + while ((time = tier.step(time)) < stop) flat_callback(time, [], group_name); } - }); + } + + Event.find(db, query_measurement, process); } } } @@ -189,47 +249,60 @@ function findOrComputeUnary(db, options, callback) { // the value for left appears first, it parks that value as left[time], where // the result for right will eventually find it. function binary(db, options, callback) { - var self = this, expression = this.expression, value; - var left = new Measurement(expression.left, this.start, this.stop, this.tier), - right = new Measurement(expression.right, this.start, this.stop, this.tier); - - left.on("complete", function(){ - var time = self.stop; - if (time in right){ - self.emit("complete"); - } else { - left[time] = undefined; + var self = this, expression = this.expression, value; + var left = new Measurement(expression.left, this.start, this.stop, this.tier), + right = new Measurement(expression.right, this.start, this.stop, this.tier), + groups, grouped_measurement = left, other_measurement = right; + + left.left = true; + right.right = true; + + function complete(measurement, other){ + return function(){ + var time = self.stop; + if (time in other){ + self.emit("complete"); + } else { + measurement[time] = undefined; + } } - }); + } - right.on("complete", function(){ - var time = self.stop; - if (time in left){ - self.emit("complete"); - } else { - right[time] = undefined; - } - }); + function measure(measurement, other){ + return function(metric){ + var time = metric.time, value = metric.value, group = metric.group, + left_value, right_value; - left.measure(db, options, function(metric) { - var time = metric.time, value = metric.value; - if (time in right) { // right val already appeared; get a result - callback(new Metric(time, expression.op(value, right[time]))); - delete right[time]; - } else { // right val still on the way; stash the value - left[time] = value; + if (time in other) { + left_value = measurement.left ? value : other[time]; + right_value = measurement.right ? value : other[time]; + callback(new Metric({time: time, value: expression.op(left_value, right_value), group: group })); + } else { + measurement[time] = value; + } } - }); + } - right.measure(db, options, function(metric) { - var time = metric.time, value = metric.value; - if (time in left) { - callback(new Metric(time, expression.op(left[time], value))); - delete left[time]; - } else { - right[time] = value; + if(this.isGrouped){ + groups = this.isGrouped.groups; + if(right.isGrouped){ + grouped_measurement = right; + other_measurement = left; } - }); + + other_measurement.on("complete", function(){ + complete(other_measurement, grouped_measurement)(); + grouped_measurement.measure(db, options, measure(grouped_measurement, other_measurement)); + }); + grouped_measurement.on("complete", complete(grouped_measurement, other_measurement)); + other_measurement.measure(db, options, measure(other_measurement, grouped_measurement)); + } else { + left.on("complete", complete(left, right)); + right.on("complete", complete(right, left)); + + left.measure(db, options, measure(left, right)); + right.measure(db, options, measure(right, left)); + } }; // execute cb on each interval from t1 to t2 diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index 2a914c9f..1e0bc5a5 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -18,9 +18,10 @@ var tiers = require("../tiers"), metric_fields = {v: 1, vs: 1}, metric_options = {sort: {"_id.t": 1}, batchSize: 1000}; -function Metric(time, value, measurement, values){ - this.time = time; - this.value = value; +function Metric(data, measurement, values){ + this.time = data.time; + this.value = data.value; + if(data.group) this.group = data.group; this.setProperty("values", { value: values||[] }); this.setProperty("measurement", { value: measurement }); @@ -59,6 +60,8 @@ function find(db, measurement, callback){ $lt: stop } }; + if (measurement.group) query["_id.g"] = measurement.group; + collection.find(query, metric_fields, metric_options, handleResponse); }); @@ -75,18 +78,24 @@ Object.defineProperty(Metric, "find", { value: find }); Object.defineProperty(Metric, "from_wire", { value: from_wire }); function from_wire(row, measurement){ - var values = null; + var values = null, + data; if(!measurement.isPyramidal) { values = (row.vs || []).reduce(function(expanded, value){ _.times(value.c, function(){ expanded.push(value.v); }); return expanded; }, []); } - return new Metric(row._id.t, row.v, measurement, values); + + data = {time: row._id.t, value: row.v}; + if(row._id.g) data.group = row._id.g; + + return new Metric(data, measurement, values); } function to_wire(){ - var values = null; + var values = null, + id; if(!this.measurement.isPyramidal){ values = this.values.reduce(function(values, value){ var pair = _.find(values, function(pair){ return pair.v == value; }); @@ -95,7 +104,10 @@ function to_wire(){ return values; }, []); } - return { i: false, v: mongo.Double(this.value), vs: values, _id: { e: this.e, l: this.l, t: this.time } }; + id = { e: this.e, l: this.l, t: this.time }; + if(this.group) id.g = this.group; + + return { i: false, v: mongo.Double(this.value), vs: values, _id: id}; }; function report(){ From 92184170e8a30dd874c6326223f1e0c07a9c623d Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Wed, 19 Dec 2012 10:26:53 -0600 Subject: [PATCH 64/87] Board URLs no longer include organization --- lib/cube/visualizer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/visualizer.js b/lib/cube/visualizer.js index 3d6122c8..3fab43a2 100644 --- a/lib/cube/visualizer.js +++ b/lib/cube/visualizer.js @@ -8,7 +8,7 @@ var url = require("url"), exports.register = function(db, endpoints) { endpoints.ws.push( - endpoint(/^\/[a-z\-_0-9]+\/boards\/[a-z0-9\-_]+(\/edit)?$/i, viewBoard(db)) + endpoint(/^\/boards\/[a-z0-9\-_]+(\/edit)?$/i, viewBoard(db)) ); }; From b8bef170a44a81f90254bda82e0824d7a17a2ef4 Mon Sep 17 00:00:00 2001 From: Huston Hoburg Date: Wed, 19 Dec 2012 10:27:19 -0600 Subject: [PATCH 65/87] Fix bug for warmer for empty boards --- lib/cube/warmer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index 466a01ca..cf6d47d2 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -18,7 +18,7 @@ module.exports = function(options){ cursor.each(function(error, row) { if (error) throw error; if (row) { - expressions.splice.apply(expressions, [0, 0].concat(row.pieces + expressions.splice.apply(expressions, [0, 0].concat((row.pieces||[]) .map(function(piece){ return piece.query; }) .filter(function(expression){ return expression && !(expression in expressions); }) )); From 129aff8d8d1adb00b7338834a0f4bddebc40ed15 Mon Sep 17 00:00:00 2001 From: Marsup Date: Tue, 6 Aug 2013 14:05:59 +0200 Subject: [PATCH 66/87] Add queue-async dependency instead of submodule --- .gitmodules | 3 --- lib/cube/metric.js | 2 +- package.json | 11 ++++++----- test/metric-test.js | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a554f13a..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "lib/queue-async"] - path = lib/queue-async - url = https://github.com/mbostock/queue.git diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 19405096..705218be 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -4,7 +4,7 @@ var _ = require("underscore"), util = require("util"), - queuer = require("../queue-async/queue"), + queuer = require("queue-async"), parser = require("./metric-expression"), tiers = require("./tiers"), reduces = require("./reduces"), diff --git a/package.json b/package.json index a584796a..10ae9574 100644 --- a/package.json +++ b/package.json @@ -20,13 +20,14 @@ "node-static": "0.6.5", "pegjs": "0.7.0", "vows": "0.7.0", - "node-gyp": "0.6.8", + "node-gyp": "0.6.8", "websocket": "1.0.8", "websocket-server": "1.4.04", - "cookies": "0.3.1", - "bcrypt": "0.7.1", - "underscore": "1.3.3", - "jake": "0.5.6" + "cookies": "0.3.1", + "bcrypt": "0.7.1", + "underscore": "1.3.3", + "jake": "0.5.6", + "queue-async": "~1.0.4" }, "scripts": { "preinstall": "npm install mongodb --mongodb:native", diff --git a/test/metric-test.js b/test/metric-test.js index 86ce3aee..bf4de820 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -6,7 +6,7 @@ var _ = require("underscore"), vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), - queuer = require("../lib/queue-async/queue"), + queuer = require("queue-async"), tiers = require("../lib/cube/tiers"), units = tiers.units, event = require("../lib/cube/event"), From edfcbd4161a8ad17407029eb61f4998670cbd3d1 Mon Sep 17 00:00:00 2001 From: Marsup Date: Tue, 6 Aug 2013 14:06:44 +0200 Subject: [PATCH 67/87] Fix database name --- config/cube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cube.js b/config/cube.js index 489b15d4..7f982daa 100644 --- a/config/cube.js +++ b/config/cube.js @@ -11,7 +11,7 @@ metalog.send_events = false; configs.common = { "mongo-host": "127.0.0.1", "mongo-port": 27017, - "mongo-database": "dashpot_development", + "mongo-database": "cube", "mongo-username": null, "mongo-password": null, "mongo-server_options": {auto_reconnect: true, poolSize: 8, socketOptions: { noDelay: true }}, From 6de91eeb5f1d74b71408d92e33204aca320467dc Mon Sep 17 00:00:00 2001 From: Marsup Date: Thu, 1 Aug 2013 17:18:57 +0200 Subject: [PATCH 68/87] Change configuration style --- Jakefile | 13 ++-- bin/collector.js | 13 ++-- bin/evaluator.js | 15 ++--- bin/warmer.js | 5 +- config/cube.js | 110 ++++++++++++++++----------------- lib/cube/authentication.js | 9 +-- lib/cube/db.js | 58 ++++++++++------- lib/cube/event.js | 14 ++--- lib/cube/index.js | 1 + lib/cube/metric.js | 9 ++- lib/cube/models/invalidator.js | 10 +-- lib/cube/models/measurement.js | 39 ++++++------ lib/cube/server.js | 42 +++++++++---- lib/cube/warmer.js | 26 ++++---- package.json | 6 +- test/authentication-test.js | 3 +- test/collector-test.js | 4 +- test/evaluator-test.js | 3 +- test/event-test.js | 20 ++++-- test/server-test.js | 7 +-- test/test_helper.js | 72 +++++++++++---------- test/visualizer-test.js | 3 +- test/warmer-test.js | 21 +++++-- 23 files changed, 276 insertions(+), 227 deletions(-) diff --git a/Jakefile b/Jakefile index bb393c69..dbd644b0 100644 --- a/Jakefile +++ b/Jakefile @@ -1,22 +1,23 @@ -var config = require('./config/cube'), - Db = require('./lib/cube/db'); +var config = require('./config/cube'), + Db = require('./lib/cube/db'), + horizons = config.get('horizons'); namespace("db", function(){ namespace("metrics", function(){ desc("Remove metrics with times past the forced metric expiration horizon") task("remove_expired", [], function(){ - if(!config.horizons.forced_metric_expiration){ + if(!horizons.forced_metric_expiration){ throw new Error("horizons.forced_metric_expiration MUST be set in: config/cube.js"); } var db = new Db(), - expiration_date = new Date(new Date() - config.horizons.forced_metric_expiration); + expiration_date = new Date(new Date() - horizons.forced_metric_expiration); function handle(err) { if(err) throw err; } - db.open(config, function(err, db){ + db.open(function(err, db){ handle(err); var metrics_db = db._dbs.metrics; metrics_db.collectionNames({namesOnly: true}, function(err, names){ @@ -39,4 +40,4 @@ namespace("db", function(){ }) }); }); -}); \ No newline at end of file +}); diff --git a/bin/collector.js b/bin/collector.js index 463cd72e..e34e64a8 100644 --- a/bin/collector.js +++ b/bin/collector.js @@ -1,11 +1,8 @@ 'use strict'; -var options = require("../config/cube").include("collector"), - cube = require("../"), - server = cube.server(options); +var cube = require("../"), + server = cube.server('collector'); -server.register = function(db, endpoints) { - cube.collector.register(db, endpoints); -}; - -server.start(); +server + .use(cube.collector.register) + .start(); diff --git a/bin/evaluator.js b/bin/evaluator.js index 4ba5a9cb..b2ee955c 100644 --- a/bin/evaluator.js +++ b/bin/evaluator.js @@ -1,12 +1,9 @@ 'use strict'; -var options = require("../config/cube").include('evaluator'), - cube = require("../"), - server = cube.server(options); +var cube = require("../"), + server = cube.server('evaluator'); -server.register = function(db, endpoints) { - cube.evaluator.register(db, endpoints); - cube.visualizer.register(db, endpoints); -}; - -server.start(); +server + .use(cube.evaluator.register) + .use(cube.visualizer.register) + .start(); diff --git a/bin/warmer.js b/bin/warmer.js index 7c0b5b38..64b058cc 100644 --- a/bin/warmer.js +++ b/bin/warmer.js @@ -1,7 +1,6 @@ 'use strict'; -var options = require("../config/cube").include('warmer'), - cube = require("../"), - warmer = cube.warmer(options); +var cube = require("../"), + warmer = cube.warmer(); warmer.start(); diff --git a/config/cube.js b/config/cube.js index 7f982daa..00e038ad 100644 --- a/config/cube.js +++ b/config/cube.js @@ -1,74 +1,74 @@ 'use strict'; -var configs = {}, - metalog = require('../lib/cube/metalog'); +var cfg = module.exports = require('cfg').createConfig(), + metalog = require('../lib/cube/metalog'); metalog.send_events = false; // -//Shared configuration +// Common configuration // -configs.common = { - "mongo-host": "127.0.0.1", - "mongo-port": 27017, - "mongo-database": "cube", - "mongo-username": null, - "mongo-password": null, - "mongo-server_options": {auto_reconnect: true, poolSize: 8, socketOptions: { noDelay: true }}, - "mongo-metrics": {autoIndexId: true, capped: false }, - "mongo-events": {autoIndexId: true, capped: true, size: 1e9 }, +cfg.set('mongodb', { + 'mongo-host': '127.0.0.1', + 'mongo-port': 27017, + 'mongo-database': 'cube', + 'mongo-username': null, + 'mongo-password': null, + 'mongo-server-options': { + auto_reconnect: true, + poolSize: 8, + socketOptions: { + noDelay: true + } + }, - "separate-events-database": true, + 'mongo-metrics': { + autoIndexId: true, + capped: false, + safe: false + }, - "horizons": { - "calculation": 1000 * 60 * 60 * 2, // 2 hours - "invalidation": 1000 * 60 * 60 * 1, // 1 hour - "forced_metric_expiration": 1000 * 60 * 60 * 24 * 7, // 7 days - } -}; + 'mongo-events': { + autoIndexId: true, + capped: true, + size: 1e9, + safe: false + }, + 'separate-events-database': true, -// -// Collector configuration -// -configs.collector = { - "http-port": 1080, - "udp-port": 1180, - "authenticator": "allow_all" -} + 'authentication-collection': 'users' +}); +cfg.set('horizons', { + 'calculation': 1000 * 60 * 60 * 2, // 2 hours + 'invalidation': 1000 * 60 * 60 * 1, // 1 hour + 'forced_metric_expiration': 1000 * 60 * 60 * 24 * 7, // 7 days +}); -// -// Evaluator configuration -// -configs.evaluator = { - "http-port": 1081, - // "authenticator": "mongo_cookie" - "authenticator": "allow_all" -} +cfg.set('collectd-mappings', { + 'snmp': { + 'if_octets': 'interface', + 'disk_octets': 'disk', + 'swap_io': 'swap', + 'swap': 'swap' + } +}); +cfg.set('collector', { + 'http-port': 1080, + 'udp-port': 1180, + 'authenticator': 'allow_all' +}); -// -// Warmer configuration -// -configs.warmer = { - "warmer-interval": 1000 * 30, - "warmer-tier": 1000 * 10 -} +cfg.set('evaluator', { + 'http-port': 1081, + 'authenticator': 'allow_all' +}); -var options = {}; -Object.defineProperty(options, "include", { - enumerable: false, - value: function() { - for(var config_name in arguments){ - var config = configs[arguments[config_name]]; - for(var prop in config) - if(config.hasOwnProperty(prop)) - options[prop] = config[prop]; - }; - return options; - } +cfg.set('warmer', { + 'warmer-interval': 1000 * 30, + 'warmer-tier': 1000 * 10 }); -module.exports = options.include('common'); diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js index a0b5c750..9c5d97fc 100644 --- a/lib/cube/authentication.js +++ b/lib/cube/authentication.js @@ -22,11 +22,12 @@ var mongodb = require("mongodb"), cookies = require("cookies"), bcrypt = require("bcrypt"), metalog = require('./metalog'), + options = require('../../config/cube'), authentication = {}; -authentication.authenticator = function(strategy, db, options){ +authentication.authenticator = function(strategy, db){ metalog.minor('cube_auth', { strategy: strategy }); - return authentication[strategy](db, options); + return authentication[strategy](db); }; authentication.allow_all = function(){ @@ -47,10 +48,10 @@ authentication.read_only = function(){ return { check: check }; }; -authentication.mongo_cookie = function(db, options){ +authentication.mongo_cookie = function(db){ var users; - db.collection(options["collection"]||"users", function(error, clxn){ + db.collection(options.get('mongodb')["authentication-collection"] || "users", function(error, clxn){ if (error) throw(error); // TODO: check if not collection? users = clxn; diff --git a/lib/cube/db.js b/lib/cube/db.js index e99dc19f..00123d25 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -3,8 +3,31 @@ var util = require("util"), mongodb = require("mongodb"), metalog = require("./metalog"), + options = require("../../config/cube"), db_index = 0; +function mongoUrl(options) { + if ("mongo-url" in options) { + return options["mongo-url"]; + } + + var user = options["mongo-username"], + pass = options["mongo-password"], + host = options["mongo-host"] || "localhost", + port = options["mongo-port"] || 27017, + name = options["mongo-database"] || "cube", + auth = user ? user + ":" + pass + "@" : ""; + return "mongodb://" + auth + host + ":" + port + "/" + name; +} + +function mongoOptions(options) { + return { + db: options["mongo-database-options"] || { native_parser: true, safe: false }, + server: options["mongo-server-options"] || { auto_reconnect: true }, + replSet: { read_secondary: true } + } +} + function Db(){ var metric_options, event_options, db_client, events_client, metrics_client; var db_id = db_index++; @@ -24,32 +47,26 @@ function Db(){ // Connect to mongodb. // - function open(options, callback){ - var server_options = options["mongo-server_options"], // MongoDB server configuration - db_options = { native_parser: true }, // MongoDB driver configuration. - mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], server_options), - database_name = options["mongo-database"], - mongo_password = options["mongo-password"]; - + function open(callback){ if (db_client) { metalog.minor('mongo_already_open', db.report()); return callback(null, db); } - collection_prefix = options["collection_prefix"] || ''; - metric_options = options["mongo-metrics"], - event_options = options["mongo-events"], - db_client = new mongodb.Db(database_name, mongo, db_options); + var mongoOpts = options.get('mongodb'); + collection_prefix = mongoOpts["collection_prefix"] || ''; + metric_options = mongoOpts["mongo-metrics"]; + event_options = mongoOpts["mongo-events"]; - delete options["mongo-password"]; metalog.info('mongo_connect', db.report()); - - db_client.open(function(error){ + mongodb.Db.connect(mongoUrl(mongoOpts), mongoOptions(mongoOpts), function(error, dbHandle) { if (error) return callback(error); + db_client = dbHandle; + // Open separate events database if configured - if (options["separate-events-database"]) events_client = db_client.db(database_name + '-events'); + if (mongoOpts["separate-events-database"]) events_client = db_client.db(db_client.databaseName + '-events'); else events_client = db_client; // Open separate metrics database if configured - if (options["separate-metrics-database"]) metrics_client = db_client.db(database_name + '-metrics'); + if (mongoOpts["separate-metrics-database"]) metrics_client = db_client.db(db_client.databaseName + '-metrics'); else metrics_client = db_client; Object.defineProperty(db, "_dbs", { @@ -66,14 +83,7 @@ function Db(){ db.types = types(events_client); db.collection = collection(db_client); db.clearCache = function(callback){ collections = {}; if(callback) callback(null, db); }; - - if (! options["mongo-username"]) return callback(null, db); - - db_client.authenticate(options["mongo-username"], mongo_password, function(error, success) { - if (error) return callback(error); - if (!success) return callback(new Error("authentication failed")); - callback(null, db); - }); + callback(null, db); }); } diff --git a/lib/cube/event.js b/lib/cube/event.js index 98e3c492..51ef1850 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -12,7 +12,8 @@ var _ = require("underscore"), Invalidator = Metric = require("./models/invalidator"), parser = require("./event-expression"), bisect = require("./bisect"), - metalog = require("./metalog"); + metalog = require("./metalog"), + config = require("../../config/cube"); // When streaming events, we should allow a delay for events to arrive, or else // we risk skipping events that arrive after their event.time. This delay can be @@ -32,8 +33,8 @@ var putter_id = 0; // - data, the event's payload // -exports.putter = function(db, config){ - var options = (config || options || {}); +exports.putter = function(db){ + var horizons = config.get('horizons'); var invalidator = new Invalidator(); @@ -43,7 +44,7 @@ exports.putter = function(db, config){ callback = callback || function(){}; // // Drop events from before invalidation horizon - if ((! request.force) && options.horizons && (time < new Date(new Date() - options.horizons.invalidation))) { + if ((! request.force) && horizons && (time < new Date(new Date() - horizons.invalidation))) { metalog.info('cube_compute', {error: "event before invalidation horizon"}); return callback({error: "event before invalidation horizon"}), -1; } @@ -100,9 +101,8 @@ exports.putter = function(db, config){ // * Issue the query; // * if streaming, register the query to be run at a regular interval // -exports.getter = function(db, config) { - var options = (config || options || {}), - streamsBySource = {}; +exports.getter = function(db) { + var streamsBySource = {}; function getter(request, callback) { var stream = !("stop" in request), diff --git a/lib/cube/index.js b/lib/cube/index.js index ef9c31b8..601b7b45 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -2,6 +2,7 @@ process.env.TZ = 'UTC'; exports.authentication = require("./authentication"); +exports.config = require("../../config/cube"); exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); exports.server = require("./server"); diff --git a/lib/cube/metric.js b/lib/cube/metric.js index 705218be..b4585ee4 100644 --- a/lib/cube/metric.js +++ b/lib/cube/metric.js @@ -11,13 +11,12 @@ var _ = require("underscore"), Metric = require("./models/metric"), Measurement = require("./models/measurement"), event = require("./event"), - metalog = require('./metalog'), - options = require('../../config/cube'); + metalog = require('./metalog'); // Query for metrics. -exports.getter = function(db, config) { - var options = (config || options || {}); +exports.getter = function(db) { + var streamsBySource = {}; function getter(request, callback) { var measurement, expression, @@ -44,7 +43,7 @@ exports.getter = function(db, config) { - measurement.measure(db, options, callback); + measurement.measure(db, callback); } return getter; diff --git a/lib/cube/models/invalidator.js b/lib/cube/models/invalidator.js index a9d7a656..dd9c4621 100644 --- a/lib/cube/models/invalidator.js +++ b/lib/cube/models/invalidator.js @@ -11,9 +11,9 @@ var invalidateInterval = 5000; // Schedule deferred invalidation of metrics by type and tier. function Invalidator(){ - this.type_tsets = {}, - this.invalidate = { $set: {i: true} }, - this.multi = { multi: true }; + this.type_tsets = {}, + this.invalidate = { $set: {i: true} }, + this.invalidationOptions = { multi: true, w: 0 }; } Model.modelize(Invalidator); @@ -32,7 +32,7 @@ function flush(db, callback){ _.each(type_tset, function(tset, tier){ var times = dateify(tset); metalog.info("event_flush", { type: type, tier: tier, times: times }); - collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, _this.invalidate, _this.multi); + collection.update({ i: false, "_id.l": +tier, "_id.t": {$in: times}}, _this.invalidate, _this.invalidationOptions); }); }); }); @@ -65,4 +65,4 @@ Invalidator.stop_flusher = function(id, on_stop){ if (on_stop) on_stop(); }; -module.exports = Invalidator; \ No newline at end of file +module.exports = Invalidator; diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index bc872932..95df2290 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -6,6 +6,7 @@ var metalog = require('../metalog'), Metric = require('./metric'), Event = require('./event'), _ = require('underscore'), + config = require('../../../config/cube'), compute = {constant: constant, binary: binary, unary: unary}, queueByName = {}; @@ -43,21 +44,21 @@ Measurement.prototype.report = function report(){ // by the specified tier, such as daily or hourly. The callback is invoked // repeatedly for each metric value. The values may be out of order due // to partial cache hits. -Measurement.prototype.measure = function measure(db, options, callback) { +Measurement.prototype.measure = function measure(db, callback) { var _this = this; if (this.isGrouped && !Array.isArray(this.isGrouped.groups)) { Event.groups(db, this, function(error, group_names){ handle(error); _this.isGrouped.groups = group_names.sort(); - compute[_this.flavor].call(_this, db, options, callback); + compute[_this.flavor].call(_this, db, callback); }); } else { - compute[this.flavor].call(this, db, options, callback); + compute[this.flavor].call(this, db, callback); } }; // Computes a constant expression like the "7" in "x * 7" -function constant(db, options, callback) { +function constant(db, callback) { var _this = this, value = this.expression.value(); walk(this.start, this.stop, this.tier, function(time){ callback(new Metric({time: time, value: value}, _this)); @@ -66,7 +67,7 @@ function constant(db, options, callback) { }; // Serializes a unary expression for computation. -function unary(db, options, callback) { +function unary(db, callback) { var self = this, total_remaining = 0, remaining = 0, @@ -115,7 +116,7 @@ function unary(db, options, callback) { query_measurement.group = group_name; - findOrComputeUnary.call(query_measurement, db, options, function(metric){ + findOrComputeUnary.call(query_measurement, db, function(metric){ onMetric(metric); if(!--group_remaining) nextGroup(group_name); }); @@ -124,13 +125,13 @@ function unary(db, options, callback) { if (self.expression.group) { nextGroup(); } else { - findOrComputeUnary.call(self, db, options, onMetric); + findOrComputeUnary.call(self, db, onMetric); } } } // Finds or computes a unary (primary) expression. -function findOrComputeUnary(db, options, callback) { +function findOrComputeUnary(db, callback) { var expression = this.expression, group_name = this.group, map = expression.value, @@ -191,17 +192,19 @@ function findOrComputeUnary(db, options, callback) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { + var horizons = config.get('horizons'); + // Reset start time to calculation horizon if requested time span goes past it - if (options.horizons && tier.floor(start) < new Date(new Date() - options.horizons.calculation)){ + if (horizons && tier.floor(start) < new Date(new Date() - horizons.calculation)){ var old_start = start, - start = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + start = tier.step(tier.floor(new Date(new Date() - horizons.calculation))) metalog.info('cube_compute', {is: 'past_horizon', start: {was: old_start, updated_to: start}, stop: stop, tier: tier, expression: expression.source }); } // Reset stop time to calculation horizon if requested time span goes past it - if (options.horizons && tier.floor(stop) < new Date(new Date() - options.horizons.calculation)){ + if (horizons && tier.floor(stop) < new Date(new Date() - horizons.calculation)){ var old_stop = stop, - stop = tier.step(tier.floor(new Date(new Date() - options.horizons.calculation))) + stop = tier.step(tier.floor(new Date(new Date() - horizons.calculation))) metalog.info('cube_compute', {is: 'past_horizon', metric: {start: start, stop: {was: old_stop, updated_to: stop}, tier: tier, expression: expression.source } }); } @@ -248,7 +251,7 @@ function findOrComputeUnary(db, options, callback) { // unary "sum(resp)". We don't know what order they'll show up in, so if say // the value for left appears first, it parks that value as left[time], where // the result for right will eventually find it. -function binary(db, options, callback) { +function binary(db, callback) { var self = this, expression = this.expression, value; var left = new Measurement(expression.left, this.start, this.stop, this.tier), right = new Measurement(expression.right, this.start, this.stop, this.tier), @@ -292,16 +295,16 @@ function binary(db, options, callback) { other_measurement.on("complete", function(){ complete(other_measurement, grouped_measurement)(); - grouped_measurement.measure(db, options, measure(grouped_measurement, other_measurement)); + grouped_measurement.measure(db, measure(grouped_measurement, other_measurement)); }); grouped_measurement.on("complete", complete(grouped_measurement, other_measurement)); - other_measurement.measure(db, options, measure(other_measurement, grouped_measurement)); + other_measurement.measure(db, measure(other_measurement, grouped_measurement)); } else { left.on("complete", complete(left, right)); right.on("complete", complete(right, left)); - left.measure(db, options, measure(left, right)); - right.measure(db, options, measure(right, left)); + left.measure(db, measure(left, right)); + right.measure(db, measure(right, left)); } }; @@ -319,4 +322,4 @@ function handle(error) { throw error; } -module.exports = Measurement; \ No newline at end of file +module.exports = Measurement; diff --git a/lib/cube/server.js b/lib/cube/server.js index d937bb8c..7cf061e1 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -23,7 +23,8 @@ var util = require("util"), authentication = require("./authentication"), event = require("./event"), metalog = require("./metalog"), - Db = require("./db"); + Db = require("./db"), + config = require("../../config/cube"); // Don't crash on errors. process.on("uncaughtException", function(error) { @@ -31,7 +32,7 @@ process.on("uncaughtException", function(error) { }); // And then this happened: -websprocket.Connection = require("../../node_modules/websocket-server/lib/ws/connection"); +websprocket.Connection = require("websocket-server/lib/ws/connection"); // Configuration for WebSocket requests. var wsOptions = { @@ -46,8 +47,9 @@ var wsOptions = { closeTimeout: 5000 }; -module.exports = function(options, db) { - var server = {}, +module.exports = function(kind, db) { + var options = config.get(kind), + server = {}, primary = http.createServer(), secondary = websprocket.createServer(), file = new file_server.Server("static"), @@ -56,11 +58,17 @@ module.exports = function(options, db) { id = 0, authenticator; + if (!options) { + throw new Error('The provided type of server does not exist in the configuration file.'); + } + // allows dependency injection from test_helper if (! db) db = new Db(); secondary.server = primary; + var uses = [] //will be used for registrating anything external, particularly endpoints. + function is_sec_ws_initiation(request){ return ("sec-websocket-version" in request.headers); } @@ -118,9 +126,14 @@ module.exports = function(options, db) { connection_callback.id = ++id; // Listen for socket disconnect. - if (e.dispatch.close) connection.socket.on("end", function() { - e.dispatch.close(connection_callback); - }); + if (e.dispatch.close) { + connection.socket.on("end", function() { + e.dispatch.close(connection_callback); + }); + connection.socket.on("close", function() { + e.dispatch.close(connection_callback); + }); + } connection.on("message", function(message) { // parse, staple the authorization on, then process @@ -178,8 +191,13 @@ module.exports = function(options, db) { } }); + server.use = function(endpointAdder) { + uses.push(endpointAdder); + return server; + } + server.start = function(server_start_cb) { - db.open(options, function(error, db){ + db.open(function(error, db){ handle(error); ready(db); }); @@ -187,10 +205,10 @@ module.exports = function(options, db) { // Start the server! function ready(db) { metalog.putter = event.putter(db); - server.register(db, endpoints); - authenticator = authentication.authenticator(options["authenticator"], db, options); + uses.forEach(function(adder){ adder(db, endpoints); }); + authenticator = authentication.authenticator(options["authenticator"], db); metalog.event('start_http', { port: options["http-port"] }); - primary.listen(options["http-port"]); + primary.listen(options["http-port"], options["http-host"]); if (endpoints.udp) { metalog.event('start_udp', { port: options["udp-port"] }); udp = dgram.createSocket("udp4"); @@ -207,7 +225,7 @@ module.exports = function(options, db) { secondary.on("close", function(){ metalog.info('ws_close' ); }); function try_close(name, obj){ if (obj){ try { - metalog.info(name+'_stopping', options); + metalog.info(name+'_stopping', config.values); obj.close( function(){ metalog.info(name+'_stop'); } ); } catch(error){} } } diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index cf6d47d2..74d340f5 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -1,13 +1,15 @@ 'use strict'; -var cluster = require('cluster'), - metric = require('./metric'), +var metric = require('./metric'), tiers = require('./tiers'), metalog = require('./metalog'), - db = new (require('./db'))(); + db = new (require('./db'))(), + options = require('../../config/cube'); -module.exports = function(options){ - var calculate_metric, tier, timeout; +module.exports = function() { + var calculate_metric, tier, timeout, + horizons = options.get('horizons'), + warmer = options.get('warmer'); function fetch_metrics(callback){ var expressions = []; @@ -33,29 +35,29 @@ module.exports = function(options){ function process_metrics(expressions){ expressions.forEach(function(expression){ var stop = new Date(), - start = tier.step(tier.floor(new Date(stop - options.horizons.calculation))); + start = tier.step(tier.floor(new Date(stop - horizons.calculation))); metalog.info('cube_warm', {is: 'warm_metric', metric: {query: expressions}, start: start, stop: stop }); // fake metrics request calculate_metric({ step: tier.key, expression: expression, start: start, stop: stop }, noop); }); - timeout = setTimeout(function(){ fetch_metrics(process_metrics); }, options['warmer-interval']); + timeout = setTimeout(function(){ fetch_metrics(process_metrics); }, warmer['warmer-interval']); } function noop(){}; return { start: function(){ - tier = tiers[options['warmer-tier'].toString()]; + tier = tiers[warmer['warmer-tier'].toString()]; - if(typeof tier === "undefined") throw new Error("Undefined warmer tier configured: " + options['warmer-tier']); + if(typeof tier === "undefined") throw new Error("Undefined warmer tier configured: " + warmer['warmer-tier']); - metalog.event("cube_life", { is: 'start_warmer', options: options }); + metalog.event("cube_life", { is: 'start_warmer', options: options.values }); - db.open(options, function(error) { + db.open(function(error) { if (error) throw error; - calculate_metric = metric.getter(db, options); + calculate_metric = metric.getter(db); fetch_metrics(process_metrics); }); }, diff --git a/package.json b/package.json index 10ae9574..fa90a158 100644 --- a/package.json +++ b/package.json @@ -20,18 +20,18 @@ "node-static": "0.6.5", "pegjs": "0.7.0", "vows": "0.7.0", - "node-gyp": "0.6.8", "websocket": "1.0.8", "websocket-server": "1.4.04", "cookies": "0.3.1", "bcrypt": "0.7.1", "underscore": "1.3.3", "jake": "0.5.6", - "queue-async": "~1.0.4" + "queue-async": "~1.0.4", + "cfg": "0.0.2" }, "scripts": { "preinstall": "npm install mongodb --mongodb:native", - "test": "vows --isolate --spec", + "test": "NODE_ENV=test vows --isolate --spec", "remove_expired_metrics": "jake db:metrics:remove_expired" } } diff --git a/test/authentication-test.js b/test/authentication-test.js index 3743b0c4..dfa65d7b 100644 --- a/test/authentication-test.js +++ b/test/authentication-test.js @@ -53,7 +53,8 @@ suite.addBatch(test_helper.batch({ topic: function(test_db){ test_db.using_objects("test_users", test_users, this); }, "": { topic: function(test_db){ - return authentication.authenticator("mongo_cookie", test_db, { collection: "test_users" }); }, + return authentication.authenticator("mongo_cookie", test_db); + }, "authenticates": { "users with good tokens": { topic: successful_auth(dummy_request("boss_hogg")), diff --git a/test/collector-test.js b/test/collector-test.js index 908f2ef7..5b1b599a 100644 --- a/test/collector-test.js +++ b/test/collector-test.js @@ -7,10 +7,8 @@ var vows = require("vows"), var suite = vows.describe("collector"); -var server_options = { 'http-port': test_helper.get_port() }; - suite.addBatch( - test_helper.with_server(server_options, cube.collector.register, { + test_helper.with_server('collector', cube.collector.register, { "POST /event/put with invalid JSON": { topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), diff --git a/test/evaluator-test.js b/test/evaluator-test.js index 435319ed..5dc6b1fd 100644 --- a/test/evaluator-test.js +++ b/test/evaluator-test.js @@ -7,14 +7,13 @@ var vows = require("vows"), var suite = vows.describe("evaluator"); -var server_options = { 'http-port': test_helper.get_port() }; function frontend_components() { cube.evaluator.register.apply(this, arguments); cube.visualizer.register.apply(this, arguments); } suite.addBatch( - test_helper.with_server(server_options, frontend_components, { + test_helper.with_server('evaluator', frontend_components, { "POST /event/put with invalid JSON": { topic: test_helper.request({method: "GET", path: "/1.0/event/get"}, "This ain't JSON.\n"), diff --git a/test/event-test.js b/test/event-test.js index f877e1ee..e156f3d7 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -4,8 +4,8 @@ var vows = require("vows"), assert = require("assert"), test_helper = require("./test_helper"), Event = require("../lib/cube/models/event"), - Metric = require("../lib/cube/models/metric"), - event = require("../lib/cube/event"); + event = require("../lib/cube/event"), + config = require('../config/cube'); var suite = vows.describe("event"); @@ -47,9 +47,14 @@ suite.addBatch(test_helper.batch({ suite.addBatch(test_helper.batch({ topic: function(test_db) { - var horizon = new Date() - fuck_wit_dre_day + (1000 * 60), - options = this.settings = test_helper._.extend({}, test_helper.settings, {horizons: { invalidation: horizon }}); - return event.putter(test_db.db, options); + var horizon = new Date() - fuck_wit_dre_day + (1000 * 60); + + // Temporarily override horizons settings + this.oldHorizons = config.get('horizons'); + config.set('horizons', { + invalidation: horizon + }); + return event.putter(test_db.db); }, 'events past invalidation horizon': { topic: function(putter){ @@ -63,7 +68,10 @@ suite.addBatch(test_helper.batch({ 'should return -1': function(error, response){ assert.equal(this.ret, -1); } - } + }/*, + teardown: function() { + config.set('horizons', this.oldHorizons); + }*/ })); suite['export'](module); diff --git a/test/server-test.js b/test/server-test.js index feec4ecb..40f54326 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -16,10 +16,6 @@ var example = { for_http: { type: "monotreme", time: now_ish, data: { echidnae: 4 } }, for_udp: { type: "monotreme", time: now_ish, data: { platypodes: 9 } } }; -var server_options = { - 'http-port': test_helper.get_port(), - 'udp-port': test_helper.get_port() -}; function dummy_server(db, endpoints){ endpoints.udp = function(req, cb){ // metalog.info('rcvd_udp', { req: req }); @@ -33,7 +29,8 @@ function dummy_server(db, endpoints){ } suite.addBatch( - test_helper.with_server(server_options, dummy_server, { + // Set to `collector` because it's the only one with both http and udp + test_helper.with_server('collector', dummy_server, { http: { topic: test_helper.request({path: "/1.0/test", method: "POST"}, JSON.stringify([example.for_http])), diff --git a/test/test_helper.js b/test/test_helper.js index f22a1da3..b48ff5f0 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -6,7 +6,8 @@ var _ = require("underscore"), http = require("http"), dgram = require('dgram'), Db = require("../lib/cube/db"), - metalog = require("../lib/cube/metalog"); + metalog = require("../lib/cube/metalog"), + config = require("../config/cube"); // ========================================================================== // @@ -18,22 +19,38 @@ var test_collections = ["test_users", "test_events", "test_metrics", "test_boa test_helper.inspectify = metalog.inspectify; test_helper._ = require('underscore'); +config.set('mongodb', { + 'mongo-host': 'localhost', + 'mongo-port': 27017, + 'mongo-username': null, + 'mongo-password': null, + 'mongo-database': 'cube_test', + 'host': 'localhost', + 'authentication-collection': 'test_users' +}); -test_helper.settings = { - "mongo-host": "localhost", - "mongo-port": 27017, - "mongo-username": null, - "mongo-password": null, - "mongo-database": "cube_test", - "host": "localhost", - "authenticator": "allow_all", +config.set('horizons', { + calculation: +(new Date()), + invalidation: +(new Date()) +}); +var basePort = 1083; +config.set('collector', { + 'http-port': basePort++, + 'udp-port': basePort++, + 'authenticator': 'allow_all' +}); + +config.set('evaluator', { + 'http-port': basePort++, + 'authenticator': 'allow_all' +}); + +config.set('warmer', { + 'warmer-interval': 10000, + 'warmer-tier': 10000 +}); - "horizons": { - "calculation": +(new Date()), - "invalidation": +(new Date()), - } -}; // Disable logging for tests. metalog.loggers.info = metalog.silent; // log @@ -45,10 +62,6 @@ metalog.send_events = false; // client / server helpers // -var port = 1083; -// test_helper.get_port() -- get a port ID, unique to your batch. -test_helper.get_port = function(){ return ++port; }; - // test_helper.request -- make an HTTP request. // // @param options standard http client options, with these defaults: @@ -132,14 +145,14 @@ test_helper.delay = delay; // inscribes 'server', 'udp_port' and 'http_port' on the test context -- letting // you say 'this.server' in your topics, etc. // -// @param options -- overrides for the settings, above. +// @param kind -- types of server to run. // @param components -- passed to server.register() // @param batch -- the tests to run -test_helper.with_server = function(options, components, batch){ +test_helper.with_server = function(kind, components, batch){ return test_helper.batch({ '': { topic: function(test_db){ var ctxt = this, cb = ctxt.callback; - start_server(options, components, ctxt, test_db); + start_server(kind, components, ctxt, test_db); }, '': batch, teardown: function(j_, test_db){ @@ -153,14 +166,12 @@ test_helper.with_server = function(options, components, batch){ }; // @see test_helper.with_server -function start_server(options, register, ctxt, test_db){ - for (var key in test_helper.settings){ - if (! options[key]){ options[key] = test_helper.settings[key]; } - } - ctxt.http_port = options['http-port']; - ctxt.udp_port = options['udp-port']; - ctxt.server = require('../lib/cube/server')(options, test_db); - ctxt.server.register = register; +function start_server(kind, register, ctxt, test_db){ + var config = require('../config/cube').get(kind); + ctxt.http_port = config['http-port']; + ctxt.udp_port = config['udp-port']; + ctxt.server = require('../lib/cube/server')(kind, test_db); + ctxt.server.use(register); ctxt.server.start(ctxt.callback); } @@ -180,8 +191,7 @@ test_helper.batch = function(batch) { topic: function() { var ctxt = this; ctxt.db = new Db(); - var options = _.extend({}, test_helper.settings); - ctxt.db.open(options, function(error){ + ctxt.db.open(function(error){ drop_and_reopen_collections(ctxt.db, function(error){ ctxt.callback.apply(ctxt, arguments); ctxt.db.clearCache(); diff --git a/test/visualizer-test.js b/test/visualizer-test.js index 327a04af..5167c149 100644 --- a/test/visualizer-test.js +++ b/test/visualizer-test.js @@ -7,14 +7,13 @@ var vows = require("vows"), var suite = vows.describe("visualizer"); -var server_options = { 'http-port': test_helper.get_port() }; function frontend_components() { cube.evaluator.register.apply(this, arguments); cube.visualizer.register.apply(this, arguments); } // suite.addBatch( -// test_helper.with_server(server_options, frontend_components, { +// test_helper.with_server('evaluator', frontend_components, { // // "POST /event/put with invalid JSON": { // topic: test_helper.request({method: "POST", path: "/1.0/event/put"}, "This ain't JSON.\n"), diff --git a/test/warmer-test.js b/test/warmer-test.js index da7d3fdf..3b2eafe3 100644 --- a/test/warmer-test.js +++ b/test/warmer-test.js @@ -5,18 +5,26 @@ var _ = require('underscore'), assert = require("assert"), test_helper = require("./test_helper"), event = require("../lib/cube/event"), - settings = {"warmer-tier": 10000, "warmer-interval": 10000, horizons: { calculation: 30000 } }, - warmer = require("../lib/cube/warmer")(_.extend({}, test_helper.settings, settings)); + config = require("../config/cube"), + warmer = require("../lib/cube/warmer"); var suite = vows.describe("warmer"); suite.addBatch(test_helper.batch({ topic: function(test_db) { + // Temporarily override horizons settings + this.oldHorizons = config.get('horizons'); + config.set('horizons', { + calculation: 30000 + }); + var board = { pieces: [{ query: "sum(test(value))" }] }, nowish = this.nowish = (10e3 * Math.floor(new Date()/10e3)), putter = this.putter = event.putter(test_db), _this = this; + this.warmer = warmer(); + putter({type: 'test', time: nowish + 500, data: {value: 10}}, function(){ putter({type: 'test', time: nowish + 2000, data: {value: 5}}, function(){ test_db.using_objects("boards", [board], {callback: function(error){ @@ -30,11 +38,11 @@ suite.addBatch(test_helper.batch({ topic: function(test_db){ var _this = this; - warmer.start() + this.warmer.start(); setTimeout(function(){ test_db.metrics('test', function(error, collection){ - collection.find().toArray(_this.callback) - }) + collection.find().toArray(_this.callback); + }); }, 1000); }, 'correct number of metrics': function(metrics){ assert.equal(metrics.length, 3); }, @@ -45,7 +53,8 @@ suite.addBatch(test_helper.batch({ } }, teardown: function(){ - warmer.stop(); + config.set('horizons', this.oldHorizons); + this.warmer.stop(); this.putter.stop(this.callback); } })); From 6ccef0a0b4a6fe966c2e25131e9cbfd3a5cd632e Mon Sep 17 00:00:00 2001 From: Marsup Date: Tue, 6 Aug 2013 17:37:26 +0200 Subject: [PATCH 69/87] Fix metric tests by rising the timeout --- test/metric-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/metric-test.js b/test/metric-test.js index bf4de820..c66e3f7b 100644 --- a/test/metric-test.js +++ b/test/metric-test.js @@ -20,7 +20,7 @@ var step_testing_delay = 250, var suite = vows.describe("metric"); var nowish = Date.now(), - nowish_floor = (10e3 * Math.floor(nowish/10e3)), + nowish_floor = tiers[10e3].floor(nowish), nowish_stop = nowish_floor + 30e3, thenish = Date.UTC(2011, 6, 18, 0, 0, 0); var invalid_expression_error = { error: { message: 'Expected "(", "-", "distinct", "max", "median", "min", "sum" or number but "D" found.', column: 1, line: 1, name: 'SyntaxError' }}; @@ -266,7 +266,7 @@ function metricTest(request, expected) { function get_metrics_with_delay(depth){ return function(){ var actual = [], - timeout = setTimeout(function() { console.log(" TIMING OUT NOW", request ); cb(new Error("Time's up!")); }, 20000), + timeout = setTimeout(function() { console.log(" TIMING OUT NOW", request ); cb(new Error("Time's up!")); }, 40e3), cb = this.callback, req = Object.create(request), getter = arguments[depth]; From 5dc75c1f6a71a3aae0bc77d2143f615e1f972c40 Mon Sep 17 00:00:00 2001 From: Tom Carden Date: Sat, 7 Sep 2013 22:38:25 -0700 Subject: [PATCH 70/87] remove use of process.nextTick --- examples/random-emitter/random-emitter.js | 5 +++-- lib/cube/broker.js | 5 +++-- lib/cube/models/measurement.js | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/random-emitter/random-emitter.js b/examples/random-emitter/random-emitter.js index af3d405f..74aa6524 100644 --- a/examples/random-emitter/random-emitter.js +++ b/examples/random-emitter/random-emitter.js @@ -2,7 +2,8 @@ process.env.TZ = 'UTC'; var util = require("util"), cube = require("../../"), // replace with require("cube") - options = require("./random-config"); + options = require("./random-config"), + setImmediate = require("../../set-immediate"); util.log("starting emitter"); var emitter = cube.emitter(options["collector"]); @@ -28,7 +29,7 @@ function insert(start){ value: value += Math.random() - .5 } }); - process.nextTick(function(){ insert(start + step) }); + setImmediate(function(){ insert(start + step) }); } insert(start); diff --git a/lib/cube/broker.js b/lib/cube/broker.js index 7c0871e0..43ce8470 100644 --- a/lib/cube/broker.js +++ b/lib/cube/broker.js @@ -2,7 +2,8 @@ var _ = require('underscore'), util = require('util'), - metalog = require('./metalog'); + metalog = require('./metalog'), + setImmediate = require("./set-immediate"); // milliseconds to sleep, if all jobs are satisfied, until checking again var worker_sleep_ms = 100; @@ -147,7 +148,7 @@ function Job(mbox, perform, args){ Job.prototype.complete = function(result){ var on_completes = this.on_completes; for (var ii in on_completes){ - process.nextTick(function(){ on_completes[ii].apply(null, result); }); + setImmediate(function(){ on_completes[ii].apply(null, result); }); }; } Job.prototype.listen = function(on_complete){ diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 95df2290..7d1a7719 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -8,6 +8,7 @@ var metalog = require('../metalog'), _ = require('underscore'), config = require('../../../config/cube'), compute = {constant: constant, binary: binary, unary: unary}, + setImmediate = require("../set-immediate"), queueByName = {}; function isGrouped(expression){ @@ -86,7 +87,7 @@ function unary(db, callback) { // Add this task to the appropriate queue. if (queue) queue.next = task; - else process.nextTick(task); + else setImmediate(task); queueByName[name] = task; function task() { @@ -94,7 +95,7 @@ function unary(db, callback) { callback(metric); if (!--total_remaining) { self.emit('complete'); - if (task.next) process.nextTick(task.next); + if (task.next) setImmediate(task.next); else delete queueByName[name]; // Record how long it took us to compute as an event! From bf2400ff3e16e51b59d309aea9492242429e188b Mon Sep 17 00:00:00 2001 From: Tom Carden Date: Sat, 7 Sep 2013 22:45:55 -0700 Subject: [PATCH 71/87] remove unused broker --- lib/cube/broker.js | 162 -------------------------------------------- test/broker-test.js | 115 ------------------------------- 2 files changed, 277 deletions(-) delete mode 100644 lib/cube/broker.js delete mode 100644 test/broker-test.js diff --git a/lib/cube/broker.js b/lib/cube/broker.js deleted file mode 100644 index 43ce8470..00000000 --- a/lib/cube/broker.js +++ /dev/null @@ -1,162 +0,0 @@ -'use strict'; - -var _ = require('underscore'), - util = require('util'), - metalog = require('./metalog'), - setImmediate = require("./set-immediate"); - -// milliseconds to sleep, if all jobs are satisfied, until checking again -var worker_sleep_ms = 100; - -// ---- Broker ---- - -function Broker(name, interval){ - var workers = []; - add_worker(); - this.name = name; - - // `deferProxy(name, perform, *args, on_complete)` -- Issue a request with - // controlled concurrency; send its results to all interested listeners. We - // use it so that when multiple clients are interested in a metric, we only - // issue the query once, yet share the good news immediately. - // - // Worker will at some time invoke perform with the supplied args, tacking on - // its own callback (`perform(*args, worker_cb)`, essentially). The task must, - // success or failure, eventually invoke the proxy callback. - // - // When the task triggers the proxy callback with the result, every - // `on_complete` handler waiting on that name is invoked with that result. - // For the second and further tasks `defer`ing to a given mbox, the `perform` - // function is ignored -- tasks with the same name must be interchangeable. - // - // @param [String] name -- handle for the query - // @param [Function] perform(*args) -- function for worker to dispatch - // @param [Array] *args -- args sent to `perform` when dispatched - // @param [Function] complete_cb -- called when the task completes - // - this.deferProxy = function deferProxy(){ // - var args = _.toArray(arguments), - name = args.shift(), perform = args.shift(), on_complete = args.pop(); - if (! (name && perform && on_complete)) throw new TypeError('you must supply a name, perform callback and on-complete handler: got ' + [name, perform, on_complete]); - worker_for(name).add(name, perform, args, on_complete); - return this; - }; - - function worker_for(name){ - return workers[0]; - } - - function add_worker(){ - var worker = new Worker((name+'-'+workers.length), interval); - workers.push(worker); - worker.start(); - return worker; - } - - function stop(){ _.each(workers, function(worker){ worker.stop() }) }; - - function report(){ return { workers: _.map(workers, function(worker){ return worker.report() }) }; } - function toString(){ return util.inspect(this.report()); }; - - Object.defineProperties(this, { - stop: {value: stop}, - report: {value: report}, toString: {value: toString}, - }); -} - -// ---- Worker ---- - -// A worker executes a set of tasks with parallelism 1 -function Worker(qname, interval){ - var queue = {}, - active = null, - self = this, - clock = null; - this.qname = qname; - - this.add = function add(mbox, perform, args, on_complete) { - var job = ((active && (active.mbox === mbox)) ? active : queue[mbox]); - if (! job){ job = queue[mbox] = new Job(mbox, perform, args); } - // - job.listen(on_complete); - metalog.minor('q_worker', {is: 'add', mbox: job.mbox, am: self.report(), job: job.report() }); - return job; - }; - - function invoke(job) { - // move this job to be active - active = job; - delete queue[job.mbox]; - // add our callback as last arg. when triggered, it takes the arguments it - // was triggered with and has job fire that at all its `on_complete`s, and - // clears the active job (letting the next task start). - job.args.push(function _completed(){ - var result = arguments; - metalog.minor('q_worker', {is: '<-!', mbox: job.mbox, am: self.report(), result: util.inspect(result).slice(0,80) }); - if (job !== active) metalog.warn('q_worker', {is: 'ERR', am: qname, error: 'job was missing when callback triggered', self: self.report() }); - active = null; - job.complete(result); - }); - // start the task - metalog.minor('q_worker', {is: '?->', mbox: job.mbox, am: self.report(), perform_args: util.inspect(job.args).slice(0,80) }); - try{ job.perform.apply(null, job.args); } catch(err){ metalog.warn('q_worker', {is: 'ERR', am: qname, as: 'performing job '+job.mbox, error: err.message }) }; - } - - this.start = function start(){ - if (clock){ return metalog.warn('q_worker', {is: 'ERR', am: self.report(), error: 'tried to start an already-running worker' }); } - clock = setInterval(work.bind(self), interval); - } - this.stop = function stop(){ - metalog.info('q_worker', {is: 'stp'}); - if (! clock){ return metalog.warn('q_worker', {is: 'ERR', am: self.report(), error: 'tried to stop an already-stopped worker' }); } - clearInterval(clock); - clock = null; - } - - function work (){ - var job; - if (active) { self.onWait(); } - else if (job = next()){ invoke(job); } - else { self.onIdle(); } - }; - - - function size(){ return _.size(queue); } - function state(){ return ((clock === null) ? 'stop' : (active ? 'busy' : 'idle')); } - function next(){ return queue[ _.keys(queue).sort()[0] ]; } - - // function onIdle(){ util.print(' '+self.qname+'!'); }; - // function onWait(){ util.print(' '+self.qname+'@'); }; - function onIdle(){ }; - function onWait(){ }; - - function report(){ return { qname: this.qname, size: this.size, state: this.state, queue: _.keys(queue) }; }; - function toString(){ return util.inspect(this.report()); }; - - Object.defineProperties(this, { - size: {get: size}, state: {get: state}, - report: {value: report}, toString: {value: toString}, - onWait: {value: onWait}, onIdle: {value: onIdle} - }); -} - -// ---- Job ---- - -function Job(mbox, perform, args){ - _.extend(this, { mbox: mbox, perform: perform, args: args, on_completes: [] }); -} -Job.prototype.complete = function(result){ - var on_completes = this.on_completes; - for (var ii in on_completes){ - setImmediate(function(){ on_completes[ii].apply(null, result); }); - }; -} -Job.prototype.listen = function(on_complete){ - this.on_completes.push(on_complete); -}; -Job.prototype.toString = function (){ return util.inspect(this.report()); }; -Job.prototype.report = function (){ return { mbox: this.mbox, args: this.args, on_completes: this.on_completes.length }; } - -// ---- - -module.exports = { Broker: Broker, Job: Job, Worker: Worker }; diff --git a/test/broker-test.js b/test/broker-test.js deleted file mode 100644 index 8cbb8a63..00000000 --- a/test/broker-test.js +++ /dev/null @@ -1,115 +0,0 @@ -'use strict'; - -var _ = require('underscore'), - util = require("util"), - vows = require("vows"), - assert = require("assert"), - test_helper = require("./test_helper"), - broker = require("../lib/cube/broker"), - Job = broker.Job, Broker = broker.Broker, Worker = broker.Worker, - metalog = require('../lib/cube/metalog'); - -var suite = vows.describe("broker"); - -var squarer = function(ii, cb){ cb(null, ii*ii, 'squarer'); }; - -function example_worker(){ var worker = new Worker('test', 50); worker.start(); return worker; } -function example_job(){ return (new Job('smurf', squarer, [7])); } - -// suite.addBatch({ -// 'Worker': { -// '.new': { -// topic: function(){ return new Worker('test', 50); }, -// 'has an empty queue': function(worker){ assert.deepEqual(worker.size, 0); }, -// 'is stopped': function(worker){ assert.equal(worker.state, 'stop'); } -// }, -// '.add': { -// topic: example_worker, -// 'pushes a job onto the queue': function(worker){ -// worker.add('smurfette', function(){ setTimeout(_.identity, 50) }, [3], _.identity); -// assert.deepEqual(worker.report(), { qname: 'test', size: 1, state: 'idle', queue: [ 'smurfette' ] }); -// } -// }, -// '.invoke': { -// topic: function(){ -// var worker = this.worker = example_worker(); -// var ctxt = this; -// ctxt.checker = assert.isCalledTimes(ctxt, 3); -// ctxt.performances = 0; -// // shortly, worker will invoke perfom (once). 200 ms later, `perform` -// // will call `worker`'s proxy callback, which invokes all 3 callbacks. -// var perform = function(cb){ ctxt.performances++; setTimeout(function(){cb('hi');}, 20); }; -// worker.add('thrice', perform, [], ctxt.checker); -// worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); -// worker.add('thrice', assert.isNotCalled('perform'), [], ctxt.checker); -// }, -// 'calls perform exactly once': function(){ assert.equal(this.performances, 1); }, -// 'calls all registered callbacks': function(results){ assert.deepEqual(results, [['hi'], ['hi'], ['hi']]); }, -// teardown: function(){ -// this.worker.stop(); -// } -// } -// }, -// -// 'Job': { -// '.new': { -// topic: example_job, -// '': function(job){ -// assert.deepEqual(job, {mbox: 'smurf', perform: squarer, args: [7], on_completes: []}); -// } -// }, -// '.listen': { -// topic: function(){ -// var job = example_job(); -// job.listen(squarer); -// return job; -// } -// } -// }, -// -// 'Broker': { -// 'handles interleaved jobs': { -// topic: function(){ -// var ctxt = this, -// broker = this.broker = new Broker('test', 10), -// ignored = assert.isNotCalled('perform'); -// ctxt.perfs = {a: 0, b: 0, c:0}; -// ctxt.checker = assert.isCalledTimes(ctxt, 8); -// var task_a = function(ii, a2, cb){ ctxt.perfs.a++; setTimeout(function(){cb('result_a', ii*ii, a2);}, 10); }; -// var task_b = function(ii, cb){ ctxt.perfs.b++; setTimeout(function(){cb('result_b', ii*ii );}, 20); }; -// var task_c = function(ii, cb){ ctxt.perfs.c++; setTimeout(function(){cb('result_c', ii*ii );}, 300); }; -// // will go second: jobs are sorted -// broker.deferProxy('task_b', task_b, 1, ctxt.checker); -// broker.deferProxy('task_b', ignored, '<>', ctxt.checker); -// // will go first -// broker.deferProxy('task_a', task_a, 0, '?', ctxt.checker); -// broker.deferProxy('task_a', ignored, '<>', ctxt.checker); -// // will go third -// broker.deferProxy('task_c', task_c, 2, ctxt.checker); -// // a & b will be done; c (takes 300ms) will still be running. -// setTimeout(function(){ -// broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); -// broker.deferProxy('task_c', ignored, '<>', ctxt.checker); -// broker.deferProxy('task_a', task_a, 3, '!', ctxt.checker); -// }, 200); -// }, -// 'calls perform exactly once': function(){ assert.deepEqual(this.perfs, {a: 2, b: 1, c: 1}); }, -// 'calls all registered callbacks': function(results){ -// assert.deepEqual(results, [ -// ['result_a', 0, '?'], ['result_a', 0, '?'], -// ['result_b', 1], ['result_b', 1], -// ['result_c', 4], ['result_c', 4], -// ['result_a', 9, '!'], ['result_a', 9, '!'] -// ]); -// }, -// teardown: function(){ -// this.broker.stop(); -// } -// } -// } // broker -// -// }); - - - -suite['export'](module); From 0b1a0184d2763f2a84e98388513a438fa456e822 Mon Sep 17 00:00:00 2001 From: Tom Carden Date: Sat, 7 Sep 2013 22:46:10 -0700 Subject: [PATCH 72/87] bump bcrypt --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 70d5f220..4262db29 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "websocket": "1.0.8", "websocket-server": "1.4.04", "cookies": "0.3.1", - "bcrypt": "0.7.1", + "bcrypt": "0.7.7", "underscore": "1.3.3", "jake": "0.5.6", "queue-async": "~1.0.4", From c0b8972d781927a3b3bdd0a016fee742bd9b8fc0 Mon Sep 17 00:00:00 2001 From: Marsup Date: Wed, 14 Aug 2013 11:37:11 +0200 Subject: [PATCH 73/87] Fix collector loop, forEach provides numbers as cb --- lib/cube/collector.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cube/collector.js b/lib/cube/collector.js index ea2dbb99..165b3b11 100644 --- a/lib/cube/collector.js +++ b/lib/cube/collector.js @@ -53,7 +53,9 @@ function post(putter) { }); request.on("end", function() { try { - JSON.parse(content).forEach(putter); + JSON.parse(content).forEach(function(event) { + putter(event); + }); } catch (e) { metalog.event("cube_request", { at: "c", res: "collector_post_error", error: e, code: 400 }); response.writeHead(400, headers); From ad9a0d3414ab37e7d36301946d5e18c17b49885a Mon Sep 17 00:00:00 2001 From: Marsup Date: Tue, 13 Aug 2013 18:11:13 +0200 Subject: [PATCH 74/87] One connection per db, that fixes auth problems --- lib/cube/db.js | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/cube/db.js b/lib/cube/db.js index 00123d25..4073f1d3 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -2,8 +2,9 @@ var util = require("util"), mongodb = require("mongodb"), + queue = require("queue-async"), metalog = require("./metalog"), - options = require("../../config/cube"), + config = require("../../config/cube"), db_index = 0; function mongoUrl(options) { @@ -50,23 +51,36 @@ function Db(){ function open(callback){ if (db_client) { metalog.minor('mongo_already_open', db.report()); return callback(null, db); } - var mongoOpts = options.get('mongodb'); - collection_prefix = mongoOpts["collection_prefix"] || ''; - metric_options = mongoOpts["mongo-metrics"]; - event_options = mongoOpts["mongo-events"]; + var mongoConfig = config.get('mongodb'), + url = mongoUrl(mongoConfig), + options = mongoOptions(mongoConfig); + + collection_prefix = mongoConfig["collection_prefix"] || ''; + metric_options = mongoConfig["mongo-metrics"]; + event_options = mongoConfig["mongo-events"]; metalog.info('mongo_connect', db.report()); - mongodb.Db.connect(mongoUrl(mongoOpts), mongoOptions(mongoOpts), function(error, dbHandle) { - if (error) return callback(error); - db_client = dbHandle; + var q = queue(); + + q.defer(mongodb.Db.connect, url, options); + + // Open separate events database if configured + if (mongoConfig["separate-events-database"]) q.defer(mongodb.Db.connect, url + '-events', options); + + // Open separate metrics database if configured + if (mongoConfig["separate-metrics-database"]) q.defer(mongodb.Db.connect, url + '-metrics', options); - // Open separate events database if configured - if (mongoOpts["separate-events-database"]) events_client = db_client.db(db_client.databaseName + '-events'); + q.awaitAll(function(err, results) { + if (err) return callback(err); + + var connId = 0; + db_client = results[connId++]; + + if (mongoConfig["separate-events-database"]) events_client = results[connId++]; else events_client = db_client; - // Open separate metrics database if configured - if (mongoOpts["separate-metrics-database"]) metrics_client = db_client.db(db_client.databaseName + '-metrics'); + if (mongoConfig["separate-metrics-database"]) metrics_client = results[connId++]; else metrics_client = db_client; Object.defineProperty(db, "_dbs", { @@ -83,6 +97,7 @@ function Db(){ db.types = types(events_client); db.collection = collection(db_client); db.clearCache = function(callback){ collections = {}; if(callback) callback(null, db); }; + callback(null, db); }); } From 137f278097e3230656e380b9c53ce2bf035560c0 Mon Sep 17 00:00:00 2001 From: Marsup Date: Fri, 9 Aug 2013 15:10:03 +0200 Subject: [PATCH 75/87] Store metrics with undefined values as well --- lib/cube/models/measurement.js | 4 ++-- lib/cube/models/metric.js | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 7d1a7719..32fa9ef3 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -182,7 +182,7 @@ function findOrComputeUnary(db, callback) { if (!--bin.size) { var metric = new Metric({time: time, value: reduce(bin.values), group: group_name}, measurement, bin.values); - if (metric.value || metric.value === 0) metric.save(db, handle); + if (metric.value || metric.value === 0 || metric.value === undefined) metric.save(db, handle); callback(metric); delete bins[time]; } @@ -219,7 +219,7 @@ function findOrComputeUnary(db, callback) { metric = new Metric({ time: time, value: value, group: group_name }, measurement, values); callback(metric); - if (metric.value || metric.value === 0) metric.save(db, handle); + if (metric.value || metric.value === 0 || metric.value === undefined) metric.save(db, handle); } function process(error, event){ diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index 1e0bc5a5..cd43318b 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -107,7 +107,11 @@ function to_wire(){ id = { e: this.e, l: this.l, t: this.time }; if(this.group) id.g = this.group; - return { i: false, v: mongo.Double(this.value), vs: values, _id: id}; + if (this.value === undefined) { + return { i: false, vs: values, _id: id}; + } else { + return { i: false, v: mongo.Double(this.value), vs: values, _id: id}; + } }; function report(){ From 201104bcf74cdfb1db7ea6366777497d85ac7342 Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 11:31:46 +0200 Subject: [PATCH 76/87] Improve collectors packets aggregation --- lib/cube/collectd.js | 6 +++--- lib/cube/collector.js | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/cube/collectd.js b/lib/cube/collectd.js index 654c9ab4..23c97661 100644 --- a/lib/cube/collectd.js +++ b/lib/cube/collectd.js @@ -49,13 +49,13 @@ exports.putter = function(putter) { } return function(request, response) { - var content = ""; + var content = []; request.on("data", function(chunk) { - content += chunk; + content.push(chunk); }); request.on("end", function() { var future = Date.now() / 1e3 + 1e9; - JSON.parse(content).forEach(function(values) { + JSON.parse(content.join('')).forEach(function(values) { var time = values.time; if (time > future) time /= 1073741824; values.time = Math.round(time) * 1e3; diff --git a/lib/cube/collector.js b/lib/cube/collector.js index 165b3b11..37d3918c 100644 --- a/lib/cube/collector.js +++ b/lib/cube/collector.js @@ -47,13 +47,13 @@ exports.register = function collector(db, endpoints) { // function post(putter) { return function(request, response) { - var content = ""; + var content = []; request.on("data", function(chunk) { - content += chunk; + content.push(chunk); }); request.on("end", function() { try { - JSON.parse(content).forEach(function(event) { + JSON.parse(content.join('')).forEach(function(event) { putter(event); }); } catch (e) { From 16eb1024cd2e597c0043492c08356028cb0f63bc Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 11:38:19 +0200 Subject: [PATCH 77/87] Remove unused files --- lib/cube/database.js | 55 -------------------------------------------- lib/queue-async | 1 - 2 files changed, 56 deletions(-) delete mode 100644 lib/cube/database.js delete mode 160000 lib/queue-async diff --git a/lib/cube/database.js b/lib/cube/database.js deleted file mode 100644 index a7de1ad2..00000000 --- a/lib/cube/database.js +++ /dev/null @@ -1,55 +0,0 @@ -var mongodb = require("mongodb"); - -var database = module.exports = {}; - -// Opens MongoDB driver given connection URL and optional options: -// -// { -// "mongo-url": "", -// "mongo-options": { -// "db": { "safe": false }, -// "server": { "auto_reconnect": true }, -// "replSet": { "read_secondary": true } -// } -// } -// -// See http://docs.mongodb.org/manual/reference/connection-string/ for details. -// You can also specify a Replica Set this way. -// -database.open = function(config, callback) { - var url = config["mongo-url"] || database.config2url(config), - options = config["mongo-options"] || database.config2options(config); - return mongodb.Db.connect(url, options, callback); -}; - -// -// For backwards-compatibility you can specify a connection to a single Mongo(s) as follows: -// -// { -// "mongo-host": "localhost", -// "mongo-port": "27017", -// "mongo-server-options": { "auto_reconnect": true }, -// "mongo-database": "cube", -// "mongo-database-options": { "safe": false }, -// "mongo-username": null, -// "mongo-password": null, -// } -// (defaults are shown) -// -database.config2url = function(config) { - var user = config["mongo-username"], - pass = config["mongo-password"], - host = config["mongo-host"] || "localhost", - port = config["mongo-port"] || 27017, - name = config["mongo-database"] || "cube", - auth = user ? user+":"+pass+"@" : ""; - return "mongodb://"+auth+host+":"+port+"/"+name; -}; - -database.config2options = function(config) { - return { - db: config["mongo-database-options"] || { safe: false }, - server: config["mongo-server-options"] || { auto_reconnect: true }, - replSet: { read_secondary: true } - }; -}; \ No newline at end of file diff --git a/lib/queue-async b/lib/queue-async deleted file mode 160000 index 5edd2096..00000000 --- a/lib/queue-async +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5edd2096b64954c3927ac0185974949499decb59 From 976d681e8b883a628216663d37d6d1b1a389c5b7 Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 11:45:45 +0200 Subject: [PATCH 78/87] Remove bins persistence --- lib/cube/models/event.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/models/event.js b/lib/cube/models/event.js index d8f55968..cfdf2c19 100644 --- a/lib/cube/models/event.js +++ b/lib/cube/models/event.js @@ -53,7 +53,7 @@ Event.setProperties({ return { time: this.time, type: this.type, bin: this.bins(), ago: this.agos() }; }}, - to_wire: { value: function(){ var event = { t: this.time, d: this.data, b: this.bins() }; if (this.id) event._id = this.id; return event; }}, + to_wire: { value: function(){ var event = { t: this.time, d: this.data }; if (this.id) event._id = this.id; return event; }}, save: { value: save }, validate: { value: validate }, From 44b16fd537c5bb8acd366b2fe0c7313497a6ad6d Mon Sep 17 00:00:00 2001 From: Marsup Date: Tue, 20 Aug 2013 19:32:20 +0200 Subject: [PATCH 79/87] Allow group-based queries to have empty answers --- lib/cube/models/measurement.js | 72 +++++++++++++++++++++++++--------- lib/cube/models/metric.js | 8 ++-- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index 32fa9ef3..dddcb975 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -83,7 +83,7 @@ function unary(db, callback) { if (!remaining) return this.emit('complete'); total_remaining = remaining; - if(this.isGrouped) total_remaining *= this.isGrouped.groups.length; + if(this.isGrouped) total_remaining *= (this.isGrouped.groups.length || 1); // Add this task to the appropriate queue. if (queue) queue.next = task; @@ -93,7 +93,7 @@ function unary(db, callback) { function task() { function onMetric(metric){ callback(metric); - if (!--total_remaining) { + if (--total_remaining <= 0) { self.emit('complete'); if (task.next) setImmediate(task.next); else delete queueByName[name]; @@ -113,7 +113,9 @@ function unary(db, callback) { group_name = self.isGrouped.groups[group_idx], group_remaining = remaining; - if (group_idx >= self.isGrouped.groups.length) return; + // Stop there if it is the last group, unless we are in the 1st pass, + // which would mean we have no group at all, so let's compute empty values. + if (group_idx >= self.isGrouped.groups.length && arguments.length) return; query_measurement.group = group_name; @@ -135,6 +137,7 @@ function unary(db, callback) { function findOrComputeUnary(db, callback) { var expression = this.expression, group_name = this.group, + isGrouped = 'group' in this, map = expression.value, reduce = reduces[expression.reduce], measurement = this; @@ -170,7 +173,7 @@ function findOrComputeUnary(db, callback) { var query_measurement = new Measurement(expression, start, stop, tier.next), bins = {}; - if(group_name) query_measurement.group = group_name; + if(isGrouped) query_measurement.group = group_name; find(query_measurement, function(metric) { var value = metric.value, time = metric.time, values = metric.values; @@ -181,7 +184,13 @@ function findOrComputeUnary(db, callback) { else bin.values = bin.values.concat(values||[]); if (!--bin.size) { - var metric = new Metric({time: time, value: reduce(bin.values), group: group_name}, measurement, bin.values); + var metric; + if (isGrouped) { + metric = new Metric({time: time, value: reduce(bin.values), group: group_name}, measurement, bin.values) + } else { + metric = new Metric({time: time, value: reduce(bin.values)}, measurement, bin.values) + } + if (metric.value || metric.value === 0 || metric.value === undefined) metric.save(db, handle); callback(metric); delete bins[time]; @@ -193,31 +202,51 @@ function findOrComputeUnary(db, callback) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { - var horizons = config.get('horizons'); + var horizons = config.get('horizons'), pastHorizon = { + is: 'past_horizon', + tier: tier, + expression: expression.source + }; // Reset start time to calculation horizon if requested time span goes past it if (horizons && tier.floor(start) < new Date(new Date() - horizons.calculation)){ var old_start = start, start = tier.step(tier.floor(new Date(new Date() - horizons.calculation))) - metalog.info('cube_compute', {is: 'past_horizon', start: {was: old_start, updated_to: start}, stop: stop, tier: tier, expression: expression.source }); + + pastHorizon.start = { was: old_start, updated_to: start }; } // Reset stop time to calculation horizon if requested time span goes past it if (horizons && tier.floor(stop) < new Date(new Date() - horizons.calculation)){ var old_stop = stop, stop = tier.step(tier.floor(new Date(new Date() - horizons.calculation))) - metalog.info('cube_compute', {is: 'past_horizon', metric: {start: start, stop: {was: old_stop, updated_to: stop}, tier: tier, expression: expression.source } }); + + pastHorizon.stop = { was: old_stop, updated_to: stop }; + } + + if (old_start || old_stop) { + metalog.info('cube_compute', pastHorizon); } - var query_measurement = new Measurement(expression, start, stop, tier), - time = start, values = []; + var time = start, values = []; + + if (isGrouped && group_name === undefined) { + // We do have grouping enabled, but no group at all, so answer right away with empty metrics. + return process(); + } + + var query_measurement = new Measurement(expression, start, stop, tier); if (group_name) query_measurement.group = group_name; function flat_callback(time, values){ var value = (values.length ? reduce(values) : reduce.empty), - metric = new Metric({ time: time, value: value, group: group_name }, measurement, values); - + metric; + if (isGrouped) { + metric = new Metric({ time: time, value: value, group: group_name }, measurement, values); + } else { + metric = new Metric({ time: time, value: value }, measurement, values); + } callback(metric); if (metric.value || metric.value === 0 || metric.value === undefined) metric.save(db, handle); } @@ -229,15 +258,15 @@ function findOrComputeUnary(db, callback) { var then = tier.floor(event.time); if (time < then) { - flat_callback(time, values, group_name); - while ((time = tier.step(time)) < then) flat_callback(time, [], group_name); + flat_callback(time, values); + while ((time = tier.step(time)) < then) flat_callback(time, []); values = [map(event.to_wire())]; } else { values.push(map(event.to_wire())); } } else { - flat_callback(time, values, group_name); - while ((time = tier.step(time)) < stop) flat_callback(time, [], group_name); + flat_callback(time, values); + while ((time = tier.step(time)) < stop) flat_callback(time, []); } } @@ -274,13 +303,18 @@ function binary(db, callback) { function measure(measurement, other){ return function(metric){ - var time = metric.time, value = metric.value, group = metric.group, - left_value, right_value; + var time = metric.time, value = metric.value, isGrouped = 'group' in metric, + group = metric.group, left_value, right_value; if (time in other) { left_value = measurement.left ? value : other[time]; right_value = measurement.right ? value : other[time]; - callback(new Metric({time: time, value: expression.op(left_value, right_value), group: group })); + if (isGrouped) { + metric = { time: time, value: expression.op(left_value, right_value), group: group }; + } else { + metric = { time: time, value: expression.op(left_value, right_value) }; + } + callback(new Metric(metric)); } else { measurement[time] = value; } diff --git a/lib/cube/models/metric.js b/lib/cube/models/metric.js index cd43318b..ef253b2f 100644 --- a/lib/cube/models/metric.js +++ b/lib/cube/models/metric.js @@ -21,7 +21,7 @@ var tiers = require("../tiers"), function Metric(data, measurement, values){ this.time = data.time; this.value = data.value; - if(data.group) this.group = data.group; + if('group' in data) this.group = data.group === undefined ? null : data.group; this.setProperty("values", { value: values||[] }); this.setProperty("measurement", { value: measurement }); @@ -60,7 +60,7 @@ function find(db, measurement, callback){ $lt: stop } }; - if (measurement.group) query["_id.g"] = measurement.group; + if ('group' in measurement) query["_id.g"] = measurement.group; collection.find(query, metric_fields, metric_options, handleResponse); }); @@ -88,7 +88,7 @@ function from_wire(row, measurement){ } data = {time: row._id.t, value: row.v}; - if(row._id.g) data.group = row._id.g; + if('g' in row._id) data.group = row._id.g; return new Metric(data, measurement, values); } @@ -105,7 +105,7 @@ function to_wire(){ }, []); } id = { e: this.e, l: this.l, t: this.time }; - if(this.group) id.g = this.group; + if('group' in this) id.g = this.group; if (this.value === undefined) { return { i: false, vs: values, _id: id}; From 7e5954f38935ab5bf7a9a6785047d6c3afb17978 Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 12:11:04 +0200 Subject: [PATCH 80/87] Upgrade node-static --- package.json | 2 +- test/server-test.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 4262db29..33bcb604 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "main": "./lib/cube", "dependencies": { "mongodb": "~1.3.18", - "node-static": "0.6.5", + "node-static": "~0.7.0", "pegjs": "0.7.0", "vows": "0.7.0", "websocket": "1.0.8", diff --git a/test/server-test.js b/test/server-test.js index 40f54326..84b7fd20 100644 --- a/test/server-test.js +++ b/test/server-test.js @@ -58,7 +58,7 @@ suite.addBatch( assert.equal(response.statusCode, 200); }, "the expected headers should be set": function(response) { - assert.equal(response.headers["content-type"], "text/javascript"); + assert.equal(response.headers["content-type"], "application/javascript"); assert.equal(response.headers["content-length"], 1); assert.ok(new Date(response.headers["date"]) > Date.UTC(2011, 0, 1)); assert.ok(new Date(response.headers["last-modified"]) > Date.UTC(2011, 0, 1)); @@ -75,7 +75,7 @@ suite.addBatch( "the expected headers should be set": function(response) { assert.ok(!("Content-Length" in response.headers)); assert.ok(new Date(response.headers["date"]) > Date.UTC(2011, 0, 1)); - assert.ok(new Date(response.headers["last-modified"]) > Date.UTC(2011, 0, 1)); + assert.ok(!("last-modified" in response.headers)); }, "no content should be returned": function(response) { assert.equal(response.body, ""); @@ -87,7 +87,7 @@ suite.addBatch( assert.equal(response.statusCode, 200); }, "the expected headers should be set": function(response) { - assert.equal(response.headers["content-type"], "text/javascript"); + assert.equal(response.headers["content-type"], "application/javascript"); assert.ok(!("Content-Length" in response.headers)); assert.ok(new Date(response.headers["date"]) > Date.UTC(2011, 0, 1)); assert.ok(new Date(response.headers["last-modified"]) > Date.UTC(2011, 0, 1)); From 1963e47a2131f924885e2a7b5f7685da96b467d3 Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 12:19:00 +0200 Subject: [PATCH 81/87] Update parsers with latest peg --- lib/cube/event-expression.js | 3745 ++++++++++++++-------------------- 1 file changed, 1518 insertions(+), 2227 deletions(-) diff --git a/lib/cube/event-expression.js b/lib/cube/event-expression.js index baaf39e7..c4950288 100644 --- a/lib/cube/event-expression.js +++ b/lib/cube/event-expression.js @@ -1,5 +1,32 @@ module.exports = (function(){ - /* Generated by PEG.js 0.6.2 (http://pegjs.majda.cz/). */ + /* + * Generated by PEG.js 0.7.0. + * + * http://pegjs.majda.cz/ + */ + + function quote(s) { + /* + * ECMA-262, 5th ed., 7.8.4: All characters may appear literally in a + * string literal except for the closing quote character, backslash, + * carriage return, line separator, paragraph separator, and line feed. + * Any character may appear in the form of an escape sequence. + * + * For portability, we also escape escape all control and non-ASCII + * characters. Note that "\0" and "\v" escape sequences are not used + * because JSHint does not like the first and IE the second. + */ + return '"' + s + .replace(/\\/g, '\\\\') // backslash + .replace(/"/g, '\\"') // closing quote character + .replace(/\x08/g, '\\b') // backspace + .replace(/\t/g, '\\t') // horizontal tab + .replace(/\n/g, '\\n') // line feed + .replace(/\f/g, '\\f') // form feed + .replace(/\r/g, '\\r') // carriage return + .replace(/[\x00-\x07\x0B\x0E-\x1F\x80-\uFFFF]/g, escape) + + '"'; + } var result = { /* @@ -10,36 +37,36 @@ module.exports = (function(){ */ parse: function(input, startRule) { var parseFunctions = { - "_": parse__, - "array_literal": parse_array_literal, - "character_escape_sequence": parse_character_escape_sequence, - "digit": parse_digit, - "digit19": parse_digit19, - "digits": parse_digits, - "double_string_char": parse_double_string_char, - "e": parse_e, - "escape_character": parse_escape_character, - "escape_sequence": parse_escape_sequence, + "start": parse_start, "event_expression": parse_event_expression, "event_filter_expression": parse_event_filter_expression, - "event_member_expression": parse_event_member_expression, "event_value_expression": parse_event_value_expression, - "exp": parse_exp, + "event_member_expression": parse_event_member_expression, "filter_operator": parse_filter_operator, - "frac": parse_frac, - "hex_digit": parse_hex_digit, - "hex_escape_sequence": parse_hex_escape_sequence, + "type": parse_type, "identifier": parse_identifier, - "int": parse_int, "literal": parse_literal, - "non_escape_character": parse_non_escape_character, - "number": parse_number, - "single_escape_character": parse_single_escape_character, - "single_string_char": parse_single_string_char, - "start": parse_start, + "array_literal": parse_array_literal, "string": parse_string, - "type": parse_type, + "double_string_char": parse_double_string_char, + "single_string_char": parse_single_string_char, + "escape_sequence": parse_escape_sequence, + "character_escape_sequence": parse_character_escape_sequence, + "single_escape_character": parse_single_escape_character, + "non_escape_character": parse_non_escape_character, + "escape_character": parse_escape_character, + "hex_escape_sequence": parse_hex_escape_sequence, "unicode_escape_sequence": parse_unicode_escape_sequence, + "number": parse_number, + "int": parse_int, + "frac": parse_frac, + "exp": parse_exp, + "digits": parse_digits, + "e": parse_e, + "digit": parse_digit, + "digit19": parse_digit19, + "hex_digit": parse_hex_digit, + "_": parse__, "whitespace": parse_whitespace }; @@ -52,10 +79,9 @@ module.exports = (function(){ } var pos = 0; - var reportMatchFailures = true; - var rightmostMatchFailuresPos = 0; - var rightmostMatchFailuresExpected = []; - var cache = {}; + var reportFailures = 0; + var rightmostFailuresPos = 0; + var rightmostFailuresExpected = []; function padLeft(input, padding, length) { var result = input; @@ -70,2830 +96,2136 @@ module.exports = (function(){ function escape(ch) { var charCode = ch.charCodeAt(0); + var escapeChar; + var length; if (charCode <= 0xFF) { - var escapeChar = 'x'; - var length = 2; + escapeChar = 'x'; + length = 2; } else { - var escapeChar = 'u'; - var length = 4; + escapeChar = 'u'; + length = 4; } return '\\' + escapeChar + padLeft(charCode.toString(16).toUpperCase(), '0', length); } - function quote(s) { - /* - * ECMA-262, 5th ed., 7.8.4: All characters may appear literally in a - * string literal except for the closing quote character, backslash, - * carriage return, line separator, paragraph separator, and line feed. - * Any character may appear in the form of an escape sequence. - */ - return '"' + s - .replace(/\\/g, '\\\\') // backslash - .replace(/"/g, '\\"') // closing quote character - .replace(/\r/g, '\\r') // carriage return - .replace(/\n/g, '\\n') // line feed - .replace(/[\x80-\uFFFF]/g, escape) // non-ASCII characters - + '"'; - } - function matchFailed(failure) { - if (pos < rightmostMatchFailuresPos) { + if (pos < rightmostFailuresPos) { return; } - if (pos > rightmostMatchFailuresPos) { - rightmostMatchFailuresPos = pos; - rightmostMatchFailuresExpected = []; + if (pos > rightmostFailuresPos) { + rightmostFailuresPos = pos; + rightmostFailuresExpected = []; } - rightmostMatchFailuresExpected.push(failure); + rightmostFailuresExpected.push(failure); } function parse_start() { - var cacheKey = 'start@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse__(); - if (result3 !== null) { - var result4 = parse_event_expression(); - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - var result1 = [result3, result4, result5]; + var result0, result1, result2; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse__(); + if (result0 !== null) { + result1 = parse_event_expression(); + if (result1 !== null) { + result2 = parse__(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(expression) { expression.source = input; return expression; })(result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, expression) { expression.source = input; return expression; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_expression() { - var cacheKey = 'event_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_event_value_expression(); - if (result3 !== null) { - var result4 = []; - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - if (input.substr(pos, 1) === ".") { - var result7 = "."; - pos += 1; + var result0, result1, result2, result3, result4, result5; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_event_value_expression(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result7 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_filter_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_filter_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; - } - while (result5 !== null) { - result4.push(result5); - var savedPos2 = pos; - var result6 = parse__(); - if (result6 !== null) { - if (input.substr(pos, 1) === ".") { - var result7 = "."; - pos += 1; + result2 = null; + pos = pos2; + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result7 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - var result9 = parse_event_filter_expression(); - if (result9 !== null) { - var result5 = [result6, result7, result8, result9]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_event_filter_expression(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } else { - var result5 = null; - pos = savedPos2; + result2 = null; + pos = pos2; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(value, filters) { - value.filter = function(filter) { - var i = -1, n = filters.length; - while (++i < n) filters[i][3](filter); - value.exists(filter); - }; - return value; - })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, value, filters) { + value.filter = function(filter) { + var i = -1, n = filters.length; + while (++i < n) filters[i][3](filter); + value.exists(filter); + }; + return value; + })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_filter_expression() { - var cacheKey = 'event_filter_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_filter_operator(); - if (result3 !== null) { - var result4 = parse__(); - if (result4 !== null) { - if (input.substr(pos, 1) === "(") { - var result5 = "("; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6, result7, result8, result9, result10; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_filter_operator(); + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 40) { + result2 = "("; + pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result5 !== null) { - var result6 = parse__(); - if (result6 !== null) { - var result7 = parse_event_member_expression(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - if (input.substr(pos, 1) === ",") { - var result9 = ","; - pos += 1; + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + result4 = parse_event_member_expression(); + if (result4 !== null) { + result5 = parse__(); + if (result5 !== null) { + if (input.charCodeAt(pos) === 44) { + result6 = ","; + pos++; } else { - var result9 = null; - if (reportMatchFailures) { + result6 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result9 !== null) { - var result10 = parse__(); - if (result10 !== null) { - var result11 = parse_literal(); - if (result11 !== null) { - var result12 = parse__(); - if (result12 !== null) { - if (input.substr(pos, 1) === ")") { - var result13 = ")"; - pos += 1; + if (result6 !== null) { + result7 = parse__(); + if (result7 !== null) { + result8 = parse_literal(); + if (result8 !== null) { + result9 = parse__(); + if (result9 !== null) { + if (input.charCodeAt(pos) === 41) { + result10 = ")"; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result10 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result13 !== null) { - var result1 = [result3, result4, result5, result6, result7, result8, result9, result10, result11, result12, result13]; + if (result10 !== null) { + result0 = [result0, result1, result2, result3, result4, result5, result6, result7, result8, result9, result10]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(op, member, value) { return function(o) { op(o, member.field, value); }; })(result1[0], result1[4], result1[8]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, op, member, value) { return function(o) { op(o, member.field, value); }; })(pos0, result0[0], result0[4], result0[8]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_value_expression() { - var cacheKey = 'event_value_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos1 = pos; - var savedPos2 = pos; - var result7 = parse_type(); - if (result7 !== null) { - var result8 = parse__(); - if (result8 !== null) { - if (input.substr(pos, 1) === "(") { - var result9 = "("; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6, result7, result8, result9; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + result0 = parse_type(); + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 40) { + result2 = "("; + pos++; } else { - var result9 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"(\""); } } - if (result9 !== null) { - var result10 = parse__(); - if (result10 !== null) { - var result11 = parse_event_member_expression(); - if (result11 !== null) { - var result12 = []; - var savedPos3 = pos; - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === ",") { - var result17 = ","; - pos += 1; + if (result2 !== null) { + result3 = parse__(); + if (result3 !== null) { + result4 = parse_event_member_expression(); + if (result4 !== null) { + result5 = []; + pos2 = pos; + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 44) { + result7 = ","; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_event_member_expression(); - if (result19 !== null) { - var result15 = [result16, result17, result18, result19]; + if (result7 !== null) { + result8 = parse__(); + if (result8 !== null) { + result9 = parse_event_member_expression(); + if (result9 !== null) { + result6 = [result6, result7, result8, result9]; } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } - while (result15 !== null) { - result12.push(result15); - var savedPos3 = pos; - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === ",") { - var result17 = ","; - pos += 1; + while (result6 !== null) { + result5.push(result6); + pos2 = pos; + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 44) { + result7 = ","; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_event_member_expression(); - if (result19 !== null) { - var result15 = [result16, result17, result18, result19]; + if (result7 !== null) { + result8 = parse__(); + if (result8 !== null) { + result9 = parse_event_member_expression(); + if (result9 !== null) { + result6 = [result6, result7, result8, result9]; } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } else { - var result15 = null; - pos = savedPos3; + result6 = null; + pos = pos2; } } - if (result12 !== null) { - var result13 = parse__(); - if (result13 !== null) { - if (input.substr(pos, 1) === ")") { - var result14 = ")"; - pos += 1; + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 41) { + result7 = ")"; + pos++; } else { - var result14 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\")\""); } } - if (result14 !== null) { - var result5 = [result7, result8, result9, result10, result11, result12, result13, result14]; + if (result7 !== null) { + result0 = [result0, result1, result2, result3, result4, result5, result6, result7]; } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } } else { - var result5 = null; - pos = savedPos2; - } - var result6 = result5 !== null - ? (function(type, head, tail) { return compoundFields(type, head, tail); })(result5[0], result5[4], result5[5]) - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; - var result2 = parse_type(); - var result3 = result2 !== null - ? (function(type) { return {type: type, exists: noop, fields: noop}; })(result2) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, type, head, tail) { return compoundFields(type, head, tail); })(pos0, result0[0], result0[4], result0[5]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_type(); + if (result0 !== null) { + result0 = (function(offset, type) { return {type: type, exists: noop, fields: noop}; })(pos0, result0); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_event_member_expression() { - var cacheKey = 'event_member_expression@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_identifier(); - if (result3 !== null) { - var result4 = []; - var savedPos4 = pos; - var savedPos5 = pos; - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === "[") { - var result17 = "["; - pos += 1; + var result0, result1, result2, result3, result4, result5, result6, result7; + var pos0, pos1, pos2, pos3; + + pos0 = pos; + pos1 = pos; + result0 = parse_identifier(); + if (result0 !== null) { + result1 = []; + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 91) { + result3 = "["; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_number(); - if (result19 !== null) { - var result20 = parse__(); - if (result20 !== null) { - if (input.substr(pos, 1) === "]") { - var result21 = "]"; - pos += 1; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_number(); + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 93) { + result7 = "]"; + pos++; } else { - var result21 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result21 !== null) { - var result14 = [result16, result17, result18, result19, result20, result21]; + if (result7 !== null) { + result2 = [result2, result3, result4, result5, result6, result7]; } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; - } - var result15 = result14 !== null - ? (function(name) { return arrayAccessor(name); })(result14[3]) - : null; - if (result15 !== null) { - var result13 = result15; - } else { - var result13 = null; - pos = savedPos4; + result2 = null; + pos = pos3; } - if (result13 !== null) { - var result5 = result13; - } else { - var savedPos2 = pos; - var savedPos3 = pos; - var result9 = parse__(); - if (result9 !== null) { - if (input.substr(pos, 1) === ".") { - var result10 = "."; - pos += 1; + if (result2 !== null) { + result2 = (function(offset, name) { return arrayAccessor(name); })(pos2, result2[3]); + } + if (result2 === null) { + pos = pos2; + } + if (result2 === null) { + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result10 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result10 !== null) { - var result11 = parse__(); - if (result11 !== null) { - var result12 = parse_identifier(); - if (result12 !== null) { - var result7 = [result9, result10, result11, result12]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_identifier(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } - var result8 = result7 !== null - ? (function(name) { return objectAccessor(name); })(result7[3]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + if (result2 !== null) { + result2 = (function(offset, name) { return objectAccessor(name); })(pos2, result2[3]); } - if (result6 !== null) { - var result5 = result6; - } else { - var result5 = null;; - }; - } - while (result5 !== null) { - result4.push(result5); - var savedPos4 = pos; - var savedPos5 = pos; - var result16 = parse__(); - if (result16 !== null) { - if (input.substr(pos, 1) === "[") { - var result17 = "["; - pos += 1; + if (result2 === null) { + pos = pos2; + } + } + while (result2 !== null) { + result1.push(result2); + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 91) { + result3 = "["; + pos++; } else { - var result17 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result17 !== null) { - var result18 = parse__(); - if (result18 !== null) { - var result19 = parse_number(); - if (result19 !== null) { - var result20 = parse__(); - if (result20 !== null) { - if (input.substr(pos, 1) === "]") { - var result21 = "]"; - pos += 1; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_number(); + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + if (input.charCodeAt(pos) === 93) { + result7 = "]"; + pos++; } else { - var result21 = null; - if (reportMatchFailures) { + result7 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result21 !== null) { - var result14 = [result16, result17, result18, result19, result20, result21]; + if (result7 !== null) { + result2 = [result2, result3, result4, result5, result6, result7]; } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } } else { - var result14 = null; - pos = savedPos5; + result2 = null; + pos = pos3; } - var result15 = result14 !== null - ? (function(name) { return arrayAccessor(name); })(result14[3]) - : null; - if (result15 !== null) { - var result13 = result15; - } else { - var result13 = null; - pos = savedPos4; + if (result2 !== null) { + result2 = (function(offset, name) { return arrayAccessor(name); })(pos2, result2[3]); } - if (result13 !== null) { - var result5 = result13; - } else { - var savedPos2 = pos; - var savedPos3 = pos; - var result9 = parse__(); - if (result9 !== null) { - if (input.substr(pos, 1) === ".") { - var result10 = "."; - pos += 1; + if (result2 === null) { + pos = pos2; + } + if (result2 === null) { + pos2 = pos; + pos3 = pos; + result2 = parse__(); + if (result2 !== null) { + if (input.charCodeAt(pos) === 46) { + result3 = "."; + pos++; } else { - var result10 = null; - if (reportMatchFailures) { + result3 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result10 !== null) { - var result11 = parse__(); - if (result11 !== null) { - var result12 = parse_identifier(); - if (result12 !== null) { - var result7 = [result9, result10, result11, result12]; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + result5 = parse_identifier(); + if (result5 !== null) { + result2 = [result2, result3, result4, result5]; } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } } else { - var result7 = null; - pos = savedPos3; + result2 = null; + pos = pos3; } - var result8 = result7 !== null - ? (function(name) { return objectAccessor(name); })(result7[3]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + if (result2 !== null) { + result2 = (function(offset, name) { return objectAccessor(name); })(pos2, result2[3]); + } + if (result2 === null) { + pos = pos2; } - if (result6 !== null) { - var result5 = result6; - } else { - var result5 = null;; - }; } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(head, tail) { return member(head, tail); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, head, tail) { return member(head, tail); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_filter_operator() { - var cacheKey = 'filter_operator@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; + var pos0; - var savedPos7 = pos; + pos0 = pos; if (input.substr(pos, 2) === "eq") { - var result23 = "eq"; + result0 = "eq"; pos += 2; } else { - var result23 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"eq\""); } } - var result24 = result23 !== null - ? (function() { return filterEqual; })() - : null; - if (result24 !== null) { - var result22 = result24; - } else { - var result22 = null; - pos = savedPos7; + if (result0 !== null) { + result0 = (function(offset) { return filterEqual; })(pos0); } - if (result22 !== null) { - var result0 = result22; - } else { - var savedPos6 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "gt") { - var result20 = "gt"; + result0 = "gt"; pos += 2; } else { - var result20 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"gt\""); } } - var result21 = result20 !== null - ? (function() { return filterGreater; })() - : null; - if (result21 !== null) { - var result19 = result21; - } else { - var result19 = null; - pos = savedPos6; + if (result0 !== null) { + result0 = (function(offset) { return filterGreater; })(pos0); } - if (result19 !== null) { - var result0 = result19; - } else { - var savedPos5 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "ge") { - var result17 = "ge"; + result0 = "ge"; pos += 2; } else { - var result17 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"ge\""); } } - var result18 = result17 !== null - ? (function() { return filterGreaterOrEqual; })() - : null; - if (result18 !== null) { - var result16 = result18; - } else { - var result16 = null; - pos = savedPos5; + if (result0 !== null) { + result0 = (function(offset) { return filterGreaterOrEqual; })(pos0); } - if (result16 !== null) { - var result0 = result16; - } else { - var savedPos4 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "lt") { - var result14 = "lt"; + result0 = "lt"; pos += 2; } else { - var result14 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"lt\""); } } - var result15 = result14 !== null - ? (function() { return filterLess; })() - : null; - if (result15 !== null) { - var result13 = result15; - } else { - var result13 = null; - pos = savedPos4; + if (result0 !== null) { + result0 = (function(offset) { return filterLess; })(pos0); } - if (result13 !== null) { - var result0 = result13; - } else { - var savedPos3 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "le") { - var result11 = "le"; + result0 = "le"; pos += 2; } else { - var result11 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"le\""); } } - var result12 = result11 !== null - ? (function() { return filterLessOrEqual; })() - : null; - if (result12 !== null) { - var result10 = result12; - } else { - var result10 = null; - pos = savedPos3; + if (result0 !== null) { + result0 = (function(offset) { return filterLessOrEqual; })(pos0); } - if (result10 !== null) { - var result0 = result10; - } else { - var savedPos2 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "ne") { - var result8 = "ne"; + result0 = "ne"; pos += 2; } else { - var result8 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"ne\""); } } - var result9 = result8 !== null - ? (function() { return filterNotEqual; })() - : null; - if (result9 !== null) { - var result7 = result9; - } else { - var result7 = null; - pos = savedPos2; + if (result0 !== null) { + result0 = (function(offset) { return filterNotEqual; })(pos0); } - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos1 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "re") { - var result5 = "re"; + result0 = "re"; pos += 2; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"re\""); } } - var result6 = result5 !== null - ? (function() { return filterRegularExpression; })() - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 !== null) { + result0 = (function(offset) { return filterRegularExpression; })(pos0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 2) === "in") { - var result2 = "in"; + result0 = "in"; pos += 2; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"in\""); } } - var result3 = result2 !== null - ? (function() { return filterIn; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset) { return filterIn; })(pos0); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; - }; - }; - }; + if (result0 === null) { + pos = pos0; + } + } + } + } + } + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_type() { - var cacheKey = 'type@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1, result2; + var pos0, pos1; - - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos).match(/^[a-z]/) !== null) { - var result3 = input.charAt(pos); + pos0 = pos; + pos1 = pos; + if (/^[a-z]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[a-z]"); } } - if (result3 !== null) { - if (input.substr(pos).match(/^[a-zA-Z0-9_]/) !== null) { - var result5 = input.charAt(pos); + if (result0 !== null) { + if (/^[a-zA-Z0-9_]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_]"); } } - if (result5 !== null) { - var result4 = []; - while (result5 !== null) { - result4.push(result5); - if (input.substr(pos).match(/^[a-zA-Z0-9_]/) !== null) { - var result5 = input.charAt(pos); + if (result2 !== null) { + result1 = []; + while (result2 !== null) { + result1.push(result2); + if (/^[a-zA-Z0-9_]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_]"); } } } } else { - var result4 = null; + result1 = null; } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(first, rest) { return first + rest.join(""); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, first, rest) { return first + rest.join(""); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_identifier() { - var cacheKey = 'identifier@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos).match(/^[a-zA-Z_]/) !== null) { - var result3 = input.charAt(pos); + pos0 = pos; + pos1 = pos; + if (/^[a-zA-Z_]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z_]"); } } - if (result3 !== null) { - var result4 = []; - if (input.substr(pos).match(/^[a-zA-Z0-9_$]/) !== null) { - var result5 = input.charAt(pos); + if (result0 !== null) { + result1 = []; + if (/^[a-zA-Z0-9_$]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_$]"); } } - while (result5 !== null) { - result4.push(result5); - if (input.substr(pos).match(/^[a-zA-Z0-9_$]/) !== null) { - var result5 = input.charAt(pos); + while (result2 !== null) { + result1.push(result2); + if (/^[a-zA-Z0-9_$]/.test(input.charAt(pos))) { + result2 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("[a-zA-Z0-9_$]"); } } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(first, rest) { return first + rest.join(""); })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, first, rest) { return first + rest.join(""); })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_literal() { - var cacheKey = 'literal@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result9 = parse_array_literal(); - if (result9 !== null) { - var result0 = result9; - } else { - var result8 = parse_string(); - if (result8 !== null) { - var result0 = result8; - } else { - var result7 = parse_number(); - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos1 = pos; + var result0; + var pos0; + + result0 = parse_array_literal(); + if (result0 === null) { + result0 = parse_string(); + if (result0 === null) { + result0 = parse_number(); + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 4) === "true") { - var result5 = "true"; + result0 = "true"; pos += 4; } else { - var result5 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"true\""); } } - var result6 = result5 !== null - ? (function() { return true; })() - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 !== null) { + result0 = (function(offset) { return true; })(pos0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; if (input.substr(pos, 5) === "false") { - var result2 = "false"; + result0 = "false"; pos += 5; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"false\""); } } - var result3 = result2 !== null - ? (function() { return false; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset) { return false; })(pos0); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; + if (result0 === null) { + pos = pos0; + } + } + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_array_literal() { - var cacheKey = 'array_literal@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1, result2, result3, result4, result5, result6, result7; + var pos0, pos1, pos2; - - var savedPos2 = pos; - var savedPos3 = pos; - if (input.substr(pos, 1) === "[") { - var result10 = "["; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 91) { + result0 = "["; + pos++; } else { - var result10 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result10 !== null) { - var result11 = parse__(); - if (result11 !== null) { - var result12 = parse_literal(); - if (result12 !== null) { - var result13 = []; - var savedPos4 = pos; - var result17 = parse__(); - if (result17 !== null) { - if (input.substr(pos, 1) === ",") { - var result18 = ","; - pos += 1; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_literal(); + if (result2 !== null) { + result3 = []; + pos2 = pos; + result4 = parse__(); + if (result4 !== null) { + if (input.charCodeAt(pos) === 44) { + result5 = ","; + pos++; } else { - var result18 = null; - if (reportMatchFailures) { + result5 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result18 !== null) { - var result19 = parse__(); - if (result19 !== null) { - var result20 = parse_literal(); - if (result20 !== null) { - var result16 = [result17, result18, result19, result20]; + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + result7 = parse_literal(); + if (result7 !== null) { + result4 = [result4, result5, result6, result7]; } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } - while (result16 !== null) { - result13.push(result16); - var savedPos4 = pos; - var result17 = parse__(); - if (result17 !== null) { - if (input.substr(pos, 1) === ",") { - var result18 = ","; - pos += 1; + while (result4 !== null) { + result3.push(result4); + pos2 = pos; + result4 = parse__(); + if (result4 !== null) { + if (input.charCodeAt(pos) === 44) { + result5 = ","; + pos++; } else { - var result18 = null; - if (reportMatchFailures) { + result5 = null; + if (reportFailures === 0) { matchFailed("\",\""); } } - if (result18 !== null) { - var result19 = parse__(); - if (result19 !== null) { - var result20 = parse_literal(); - if (result20 !== null) { - var result16 = [result17, result18, result19, result20]; + if (result5 !== null) { + result6 = parse__(); + if (result6 !== null) { + result7 = parse_literal(); + if (result7 !== null) { + result4 = [result4, result5, result6, result7]; } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } else { - var result16 = null; - pos = savedPos4; + result4 = null; + pos = pos2; } } - if (result13 !== null) { - var result14 = parse__(); - if (result14 !== null) { - if (input.substr(pos, 1) === "]") { - var result15 = "]"; - pos += 1; + if (result3 !== null) { + result4 = parse__(); + if (result4 !== null) { + if (input.charCodeAt(pos) === 93) { + result5 = "]"; + pos++; } else { - var result15 = null; - if (reportMatchFailures) { + result5 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result15 !== null) { - var result8 = [result10, result11, result12, result13, result14, result15]; + if (result5 !== null) { + result0 = [result0, result1, result2, result3, result4, result5]; } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; - } - var result9 = result8 !== null - ? (function(first, rest) { return [first].concat(rest.map(function(d) { return d[3]; })); })(result8[2], result8[3]) - : null; - if (result9 !== null) { - var result7 = result9; - } else { - var result7 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "[") { - var result4 = "["; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, first, rest) { return [first].concat(rest.map(function(d) { return d[3]; })); })(pos0, result0[2], result0[3]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 91) { + result0 = "["; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"[\""); } } - if (result4 !== null) { - var result5 = parse__(); - if (result5 !== null) { - if (input.substr(pos, 1) === "]") { - var result6 = "]"; - pos += 1; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + if (input.charCodeAt(pos) === 93) { + result2 = "]"; + pos++; } else { - var result6 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"]\""); } } - if (result6 !== null) { - var result2 = [result4, result5, result6]; + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function() { return []; })() - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset) { return []; })(pos0); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_string() { - var cacheKey = 'string@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - var savedReportMatchFailures = reportMatchFailures; - reportMatchFailures = false; - var savedPos2 = pos; - var savedPos3 = pos; - if (input.substr(pos, 1) === "\"") { - var result11 = "\""; - pos += 1; + var result0, result1, result2; + var pos0, pos1; + + reportFailures++; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 34) { + result0 = "\""; + pos++; } else { - var result11 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\"\""); } } - if (result11 !== null) { - var result12 = []; - var result14 = parse_double_string_char(); - while (result14 !== null) { - result12.push(result14); - var result14 = parse_double_string_char(); + if (result0 !== null) { + result1 = []; + result2 = parse_double_string_char(); + while (result2 !== null) { + result1.push(result2); + result2 = parse_double_string_char(); } - if (result12 !== null) { - if (input.substr(pos, 1) === "\"") { - var result13 = "\""; - pos += 1; + if (result1 !== null) { + if (input.charCodeAt(pos) === 34) { + result2 = "\""; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"\\\"\""); } } - if (result13 !== null) { - var result9 = [result11, result12, result13]; + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result9 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result9 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result9 = null; - pos = savedPos3; - } - var result10 = result9 !== null - ? (function(chars) { return chars.join(""); })(result9[1]) - : null; - if (result10 !== null) { - var result8 = result10; - } else { - var result8 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result8 !== null) { - var result0 = result8; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "'") { - var result4 = "'"; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, chars) { return chars.join(""); })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 39) { + result0 = "'"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"'\""); } } - if (result4 !== null) { - var result5 = []; - var result7 = parse_single_string_char(); - while (result7 !== null) { - result5.push(result7); - var result7 = parse_single_string_char(); + if (result0 !== null) { + result1 = []; + result2 = parse_single_string_char(); + while (result2 !== null) { + result1.push(result2); + result2 = parse_single_string_char(); } - if (result5 !== null) { - if (input.substr(pos, 1) === "'") { - var result6 = "'"; - pos += 1; + if (result1 !== null) { + if (input.charCodeAt(pos) === 39) { + result2 = "'"; + pos++; } else { - var result6 = null; - if (reportMatchFailures) { + result2 = null; + if (reportFailures === 0) { matchFailed("\"'\""); } } - if (result6 !== null) { - var result2 = [result4, result5, result6]; + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(chars) { return chars.join(""); })(result2[1]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, chars) { return chars.join(""); })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - reportMatchFailures = savedReportMatchFailures; - if (reportMatchFailures && result0 === null) { + reportFailures--; + if (reportFailures === 0 && result0 === null) { matchFailed("string"); } - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_double_string_char() { - var cacheKey = 'double_string_char@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos2 = pos; - var savedPos3 = pos; - var savedPos4 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - if (input.substr(pos, 1) === "\"") { - var result13 = "\""; - pos += 1; + var result0, result1; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + pos2 = pos; + reportFailures++; + if (input.charCodeAt(pos) === 34) { + result0 = "\""; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\"\""); } } - if (result13 !== null) { - var result11 = result13; - } else { - if (input.substr(pos, 1) === "\\") { - var result12 = "\\"; - pos += 1; + if (result0 === null) { + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result12 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result12 !== null) { - var result11 = result12; - } else { - var result11 = null;; - }; } - reportMatchFailures = savedReportMatchFailuresVar0; - if (result11 === null) { - var result9 = ''; + reportFailures--; + if (result0 === null) { + result0 = ""; } else { - var result9 = null; - pos = savedPos4; + result0 = null; + pos = pos2; } - if (result9 !== null) { + if (result0 !== null) { if (input.length > pos) { - var result10 = input.charAt(pos); + result1 = input.charAt(pos); pos++; } else { - var result10 = null; - if (reportMatchFailures) { - matchFailed('any character'); + result1 = null; + if (reportFailures === 0) { + matchFailed("any character"); } } - if (result10 !== null) { - var result7 = [result9, result10]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result7 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result7 = null; - pos = savedPos3; - } - var result8 = result7 !== null - ? (function(char_) { return char_; })(result7[1]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result6 !== null) { - var result0 = result6; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "\\") { - var result4 = "\\"; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, char_) { return char_; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result4 !== null) { - var result5 = parse_escape_sequence(); - if (result5 !== null) { - var result2 = [result4, result5]; + if (result0 !== null) { + result1 = parse_escape_sequence(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(sequence) { return sequence; })(result2[1]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, sequence) { return sequence; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_single_string_char() { - var cacheKey = 'single_string_char@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos2 = pos; - var savedPos3 = pos; - var savedPos4 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - if (input.substr(pos, 1) === "'") { - var result13 = "'"; - pos += 1; + var result0, result1; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + pos2 = pos; + reportFailures++; + if (input.charCodeAt(pos) === 39) { + result0 = "'"; + pos++; } else { - var result13 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"'\""); } } - if (result13 !== null) { - var result11 = result13; - } else { - if (input.substr(pos, 1) === "\\") { - var result12 = "\\"; - pos += 1; + if (result0 === null) { + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result12 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result12 !== null) { - var result11 = result12; - } else { - var result11 = null;; - }; } - reportMatchFailures = savedReportMatchFailuresVar0; - if (result11 === null) { - var result9 = ''; + reportFailures--; + if (result0 === null) { + result0 = ""; } else { - var result9 = null; - pos = savedPos4; + result0 = null; + pos = pos2; } - if (result9 !== null) { + if (result0 !== null) { if (input.length > pos) { - var result10 = input.charAt(pos); + result1 = input.charAt(pos); pos++; } else { - var result10 = null; - if (reportMatchFailures) { - matchFailed('any character'); + result1 = null; + if (reportFailures === 0) { + matchFailed("any character"); } } - if (result10 !== null) { - var result7 = [result9, result10]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result7 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result7 = null; - pos = savedPos3; - } - var result8 = result7 !== null - ? (function(char_) { return char_; })(result7[1]) - : null; - if (result8 !== null) { - var result6 = result8; - } else { - var result6 = null; - pos = savedPos2; + result0 = null; + pos = pos1; } - if (result6 !== null) { - var result0 = result6; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "\\") { - var result4 = "\\"; - pos += 1; + if (result0 !== null) { + result0 = (function(offset, char_) { return char_; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 92) { + result0 = "\\"; + pos++; } else { - var result4 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"\\\\\""); } } - if (result4 !== null) { - var result5 = parse_escape_sequence(); - if (result5 !== null) { - var result2 = [result4, result5]; + if (result0 !== null) { + result1 = parse_escape_sequence(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result2 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result2 = null; - pos = savedPos1; - } - var result3 = result2 !== null - ? (function(sequence) { return sequence; })(result2[1]) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, sequence) { return sequence; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_escape_sequence() { - var cacheKey = 'escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result9 = parse_character_escape_sequence(); - if (result9 !== null) { - var result0 = result9; - } else { - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "0") { - var result6 = "0"; - pos += 1; + var result0, result1; + var pos0, pos1, pos2; + + result0 = parse_character_escape_sequence(); + if (result0 === null) { + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 48) { + result0 = "0"; + pos++; } else { - var result6 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"0\""); } } - if (result6 !== null) { - var savedPos2 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - var result8 = parse_digit(); - reportMatchFailures = savedReportMatchFailuresVar0; - if (result8 === null) { - var result7 = ''; + if (result0 !== null) { + pos2 = pos; + reportFailures++; + result1 = parse_digit(); + reportFailures--; + if (result1 === null) { + result1 = ""; } else { - var result7 = null; - pos = savedPos2; + result1 = null; + pos = pos2; } - if (result7 !== null) { - var result4 = [result6, result7]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result4 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result4 = null; - pos = savedPos1; - } - var result5 = result4 !== null - ? (function() { return "\0"; })() - : null; - if (result5 !== null) { - var result3 = result5; - } else { - var result3 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset) { return "\0"; })(pos0); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + result0 = parse_hex_escape_sequence(); + if (result0 === null) { + result0 = parse_unicode_escape_sequence(); + } } - if (result3 !== null) { - var result0 = result3; - } else { - var result2 = parse_hex_escape_sequence(); - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_unicode_escape_sequence(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_character_escape_sequence() { - var cacheKey = 'character_escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - var result2 = parse_single_escape_character(); - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_non_escape_character(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; + result0 = parse_single_escape_character(); + if (result0 === null) { + result0 = parse_non_escape_character(); } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_single_escape_character() { - var cacheKey = 'single_escape_character@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0; + var pos0; - - var savedPos0 = pos; - if (input.substr(pos).match(/^['"\\bfnrtv]/) !== null) { - var result1 = input.charAt(pos); + pos0 = pos; + if (/^['"\\bfnrtv]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result1 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("['\"\\\\bfnrtv]"); } } - var result2 = result1 !== null - ? (function(char_) { return char_.replace("b", "\b").replace("f", "\f").replace("n", "\n").replace("r", "\r").replace("t", "\t").replace("v", "\x0B"); })(result1) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, char_) { return char_.replace("b", "\b").replace("f", "\f").replace("n", "\n").replace("r", "\r").replace("t", "\t").replace("v", "\x0B"); })(pos0, result0); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_non_escape_character() { - var cacheKey = 'non_escape_character@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var savedPos2 = pos; - var savedReportMatchFailuresVar0 = reportMatchFailures; - reportMatchFailures = false; - var result5 = parse_escape_character(); - reportMatchFailures = savedReportMatchFailuresVar0; - if (result5 === null) { - var result3 = ''; - } else { - var result3 = null; - pos = savedPos2; - } - if (result3 !== null) { + var result0, result1; + var pos0, pos1, pos2; + + pos0 = pos; + pos1 = pos; + pos2 = pos; + reportFailures++; + result0 = parse_escape_character(); + reportFailures--; + if (result0 === null) { + result0 = ""; + } else { + result0 = null; + pos = pos2; + } + if (result0 !== null) { if (input.length > pos) { - var result4 = input.charAt(pos); + result1 = input.charAt(pos); pos++; } else { - var result4 = null; - if (reportMatchFailures) { - matchFailed('any character'); + result1 = null; + if (reportFailures === 0) { + matchFailed("any character"); } } - if (result4 !== null) { - var result1 = [result3, result4]; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(char_) { return char_; })(result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, char_) { return char_; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_escape_character() { - var cacheKey = 'escape_character@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var result4 = parse_single_escape_character(); - if (result4 !== null) { - var result0 = result4; - } else { - var result3 = parse_digit(); - if (result3 !== null) { - var result0 = result3; - } else { - if (input.substr(pos, 1) === "x") { - var result2 = "x"; - pos += 1; + var result0; + + result0 = parse_single_escape_character(); + if (result0 === null) { + result0 = parse_digit(); + if (result0 === null) { + if (input.charCodeAt(pos) === 120) { + result0 = "x"; + pos++; } else { - var result2 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"x\""); } } - if (result2 !== null) { - var result0 = result2; - } else { - if (input.substr(pos, 1) === "u") { - var result1 = "u"; - pos += 1; + if (result0 === null) { + if (input.charCodeAt(pos) === 117) { + result0 = "u"; + pos++; } else { - var result1 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"u\""); } } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; + } + } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_hex_escape_sequence() { - var cacheKey = 'hex_escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "x") { - var result3 = "x"; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 120) { + result0 = "x"; + pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"x\""); } } - if (result3 !== null) { - var result4 = parse_hex_digit(); - if (result4 !== null) { - var result5 = parse_hex_digit(); - if (result5 !== null) { - var result1 = [result3, result4, result5]; + if (result0 !== null) { + result1 = parse_hex_digit(); + if (result1 !== null) { + result2 = parse_hex_digit(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(h1, h2) { return String.fromCharCode(+("0x" + h1 + h2)); })(result1[1], result1[2]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, h1, h2) { return String.fromCharCode(+("0x" + h1 + h2)); })(pos0, result0[1], result0[2]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_unicode_escape_sequence() { - var cacheKey = 'unicode_escape_sequence@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1, result2, result3, result4; + var pos0, pos1; - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === "u") { - var result3 = "u"; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 117) { + result0 = "u"; + pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"u\""); } } - if (result3 !== null) { - var result4 = parse_hex_digit(); - if (result4 !== null) { - var result5 = parse_hex_digit(); - if (result5 !== null) { - var result6 = parse_hex_digit(); - if (result6 !== null) { - var result7 = parse_hex_digit(); - if (result7 !== null) { - var result1 = [result3, result4, result5, result6, result7]; + if (result0 !== null) { + result1 = parse_hex_digit(); + if (result1 !== null) { + result2 = parse_hex_digit(); + if (result2 !== null) { + result3 = parse_hex_digit(); + if (result3 !== null) { + result4 = parse_hex_digit(); + if (result4 !== null) { + result0 = [result0, result1, result2, result3, result4]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(h1, h2, h3, h4) { return String.fromCharCode(+("0x" + h1 + h2 + h3 + h4)); })(result1[1], result1[2], result1[3], result1[4]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, h1, h2, h3, h4) { return String.fromCharCode(+("0x" + h1 + h2 + h3 + h4)); })(pos0, result0[1], result0[2], result0[3], result0[4]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_number() { - var cacheKey = 'number@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - var savedReportMatchFailures = reportMatchFailures; - reportMatchFailures = false; - var savedPos8 = pos; - var savedPos9 = pos; - if (input.substr(pos, 1) === "-") { - var result26 = "-"; - pos += 1; + var result0, result1, result2; + var pos0, pos1; + + reportFailures++; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 45) { + result0 = "-"; + pos++; } else { - var result26 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\"-\""); } } - if (result26 !== null) { - var result27 = parse__(); - if (result27 !== null) { - var result28 = parse_number(); - if (result28 !== null) { - var result24 = [result26, result27, result28]; + if (result0 !== null) { + result1 = parse__(); + if (result1 !== null) { + result2 = parse_number(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result24 = null; - pos = savedPos9; + result0 = null; + pos = pos1; } } else { - var result24 = null; - pos = savedPos9; + result0 = null; + pos = pos1; } } else { - var result24 = null; - pos = savedPos9; - } - var result25 = result24 !== null - ? (function(number) { return -number; })(result24[2]) - : null; - if (result25 !== null) { - var result23 = result25; - } else { - var result23 = null; - pos = savedPos8; + result0 = null; + pos = pos1; } - if (result23 !== null) { - var result0 = result23; - } else { - var savedPos6 = pos; - var savedPos7 = pos; - var result20 = parse_int(); - if (result20 !== null) { - var result21 = parse_frac(); - if (result21 !== null) { - var result22 = parse_exp(); - if (result22 !== null) { - var result18 = [result20, result21, result22]; + if (result0 !== null) { + result0 = (function(offset, number) { return -number; })(pos0, result0[2]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + result0 = parse_int(); + if (result0 !== null) { + result1 = parse_frac(); + if (result1 !== null) { + result2 = parse_exp(); + if (result2 !== null) { + result0 = [result0, result1, result2]; } else { - var result18 = null; - pos = savedPos7; + result0 = null; + pos = pos1; } } else { - var result18 = null; - pos = savedPos7; + result0 = null; + pos = pos1; } } else { - var result18 = null; - pos = savedPos7; - } - var result19 = result18 !== null - ? (function(int_, frac, exp) { return +(int_ + frac + exp); })(result18[0], result18[1], result18[2]) - : null; - if (result19 !== null) { - var result17 = result19; - } else { - var result17 = null; - pos = savedPos6; + result0 = null; + pos = pos1; } - if (result17 !== null) { - var result0 = result17; - } else { - var savedPos4 = pos; - var savedPos5 = pos; - var result15 = parse_int(); - if (result15 !== null) { - var result16 = parse_frac(); - if (result16 !== null) { - var result13 = [result15, result16]; + if (result0 !== null) { + result0 = (function(offset, int_, frac, exp) { return +(int_ + frac + exp); })(pos0, result0[0], result0[1], result0[2]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + result0 = parse_int(); + if (result0 !== null) { + result1 = parse_frac(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result13 = null; - pos = savedPos5; + result0 = null; + pos = pos1; } } else { - var result13 = null; - pos = savedPos5; + result0 = null; + pos = pos1; } - var result14 = result13 !== null - ? (function(int_, frac) { return +(int_ + frac); })(result13[0], result13[1]) - : null; - if (result14 !== null) { - var result12 = result14; - } else { - var result12 = null; - pos = savedPos4; + if (result0 !== null) { + result0 = (function(offset, int_, frac) { return +(int_ + frac); })(pos0, result0[0], result0[1]); } - if (result12 !== null) { - var result0 = result12; - } else { - var savedPos2 = pos; - var savedPos3 = pos; - var result10 = parse_int(); - if (result10 !== null) { - var result11 = parse_exp(); - if (result11 !== null) { - var result8 = [result10, result11]; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + pos1 = pos; + result0 = parse_int(); + if (result0 !== null) { + result1 = parse_exp(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } } else { - var result8 = null; - pos = savedPos3; + result0 = null; + pos = pos1; } - var result9 = result8 !== null - ? (function(int_, exp) { return +(int_ + exp); })(result8[0], result8[1]) - : null; - if (result9 !== null) { - var result7 = result9; - } else { - var result7 = null; - pos = savedPos2; + if (result0 !== null) { + result0 = (function(offset, int_, exp) { return +(int_ + exp); })(pos0, result0[0], result0[1]); } - if (result7 !== null) { - var result0 = result7; - } else { - var savedPos1 = pos; - var result5 = parse_frac(); - var result6 = result5 !== null - ? (function(frac) { return +frac; })(result5) - : null; - if (result6 !== null) { - var result4 = result6; - } else { - var result4 = null; - pos = savedPos1; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_frac(); + if (result0 !== null) { + result0 = (function(offset, frac) { return +frac; })(pos0, result0); } - if (result4 !== null) { - var result0 = result4; - } else { - var savedPos0 = pos; - var result2 = parse_int(); - var result3 = result2 !== null - ? (function(int_) { return +int_; })(result2) - : null; - if (result3 !== null) { - var result1 = result3; - } else { - var result1 = null; - pos = savedPos0; + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + pos0 = pos; + result0 = parse_int(); + if (result0 !== null) { + result0 = (function(offset, int_) { return +int_; })(pos0, result0); } - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; - }; - }; - }; - }; + if (result0 === null) { + pos = pos0; + } + } + } + } + } } - reportMatchFailures = savedReportMatchFailures; - if (reportMatchFailures && result0 === null) { + reportFailures--; + if (reportFailures === 0 && result0 === null) { matchFailed("number"); } - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_int() { - var cacheKey = 'int@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result5 = parse_digit19(); - if (result5 !== null) { - var result6 = parse_digits(); - if (result6 !== null) { - var result3 = [result5, result6]; + var result0, result1; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_digit19(); + if (result0 !== null) { + result1 = parse_digits(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result3 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result3 = null; - pos = savedPos1; - } - var result4 = result3 !== null - ? (function(digit19, digits) { return digit19 + digits; })(result3[0], result3[1]) - : null; - if (result4 !== null) { - var result2 = result4; - } else { - var result2 = null; - pos = savedPos0; + result0 = null; + pos = pos1; } - if (result2 !== null) { - var result0 = result2; - } else { - var result1 = parse_digit(); - if (result1 !== null) { - var result0 = result1; - } else { - var result0 = null;; - }; + if (result0 !== null) { + result0 = (function(offset, digit19, digits) { return digit19 + digits; })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; + } + if (result0 === null) { + result0 = parse_digit(); } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_frac() { - var cacheKey = 'frac@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1; + var pos0, pos1; - - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos, 1) === ".") { - var result3 = "."; - pos += 1; + pos0 = pos; + pos1 = pos; + if (input.charCodeAt(pos) === 46) { + result0 = "."; + pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("\".\""); } } - if (result3 !== null) { - var result4 = parse_digits(); - if (result4 !== null) { - var result1 = [result3, result4]; + if (result0 !== null) { + result1 = parse_digits(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(digits) { return "." + digits; })(result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, digits) { return "." + digits; })(pos0, result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_exp() { - var cacheKey = 'exp@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - - - var savedPos0 = pos; - var savedPos1 = pos; - var result3 = parse_e(); - if (result3 !== null) { - var result4 = parse_digits(); - if (result4 !== null) { - var result1 = [result3, result4]; + var result0, result1; + var pos0, pos1; + + pos0 = pos; + pos1 = pos; + result0 = parse_e(); + if (result0 !== null) { + result1 = parse_digits(); + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(e, digits) { return e + digits; })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, e, digits) { return e + digits; })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_digits() { - var cacheKey = 'digits@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0, result1; + var pos0; - var savedPos0 = pos; - var result3 = parse_digit(); - if (result3 !== null) { - var result1 = []; - while (result3 !== null) { - result1.push(result3); - var result3 = parse_digit(); + pos0 = pos; + result1 = parse_digit(); + if (result1 !== null) { + result0 = []; + while (result1 !== null) { + result0.push(result1); + result1 = parse_digit(); } } else { - var result1 = null; + result0 = null; } - var result2 = result1 !== null - ? (function(digits) { return digits.join(""); })(result1) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + if (result0 !== null) { + result0 = (function(offset, digits) { return digits.join(""); })(pos0, result0); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_e() { - var cacheKey = 'e@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1; + var pos0, pos1; - - var savedPos0 = pos; - var savedPos1 = pos; - if (input.substr(pos).match(/^[eE]/) !== null) { - var result3 = input.charAt(pos); + pos0 = pos; + pos1 = pos; + if (/^[eE]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result3 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[eE]"); } } - if (result3 !== null) { - if (input.substr(pos).match(/^[+\-]/) !== null) { - var result5 = input.charAt(pos); + if (result0 !== null) { + if (/^[+\-]/.test(input.charAt(pos))) { + result1 = input.charAt(pos); pos++; } else { - var result5 = null; - if (reportMatchFailures) { + result1 = null; + if (reportFailures === 0) { matchFailed("[+\\-]"); } } - var result4 = result5 !== null ? result5 : ''; - if (result4 !== null) { - var result1 = [result3, result4]; + result1 = result1 !== null ? result1 : ""; + if (result1 !== null) { + result0 = [result0, result1]; } else { - var result1 = null; - pos = savedPos1; + result0 = null; + pos = pos1; } } else { - var result1 = null; - pos = savedPos1; - } - var result2 = result1 !== null - ? (function(e, sign) { return e + sign; })(result1[0], result1[1]) - : null; - if (result2 !== null) { - var result0 = result2; - } else { - var result0 = null; - pos = savedPos0; + result0 = null; + pos = pos1; + } + if (result0 !== null) { + result0 = (function(offset, e, sign) { return e + sign; })(pos0, result0[0], result0[1]); + } + if (result0 === null) { + pos = pos0; } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_digit() { - var cacheKey = 'digit@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0; - - if (input.substr(pos).match(/^[0-9]/) !== null) { - var result0 = input.charAt(pos); + if (/^[0-9]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[0-9]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_digit19() { - var cacheKey = 'digit19@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0; - - if (input.substr(pos).match(/^[1-9]/) !== null) { - var result0 = input.charAt(pos); + if (/^[1-9]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[1-9]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_hex_digit() { - var cacheKey = 'hex_digit@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - if (input.substr(pos).match(/^[0-9a-fA-F]/) !== null) { - var result0 = input.charAt(pos); + if (/^[0-9a-fA-F]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { + result0 = null; + if (reportFailures === 0) { matchFailed("[0-9a-fA-F]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse__() { - var cacheKey = '_@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } + var result0, result1; - var savedReportMatchFailures = reportMatchFailures; - reportMatchFailures = false; - var result0 = []; - var result1 = parse_whitespace(); + reportFailures++; + result0 = []; + result1 = parse_whitespace(); while (result1 !== null) { result0.push(result1); - var result1 = parse_whitespace(); + result1 = parse_whitespace(); } - reportMatchFailures = savedReportMatchFailures; - if (reportMatchFailures && result0 === null) { + reportFailures--; + if (reportFailures === 0 && result0 === null) { matchFailed("whitespace"); } - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } function parse_whitespace() { - var cacheKey = 'whitespace@' + pos; - var cachedResult = cache[cacheKey]; - if (cachedResult) { - pos = cachedResult.nextPos; - return cachedResult.result; - } - + var result0; - if (input.substr(pos).match(/^[ \n\r]/) !== null) { - var result0 = input.charAt(pos); + if (/^[ \t\n\r]/.test(input.charAt(pos))) { + result0 = input.charAt(pos); pos++; } else { - var result0 = null; - if (reportMatchFailures) { - matchFailed("[ \\n\\r]"); + result0 = null; + if (reportFailures === 0) { + matchFailed("[ \\t\\n\\r]"); } } - - - - cache[cacheKey] = { - nextPos: pos, - result: result0 - }; return result0; } - function buildErrorMessage() { - function buildExpected(failuresExpected) { - failuresExpected.sort(); - - var lastFailure = null; - var failuresExpectedUnique = []; - for (var i = 0; i < failuresExpected.length; i++) { - if (failuresExpected[i] !== lastFailure) { - failuresExpectedUnique.push(failuresExpected[i]); - lastFailure = failuresExpected[i]; - } - } - - switch (failuresExpectedUnique.length) { - case 0: - return 'end of input'; - case 1: - return failuresExpectedUnique[0]; - default: - return failuresExpectedUnique.slice(0, failuresExpectedUnique.length - 1).join(', ') - + ' or ' - + failuresExpectedUnique[failuresExpectedUnique.length - 1]; + + function cleanupExpected(expected) { + expected.sort(); + + var lastExpected = null; + var cleanExpected = []; + for (var i = 0; i < expected.length; i++) { + if (expected[i] !== lastExpected) { + cleanExpected.push(expected[i]); + lastExpected = expected[i]; } } - - var expected = buildExpected(rightmostMatchFailuresExpected); - var actualPos = Math.max(pos, rightmostMatchFailuresPos); - var actual = actualPos < input.length - ? quote(input.charAt(actualPos)) - : 'end of input'; - - return 'Expected ' + expected + ' but ' + actual + ' found.'; + return cleanExpected; } function computeErrorPosition() { @@ -2908,13 +2240,13 @@ module.exports = (function(){ var column = 1; var seenCR = false; - for (var i = 0; i < rightmostMatchFailuresPos; i++) { + for (var i = 0; i < Math.max(pos, rightmostFailuresPos); i++) { var ch = input.charAt(i); - if (ch === '\n') { + if (ch === "\n") { if (!seenCR) { line++; } column = 1; seenCR = false; - } else if (ch === '\r' | ch === '\u2028' || ch === '\u2029') { + } else if (ch === "\r" || ch === "\u2028" || ch === "\u2029") { line++; column = 1; seenCR = true; @@ -2928,146 +2260,76 @@ module.exports = (function(){ } - - var filterEqual = function(o, k, v) { o[k] = v; }, - - filterGreater = filter("$gt"), - - filterGreaterOrEqual = filter("$gte"), - - filterLess = filter("$lt"), - - filterLessOrEqual = filter("$lte"), - - filterNotEqual = filter("$ne"), - - filterRegularExpression = filter("$regex"), - - filterIn = filter("$in"), - - exists = {$exists: true}; - - - - function noop() {} - - - - function filter(op) { - - return function(o, k, v) { - - var f = o[k]; - - switch (typeof f) { - - case "undefined": o[k] = f = {}; // continue - - case "object": f[op] = v; break; - - // otherwise, observe the existing equals (literal) filter - + var filterEqual = function(o, k, v) { o[k] = v; }, + filterGreater = filter("$gt"), + filterGreaterOrEqual = filter("$gte"), + filterLess = filter("$lt"), + filterLessOrEqual = filter("$lte"), + filterNotEqual = filter("$ne"), + filterRegularExpression = filter("$regex"), + filterIn = filter("$in"), + exists = {$exists: true}; + + function noop() {} + + function filter(op) { + return function(o, k, v) { + var f = o[k]; + switch (typeof f) { + case "undefined": o[k] = f = {}; // continue + case "object": f[op] = v; break; + // otherwise, observe the existing equals (literal) filter + } + }; } - }; - - } - - - - function arrayAccessor(name) { - - name = new String(name); - - name.array = true; - - return name; - - } - - - - function objectAccessor(name) { - - return name; - - } - - - - function compoundFields(type, head, tail) { - - var n = tail.length; - - return { - - type: type, - - exists: function(o) { - - var i = -1; - - head.exists(o); - - while (++i < n) tail[i][3].exists(o); - - }, - - fields: function(o) { - - var i = -1; - - head.fields(o); - - while (++i < n) tail[i][3].fields(o); - + function arrayAccessor(name) { + name = new String(name); + name.array = true; + return name; } - }; - - } - - - - function member(head, tail) { - - var fields = ["d", head].concat(tail), - - shortName = fields.filter(function(d) { return !d.array; }).join("."), - - longName = fields.join("."), - - i = -1, - - n = fields.length; - - return { - - field: longName, - - exists: function(o) { - - if (!(shortName in o)) { - - o[shortName] = exists; - - } - - }, - - fields: function(o) { - - o[shortName] = 1; - + function objectAccessor(name) { + return name; } - }; + function compoundFields(type, head, tail) { + var n = tail.length; + return { + type: type, + exists: function(o) { + var i = -1; + head.exists(o); + while (++i < n) tail[i][3].exists(o); + }, + fields: function(o) { + var i = -1; + head.fields(o); + while (++i < n) tail[i][3].fields(o); + } + }; + } - } + function member(head, tail) { + var fields = ["d", head].concat(tail), + shortName = fields.filter(function(d) { return !d.array; }).join("."), + longName = fields.join("."), + i = -1, + n = fields.length; + return { + field: longName, + exists: function(o) { + if (!(shortName in o)) { + o[shortName] = exists; + } + }, + fields: function(o) { + o[shortName] = 1; + } + }; + } - - var result = parseFunctions[startRule](); @@ -3078,27 +2340,32 @@ module.exports = (function(){ * * - |result !== null| * - |pos === input.length| - * - |rightmostMatchFailuresExpected| may or may not contain something + * - |rightmostFailuresExpected| may or may not contain something * * 2. The parser successfully parsed only a part of the input. * * - |result !== null| * - |pos < input.length| - * - |rightmostMatchFailuresExpected| may or may not contain something + * - |rightmostFailuresExpected| may or may not contain something * * 3. The parser did not successfully parse any part of the input. * * - |result === null| * - |pos === 0| - * - |rightmostMatchFailuresExpected| contains at least one failure + * - |rightmostFailuresExpected| contains at least one failure * * All code following this comment (including called functions) must * handle these states. */ if (result === null || pos !== input.length) { + var offset = Math.max(pos, rightmostFailuresPos); + var found = offset < input.length ? input.charAt(offset) : null; var errorPosition = computeErrorPosition(); + throw new this.SyntaxError( - buildErrorMessage(), + cleanupExpected(rightmostFailuresExpected), + found, + offset, errorPosition.line, errorPosition.column ); @@ -3113,9 +2380,33 @@ module.exports = (function(){ /* Thrown when a parser encounters a syntax error. */ - result.SyntaxError = function(message, line, column) { - this.name = 'SyntaxError'; - this.message = message; + result.SyntaxError = function(expected, found, offset, line, column) { + function buildMessage(expected, found) { + var expectedHumanized, foundHumanized; + + switch (expected.length) { + case 0: + expectedHumanized = "end of input"; + break; + case 1: + expectedHumanized = expected[0]; + break; + default: + expectedHumanized = expected.slice(0, expected.length - 1).join(", ") + + " or " + + expected[expected.length - 1]; + } + + foundHumanized = found ? quote(found) : "end of input"; + + return "Expected " + expectedHumanized + " but " + foundHumanized + " found."; + } + + this.name = "SyntaxError"; + this.expected = expected; + this.found = found; + this.message = buildMessage(expected, found); + this.offset = offset; this.line = line; this.column = column; }; From 0b9c7e7bc38730e3118adf203e518d6f9d372b08 Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 12:12:23 +0200 Subject: [PATCH 82/87] Fix Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 759e5661..544ca40c 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,4 @@ all: \ lib/cube/metric-expression.js test: all - @$(JS_TESTER) + NODE_ENV=test $(JS_TESTER) --isolate --spec From 51f129061f2fbe83db92bc6a85d4c823f749330f Mon Sep 17 00:00:00 2001 From: Marsup Date: Mon, 9 Sep 2013 14:16:14 +0200 Subject: [PATCH 83/87] Fix example's configuration key --- examples/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helper.js b/examples/helper.js index f46d34ba..4561128f 100644 --- a/examples/helper.js +++ b/examples/helper.js @@ -10,7 +10,7 @@ var cube = require("../"), var options = require("../config/cube").include('evaluator'), mongodb = require("mongodb"), - mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], options["mongo-server_options"]), + mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], options["mongo-server-options"]), db = new mongodb.Db(options["mongo-database"], mongo, { native_parser: true }), putter, getter; From f237776435d3cc5bb996b4f6a5b36ee897f5a6bb Mon Sep 17 00:00:00 2001 From: Marsup Date: Fri, 13 Sep 2013 17:33:49 +0200 Subject: [PATCH 84/87] Move configuration file into lib --- config/cube.js => lib/cube/config.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config/cube.js => lib/cube/config.js (100%) diff --git a/config/cube.js b/lib/cube/config.js similarity index 100% rename from config/cube.js rename to lib/cube/config.js From 05ab0b49d476f9106b930b55a62e60cf28c3569e Mon Sep 17 00:00:00 2001 From: Marsup Date: Fri, 13 Sep 2013 19:31:15 +0200 Subject: [PATCH 85/87] Revert configuration style to POJO --- Jakefile | 11 ++--- bin/collector-config.js | 50 ++++++++++++++++++++++ bin/collector.js | 2 +- bin/evaluator-config.js | 49 ++++++++++++++++++++++ bin/evaluator.js | 2 +- bin/warmer-config.js | 51 ++++++++++++++++++++++ bin/warmer.js | 2 +- examples/helper.js | 21 +++++++--- lib/cube/authentication.js | 4 +- lib/cube/config.js | 77 ++++------------------------------ lib/cube/db.js | 4 +- lib/cube/event.js | 5 +-- lib/cube/index.js | 1 - lib/cube/models/measurement.js | 4 +- lib/cube/server.js | 17 ++++---- lib/cube/warmer.js | 12 +++--- package.json | 3 +- test/event-test.js | 14 +++---- test/test_helper.js | 66 ++++++++++++++--------------- test/warmer-test.js | 12 +++--- 20 files changed, 247 insertions(+), 160 deletions(-) create mode 100644 bin/collector-config.js create mode 100644 bin/evaluator-config.js create mode 100644 bin/warmer-config.js diff --git a/Jakefile b/Jakefile index dbd644b0..6e87c501 100644 --- a/Jakefile +++ b/Jakefile @@ -1,17 +1,12 @@ -var config = require('./config/cube'), - Db = require('./lib/cube/db'), - horizons = config.get('horizons'); +var forced_metric_expiration = 1000 * 60 * 60 * 24 * 7, // 7 days + Db = require('./lib/cube/db'); namespace("db", function(){ namespace("metrics", function(){ desc("Remove metrics with times past the forced metric expiration horizon") task("remove_expired", [], function(){ - if(!horizons.forced_metric_expiration){ - throw new Error("horizons.forced_metric_expiration MUST be set in: config/cube.js"); - } - var db = new Db(), - expiration_date = new Date(new Date() - horizons.forced_metric_expiration); + expiration_date = new Date(new Date() - forced_metric_expiration); function handle(err) { if(err) throw err; diff --git a/bin/collector-config.js b/bin/collector-config.js new file mode 100644 index 00000000..da59e974 --- /dev/null +++ b/bin/collector-config.js @@ -0,0 +1,50 @@ +module.exports = { + mongodb: { + 'mongo-host': '127.0.0.1', + 'mongo-port': 27017, + 'mongo-database': 'cube', + 'mongo-username': null, + 'mongo-password': null, + 'mongo-server-options': { + auto_reconnect: true, + poolSize: 8, + socketOptions: { + noDelay: true + } + }, + + 'mongo-metrics': { + autoIndexId: true, + capped: false, + safe: false + }, + + 'mongo-events': { + autoIndexId: true, + capped: true, + size: 1e9, + safe: false + }, + + 'separate-events-database': true, + + 'authentication-collection': 'users' + }, + horizons: { + 'calculation': 1000 * 60 * 60 * 2, // 2 hours + 'invalidation': 1000 * 60 * 60 * 1, // 1 hour + 'forced_metric_expiration': 1000 * 60 * 60 * 24 * 7, // 7 days + }, + 'collectd-mappings': { + 'snmp': { + 'if_octets': 'interface', + 'disk_octets': 'disk', + 'swap_io': 'swap', + 'swap': 'swap' + } + }, + + 'http-port': 1080, + 'udp-port': 1180, + 'authenticator': 'allow_all' +}; diff --git a/bin/collector.js b/bin/collector.js index e34e64a8..62a9a994 100644 --- a/bin/collector.js +++ b/bin/collector.js @@ -1,7 +1,7 @@ 'use strict'; var cube = require("../"), - server = cube.server('collector'); + server = cube.server(require('./collector-config')); server .use(cube.collector.register) diff --git a/bin/evaluator-config.js b/bin/evaluator-config.js new file mode 100644 index 00000000..29e83bc0 --- /dev/null +++ b/bin/evaluator-config.js @@ -0,0 +1,49 @@ +module.exports = { + mongodb: { + 'mongo-host': '127.0.0.1', + 'mongo-port': 27017, + 'mongo-database': 'cube', + 'mongo-username': null, + 'mongo-password': null, + 'mongo-server-options': { + auto_reconnect: true, + poolSize: 8, + socketOptions: { + noDelay: true + } + }, + + 'mongo-metrics': { + autoIndexId: true, + capped: false, + safe: false + }, + + 'mongo-events': { + autoIndexId: true, + capped: true, + size: 1e9, + safe: false + }, + + 'separate-events-database': true, + + 'authentication-collection': 'users' + }, + horizons: { + 'calculation': 1000 * 60 * 60 * 2, // 2 hours + 'invalidation': 1000 * 60 * 60 * 1, // 1 hour + 'forced_metric_expiration': 1000 * 60 * 60 * 24 * 7, // 7 days + }, + 'collectd-mappings': { + 'snmp': { + 'if_octets': 'interface', + 'disk_octets': 'disk', + 'swap_io': 'swap', + 'swap': 'swap' + } + }, + + 'http-port': 1081, + 'authenticator': 'allow_all' +}; diff --git a/bin/evaluator.js b/bin/evaluator.js index b2ee955c..97038d1f 100644 --- a/bin/evaluator.js +++ b/bin/evaluator.js @@ -1,7 +1,7 @@ 'use strict'; var cube = require("../"), - server = cube.server('evaluator'); + server = cube.server(require('./evaluator-config')); server .use(cube.evaluator.register) diff --git a/bin/warmer-config.js b/bin/warmer-config.js new file mode 100644 index 00000000..cf7e3906 --- /dev/null +++ b/bin/warmer-config.js @@ -0,0 +1,51 @@ +module.exports = { + mongodb: { + 'mongo-host': '127.0.0.1', + 'mongo-port': 27017, + 'mongo-database': 'cube', + 'mongo-username': null, + 'mongo-password': null, + 'mongo-server-options': { + auto_reconnect: true, + poolSize: 8, + socketOptions: { + noDelay: true + } + }, + + 'mongo-metrics': { + autoIndexId: true, + capped: false, + safe: false + }, + + 'mongo-events': { + autoIndexId: true, + capped: true, + size: 1e9, + safe: false + }, + + 'separate-events-database': true, + + 'authentication-collection': 'users' + }, + horizons: { + 'calculation': 1000 * 60 * 60 * 2, // 2 hours + 'invalidation': 1000 * 60 * 60 * 1, // 1 hour + 'forced_metric_expiration': 1000 * 60 * 60 * 24 * 7, // 7 days + }, + 'collectd-mappings': { + 'snmp': { + 'if_octets': 'interface', + 'disk_octets': 'disk', + 'swap_io': 'swap', + 'swap': 'swap' + } + }, + + warmer: { + 'warmer-interval': 1000 * 30, + 'warmer-tier': 1000 * 10 + } +}; diff --git a/bin/warmer.js b/bin/warmer.js index 64b058cc..3e9beddd 100644 --- a/bin/warmer.js +++ b/bin/warmer.js @@ -1,6 +1,6 @@ 'use strict'; var cube = require("../"), - warmer = cube.warmer(); + warmer = cube.warmer(require('./warmer-config')); warmer.start(); diff --git a/examples/helper.js b/examples/helper.js index 4561128f..a18b1881 100644 --- a/examples/helper.js +++ b/examples/helper.js @@ -8,11 +8,22 @@ var cube = require("../"), event_mod = require("../lib/cube/event"), models = require("../lib/cube/models"), Event = models.Event; -var options = require("../config/cube").include('evaluator'), - mongodb = require("mongodb"), - mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], options["mongo-server-options"]), - db = new mongodb.Db(options["mongo-database"], mongo, { native_parser: true }), - putter, getter; +var options = { + 'mongo-host': '127.0.0.1', + 'mongo-port': 27017, + 'mongo-server-options': { + auto_reconnect: true, + poolSize: 8, + socketOptions: { + noDelay: true + } + }, + 'mongo-database': 'cube' + }, + mongodb = require("mongodb"), + mongo = new mongodb.Server(options["mongo-host"], options["mongo-port"], options["mongo-server-options"]), + db = new mongodb.Db(options["mongo-database"], mongo, { native_parser: true }), + putter, getter; var type = 'doh'; diff --git a/lib/cube/authentication.js b/lib/cube/authentication.js index 9c5d97fc..4ea28aa2 100644 --- a/lib/cube/authentication.js +++ b/lib/cube/authentication.js @@ -22,7 +22,7 @@ var mongodb = require("mongodb"), cookies = require("cookies"), bcrypt = require("bcrypt"), metalog = require('./metalog'), - options = require('../../config/cube'), + config = require('./config'), authentication = {}; authentication.authenticator = function(strategy, db){ @@ -51,7 +51,7 @@ authentication.read_only = function(){ authentication.mongo_cookie = function(db){ var users; - db.collection(options.get('mongodb')["authentication-collection"] || "users", function(error, clxn){ + db.collection(config.mongodb["authentication-collection"] || "users", function(error, clxn) { if (error) throw(error); // TODO: check if not collection? users = clxn; diff --git a/lib/cube/config.js b/lib/cube/config.js index 00e038ad..108d03d2 100644 --- a/lib/cube/config.js +++ b/lib/cube/config.js @@ -1,74 +1,13 @@ 'use strict'; -var cfg = module.exports = require('cfg').createConfig(), - metalog = require('../lib/cube/metalog'); - +var metalog = require('./metalog'); metalog.send_events = false; -// -// Common configuration -// - -cfg.set('mongodb', { - 'mongo-host': '127.0.0.1', - 'mongo-port': 27017, - 'mongo-database': 'cube', - 'mongo-username': null, - 'mongo-password': null, - 'mongo-server-options': { - auto_reconnect: true, - poolSize: 8, - socketOptions: { - noDelay: true - } - }, - - 'mongo-metrics': { - autoIndexId: true, - capped: false, - safe: false - }, - - 'mongo-events': { - autoIndexId: true, - capped: true, - size: 1e9, - safe: false - }, - - 'separate-events-database': true, - - 'authentication-collection': 'users' -}); - -cfg.set('horizons', { - 'calculation': 1000 * 60 * 60 * 2, // 2 hours - 'invalidation': 1000 * 60 * 60 * 1, // 1 hour - 'forced_metric_expiration': 1000 * 60 * 60 * 24 * 7, // 7 days -}); - -cfg.set('collectd-mappings', { - 'snmp': { - 'if_octets': 'interface', - 'disk_octets': 'disk', - 'swap_io': 'swap', - 'swap': 'swap' +module.exports = { + load: function(options) { + Object.keys(options).forEach(function (key) { + module.exports[key] = options[key]; + }); + return module.exports; } -}); - -cfg.set('collector', { - 'http-port': 1080, - 'udp-port': 1180, - 'authenticator': 'allow_all' -}); - -cfg.set('evaluator', { - 'http-port': 1081, - 'authenticator': 'allow_all' -}); - -cfg.set('warmer', { - 'warmer-interval': 1000 * 30, - 'warmer-tier': 1000 * 10 -}); - +} diff --git a/lib/cube/db.js b/lib/cube/db.js index 4073f1d3..ea100c3e 100644 --- a/lib/cube/db.js +++ b/lib/cube/db.js @@ -4,7 +4,7 @@ var util = require("util"), mongodb = require("mongodb"), queue = require("queue-async"), metalog = require("./metalog"), - config = require("../../config/cube"), + config = require("./config"), db_index = 0; function mongoUrl(options) { @@ -51,7 +51,7 @@ function Db(){ function open(callback){ if (db_client) { metalog.minor('mongo_already_open', db.report()); return callback(null, db); } - var mongoConfig = config.get('mongodb'), + var mongoConfig = config.mongodb, url = mongoUrl(mongoConfig), options = mongoOptions(mongoConfig); diff --git a/lib/cube/event.js b/lib/cube/event.js index 51ef1850..cc90cbd0 100644 --- a/lib/cube/event.js +++ b/lib/cube/event.js @@ -13,7 +13,7 @@ var _ = require("underscore"), parser = require("./event-expression"), bisect = require("./bisect"), metalog = require("./metalog"), - config = require("../../config/cube"); + config = require("./config"); // When streaming events, we should allow a delay for events to arrive, or else // we risk skipping events that arrive after their event.time. This delay can be @@ -34,8 +34,7 @@ var putter_id = 0; // exports.putter = function(db){ - var horizons = config.get('horizons'); - + var horizons = config.horizons; var invalidator = new Invalidator(); function putter(request, callback){ diff --git a/lib/cube/index.js b/lib/cube/index.js index 601b7b45..ef9c31b8 100644 --- a/lib/cube/index.js +++ b/lib/cube/index.js @@ -2,7 +2,6 @@ process.env.TZ = 'UTC'; exports.authentication = require("./authentication"); -exports.config = require("../../config/cube"); exports.metalog = require("./metalog"); exports.emitter = require("./emitter"); exports.server = require("./server"); diff --git a/lib/cube/models/measurement.js b/lib/cube/models/measurement.js index dddcb975..2f1b51d8 100644 --- a/lib/cube/models/measurement.js +++ b/lib/cube/models/measurement.js @@ -6,7 +6,7 @@ var metalog = require('../metalog'), Metric = require('./metric'), Event = require('./event'), _ = require('underscore'), - config = require('../../../config/cube'), + config = require('../config'), compute = {constant: constant, binary: binary, unary: unary}, setImmediate = require("../set-immediate"), queueByName = {}; @@ -202,7 +202,7 @@ function findOrComputeUnary(db, callback) { // the order in which rows are returned from the database. Thus, we know // when we've seen all of the events for a given time interval. function computeFlat(start, stop) { - var horizons = config.get('horizons'), pastHorizon = { + var horizons = config.horizons, pastHorizon = { is: 'past_horizon', tier: tier, expression: expression.source diff --git a/lib/cube/server.js b/lib/cube/server.js index 7cf061e1..6d4a0b42 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -24,7 +24,7 @@ var util = require("util"), event = require("./event"), metalog = require("./metalog"), Db = require("./db"), - config = require("../../config/cube"); + config = require("./config"); // Don't crash on errors. process.on("uncaughtException", function(error) { @@ -47,9 +47,10 @@ var wsOptions = { closeTimeout: 5000 }; -module.exports = function(kind, db) { - var options = config.get(kind), - server = {}, +module.exports = function(options, db) { + config.load(options); + + var server = {}, primary = http.createServer(), secondary = websprocket.createServer(), file = new file_server.Server("static"), @@ -58,13 +59,9 @@ module.exports = function(kind, db) { id = 0, authenticator; - if (!options) { - throw new Error('The provided type of server does not exist in the configuration file.'); - } - // allows dependency injection from test_helper if (! db) db = new Db(); - + secondary.server = primary; var uses = [] //will be used for registrating anything external, particularly endpoints. @@ -225,7 +222,7 @@ module.exports = function(kind, db) { secondary.on("close", function(){ metalog.info('ws_close' ); }); function try_close(name, obj){ if (obj){ try { - metalog.info(name+'_stopping', config.values); + metalog.info(name+'_stopping', config); obj.close( function(){ metalog.info(name+'_stop'); } ); } catch(error){} } } diff --git a/lib/cube/warmer.js b/lib/cube/warmer.js index 74d340f5..9354f2b9 100644 --- a/lib/cube/warmer.js +++ b/lib/cube/warmer.js @@ -4,12 +4,14 @@ var metric = require('./metric'), tiers = require('./tiers'), metalog = require('./metalog'), db = new (require('./db'))(), - options = require('../../config/cube'); + config = require('./config'); + +module.exports = function(options) { + config.load(options); -module.exports = function() { var calculate_metric, tier, timeout, - horizons = options.get('horizons'), - warmer = options.get('warmer'); + horizons = config.horizons, + warmer = config.warmer; function fetch_metrics(callback){ var expressions = []; @@ -53,7 +55,7 @@ module.exports = function() { if(typeof tier === "undefined") throw new Error("Undefined warmer tier configured: " + warmer['warmer-tier']); - metalog.event("cube_life", { is: 'start_warmer', options: options.values }); + metalog.event("cube_life", { is: 'start_warmer', options: config }); db.open(function(error) { if (error) throw error; diff --git a/package.json b/package.json index 33bcb604..ad782ebf 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,7 @@ "bcrypt": "0.7.7", "underscore": "1.3.3", "jake": "0.5.6", - "queue-async": "~1.0.4", - "cfg": "0.0.2" + "queue-async": "~1.0.4" }, "scripts": { "preinstall": "npm install mongodb --mongodb:native", diff --git a/test/event-test.js b/test/event-test.js index e156f3d7..70cf70f5 100644 --- a/test/event-test.js +++ b/test/event-test.js @@ -5,7 +5,7 @@ var vows = require("vows"), test_helper = require("./test_helper"), Event = require("../lib/cube/models/event"), event = require("../lib/cube/event"), - config = require('../config/cube'); + config = require('../lib/cube/config'); var suite = vows.describe("event"); @@ -50,10 +50,8 @@ suite.addBatch(test_helper.batch({ var horizon = new Date() - fuck_wit_dre_day + (1000 * 60); // Temporarily override horizons settings - this.oldHorizons = config.get('horizons'); - config.set('horizons', { - invalidation: horizon - }); + this.oldHorizons = config.horizons; + config.horizons = { invalidation: horizon }; return event.putter(test_db.db); }, 'events past invalidation horizon': { @@ -68,10 +66,10 @@ suite.addBatch(test_helper.batch({ 'should return -1': function(error, response){ assert.equal(this.ret, -1); } - }/*, + }, teardown: function() { - config.set('horizons', this.oldHorizons); - }*/ + config.horizons = this.oldHorizons; + } })); suite['export'](module); diff --git a/test/test_helper.js b/test/test_helper.js index b48ff5f0..dc3720f7 100644 --- a/test/test_helper.js +++ b/test/test_helper.js @@ -7,7 +7,7 @@ var _ = require("underscore"), dgram = require('dgram'), Db = require("../lib/cube/db"), metalog = require("../lib/cube/metalog"), - config = require("../config/cube"); + config = require("../lib/cube/config"); // ========================================================================== // @@ -19,38 +19,38 @@ var test_collections = ["test_users", "test_events", "test_metrics", "test_boa test_helper.inspectify = metalog.inspectify; test_helper._ = require('underscore'); -config.set('mongodb', { - 'mongo-host': 'localhost', - 'mongo-port': 27017, - 'mongo-username': null, - 'mongo-password': null, - 'mongo-database': 'cube_test', - 'host': 'localhost', - 'authentication-collection': 'test_users' -}); - -config.set('horizons', { - calculation: +(new Date()), - invalidation: +(new Date()) +config.load({ + mongodb: { + 'mongo-host': 'localhost', + 'mongo-port': 27017, + 'mongo-username': null, + 'mongo-password': null, + 'mongo-database': 'cube_test', + 'host': 'localhost', + 'authentication-collection': 'test_users' + }, + horizons: { + calculation: +(new Date()), + invalidation: +(new Date()) + }, + warmer: { + 'warmer-interval': 10000, + 'warmer-tier': 10000 + } }); var basePort = 1083; -config.set('collector', { - 'http-port': basePort++, - 'udp-port': basePort++, - 'authenticator': 'allow_all' -}); - -config.set('evaluator', { - 'http-port': basePort++, - 'authenticator': 'allow_all' -}); - -config.set('warmer', { - 'warmer-interval': 10000, - 'warmer-tier': 10000 -}); - +var kinds = { + collector: { + 'http-port': basePort++, + 'udp-port': basePort++, + 'authenticator': 'allow_all' + }, + evaluator: { + 'http-port': basePort++, + 'authenticator': 'allow_all' + } +}; // Disable logging for tests. metalog.loggers.info = metalog.silent; // log @@ -151,7 +151,7 @@ test_helper.delay = delay; test_helper.with_server = function(kind, components, batch){ return test_helper.batch({ '': { topic: function(test_db){ - var ctxt = this, cb = ctxt.callback; + var ctxt = this; start_server(kind, components, ctxt, test_db); }, '': batch, @@ -167,10 +167,10 @@ test_helper.with_server = function(kind, components, batch){ // @see test_helper.with_server function start_server(kind, register, ctxt, test_db){ - var config = require('../config/cube').get(kind); + var config = require('../lib/cube/config').load(kinds[kind]); ctxt.http_port = config['http-port']; ctxt.udp_port = config['udp-port']; - ctxt.server = require('../lib/cube/server')(kind, test_db); + ctxt.server = require('../lib/cube/server')(config, test_db); ctxt.server.use(register); ctxt.server.start(ctxt.callback); } diff --git a/test/warmer-test.js b/test/warmer-test.js index 3b2eafe3..515c70c2 100644 --- a/test/warmer-test.js +++ b/test/warmer-test.js @@ -5,7 +5,7 @@ var _ = require('underscore'), assert = require("assert"), test_helper = require("./test_helper"), event = require("../lib/cube/event"), - config = require("../config/cube"), + config = require("../lib/cube/config"), warmer = require("../lib/cube/warmer"); var suite = vows.describe("warmer"); @@ -13,17 +13,15 @@ var suite = vows.describe("warmer"); suite.addBatch(test_helper.batch({ topic: function(test_db) { // Temporarily override horizons settings - this.oldHorizons = config.get('horizons'); - config.set('horizons', { - calculation: 30000 - }); + this.oldHorizons = config.horizons; + config.horizons = { calculation: 30000 }; var board = { pieces: [{ query: "sum(test(value))" }] }, nowish = this.nowish = (10e3 * Math.floor(new Date()/10e3)), putter = this.putter = event.putter(test_db), _this = this; - this.warmer = warmer(); + this.warmer = warmer(config); putter({type: 'test', time: nowish + 500, data: {value: 10}}, function(){ putter({type: 'test', time: nowish + 2000, data: {value: 5}}, function(){ @@ -53,7 +51,7 @@ suite.addBatch(test_helper.batch({ } }, teardown: function(){ - config.set('horizons', this.oldHorizons); + config.horizons = this.oldHorizons; this.warmer.stop(); this.putter.stop(this.callback); } From f8de9457a9d168c7324e276cfa20599d344d78e8 Mon Sep 17 00:00:00 2001 From: Marsup Date: Thu, 24 Oct 2013 10:34:39 +0200 Subject: [PATCH 86/87] Complete the interface binding work --- lib/cube/server.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cube/server.js b/lib/cube/server.js index 6d4a0b42..3d271d6e 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -204,15 +204,15 @@ module.exports = function(options, db) { metalog.putter = event.putter(db); uses.forEach(function(adder){ adder(db, endpoints); }); authenticator = authentication.authenticator(options["authenticator"], db); - metalog.event('start_http', { port: options["http-port"] }); + metalog.event('start_http', { port: options["http-port"], host: options["http-host"] }); primary.listen(options["http-port"], options["http-host"]); if (endpoints.udp) { - metalog.event('start_udp', { port: options["udp-port"] }); + metalog.event('start_udp', { port: options["udp-port"], host: options["udp-host"] }); udp = dgram.createSocket("udp4"); udp.on("message", function(message) { endpoints.udp(JSON.parse(message.toString("utf8")), ignore); }); - udp.bind(options["udp-port"]); + udp.bind(options["udp-port"], options["udp-host"]); } if (server_start_cb) server_start_cb(null, options); } From e0b19e86d973ed1577b2dae70d2233e22746763d Mon Sep 17 00:00:00 2001 From: Alfred Perlstein Date: Fri, 25 Oct 2013 10:44:48 +0200 Subject: [PATCH 87/87] If udp is not configured then don't bind udp. --- lib/cube/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cube/server.js b/lib/cube/server.js index 3d271d6e..e61c6e79 100644 --- a/lib/cube/server.js +++ b/lib/cube/server.js @@ -206,7 +206,7 @@ module.exports = function(options, db) { authenticator = authentication.authenticator(options["authenticator"], db); metalog.event('start_http', { port: options["http-port"], host: options["http-host"] }); primary.listen(options["http-port"], options["http-host"]); - if (endpoints.udp) { + if (endpoints.udp && options["udp-port"] !== undefined) { metalog.event('start_udp', { port: options["udp-port"], host: options["udp-host"] }); udp = dgram.createSocket("udp4"); udp.on("message", function(message) {