Fix #244, normalize for inconsistent insert rows

This commit is contained in:
Tim Griesser 2014-06-03 00:47:54 -04:00
parent c1d0f3fd44
commit af1f9e83ae
6 changed files with 144 additions and 110 deletions

View File

@ -1,5 +1,5 @@
!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var f;"undefined"!=typeof window?f=window:"undefined"!=typeof global?f=global:"undefined"!=typeof self&&(f=self),f.Knex=e()}}(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(_dereq_,module,exports){
// Knex.js 0.6.0-alpha
// Knex.js 0.6.0
// --------------
// (c) 2014 Tim Griesser
@ -159,10 +159,9 @@ Knex.initialize = function(config) {
// knex.migrate.latest().then(...
// knex.migrate.currentVersion(...
_.each(MigrateInterface, function(method) {
migrate[method] = function(config) {
migrate[method] = function() {
if (!client.Migrator) client.initMigrator();
config.knex = knex;
var migrator = new client.Migrator(config);
var migrator = new client.Migrator(knex);
return migrator[method].apply(migrator, arguments);
};
});
@ -178,16 +177,18 @@ module.exports = Knex;
// "Base Client"
// ------
var Promise = _dereq_('./promise');
var _ = _dereq_('lodash');
// The base client provides the general structure
// for a dialect specific client object. The client
// object attaches fresh constructors for each component
// of the library.
function Client() {
function Client(config) {
this.initFormatter();
this.initRaw();
this.initTransaction();
this.initQuery();
this.migrationConfig = _.clone(config && config.migrations);
}
// Set the "isDebugging" flag on the client to "true" to log
@ -224,7 +225,7 @@ Client.prototype.database = function() {
};
module.exports = Client;
},{"./promise":49}],3:[function(_dereq_,module,exports){
},{"./promise":49,"lodash":"K2RcUv"}],3:[function(_dereq_,module,exports){
// MySQL Formatter
// ------
module.exports = function(client) {
@ -681,12 +682,10 @@ ColumnCompiler_MySQL.prototype.after = function(column) {
return 'after ' + this.formatter.wrap(column);
};
ColumnCompiler_MySQL.prototype.comment = function(comment) {
if (comment) {
if (comment.length > 255) {
helpers.warn('Your comment is longer than the max comment length for MySQL');
}
return "comment '" + comment + "'";
if (comment && comment.length > 255) {
helpers.warn('Your comment is longer than the max comment length for MySQL');
}
return comment && "comment '" + comment + "'";
};
client.ColumnBuilder = ColumnBuilder_MySQL;
@ -1089,7 +1088,10 @@ Formatter_PG.prototype.operators = [
// Wraps a value (column, tableName) with the correct ticks.
Formatter_PG.prototype.wrapValue = function(value) {
return (value !== '*' ? '"' + value + '"' : '*');
if (value === '*') return value;
var matched = value.match(/(.*?)(\[[0-9]\])/);
if (matched) return this.wrapValue(matched[1]) + matched[2];
return '"' + value + '"';
};
// Memoize the calls to "wrap" for a little extra perf.
@ -1485,7 +1487,7 @@ function ColumnBuilder_PG() {
inherits(ColumnBuilder_PG, Schema.ColumnBuilder);
function ColumnCompiler_PG() {
this.modifiers = ['nullable', 'defaultTo'];
this.modifiers = ['nullable', 'defaultTo', 'comment'];
this.Formatter = client.Formatter;
Schema.ColumnCompiler.apply(this, arguments);
}
@ -1523,8 +1525,10 @@ ColumnCompiler_PG.prototype.uuid = 'uuid';
// Modifiers:
// ------
ColumnCompiler_PG.prototype.comment = function(comment) {
this.pushQuery('comment on column ' + this.tableName + '.' +
this.formatter.wrap(this.args[0]) + " is " + (comment ? "'" + comment + "'" : 'NULL'));
this.pushAdditional(function() {
this.pushQuery('comment on column ' + this.tableCompiler.tableName() + '.' +
this.formatter.wrap(this.args[0]) + " is " + (comment ? "'" + comment + "'" : 'NULL'));
}, comment);
};
client.ColumnBuilder = ColumnBuilder_PG;
@ -1637,10 +1641,8 @@ TableCompiler_PG.prototype.createQuery = function(columns) {
sql: 'create table ' + this.tableName() + ' (' + columns.sql.join(', ') + ')',
bindings: columns.bindings
});
var hasComment = _.has(this._single, 'comment');
if (hasComment) {
this.pushQuery('comment on table ' + this.tableName() + ' is ' + "'" + this.attributes.comment + "'");
}
var hasComment = _.has(this.single, 'comment');
if (hasComment) this.comment(this.single.comment);
};
// Compile a foreign key command.
@ -1649,6 +1651,11 @@ TableCompiler_PG.prototype.foreign = function(foreignData) {
if (sql) this.pushQuery(sql);
};
// Compiles the comment on the table.
TableCompiler_PG.prototype.comment = function(comment) {
this.pushQuery('comment on table ' + this.tableName() + ' is ' + "'" + (this.single.comment || '') + "'");
};
// Indexes:
// -------
@ -4036,11 +4043,15 @@ QueryCompiler.prototype._prepInsert = function(data) {
var isRaw = this.formatter.rawOrFn(data);
if (isRaw) return isRaw;
var values = [];
var columns;
var columns, colList;
if (!_.isArray(data)) data = data ? [data] : [];
for (var i = 0, l = data.length; i<l; i++) {
var sorted = helpers.sortObject(data[i]);
if (i === 0) columns = _.pluck(sorted, 0);
columns = _.pluck(sorted, 0);
colList = colList || columns;
if (!_.isEqual(columns, colList)) {
return this._prepInsert(this._normalizeInsert(data));
}
values.push(_.pluck(sorted, 1));
}
return {
@ -4049,6 +4060,19 @@ QueryCompiler.prototype._prepInsert = function(data) {
};
};
// If we are running an insert with variable object keys, we need to normalize
// for the missing keys, presumably setting the values to undefined.
QueryCompiler.prototype._normalizeInsert = function(data) {
if (!_.isArray(data)) return _.clone(data);
var defaultObj = _.reduce(_.union.apply(_, _.map(data, function(val) {
return _.keys(val);
})), function(memo, key) {
memo[key] = void 0;
return memo;
}, {});
return _.map(data, function(row) { return _.defaults(row, defaultObj); });
};
// "Preps" the update.
QueryCompiler.prototype._prepUpdate = function(data) {
var vals = [];
@ -4209,7 +4233,6 @@ Raw.prototype.wrap = function(before, after) {
};
// Calls `toString` on the Knex object.
// return '[object Knex$raw]';
Raw.prototype.toString = function() {
return this.toQuery();
};
@ -4477,7 +4500,6 @@ _.each([
};
});
// return '[object Knex:SchemaBuilder]';
SchemaBuilder.prototype.toString = function() {
return this.toQuery();
};
@ -4593,6 +4615,7 @@ var Raw = _dereq_('../raw');
function ColumnCompiler(tableCompiler, columnBuilder) {
this.tableCompiler = tableCompiler;
this.columnBuilder = columnBuilder;
this.args = columnBuilder._args;
this.type = columnBuilder._type;
this.grouped = _.groupBy(columnBuilder._statements, 'grouping');
@ -4605,6 +4628,9 @@ function ColumnCompiler(tableCompiler, columnBuilder) {
// column as it would be in the insert statement
ColumnCompiler.prototype.toSQL = function() {
this.pushQuery(this.compileColumn());
if (this.sequence.additional) {
this.sequence = this.sequence.concat(this.sequence.additional);
}
return this.sequence;
};
@ -4782,6 +4808,13 @@ ColumnCompiler.prototype.pushQuery = function(query) {
this.formatter = new this.Formatter();
};
// Used in cases where we need to push some additional column specific statements.
ColumnCompiler.prototype.pushAdditional = function(fn) {
var child = new this.constructor(this.tableCompiler, this.columnBuilder);
fn.call(child, _.rest(arguments));
this.sequence.additional = (this.sequence.additional || []).concat(child.sequence);
};
module.exports = {
Builder: Builder,
Compiler: Compiler,
@ -5073,6 +5106,7 @@ TableCompiler.prototype.create = function() {
var columnTypes = this.getColumnTypes(columns);
this.createQuery(columnTypes);
this.columnQueries(columns);
delete this.single.comment;
this.alterTable();
};
@ -5109,14 +5143,13 @@ TableCompiler.prototype.getColumnTypes = function(columns) {
// Adds all of the additional queries from the "column"
TableCompiler.prototype.columnQueries = function(columns) {
var queries = _.reduce(columns, function(memo, column) {
memo.concat(_.rest(column));
var queries = _.reduce(_.map(columns, _.rest), function(memo, column) {
if (!_.isEmpty(column)) return memo.concat(column);
return memo;
}, []);
for (var i = 0, l = queries.length; i < l; i++) {
this.pushQuery(queries[i]);
}
return queries;
};
// Add a new column.
@ -5160,6 +5193,9 @@ TableCompiler.prototype.alterTable = function() {
console.error('Debug: ' + statement.method + ' does not exist');
}
}
for (var item in this.single) {
if (_.isFunction(this[item])) this[item](this.single[item]);
}
};
// Drop the index on the current table.
@ -5284,23 +5320,8 @@ Transaction.prototype.then = function(onFulfilled, onRejected) {
module.exports = Transaction;
},{"../knex":1,"./interface":46,"./promise":49,"events":69,"inherits":70}],65:[function(_dereq_,module,exports){
var _ = _dereq_('lodash');
module.exports = {
// If we are running an insert with variable object keys, we need to normalize
// for the missing keys, presumably setting the values to undefined.
prepInsert: function(data) {
if (!_.isArray(data)) return _.clone(data);
var defaultObj = _.reduce(_.union.apply(_, _.map(data, function(val) {
return _.keys(val);
})), function(memo, key) {
memo[key] = void 0;
return memo;
}, {});
return _.map(data, function(row) { return _.defaults(row, defaultObj); });
},
pgBindings: function(sql) {
var questionCount = 0;
return sql.replace(/\?/g, function() {
@ -5310,7 +5331,7 @@ module.exports = {
}
};
},{"lodash":"K2RcUv"}],66:[function(_dereq_,module,exports){
},{}],66:[function(_dereq_,module,exports){
/*!
* The buffer module from node.js, for the browser.
*
@ -5403,11 +5424,12 @@ function Buffer (subject, encoding, noZero) {
buf._set(subject)
} else if (isArrayish(subject)) {
// Treat array-ish objects as a byte array
for (i = 0; i < length; i++) {
if (Buffer.isBuffer(subject))
if (Buffer.isBuffer(subject)) {
for (i = 0; i < length; i++)
buf[i] = subject.readUInt8(i)
else
buf.writeInt8(subject[i], i)
} else {
for (i = 0; i < length; i++)
buf[i] = ((subject[i] % 256) + 256) % 256
}
} else if (type === 'string') {
buf.write(subject, 0, encoding)

View File

@ -1,5 +1,5 @@
!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var f;"undefined"!=typeof window?f=window:"undefined"!=typeof global?f=global:"undefined"!=typeof self&&(f=self),f.Knex=e()}}(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(_dereq_,module,exports){
// Knex.js 0.6.0-alpha
// Knex.js 0.6.0
// --------------
// (c) 2014 Tim Griesser
@ -159,10 +159,9 @@ Knex.initialize = function(config) {
// knex.migrate.latest().then(...
// knex.migrate.currentVersion(...
_.each(MigrateInterface, function(method) {
migrate[method] = function(config) {
migrate[method] = function() {
if (!client.Migrator) client.initMigrator();
config.knex = knex;
var migrator = new client.Migrator(config);
var migrator = new client.Migrator(knex);
return migrator[method].apply(migrator, arguments);
};
});
@ -178,16 +177,18 @@ module.exports = Knex;
// "Base Client"
// ------
var Promise = _dereq_('./promise');
var _ = _dereq_('lodash');
// The base client provides the general structure
// for a dialect specific client object. The client
// object attaches fresh constructors for each component
// of the library.
function Client() {
function Client(config) {
this.initFormatter();
this.initRaw();
this.initTransaction();
this.initQuery();
this.migrationConfig = _.clone(config && config.migrations);
}
// Set the "isDebugging" flag on the client to "true" to log
@ -224,7 +225,7 @@ Client.prototype.database = function() {
};
module.exports = Client;
},{"./promise":24}],3:[function(_dereq_,module,exports){
},{"./promise":24,"lodash":"K2RcUv"}],3:[function(_dereq_,module,exports){
(function (Buffer){
var SqlString = exports;
@ -2605,11 +2606,15 @@ QueryCompiler.prototype._prepInsert = function(data) {
var isRaw = this.formatter.rawOrFn(data);
if (isRaw) return isRaw;
var values = [];
var columns;
var columns, colList;
if (!_.isArray(data)) data = data ? [data] : [];
for (var i = 0, l = data.length; i<l; i++) {
var sorted = helpers.sortObject(data[i]);
if (i === 0) columns = _.pluck(sorted, 0);
columns = _.pluck(sorted, 0);
colList = colList || columns;
if (!_.isEqual(columns, colList)) {
return this._prepInsert(this._normalizeInsert(data));
}
values.push(_.pluck(sorted, 1));
}
return {
@ -2618,6 +2623,19 @@ QueryCompiler.prototype._prepInsert = function(data) {
};
};
// If we are running an insert with variable object keys, we need to normalize
// for the missing keys, presumably setting the values to undefined.
QueryCompiler.prototype._normalizeInsert = function(data) {
if (!_.isArray(data)) return _.clone(data);
var defaultObj = _.reduce(_.union.apply(_, _.map(data, function(val) {
return _.keys(val);
})), function(memo, key) {
memo[key] = void 0;
return memo;
}, {});
return _.map(data, function(row) { return _.defaults(row, defaultObj); });
};
// "Preps" the update.
QueryCompiler.prototype._prepUpdate = function(data) {
var vals = [];
@ -2778,7 +2796,6 @@ Raw.prototype.wrap = function(before, after) {
};
// Calls `toString` on the Knex object.
// return '[object Knex$raw]';
Raw.prototype.toString = function() {
return this.toQuery();
};
@ -3046,7 +3063,6 @@ _.each([
};
});
// return '[object Knex:SchemaBuilder]';
SchemaBuilder.prototype.toString = function() {
return this.toQuery();
};
@ -3162,6 +3178,7 @@ var Raw = _dereq_('../raw');
function ColumnCompiler(tableCompiler, columnBuilder) {
this.tableCompiler = tableCompiler;
this.columnBuilder = columnBuilder;
this.args = columnBuilder._args;
this.type = columnBuilder._type;
this.grouped = _.groupBy(columnBuilder._statements, 'grouping');
@ -3174,6 +3191,9 @@ function ColumnCompiler(tableCompiler, columnBuilder) {
// column as it would be in the insert statement
ColumnCompiler.prototype.toSQL = function() {
this.pushQuery(this.compileColumn());
if (this.sequence.additional) {
this.sequence = this.sequence.concat(this.sequence.additional);
}
return this.sequence;
};
@ -3351,6 +3371,13 @@ ColumnCompiler.prototype.pushQuery = function(query) {
this.formatter = new this.Formatter();
};
// Used in cases where we need to push some additional column specific statements.
ColumnCompiler.prototype.pushAdditional = function(fn) {
var child = new this.constructor(this.tableCompiler, this.columnBuilder);
fn.call(child, _.rest(arguments));
this.sequence.additional = (this.sequence.additional || []).concat(child.sequence);
};
module.exports = {
Builder: Builder,
Compiler: Compiler,
@ -3642,6 +3669,7 @@ TableCompiler.prototype.create = function() {
var columnTypes = this.getColumnTypes(columns);
this.createQuery(columnTypes);
this.columnQueries(columns);
delete this.single.comment;
this.alterTable();
};
@ -3678,14 +3706,13 @@ TableCompiler.prototype.getColumnTypes = function(columns) {
// Adds all of the additional queries from the "column"
TableCompiler.prototype.columnQueries = function(columns) {
var queries = _.reduce(columns, function(memo, column) {
memo.concat(_.rest(column));
var queries = _.reduce(_.map(columns, _.rest), function(memo, column) {
if (!_.isEmpty(column)) return memo.concat(column);
return memo;
}, []);
for (var i = 0, l = queries.length; i < l; i++) {
this.pushQuery(queries[i]);
}
return queries;
};
// Add a new column.
@ -3729,6 +3756,9 @@ TableCompiler.prototype.alterTable = function() {
console.error('Debug: ' + statement.method + ' does not exist');
}
}
for (var item in this.single) {
if (_.isFunction(this[item])) this[item](this.single[item]);
}
};
// Drop the index on the current table.
@ -3853,23 +3883,8 @@ Transaction.prototype.then = function(onFulfilled, onRejected) {
module.exports = Transaction;
},{"../knex":1,"./interface":21,"./promise":24,"events":44,"inherits":45}],40:[function(_dereq_,module,exports){
var _ = _dereq_('lodash');
module.exports = {
// If we are running an insert with variable object keys, we need to normalize
// for the missing keys, presumably setting the values to undefined.
prepInsert: function(data) {
if (!_.isArray(data)) return _.clone(data);
var defaultObj = _.reduce(_.union.apply(_, _.map(data, function(val) {
return _.keys(val);
})), function(memo, key) {
memo[key] = void 0;
return memo;
}, {});
return _.map(data, function(row) { return _.defaults(row, defaultObj); });
},
pgBindings: function(sql) {
var questionCount = 0;
return sql.replace(/\?/g, function() {
@ -3879,7 +3894,7 @@ module.exports = {
}
};
},{"lodash":"K2RcUv"}],41:[function(_dereq_,module,exports){
},{}],41:[function(_dereq_,module,exports){
/*!
* The buffer module from node.js, for the browser.
*
@ -3972,11 +3987,12 @@ function Buffer (subject, encoding, noZero) {
buf._set(subject)
} else if (isArrayish(subject)) {
// Treat array-ish objects as a byte array
for (i = 0; i < length; i++) {
if (Buffer.isBuffer(subject))
if (Buffer.isBuffer(subject)) {
for (i = 0; i < length; i++)
buf[i] = subject.readUInt8(i)
else
buf.writeInt8(subject[i], i)
} else {
for (i = 0; i < length; i++)
buf[i] = ((subject[i] % 256) + 256) % 256
}
} else if (type === 'string') {
buf.write(subject, 0, encoding)

View File

@ -324,11 +324,15 @@ QueryCompiler.prototype._prepInsert = function(data) {
var isRaw = this.formatter.rawOrFn(data);
if (isRaw) return isRaw;
var values = [];
var columns;
var columns, colList;
if (!_.isArray(data)) data = data ? [data] : [];
for (var i = 0, l = data.length; i<l; i++) {
var sorted = helpers.sortObject(data[i]);
if (i === 0) columns = _.pluck(sorted, 0);
columns = _.pluck(sorted, 0);
colList = colList || columns;
if (!_.isEqual(columns, colList)) {
return this._prepInsert(this._normalizeInsert(data));
}
values.push(_.pluck(sorted, 1));
}
return {
@ -337,6 +341,19 @@ QueryCompiler.prototype._prepInsert = function(data) {
};
};
// If we are running an insert with variable object keys, we need to normalize
// for the missing keys, presumably setting the values to undefined.
QueryCompiler.prototype._normalizeInsert = function(data) {
if (!_.isArray(data)) return _.clone(data);
var defaultObj = _.reduce(_.union.apply(_, _.map(data, function(val) {
return _.keys(val);
})), function(memo, key) {
memo[key] = void 0;
return memo;
}, {});
return _.map(data, function(row) { return _.defaults(row, defaultObj); });
};
// "Preps" the update.
QueryCompiler.prototype._prepUpdate = function(data) {
var vals = [];

View File

@ -1,20 +1,5 @@
var _ = require('lodash');
module.exports = {
// If we are running an insert with variable object keys, we need to normalize
// for the missing keys, presumably setting the values to undefined.
prepInsert: function(data) {
if (!_.isArray(data)) return _.clone(data);
var defaultObj = _.reduce(_.union.apply(_, _.map(data, function(val) {
return _.keys(val);
})), function(memo, key) {
memo[key] = void 0;
return memo;
}, {});
return _.map(data, function(row) { return _.defaults(row, defaultObj); });
},
pgBindings: function(sql) {
var questionCount = 0;
return sql.replace(/\?/g, function() {

View File

@ -26,22 +26,9 @@ describe('Unit tests', function() {
require('./unit/schema/sqlite3')(sqlite3.client);
require('./unit/schema/mysql')(mysql.client);
require('./unit/query/builder')(postgres.client, mysql.client, sqlite3.client);
it('provides a knex.utils.prepInsert for normalizing js objects', function() {
var data = [{a: 1}, {b: 2}, {a: 2, c: 3}];
var dataPrepped = mysql.utils.prepInsert(data);
assert.deepEqual(dataPrepped, [
{a: 1, b: void 0, c: void 0},
{a: void 0, b: 2, c: void 0},
{a: 2, b: void 0, c: 3}
]);
assert.notEqual(data, dataPrepped);
});
});
// Integration Tests
describe('Integration Tests', function() {
var runner = require('./integration')(this);
require('./integration')(this);
});

View File

@ -422,6 +422,13 @@ module.exports = function(pgclient, mysqlclient, sqlite3client) {
expect(result.sql).to.equal('insert into "users" ("email") values (CURRENT TIMESTAMP)');
});
it("normalizes for missing keys in insert", function() {
var data = [{a: 1}, {b: 2}, {a: 2, c: 3}];
chain = sql().insert(data).into('table').toSQL();
assert.deepEqual(chain.bindings, [1, undefined, undefined, undefined, 2, undefined, 2, undefined, 3]);
assert.equal(chain.sql, 'insert into "table" ("a", "b", "c") values (?, ?, ?), (?, ?, ?), (?, ?, ?)');
});
it("update method", function() {
chain = sql().update({'email': 'foo', 'name': 'bar'}).table('users').where('id', '=', 1).toSQL();
expect(chain.sql).to.equal('update "users" set "email" = ?, "name" = ? where "id" = ?');