Support of MATERIALIZED and NOT MATERIALIZED with WITH/CTE (#4940)

lib/dialects/postgres/query/pg-querybuilder.js

@@ -5,4 +5,34 @@ module.exports = class QueryBuilder_PostgreSQL extends QueryBuilder {
     this._single.using = tables;
     return this;
   }
+
+  withMaterialized(alias, statementOrColumnList, nothingOrStatement) {
+    this._validateWithArgs(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      'with'
+    );
+    return this.withWrapped(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      true
+    );
+  }
+
+  withNotMaterialized(alias, statementOrColumnList, nothingOrStatement) {
+    this._validateWithArgs(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      'with'
+    );
+    return this.withWrapped(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      false
+    );
+  }
 };

lib/dialects/sqlite3/index.js

@@ -15,6 +15,7 @@ const TableCompiler = require('./schema/sqlite-tablecompiler');
 const ViewCompiler = require('./schema/sqlite-viewcompiler');
 const SQLite3_DDL = require('./schema/ddl');
 const Formatter = require('../../formatter');
+const QueryBuilder = require('./query/sqlite-querybuilder');
 
 class Client_SQLite3 extends Client {
   constructor(config) {
@@ -44,6 +45,10 @@ class Client_SQLite3 extends Client {
     return new SqliteQueryCompiler(this, builder, formatter);
   }
 
+  queryBuilder() {
+    return new QueryBuilder(this);
+  }
+
   viewCompiler(builder, formatter) {
     return new ViewCompiler(this, builder, formatter);
   }

lib/dialects/sqlite3/query/sqlite-querybuilder.js

@@ -0,0 +1,33 @@
+const QueryBuilder = require('../../../query/querybuilder.js');
+
+module.exports = class QueryBuilder_SQLite3 extends QueryBuilder {
+  withMaterialized(alias, statementOrColumnList, nothingOrStatement) {
+    this._validateWithArgs(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      'with'
+    );
+    return this.withWrapped(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      true
+    );
+  }
+
+  withNotMaterialized(alias, statementOrColumnList, nothingOrStatement) {
+    this._validateWithArgs(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      'with'
+    );
+    return this.withWrapped(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      false
+    );
+  }
+};

lib/query/querybuilder.js

@@ -118,25 +118,82 @@ class Builder extends EventEmitter {
 
   // With
   // ------
-
-  with(alias, statementOrColumnList, nothingOrStatement) {
-    validateWithArgs(alias, statementOrColumnList, nothingOrStatement, 'with');
-    return this.withWrapped(alias, statementOrColumnList, nothingOrStatement);
+  isValidStatementArg(statement) {
+    return (
+      typeof statement === 'function' ||
+      statement instanceof Builder ||
+      (statement && statement.isRawInstance)
+    );
   }
 
-  // Helper for compiling any advanced `with` queries.
-  withWrapped(alias, statementOrColumnList, nothingOrStatement) {
+  _validateWithArgs(alias, statementOrColumnList, nothingOrStatement, method) {
     const [query, columnList] =
       typeof nothingOrStatement === 'undefined'
         ? [statementOrColumnList, undefined]
         : [nothingOrStatement, statementOrColumnList];
-    this._statements.push({
+    if (typeof alias !== 'string') {
+      throw new Error(`${method}() first argument must be a string`);
+    }
+
+    if (this.isValidStatementArg(query) && typeof columnList === 'undefined') {
+      // Validated as two-arg variant (alias, statement).
+      return;
+    }
+
+    // Attempt to interpret as three-arg variant (alias, columnList, statement).
+    const isNonEmptyNameList =
+      Array.isArray(columnList) &&
+      columnList.length > 0 &&
+      columnList.every((it) => typeof it === 'string');
+    if (!isNonEmptyNameList) {
+      throw new Error(
+        `${method}() second argument must be a statement or non-empty column name list.`
+      );
+    }
+
+    if (this.isValidStatementArg(query)) {
+      return;
+    }
+    throw new Error(
+      `${method}() third argument must be a function / QueryBuilder or a raw when its second argument is a column name list`
+    );
+  }
+
+  with(alias, statementOrColumnList, nothingOrStatement) {
+    this._validateWithArgs(
+      alias,
+      statementOrColumnList,
+      nothingOrStatement,
+      'with'
+    );
+    return this.withWrapped(alias, statementOrColumnList, nothingOrStatement);
+  }
+
+  withMaterialized(alias, statementOrColumnList, nothingOrStatement) {
+    throw new Error('With materialized is not supported by this dialect');
+  }
+
+  withNotMaterialized(alias, statementOrColumnList, nothingOrStatement) {
+    throw new Error('With materialized is not supported by this dialect');
+  }
+
+  // Helper for compiling any advanced `with` queries.
+  withWrapped(alias, statementOrColumnList, nothingOrStatement, materialized) {
+    const [query, columnList] =
+      typeof nothingOrStatement === 'undefined'
+        ? [statementOrColumnList, undefined]
+        : [nothingOrStatement, statementOrColumnList];
+    const statement = {
       grouping: 'with',
       type: 'withWrapped',
       alias: alias,
       columnList,
       value: query,
-    });
+    };
+    if (materialized !== undefined) {
+      statement.materialized = materialized;
+    }
+    this._statements.push(statement);
     return this;
   }
 
@@ -144,7 +201,7 @@ class Builder extends EventEmitter {
   // ------
 
   withRecursive(alias, statementOrColumnList, nothingOrStatement) {
-    validateWithArgs(
+    this._validateWithArgs(
       alias,
       statementOrColumnList,
       nothingOrStatement,
@@ -1656,49 +1713,6 @@ class Builder extends EventEmitter {
   }
 }
 
-const isValidStatementArg = (statement) =>
-  typeof statement === 'function' ||
-  statement instanceof Builder ||
-  (statement && statement.isRawInstance);
-
-const validateWithArgs = function (
-  alias,
-  statementOrColumnList,
-  nothingOrStatement,
-  method
-) {
-  const [query, columnList] =
-    typeof nothingOrStatement === 'undefined'
-      ? [statementOrColumnList, undefined]
-      : [nothingOrStatement, statementOrColumnList];
-  if (typeof alias !== 'string') {
-    throw new Error(`${method}() first argument must be a string`);
-  }
-
-  if (isValidStatementArg(query) && typeof columnList === 'undefined') {
-    // Validated as two-arg variant (alias, statement).
-    return;
-  }
-
-  // Attempt to interpret as three-arg variant (alias, columnList, statement).
-  const isNonEmptyNameList =
-    Array.isArray(columnList) &&
-    columnList.length > 0 &&
-    columnList.every((it) => typeof it === 'string');
-  if (!isNonEmptyNameList) {
-    throw new Error(
-      `${method}() second argument must be a statement or non-empty column name list.`
-    );
-  }
-
-  if (isValidStatementArg(query)) {
-    return;
-  }
-  throw new Error(
-    `${method}() third argument must be a function / QueryBuilder or a raw when its second argument is a column name list`
-  );
-};
-
 Builder.prototype.select = Builder.prototype.columns;
 Builder.prototype.column = Builder.prototype.columns;
 Builder.prototype.andWhereNot = Builder.prototype.whereNot;

lib/query/querycompiler.js

@@ -1170,6 +1170,12 @@ class QueryCompiler {
         ) +
         ')'
       : '';
+    const materialized =
+      statement.materialized === undefined
+        ? ''
+        : statement.materialized
+        ? 'materialized '
+        : 'not materialized ';
     return (
       (val &&
         columnize_(
@@ -1179,7 +1185,9 @@ class QueryCompiler {
           this.bindingsHolder
         ) +
           columnList +
-          ' as (' +
+          ' as ' +
+          materialized +
+          '(' +
           val +
           ')') ||
       ''

test/integration2/query/select/selects.spec.js

@@ -1255,6 +1255,34 @@ describe('Selects', function () {
           );
       });
 
+      describe('with (not) materialized tests', () => {
+        before(async function () {
+          if (!isPostgreSQL(knex) && !isSQLite(knex)) {
+            return this.skip();
+          }
+          await knex('test_default_table').truncate();
+          await knex('test_default_table').insert([
+            { string: 'something', tinyint: 1 },
+          ]);
+        });
+
+        it('with materialized', async function () {
+          const materialized = await knex('t')
+            .withMaterialized('t', knex('test_default_table'))
+            .from('t')
+            .first();
+          expect(materialized.tinyint).to.equal(1);
+        });
+
+        it('with not materialized', async function () {
+          const notMaterialized = await knex('t')
+            .withNotMaterialized('t', knex('test_default_table'))
+            .from('t')
+            .first();
+          expect(notMaterialized.tinyint).to.equal(1);
+        });
+      });
+
       describe('json selections', () => {
         before(async () => {
           await knex.schema.dropTableIfExists('cities');

test/unit/knex.js

@@ -628,7 +628,9 @@ describe('knex', () => {
       await knex('some_nonexisten_table')
         .select()
         .catch((err) => {
-          expect(err.stack.split('\n')[1]).to.match(/at createQueryBuilder \(/); // the index 1 might need adjustment if the code is refactored
+          expect(err.stack.split('\n')[1]).to.match(
+            /at Object.queryBuilder \(/
+          ); // the index 1 might need adjustment if the code is refactored
           expect(typeof err.originalStack).to.equal('string');
         });
 

test/unit/query/builder.js

@@ -9482,6 +9482,40 @@ describe('QueryBuilder', () => {
     );
   });
 
+  it("wrapped 'withMaterialized' clause update", () => {
+    testsql(
+      qb()
+        .withMaterialized('withClause', function () {
+          this.select('foo').from('users');
+        })
+        .update({ foo: 'updatedFoo' })
+        .where('email', '=', 'foo')
+        .from('users'),
+      {
+        sqlite3:
+          'with `withClause` as materialized (select `foo` from `users`) update `users` set `foo` = ? where `email` = ?',
+        pg: 'with "withClause" as materialized (select "foo" from "users") update "users" set "foo" = ? where "email" = ?',
+      }
+    );
+  });
+
+  it("wrapped 'withNotMaterialized' clause update", () => {
+    testsql(
+      qb()
+        .withNotMaterialized('withClause', function () {
+          this.select('foo').from('users');
+        })
+        .update({ foo: 'updatedFoo' })
+        .where('email', '=', 'foo')
+        .from('users'),
+      {
+        sqlite3:
+          'with `withClause` as not materialized (select `foo` from `users`) update `users` set `foo` = ? where `email` = ?',
+        pg: 'with "withClause" as not materialized (select "foo" from "users") update "users" set "foo" = ? where "email" = ?',
+      }
+    );
+  });
+
   it("wrapped 'with' clause delete", () => {
     testsql(
       qb()

types/index.d.ts

@@ -520,6 +520,8 @@ export declare namespace Knex {
 
     // Withs
     with: With<TRecord, TResult>;
+    withMaterialized: With<TRecord, TResult>;
+    withNotMaterialized: With<TRecord, TResult>;
     withRecursive: With<TRecord, TResult>;
     withRaw: WithRaw<TRecord, TResult>;
     withSchema: WithSchema<TRecord, TResult>;