diff --git a/api/src/org/labkey/api/ApiModule.java b/api/src/org/labkey/api/ApiModule.java index 3ebd50c841b..012a2f26749 100644 --- a/api/src/org/labkey/api/ApiModule.java +++ b/api/src/org/labkey/api/ApiModule.java @@ -431,6 +431,7 @@ public void registerServlets(ServletContext servletCtx) SimpleFilter.BetweenClauseTestCase.class, SimpleFilter.FilterTestCase.class, SimpleFilter.InClauseTestCase.class, + SimpleFilter.SqlClauseTestCase.class, SqlScanner.TestCase.class, StringExpressionFactory.TestCase.class, StringUtilsLabKey.TestCase.class, diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index 87de41aff1b..63f28a9b3a0 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -313,12 +313,15 @@ public String toString() return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); } - public String toDebugString() { return JdbcUtil.format(this); } + public String toDebugString(SqlDialect dialect) + { + return JdbcUtil.format(this, dialect); + } public List getParams() { @@ -1322,6 +1325,12 @@ public boolean equals(Object obj) return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); } + @Override + public int hashCode() + { + return Objects.hash(getSQL(), getParams()); + } + /** * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string * concatenation using the provided separator. The parameters are combined to form the new parameter list. diff --git a/api/src/org/labkey/api/data/SimpleFilter.java b/api/src/org/labkey/api/data/SimpleFilter.java index 5393bbe41aa..f4fe501d8b7 100644 --- a/api/src/org/labkey/api/data/SimpleFilter.java +++ b/api/src/org/labkey/api/data/SimpleFilter.java @@ -393,6 +393,20 @@ public String getLabKeySQLWhereClause(Map column { throw new UnsupportedOperationException(); } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + SQLClause sqlClause = (SQLClause) o; + return Objects.equals(_fragment, sqlClause._fragment) && Objects.equals(_fieldKeys, sqlClause._fieldKeys); + } + + @Override + public int hashCode() + { + return Objects.hash(_fragment, _fieldKeys); + } } public static class FalseClause extends SQLClause @@ -1701,7 +1715,6 @@ protected void test(String expectedSQL, String description, FilterClause clause, { test(expectedSQL, description, clause, dialect, Collections.emptyMap()); } - } public static class InClauseTestCase extends ClauseTestCase @@ -1953,4 +1966,22 @@ public void testBetweenQueryString() new CompareType.BetweenClause(fieldKey, "a,b,c", "Z", false).toURLParam("query.")); } } + + public static class SqlClauseTestCase extends Assert + { + @Test + public void testEquals() + { + SQLClause clause1 = new SQLClause(new SQLFragment("This = That", 1, 2)); + SQLClause clause2 = new SQLClause(new SQLFragment("This = That", 1, 2)); + assertEquals(clause1, clause2); + assertEquals(clause1.hashCode(), clause2.hashCode()); + SQLClause clause3 = new SQLClause(new SQLFragment("That = This", 1, 2)); + assertNotEquals(clause1, clause3); + assertNotEquals(clause1.hashCode(), clause3.hashCode()); + SQLClause clause4 = new SQLClause(new SQLFragment("This = That", 3, 4)); + assertNotEquals(clause1, clause4); + assertNotEquals(clause1.hashCode(), clause4.hashCode()); + } + } } diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index 7014fb11941..f193447804b 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -16,7 +16,6 @@ package org.labkey.api.data.dialect; -import jakarta.servlet.ServletException; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -27,62 +26,41 @@ import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.ConnectionWrapper; import org.labkey.api.data.ConnectionWrapper.Closer; -import org.labkey.api.data.Constraint; -import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DatabaseIdentifier; import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; import org.labkey.api.data.DbScope.LabKeyDataSource; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MetadataSqlSelector; import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.PropertyStorageSpec.Index; import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.Selector; -import org.labkey.api.data.Selector.ForEachBlock; import org.labkey.api.data.SqlExecutingSelector.ConnectionFactory; -import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.Table; -import org.labkey.api.data.TableChange; import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TempTableTracker; import org.labkey.api.data.dialect.LimitRowsSqlGenerator.LimitRowsCustomizer; import org.labkey.api.data.dialect.LimitRowsSqlGenerator.StandardLimitRowsCustomizer; -import org.labkey.api.query.AliasManager; -import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.ExceptionUtil; import org.labkey.api.util.HtmlString; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.view.template.Warnings; import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; -import org.springframework.jdbc.BadSqlGrammarException; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DatabaseMetaData; -import java.sql.Driver; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; -import java.util.ArrayList; import java.util.Calendar; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; // Base dialect for PostgreSQL AND Redshift. IMPORTANT: Make sure everything added here applies to Redshift as well; // if not, put it in PostgreSql92Dialect. @@ -92,7 +70,6 @@ public abstract class BasePostgreSqlDialect extends SqlDialect public static final String POSTGRES_SCHEMA_NAME = "postgres"; private final Map _domainScaleMap = new CopyOnWriteHashMap<>(); - private final AtomicBoolean _arraySortFunctionExists = new AtomicBoolean(false); private HtmlString _adminWarning = null; @@ -232,24 +209,12 @@ public boolean isOracle() return false; } - @Override - public String getSQLScriptPath() - { - return "postgresql"; - } - @Override public String getDefaultDateTimeDataType() { return "TIMESTAMP"; } - @Override - public String getUniqueIdentType() - { - return "SERIAL"; - } - @Override public String getGuidType() { @@ -441,88 +406,6 @@ public SQLFragment wrapBooleanExpression(SQLFragment booleanSql) return booleanSql; } - @Override - public boolean supportsGroupConcat() - { - return getServerType().supportsGroupConcat(); - } - - @Override - public boolean supportsSelectConcat() - { - return true; - } - - @Override - public SQLFragment getSelectConcat(SQLFragment selectSql, String delimiter) - { - SQLFragment result = new SQLFragment("array_to_string(array("); - result.append(selectSql); - result.append("), "); - result.append(getStringHandler().quoteStringLiteral(delimiter)); - result.append(")"); - return result; - } - - @Override - public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sorted, @NotNull SQLFragment delimiterSQL, boolean includeNulls) - { - // Sort function might not exist in external datasource; skip that syntax if not - boolean useSortFunction = sorted && _arraySortFunctionExists.get(); - SQLFragment result = new SQLFragment(); - - if (useSortFunction) - { - result.append("array_to_string("); - result.append("core.sort("); // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function - result.append("array_agg("); - if (distinct) - { - result.append("DISTINCT "); - } - - if (includeNulls) - { - result.append("COALESCE(CAST("); - result.append(sql); - result.append(" AS VARCHAR), '')"); - } - else - { - result.append(sql); - } - - result.append(")"); // array_agg - result.append(")"); // core.sort - } - else - { - result.append("string_agg("); - if (distinct) - { - result.append("DISTINCT "); - } - - if (includeNulls) - { - result.append("COALESCE("); - result.append(sql); - result.append("::text, '')"); - } - else - { - result.append(sql); - result.append("::text"); - } - } - - result.append(", "); - result.append(delimiterSQL); - result.append(")"); // array_to_string | string_agg - - return result; - } - @Override protected String getSystemTableNames() { @@ -563,12 +446,6 @@ public String getBooleanFALSE() return "false"; } - @Override - public String getBinaryDataType() - { - return "BYTEA"; - } - @Override public String getTempTableKeyword() { @@ -581,12 +458,6 @@ public String getTempTablePrefix() return ""; } - @Override - public String getGlobalTempTablePrefix() - { - return DbSchema.TEMP_SCHEMA_NAME + "."; - } - @Override public boolean isNoDatabaseException(SQLException e) { @@ -599,38 +470,6 @@ public boolean isSortableDataType(String sqlDataTypeName) return !"json".equals(sqlDataTypeName) && !"jsonb".equals(sqlDataTypeName); } - @Override - public String getDropIndexCommand(String tableName, String indexName) - { - return "DROP INDEX " + indexName; - } - - @Override - public String getCreateDatabaseSql(String dbName) - { - // This will handle both mixed case and special characters on PostgreSQL - var legal = makeIdentifierFromMetaDataName(dbName); - return new SQLFragment("CREATE DATABASE ").appendIdentifier(legal).append(" WITH ENCODING 'UTF8'").getRawSQL(); - } - - @Override - public String getCreateSchemaSql(String schemaName) - { - if (!isLegalName(schemaName) || isReserved(schemaName)) - throw new IllegalArgumentException("Not a legal schema name: " + schemaName); - - //Quoted schema names are bad news - return "CREATE SCHEMA " + schemaName; - } - - @Override - public String getTruncateSql(String tableName) - { - // To be consistent with MS SQL server, always restart the sequence. Note that the default for postgres - // is to continue the sequence but we don't have this option with MS SQL Server - return "TRUNCATE TABLE " + tableName + " RESTART IDENTITY"; - } - @Override public String getDateDiff(int part, String value1, String value2) { @@ -714,49 +553,6 @@ public boolean supportsRoundDouble() return false; } - @Override - public void handleCreateDatabaseException(SQLException e) throws ServletException - { - if ("55006".equals(e.getSQLState())) - { - LOG.error("You must close down pgAdmin III and all other applications accessing PostgreSQL."); - throw (new ServletException("Close down or disconnect pgAdmin III and all other applications accessing PostgreSQL", e)); - } - else - { - super.handleCreateDatabaseException(e); - } - } - - - @Override - public void prepareDriver(Class driverClass) - { - // PostgreSQL driver 42.0.0 added logging via the Java Logging API (java.util.logging). This caused the driver to - // start logging SQLExceptions (such as the initial connection failure on bootstrap) to the console... harmless - // but annoying. This code suppresses the driver logging. - Logger pgjdbcLogger = LogManager.getLogManager().getLogger("org.postgresql"); - - if (null != pgjdbcLogger) - pgjdbcLogger.setLevel(Level.OFF); - } - - - // Make sure that the PL/pgSQL language is enabled in the associated database. If not, throw. Since 9.0, PostgreSQL has - // shipped with PL/pgSQL enabled by default, so the check is no longer critical, but continue to verify just to be safe. - @Override - public void prepareNewLabKeyDatabase(DbScope scope) - { - if (new SqlSelector(scope, "SELECT * FROM pg_language WHERE lanname = 'plpgsql'").exists()) - return; - - String dbName = scope.getDatabaseName(); - String message = "PL/pgSQL is not enabled in the \"" + dbName + "\" database because it is not enabled in your Template1 master database."; - String advice = "Use PostgreSQL's 'createlang' command line utility to enable PL/pgSQL in the \"" + dbName + "\" database then restart Tomcat."; - - throw new ConfigurationException(message, advice); - } - @Override public void prepare(LabKeyDataSource dataSource) { @@ -774,7 +570,6 @@ public String prepare(DbScope scope) { initializeUserDefinedTypes(scope); determineSettings(scope); - determineIfArraySortFunctionExists(scope); return super.prepare(scope); } @@ -817,7 +612,6 @@ private void initializeUserDefinedTypes(DbScope scope) } } - private String getDomainKey(String schemaName, String domainName) { // Domain names are returned from column metadata fully qualified and quoted, so save them that way. See #26149. @@ -834,21 +628,6 @@ protected void determineSettings(DbScope scope) } } - - // Does this datasource include our sort array function? The LabKey datasource should always have it, but external datasources might not - private void determineIfArraySortFunctionExists(DbScope scope) - { - if (getServerType().supportsSpecialMetadataQueries()) - { - Selector selector = new SqlSelector(scope, "SELECT * FROM pg_catalog.pg_namespace n INNER JOIN pg_catalog.pg_proc p ON pronamespace = n.oid WHERE nspname = 'core' AND proname = 'sort'"); - _arraySortFunctionExists.set(selector.exists()); - } - - // Array sort function should always exist in LabKey scope (for now) - assert !scope.isLabKeyScope() || _arraySortFunctionExists.get(); - } - - /** * Wrap one or more INSERT statements to allow explicit specification * of values for autoincrementing columns (e.g. IDENTITY in SQL Server @@ -895,25 +674,6 @@ public DatabaseIdentifier makeDatabaseIdentifier(String alias) } } - private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); - - @NotNull - @Override - protected Pattern getSQLScriptProcPattern() - { - return PROC_PATTERN; - } - - @Override - protected void checkSqlScript(String lowerNoComments, String lowerNoCommentsNoWhiteSpace, Collection errors) - { - if (lowerNoCommentsNoWhiteSpace.contains("setsearch_pathto")) - errors.add("Do not use \"SET search_path TO \". Instead, schema-qualify references to all objects."); - - if (!lowerNoCommentsNoWhiteSpace.endsWith(";")) - errors.add("Script must end with a semicolon"); - } - @Override public String getJDBCArrayType(Object object) { @@ -930,32 +690,6 @@ else if (object instanceof Double) return super.getJDBCArrayType(object); } - - @Override - public boolean canExecuteUpgradeScripts() - { - return true; - } - - - @Override - public String getDefaultDatabaseName() - { - return "template1"; - } - - - /* - PostgreSQL example connection URLs we need to parse: - - jdbc:postgresql:database - jdbc:postgresql://host/database - jdbc:postgresql://host:port/database - jdbc:postgresql:database?user=fred&password=secret&ssl=true - jdbc:postgresql://host/database?user=fred&password=secret&ssl=true - jdbc:postgresql://host:port/database?user=fred&password=secret&ssl=true - */ - @Override protected @Nullable String getDatabaseMaintenanceSql() { @@ -1002,427 +736,6 @@ public boolean allowSortOnSubqueryWithoutLimit() return true; } - - @Override - public List getChangeStatements(TableChange change) - { - List result = new ArrayList<>(); - switch (change.getType()) - { - case CreateTable -> result.addAll(getCreateTableStatements(change)); - case DropTable -> { - SQLFragment f = new SQLFragment("DROP TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - result.add(f); - } - case AddColumns -> result.addAll(getAddColumnsStatements(change)); - case DropColumns -> result.add(getDropColumnsStatement(change)); - case RenameColumns -> result.addAll(getRenameColumnsStatement(change)); - case DropIndicesByName -> result.addAll(getDropIndexByNameStatements(change)); - case AddIndices -> result.addAll(getCreateIndexStatements(change)); - case ResizeColumns, ChangeColumnTypes -> result.addAll(getChangeColumnTypeStatement(change)); - case DropConstraints -> result.addAll(getDropConstraintsStatement(change)); - case AddConstraints -> result.addAll(getAddConstraintsStatement(change)); - default -> throw new IllegalArgumentException("Unsupported change type: " + change.getType()); - } - - return result; - } - - private Collection getDropIndexByNameStatements(TableChange change) - { - List statements = new ArrayList<>(); - for (String indexName : change.getIndicesToBeDroppedByName()) - { - statements.add(getDropIndexCommand(change, indexName)); - } - return statements; - } - - private SQLFragment getDropIndexCommand(TableChange change, String indexName) - { - SQLFragment f = new SQLFragment("DROP INDEX "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(indexName); - return f; - } - - - /** - * Generate the Alter Table statement to change the size or type of the column - *

- * NOTE: expects data size check to be done prior, - * will throw a SQL exception if not able to change size due to existing data - */ - private List getChangeColumnTypeStatement(TableChange change) - { - List statements = new ArrayList<>(); - - // Postgres allows executing multiple ALTER COLUMN statements under one ALTER TABLE - List nonDateTimeClauses = new ArrayList<>(); - - for (PropertyStorageSpec column : change.getColumns()) - { - DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); - if (column.getJdbcType().isDateOrTime()) - { - String tempColumnName = column.getName() + "~~temp~~"; - DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); - - // 1) ADD temp column - SQLFragment addTemp = new SQLFragment("ALTER TABLE "); - addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); - statements.add(addTemp); - - // 2) UPDATE: copy casted value to temp column - SQLFragment update = new SQLFragment("UPDATE "); - update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - update.append(" SET ").appendIdentifier(tempColumnIdent); - update.append(" = CAST(").appendIdentifier(columnIdent).append(" AS ").append(getSqlTypeName(column)).append(")"); - statements.add(update); - - // 3) DROP original column - SQLFragment drop = new SQLFragment("ALTER TABLE "); - drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); - statements.add(drop); - - // 4) RENAME temp column to original column name - SQLFragment rename = new SQLFragment("ALTER TABLE "); - rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); - statements.add(rename); - } - else - { - String dbType; - if (column.getJdbcType().isText()) - { - // Using the common default max size to make type change to text - dbType = column.getSize() == -1 || column.getSize() > SqlDialect.MAX_VARCHAR_SIZE ? - getSqlTypeName(JdbcType.LONGVARCHAR) : - getSqlTypeName(column.getJdbcType()) + "(" + column.getSize().toString() + ")"; - } - else if (column.getJdbcType().isDecimal()) - { - dbType = getSqlTypeName(column.getJdbcType()) + DEFAULT_DECIMAL_SCALE_PRECISION; - } - else - { - dbType = getSqlTypeName(column.getJdbcType()); - } - - SQLFragment clause = new SQLFragment(); - clause.append("ALTER COLUMN ").appendIdentifier(columnIdent).append(" TYPE ").append(dbType); - nonDateTimeClauses.add(clause); - } - } - - if (!nonDateTimeClauses.isEmpty()) - { - SQLFragment alter = new SQLFragment("ALTER TABLE "); - alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - alter.append(" "); - String sep = ""; - for (SQLFragment c : nonDateTimeClauses) - { - alter.append(sep).append(c); - sep = ", "; - } - statements.add(alter); - } - - return statements; - } - - private List getRenameColumnsStatement(TableChange change) - { - List statements = new ArrayList<>(); - for (Map.Entry oldToNew : change.getColumnRenames().entrySet()) - { - DatabaseIdentifier oldIdentifier = makePropertyIdentifier(oldToNew.getKey()); - DatabaseIdentifier newIdentifier = makePropertyIdentifier(oldToNew.getValue()); - if (!oldIdentifier.equals(newIdentifier)) - { - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" RENAME COLUMN ").appendIdentifier(oldIdentifier).append(" TO ").appendIdentifier(newIdentifier); - statements.add(f); - } - } - - // TODO: This loop should not guess the name of the old indices; instead, it should look them up. - // TableChange.setIndexedColumns() could set _indexRenames providing the name, and then this code uses that info. - // Or maybe schemaTableInfo.getAllIndices() and then use Index.isSameIndex() to find names. Issue 53838. - for (Map.Entry oldToNew : change.getIndexRenames().entrySet()) - { - Index oldIndex = oldToNew.getKey(); - Index newIndex = oldToNew.getValue(); - String oldName = nameIndex(change.getTableName(), oldIndex.columnNames); // TODO: Look up name - String newName = nameIndex(change.getTableName(), newIndex.columnNames); - if (!oldName.equals(newName)) - { - SQLFragment f = new SQLFragment("ALTER INDEX "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(oldName); - f.append(" RENAME TO ").appendIdentifier(newName); - statements.add(f); - } - } - - return statements; - } - - private SQLFragment getDropColumnsStatement(TableChange change) - { - List sqlParts = new ArrayList<>(); - for (PropertyStorageSpec prop : change.getColumns()) - { - SQLFragment sql = new SQLFragment("DROP COLUMN "); - if (prop.getExactName()) - { - sql.append(quoteIdentifier(prop.getName())); - } - else - { - sql.appendIdentifier(makePropertyIdentifier(prop.getName())); - } - sqlParts.add(sql); - } - - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ").append(sqlParts, ", "); - return f; - } - - // TODO if there are cases where user-defined columns need indices, this method will need to support - // creating indices like getCreateTableStatement does. - - private List getAddColumnsStatements(TableChange change) - { - List statements = new ArrayList<>(); - String pkColumn = null; - Constraint constraint = null; - - List columnSpecs = new ArrayList<>(); - for (PropertyStorageSpec prop : change.getColumns()) - { - columnSpecs.add(getSqlColumnSpec(prop)); - if (prop.isPrimaryKey()) - { - assert null == pkColumn : "no more than one primary key defined"; - pkColumn = prop.getName(); - constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); - } - } - - SQLFragment alter = new SQLFragment("ALTER TABLE "); - alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - alter.append(" "); - String sep = ""; - for (SQLFragment col : columnSpecs) - { - alter.append(sep); - alter.append("ADD COLUMN "); - alter.append(col); - sep = ", "; - } - statements.add(alter); - if (null != pkColumn) - { - SQLFragment addPk = new SQLFragment("ALTER TABLE "); - addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) - .append(" ").append(constraint.getType().toString()).append(" (") - .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); - statements.add(addPk); - } - - return statements; - } - - private List getDropConstraintsStatement(TableChange change) - { - return change.getConstraints().stream().map(constraint -> { - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" DROP CONSTRAINT ").appendIdentifier(constraint.getName()); - return f; - }).collect(Collectors.toList()); - } - - private List getAddConstraintsStatement(TableChange change) - { - List statements = new ArrayList<>(); - Collection constraints = change.getConstraints(); - - if (null!=constraints && !constraints.isEmpty()) - { - statements = constraints.stream().map(constraint -> { - List columns = new ArrayList<>(); - for (String col : constraint.getColumns()) - { - columns.add(new SQLFragment().appendIdentifier(col)); - } - - SQLFragment f = new SQLFragment(); - f.append("DO $$\nBEGIN\nIF NOT EXISTS\n(SELECT 1 FROM information_schema.constraint_column_usage\nWHERE table_name = ") - .appendStringLiteral(change.getSchemaName() + "." + change.getTableName(), this) - .append(" and constraint_name = ") - .appendStringLiteral(constraint.getName(), this) - .append(") THEN\nALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()).append(" ") - .append(constraint.getType().toString()).append(" (") - .append(columns, ",") - .append(")").appendEOS().append("\nEND IF)").appendEOS().append("\nEND$$").appendEOS(); - return f; - }).collect(Collectors.toList()); - } - - return statements; - } - - private List getCreateTableStatements(TableChange change) - { - List statements = new ArrayList<>(); - List createTableSqlParts = new ArrayList<>(); - String pkColumn = null; - for (PropertyStorageSpec prop : change.getColumns()) - { - createTableSqlParts.add(getSqlColumnSpec(prop)); - if (prop.isPrimaryKey()) - { - assert null == pkColumn : "no more than one primary key defined"; - pkColumn = prop.getName(); - } - } - - for (PropertyStorageSpec.ForeignKey foreignKey : change.getForeignKeys()) - { - DbSchema schema = DbSchema.get(foreignKey.getSchemaName(), DbSchemaType.Module); - TableInfo tableInfo = foreignKey.isProvisioned() ? - foreignKey.getTableInfoProvisioned() : - schema.getTable(foreignKey.getTableName()); - String constraintName = "fk_" + foreignKey.getColumnName() + "_" + change.getTableName() + "_" + tableInfo.getName(); - SQLFragment fkFrag = new SQLFragment("CONSTRAINT "); - fkFrag.appendIdentifier(constraintName) - .append(" FOREIGN KEY (") - .appendIdentifier(makePropertyIdentifier(foreignKey.getColumnName())) - .append(") REFERENCES ") - .appendIdentifier(tableInfo.getSchema().getName()).append(".").appendIdentifier(tableInfo.getName()) - .append(" (") - .appendIdentifier(makePropertyIdentifier(foreignKey.getForeignColumnName())) - .append(")"); - createTableSqlParts.add(fkFrag); - } - - SQLFragment create = new SQLFragment("CREATE TABLE "); - create.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - create.append(" (").append(createTableSqlParts, ", ").append(")"); - statements.add(create); - if (null != pkColumn) - { - // Making this just for consistent naming - Constraint constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); - - SQLFragment addPk = new SQLFragment("ALTER TABLE "); - addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) - .append(" ").append(constraint.getType().toString()).append(" (") - .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); - statements.add(addPk); - } - - statements.addAll(getCreateIndexStatements(change)); - statements.addAll(getAddConstraintsStatement(change)); - return statements; - } - - private List getCreateIndexStatements(TableChange change) - { - List statements = new ArrayList<>(); - for (Index index : change.getIndexedColumns()) - { - String newIndexName = nameIndex(change.getTableName(), index.columnNames); - SQLFragment f = new SQLFragment("CREATE "); - if (index.isUnique) - f.append("UNIQUE "); - f.append("INDEX ").appendIdentifier(newIndexName).append(" ON "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ("); - String separator = ""; - for (String columnName : index.columnNames) - { - f.append(separator).appendIdentifier(makePropertyIdentifier(columnName)); - separator = ", "; - } - f.append(")"); - f.appendEOS(); - statements.add(f); - - if (index.isClustered) - { - SQLFragment c = new SQLFragment(); - c.append(PropertyStorageSpec.CLUSTER_TYPE.CLUSTER.toString()).append(" "); - c.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - c.append(" USING ").appendIdentifier(newIndexName); - statements.add(c); - } - } - return statements; - } - - @Override - public String nameIndex(String tableName, String[] indexedColumns) - { - return AliasManager.makeLegalName(tableName + '_' + StringUtils.join(indexedColumns, "_"), this); - } - - private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop) - { - return getSqlColumnSpec(prop, prop.getName()); - } - - private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop, String columnName) - { - SQLFragment colSpec = new SQLFragment(); - colSpec.appendIdentifier(makePropertyIdentifier(columnName)).append(" "); - colSpec.append(getSqlTypeName(prop)); - - // Apply size and precision to varchar and Decimal types - if (prop.getJdbcType() == JdbcType.VARCHAR && prop.getSize() != -1 && prop.getSize() <= SqlDialect.MAX_VARCHAR_SIZE) - { - colSpec.append("(").append(prop.getSize().toString()).append(")"); - } - else if (prop.getJdbcType() == JdbcType.DECIMAL) - { - colSpec.append(DEFAULT_DECIMAL_SCALE_PRECISION); - } - - if (prop.isPrimaryKey() || !prop.isNullable()) - colSpec.append(" NOT NULL"); - - if (null != prop.getDefaultValue()) - { - if (prop.getJdbcType() == JdbcType.BOOLEAN) - { - colSpec.append(" DEFAULT "); - colSpec.append((Boolean)prop.getDefaultValue() ? getBooleanTRUE() : getBooleanFALSE()); - } - else if (prop.getJdbcType() == JdbcType.VARCHAR) - { - colSpec.append(" DEFAULT "); - colSpec.append(getStringHandler().quoteStringLiteral(prop.getDefaultValue().toString())); - } - else - { - throw new IllegalArgumentException("Default value on type " + prop.getJdbcType().name() + " is not supported."); - } - } - return colSpec; - } - @Override public String getSqlTypeName(PropertyStorageSpec prop) { @@ -1458,75 +771,6 @@ else if (prop.getJdbcType() == JdbcType.VARCHAR && (prop.getSize() == -1 || prop } } - /** - * We've historically created lower-cased column names in provisioned tables in Postgres. Keep doing that - * for consistency, though ideally we'd stop doing this and update all existing provisioned tables. - */ - private DatabaseIdentifier makePropertyIdentifier(String name) - { - if (isIdentifierTooLong(name)) - throw new UnsupportedOperationException("Name is too long: " + name); - return new _DatabaseIdentifier(name, quoteIdentifier(name.toLowerCase()), this); - } - - @Override - public void purgeTempSchema(Map createdTableNames) - { - try - { - trackTempTables(createdTableNames); - } - catch (SQLException e) - { - LOG.warn("error cleaning up temp schema", e); - } - - DbSchema coreSchema = CoreSchema.getInstance().getSchema(); - SqlExecutor executor = new SqlExecutor(coreSchema); - - //rs = conn.getMetaData().getFunctions(dbName, tempSchemaName, "%"); - - new SqlSelector(coreSchema, "SELECT proname AS SPECIFIC_NAME, CAST(proargtypes AS VARCHAR) FROM pg_proc WHERE pronamespace=(select oid from pg_namespace where nspname = ?)", DbSchema.getTemp().getName()).forEach( - new ForEachBlock<>() - { - private Map _types = null; - - @Override - public void exec(ResultSet rs) throws SQLException - { - if (null == _types) - { - _types = new HashMap<>(); - new SqlSelector(coreSchema, "SELECT CAST(oid AS VARCHAR) as oid, typname, (select nspname from pg_namespace where oid = typnamespace) as nspname FROM pg_type").forEach(type -> - _types.put(type.getString(1), quoteIdentifier(type.getString(3)) + "." + quoteIdentifier(type.getString(2)))); - } - - String name = rs.getString(1); - String[] oids = StringUtils.split(rs.getString(2), ' '); - SQLFragment drop = new SQLFragment("DROP FUNCTION temp.").append(name); - drop.append("("); - String comma = ""; - for (String oid : oids) - { - drop.append(comma).append(_types.get(oid)); - comma = ","; - } - drop.append(")"); - - try - { - executor.execute(drop); - } - catch (BadSqlGrammarException x) - { - LOG.warn("could not clean up postgres function : temp." + name, x); - } - } - }); - - // TODO delete types in temp schema as well! search for "CREATE TYPE" in StatementUtils.java - } - @Override public boolean isCaseSensitive() { @@ -1842,23 +1086,6 @@ public String encodeLikeOpSearchString(String search) return search.replaceAll("_", "\\\\_").replaceAll("%", "\\\\%"); } - - @Override - public boolean canShowExecutionPlan(ExecutionPlanType type) - { - return true; - } - - @Override - protected Collection getQueryExecutionPlan(Connection conn, DbScope scope, SQLFragment sql, ExecutionPlanType type) - { - SQLFragment copy = new SQLFragment(sql); - copy.insert(0, type == ExecutionPlanType.Estimated ? "EXPLAIN " : "EXPLAIN ANALYZE "); - - return new SqlSelector(scope, conn, copy).getCollection(String.class); - } - - // This list is definitely not exhaustive, can be used for any function where the parameter count and // order are exactly the same as the JDBC equivalent static final CaseInsensitiveHashMap passthroughFn = new CaseInsensitiveHashMap<>(); diff --git a/api/src/org/labkey/api/data/dialect/PostgreSqlServerType.java b/api/src/org/labkey/api/data/dialect/PostgreSqlServerType.java index bd97b523489..772126a5a92 100644 --- a/api/src/org/labkey/api/data/dialect/PostgreSqlServerType.java +++ b/api/src/org/labkey/api/data/dialect/PostgreSqlServerType.java @@ -13,7 +13,7 @@ boolean shouldTest() } @Override - boolean supportsGroupConcat() + public boolean supportsGroupConcat() { return true; } @@ -33,7 +33,7 @@ boolean shouldTest() } @Override - boolean supportsGroupConcat() + public boolean supportsGroupConcat() { return false; } @@ -46,7 +46,7 @@ public boolean supportsSpecialMetadataQueries() }; abstract boolean shouldTest(); - abstract boolean supportsGroupConcat(); + public abstract boolean supportsGroupConcat(); public abstract boolean supportsSpecialMetadataQueries(); public static PostgreSqlServerType getFromParameterStatuses(Map parameterStatuses) diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index d412602a182..390adcc71a9 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -32,11 +32,11 @@ import org.labkey.api.data.CompareType; import org.labkey.api.data.ConnectionWrapper; import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseIdentifier; import org.labkey.api.data.DatabaseTableType; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.DbScope.LabKeyDataSource; -import org.labkey.api.data.DatabaseIdentifier; import org.labkey.api.data.InClauseGenerator; import org.labkey.api.data.JdbcMetaDataSelector.JdbcMetaDataResultSetFactory; import org.labkey.api.data.JdbcType; @@ -923,7 +923,7 @@ private boolean validateIdentifier(DatabaseIdentifier id) // dialect is just for debugging reference protected record _DatabaseIdentifier(String id, SQLFragment sql, SqlDialect dialect) implements DatabaseIdentifier { - _DatabaseIdentifier(String id, String sql, SqlDialect dialect) + public _DatabaseIdentifier(String id, String sql, SqlDialect dialect) { this(id, new SQLFragment().appendIdentifier(sql), dialect); assert null==dialect || dialect.validateIdentifier(this); @@ -2251,13 +2251,23 @@ void testDialectStringHandler() void testLikeOperator() { String stringLiteralPrefix = d.isSqlServer() ? " N" : " "; - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'ABC%' ESCAPE '!'", d.appendCaseInsensitiveStartsWith(new SQLFragment("SELECT * FROM A WHERE Name"), "ABC").toDebugString()); - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'a!%bc%' ESCAPE '!'", d.appendCaseInsensitiveStartsWith(new SQLFragment("SELECT * FROM A WHERE Name"), "a%bc").toDebugString()); - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'%ab!_C' ESCAPE '!'", d.appendCaseInsensitiveEndsWith(new SQLFragment("SELECT * FROM A WHERE Name"), "ab_C").toDebugString()); - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'%a![b]C%' ESCAPE '!'", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a[b]C").toDebugString()); - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'a![b]C_' ESCAPE '!'", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a[b]C", null, "_").toDebugString()); - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'_a!_![b]C%' ESCAPE '!'", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a_[b]C", "_", "%").toDebugString()); - assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'_a[_[[b]C!d%' ESCAPE '['", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a_[b]C!d", "_", "%", '[').toDebugString()); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'ABC%' ESCAPE '!'", d.appendCaseInsensitiveStartsWith(new SQLFragment("SELECT * FROM A WHERE Name"), "ABC").toDebugString(d)); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'a!%bc%' ESCAPE '!'", d.appendCaseInsensitiveStartsWith(new SQLFragment("SELECT * FROM A WHERE Name"), "a%bc").toDebugString(d)); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'%ab!_C' ESCAPE '!'", d.appendCaseInsensitiveEndsWith(new SQLFragment("SELECT * FROM A WHERE Name"), "ab_C").toDebugString(d)); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'%a![b]C%' ESCAPE '!'", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a[b]C").toDebugString(d)); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'a![b]C_' ESCAPE '!'", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a[b]C", null, "_").toDebugString(d)); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'_a!_![b]C%' ESCAPE '!'", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a_[b]C", "_", "%").toDebugString(d)); + assertEquals("SELECT * FROM A WHERE Name " + d.getCaseInsensitiveLikeOperator() + stringLiteralPrefix + "'_a[_[[b]C!d%' ESCAPE '['", d.appendCaseInsensitiveLikeClause(new SQLFragment("SELECT * FROM A WHERE Name"), "a_[b]C!d", "_", "%", '[').toDebugString(d)); + } + + @Test + public void testAutoIncrementQuery() + { + TableInfo tableInfo = CoreSchema.getInstance().getTableInfoContainers(); + Collection sequences = DbScope.getLabKeyScope().getSqlDialect().getAutoIncrementSequences(tableInfo); + assertEquals(1, sequences.size()); + Sequence seq = sequences.stream().findFirst().orElseThrow(); + assertEquals("rowid", seq.columnName().toLowerCase()); } } } diff --git a/api/src/org/labkey/api/data/dialect/SqlDialectFactory.java b/api/src/org/labkey/api/data/dialect/SqlDialectFactory.java index a36fd7d47bc..b274f84ee0b 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialectFactory.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialectFactory.java @@ -27,7 +27,7 @@ public interface SqlDialectFactory @Nullable SqlDialect createFromDriverClassName(String driverClassName); /** - * Returns null if this factory is not responsible for the specified database server. Otherwise, if the version is + * Returns null if this factory is not responsible for the specified database server. Otherwise, if the version is * supported, returns the matching implementation; if the version is not supported, throws DatabaseNotSupportedException. * @param primaryDataSource whether the data source is the primary LabKey Server database, or an external/secondary database */ diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java index 60a48a6e791..0b9af5bad34 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java @@ -10,9 +10,7 @@ import org.labkey.api.util.GUID; import org.labkey.api.util.Pair; -import java.io.PrintWriter; import java.util.Set; -import java.util.function.Predicate; public interface DatabaseMigrationConfiguration { @@ -21,7 +19,6 @@ default void beforeMigration(){} DbScope getSourceScope(); DbScope getTargetScope(); @NotNull Set getSkipSchemas(); - Predicate getColumnNameFilter(); @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler); default void copyAttachments(DbSchema sourceSchema, DbSchema targetSchema, MigrationSchemaHandler schemaHandler){} default @Nullable Pair> initializeFilePathWriter() diff --git a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java index 8b5017a0edf..54703fee67a 100644 --- a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java @@ -8,7 +8,6 @@ import org.labkey.api.data.TableSelector; import java.util.Set; -import java.util.function.Predicate; public class DefaultDatabaseMigrationConfiguration implements DatabaseMigrationConfiguration { @@ -36,12 +35,6 @@ public DbScope getTargetScope() return Set.of(); } - @Override - public Predicate getColumnNameFilter() - { - return null; - } - @Override public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler) { diff --git a/api/src/org/labkey/api/security/GroupManager.java b/api/src/org/labkey/api/security/GroupManager.java index 32eee5f07f9..a9d49fb0567 100644 --- a/api/src/org/labkey/api/security/GroupManager.java +++ b/api/src/org/labkey/api/security/GroupManager.java @@ -16,7 +16,6 @@ package org.labkey.api.security; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -49,6 +48,7 @@ import org.labkey.api.util.JunitUtil; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.HttpView; import org.labkey.api.view.ViewContext; import org.labkey.security.xml.GroupEnumType; @@ -72,7 +72,7 @@ public class GroupManager { - private static final Logger _log = LogManager.getLogger(GroupManager.class); + private static final Logger _log = LogHelper.getLogger(GroupManager.class, "Security group warnings"); private static final CoreSchema _core = CoreSchema.getInstance(); public static final String GROUP_AUDIT_EVENT = "GroupAuditEvent"; @@ -101,7 +101,7 @@ public static void bootstrapGroup(int userId, String name, PrincipalType type) { int gotUserId; if ((gotUserId = createSystemGroup(userId, name, type)) != userId) - _log.warn(name + " group exists but has an unexpected UserId (is " + gotUserId + ", should be " + userId + ")"); + _log.warn("{} group exists but has an unexpected UserId (is {}, should be {})", name, gotUserId, userId); GroupCache.uncache(userId); } diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index 72406413b06..7839e7084c0 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -86,10 +86,12 @@ public void beforeVerification() super.beforeVerification(); // Delete root and shared containers that were needed for bootstrapping - TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); - Table.delete(containers); + Table.delete(CoreSchema.getInstance().getTableInfoContainers()); DbScope targetScope = DbScope.getLabKeyScope(); new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence + + // Delete Guests and Users groups that were needed for bootstrapping + Table.delete(CoreSchema.getInstance().getTableInfoPrincipals()); } @Override diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index ee5d38b3ba9..f959bf5aa1e 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -871,12 +871,12 @@ public void afterUpdate(ModuleContext moduleContext) private void bootstrap() { + // Create the initial groups + GroupManager.bootstrapGroup(Group.groupUsers, "Users"); + GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); + if (ModuleLoader.getInstance().shouldInsertData()) { - // Create the initial groups - GroupManager.bootstrapGroup(Group.groupUsers, "Users"); - GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); - // Other containers inherit permissions from root; admins get all permissions, users & guests none Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); Role readerRole = RoleManager.getRole(ReaderRole.class); @@ -926,7 +926,7 @@ private void bootstrap() } else { - // It's very difficult to bootstrap without the root or shared containers in place; create them now and + // It's very difficult to bootstrap without the root or shared containers in place; create them now, and // we'll delete them later Container root = ContainerManager.ensureContainer("/", User.getAdminServiceUser()); Table.insert(null, CoreSchema.getInstance().getTableInfoContainers(), Map.of("Parent", root.getId(), "Name", "Shared")); diff --git a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java index 6fe789d4faa..c5fbcad586b 100644 --- a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java +++ b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java @@ -15,31 +15,61 @@ */ package org.labkey.core.dialect; +import jakarta.servlet.ServletException; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Constraint; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseIdentifier; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; import org.labkey.api.data.InClauseGenerator; +import org.labkey.api.data.JdbcType; import org.labkey.api.data.ParameterMarkerInClauseGenerator; +import org.labkey.api.data.PropertyStorageSpec; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Selector; +import org.labkey.api.data.Selector.ForEachBlock; +import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableChange; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TempTableInClauseGenerator; +import org.labkey.api.data.TempTableTracker; import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.data.dialect.DialectStringHandler; import org.labkey.api.data.dialect.JdbcHelper; +import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.data.dialect.StandardJdbcHelper; +import org.labkey.api.query.AliasManager; +import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.HtmlString; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.view.template.Warnings; import org.labkey.core.admin.sql.ScriptReorderer; +import org.springframework.jdbc.BadSqlGrammarException; import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /* * This is the base class defining PostgreSQL-specific (i.e., not Redshift) behavior. PostgreSQL 9.2 is no longer @@ -53,14 +83,57 @@ abstract class PostgreSql92Dialect extends BasePostgreSqlDialect // the future plus servers can be compiled with a different limit, so we query this setting on first connection to // each database. private int _maxIdentifierByteLength = 63; - private InClauseGenerator _inClauseGenerator; + private final TempTableInClauseGenerator _tempTableInClauseGenerator = new TempTableInClauseGenerator(); + private final AtomicBoolean _arraySortFunctionExists = new AtomicBoolean(false); + + @Override + public void handleCreateDatabaseException(SQLException e) throws ServletException + { + if ("55006".equals(e.getSQLState())) + { + LOG.error("You must close down pgAdmin III and all other applications accessing PostgreSQL."); + throw (new ServletException("Close down or disconnect pgAdmin III and all other applications accessing PostgreSQL", e)); + } + else + { + super.handleCreateDatabaseException(e); + } + } + + @Override + public void prepareDriver(Class driverClass) + { + // PostgreSQL driver 42.0.0 added logging via the Java Logging API (java.util.logging). This caused the driver to + // start logging SQLExceptions (such as the initial connection failure on bootstrap) to the console... harmless + // but annoying. This code suppresses the driver logging. + Logger pgjdbcLogger = LogManager.getLogManager().getLogger("org.postgresql"); + + if (null != pgjdbcLogger) + pgjdbcLogger.setLevel(Level.OFF); + } + + // Make sure that the PL/pgSQL language is enabled in the associated database. If not, throw. Since 9.0, PostgreSQL has + // shipped with PL/pgSQL enabled by default, so the check is no longer critical, but continue to verify just to be safe. + @Override + public void prepareNewLabKeyDatabase(DbScope scope) + { + if (new SqlSelector(scope, "SELECT * FROM pg_language WHERE lanname = 'plpgsql'").exists()) + return; + + String dbName = scope.getDatabaseName(); + String message = "PL/pgSQL is not enabled in the \"" + dbName + "\" database because it is not enabled in your Template1 master database."; + String advice = "Use PostgreSQL's 'createlang' command line utility to enable PL/pgSQL in the \"" + dbName + "\" database then restart Tomcat."; + + throw new ConfigurationException(message, advice); + } @Override public String prepare(DbScope scope) { initializeInClauseGenerator(scope); + determineIfArraySortFunctionExists(scope); return super.prepare(scope); } @@ -119,12 +192,34 @@ protected DialectStringHandler createStringHandler() return new PostgreSqlNonConformingStringHandler(); } + /* + PostgreSQL example connection URLs we need to parse: + + jdbc:postgresql:database + jdbc:postgresql://host/database + jdbc:postgresql://host:port/database + jdbc:postgresql:database?user=fred&password=secret&ssl=true + jdbc:postgresql://host/database?user=fred&password=secret&ssl=true + jdbc:postgresql://host:port/database?user=fred&password=secret&ssl=true + */ @Override public JdbcHelper getJdbcHelper() { return new StandardJdbcHelper(PostgreSqlDialectFactory.JDBC_PREFIX); } + @Override + public String getDefaultDatabaseName() + { + return "template1"; + } + + @Override + public boolean canExecuteUpgradeScripts() + { + return true; + } + @Override public Collection getScriptWarnings(String name, String sql) { @@ -145,9 +240,111 @@ public Collection getScriptWarnings(String name, String sql) return warnings; } - private void initializeInClauseGenerator(DbScope scope) + @Override + public String getSQLScriptPath() { - _inClauseGenerator = getJdbcVersion(scope) >= 4 ? new ArrayParameterInClauseGenerator(scope) : new ParameterMarkerInClauseGenerator(); + return "postgresql"; + } + + @Override + public String getUniqueIdentType() + { + return "SERIAL"; + } + + @Override + public boolean supportsGroupConcat() + { + return getServerType().supportsGroupConcat(); + } + + @Override + public boolean supportsSelectConcat() + { + return true; + } + + @Override + public SQLFragment getSelectConcat(SQLFragment selectSql, String delimiter) + { + SQLFragment result = new SQLFragment("array_to_string(array("); + result.append(selectSql); + result.append("), "); + result.append(getStringHandler().quoteStringLiteral(delimiter)); + result.append(")"); + return result; + } + + // Does this datasource include our sort array function? The LabKey datasource should always have it, but external datasources might not + private void determineIfArraySortFunctionExists(DbScope scope) + { + if (getServerType().supportsSpecialMetadataQueries()) + { + Selector selector = new SqlSelector(scope, "SELECT * FROM pg_catalog.pg_namespace n INNER JOIN pg_catalog.pg_proc p ON pronamespace = n.oid WHERE nspname = 'core' AND proname = 'sort'"); + _arraySortFunctionExists.set(selector.exists()); + } + + // Array sort function should always exist in LabKey scope (for now) + assert !scope.isLabKeyScope() || _arraySortFunctionExists.get(); + } + + @Override + public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sorted, @NotNull SQLFragment delimiterSQL, boolean includeNulls) + { + // Sort function might not exist in external datasource; skip that syntax if not + boolean useSortFunction = sorted && _arraySortFunctionExists.get(); + SQLFragment result = new SQLFragment(); + + if (useSortFunction) + { + result.append("array_to_string("); + result.append("core.sort("); // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function + result.append("array_agg("); + if (distinct) + { + result.append("DISTINCT "); + } + + if (includeNulls) + { + result.append("COALESCE(CAST("); + result.append(sql); + result.append(" AS VARCHAR), '')"); + } + else + { + result.append(sql); + } + + result.append(")"); // array_agg + result.append(")"); // core.sort + } + else + { + result.append("string_agg("); + if (distinct) + { + result.append("DISTINCT "); + } + + if (includeNulls) + { + result.append("COALESCE("); + result.append(sql); + result.append("::text, '')"); + } + else + { + result.append(sql); + result.append("::text"); + } + } + + result.append(", "); + result.append(delimiterSQL); + result.append(")"); // array_to_string | string_agg + + return result; } @Override @@ -156,6 +353,11 @@ public SQLFragment getAnalyzeCommandForTable(String tableName) return new SQLFragment("ANALYZE ").appendIdentifier(tableName); } + private void initializeInClauseGenerator(DbScope scope) + { + _inClauseGenerator = getJdbcVersion(scope) >= 4 ? new ArrayParameterInClauseGenerator(scope) : new ParameterMarkerInClauseGenerator(); + } + @Override public InClauseGenerator getDefaultInClauseGenerator() { @@ -235,6 +437,21 @@ private static String truncateBytes(String str, int maxBytes) return str; } + @Override + public boolean canShowExecutionPlan(ExecutionPlanType type) + { + return true; + } + + @Override + protected Collection getQueryExecutionPlan(Connection conn, DbScope scope, SQLFragment sql, ExecutionPlanType type) + { + SQLFragment copy = new SQLFragment(sql); + copy.insert(0, type == ExecutionPlanType.Estimated ? "EXPLAIN " : "EXPLAIN ANALYZE "); + + return new SqlSelector(scope, conn, copy).getCollection(String.class); + } + @Override // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). protected Pattern getSQLScriptSplitPattern() @@ -242,6 +459,25 @@ protected Pattern getSQLScriptSplitPattern() return null; } + private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); + + @NotNull + @Override + protected Pattern getSQLScriptProcPattern() + { + return PROC_PATTERN; + } + + @Override + protected void checkSqlScript(String lowerNoComments, String lowerNoCommentsNoWhiteSpace, Collection errors) + { + if (lowerNoCommentsNoWhiteSpace.contains("setsearch_pathto")) + errors.add("Do not use \"SET search_path TO \". Instead, schema-qualify references to all objects."); + + if (!lowerNoCommentsNoWhiteSpace.endsWith(";")) + errors.add("Script must end with a semicolon"); + } + @Override public @NotNull Collection getAutoIncrementSequences(TableInfo table) { @@ -281,6 +517,536 @@ AND d.deptype IN ('a', 'i') -- Automatic dependency for DEFAULT or index-related return new SqlSelector(table.getSchema(), sql).getCollection(Sequence.class); } + @Override + public String getBinaryDataType() + { + return "BYTEA"; + } + + @Override + public String getGlobalTempTablePrefix() + { + return DbSchema.TEMP_SCHEMA_NAME + "."; + } + + @Override + public String getDropIndexCommand(String tableName, String indexName) + { + return "DROP INDEX " + indexName; + } + + @Override + public String getCreateDatabaseSql(String dbName) + { + // This will handle both mixed case and special characters on PostgreSQL + var legal = makeIdentifierFromMetaDataName(dbName); + return new SQLFragment("CREATE DATABASE ").appendIdentifier(legal).append(" WITH ENCODING 'UTF8'").getRawSQL(); + } + + @Override + public String getCreateSchemaSql(String schemaName) + { + if (!isLegalName(schemaName) || isReserved(schemaName)) + throw new IllegalArgumentException("Not a legal schema name: " + schemaName); + + //Quoted schema names are bad news + return "CREATE SCHEMA " + schemaName; + } + + @Override + public String getTruncateSql(String tableName) + { + // To be consistent with MS SQL server, always restart the sequence. Note that the default for postgres + // is to continue the sequence but we don't have this option with MS SQL Server + return "TRUNCATE TABLE " + tableName + " RESTART IDENTITY"; + } + + @Override + public List getChangeStatements(TableChange change) + { + List result = new ArrayList<>(); + switch (change.getType()) + { + case CreateTable -> result.addAll(getCreateTableStatements(change)); + case DropTable -> { + SQLFragment f = new SQLFragment("DROP TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + result.add(f); + } + case AddColumns -> result.addAll(getAddColumnsStatements(change)); + case DropColumns -> result.add(getDropColumnsStatement(change)); + case RenameColumns -> result.addAll(getRenameColumnsStatement(change)); + case DropIndicesByName -> result.addAll(getDropIndexByNameStatements(change)); + case AddIndices -> result.addAll(getCreateIndexStatements(change)); + case ResizeColumns, ChangeColumnTypes -> result.addAll(getChangeColumnTypeStatement(change)); + case DropConstraints -> result.addAll(getDropConstraintsStatement(change)); + case AddConstraints -> result.addAll(getAddConstraintsStatement(change)); + default -> throw new IllegalArgumentException("Unsupported change type: " + change.getType()); + } + + return result; + } + + private Collection getDropIndexByNameStatements(TableChange change) + { + List statements = new ArrayList<>(); + for (String indexName : change.getIndicesToBeDroppedByName()) + { + statements.add(getDropIndexCommand(change, indexName)); + } + return statements; + } + + private SQLFragment getDropIndexCommand(TableChange change, String indexName) + { + SQLFragment f = new SQLFragment("DROP INDEX "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(indexName); + return f; + } + + /** + * We've historically created lower-cased column names in provisioned tables in Postgres. Keep doing that + * for consistency, though ideally we'd stop doing this and update all existing provisioned tables. + */ + private DatabaseIdentifier makePropertyIdentifier(String name) + { + if (isIdentifierTooLong(name)) + throw new UnsupportedOperationException("Name is too long: " + name); + return new _DatabaseIdentifier(name, quoteIdentifier(name.toLowerCase()), this); + } + + /** + * Generate the Alter Table statement to change the size or type of the column + *

+ * NOTE: expects data size check to be done prior, + * will throw a SQL exception if not able to change size due to existing data + */ + private List getChangeColumnTypeStatement(TableChange change) + { + List statements = new ArrayList<>(); + + // Postgres allows executing multiple ALTER COLUMN statements under one ALTER TABLE + List nonDateTimeClauses = new ArrayList<>(); + + for (PropertyStorageSpec column : change.getColumns()) + { + DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); + if (column.getJdbcType().isDateOrTime()) + { + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: copy casted value to temp column + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent); + update.append(" = CAST(").appendIdentifier(columnIdent).append(" AS ").append(getSqlTypeName(column)).append(")"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else + { + String dbType; + if (column.getJdbcType().isText()) + { + // Using the common default max size to make type change to text + dbType = column.getSize() == -1 || column.getSize() > SqlDialect.MAX_VARCHAR_SIZE ? + getSqlTypeName(JdbcType.LONGVARCHAR) : + getSqlTypeName(column.getJdbcType()) + "(" + column.getSize().toString() + ")"; + } + else if (column.getJdbcType().isDecimal()) + { + dbType = getSqlTypeName(column.getJdbcType()) + DEFAULT_DECIMAL_SCALE_PRECISION; + } + else + { + dbType = getSqlTypeName(column.getJdbcType()); + } + + SQLFragment clause = new SQLFragment(); + clause.append("ALTER COLUMN ").appendIdentifier(columnIdent).append(" TYPE ").append(dbType); + nonDateTimeClauses.add(clause); + } + } + + if (!nonDateTimeClauses.isEmpty()) + { + SQLFragment alter = new SQLFragment("ALTER TABLE "); + alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + alter.append(" "); + String sep = ""; + for (SQLFragment c : nonDateTimeClauses) + { + alter.append(sep).append(c); + sep = ", "; + } + statements.add(alter); + } + + return statements; + } + + private List getRenameColumnsStatement(TableChange change) + { + List statements = new ArrayList<>(); + for (Map.Entry oldToNew : change.getColumnRenames().entrySet()) + { + DatabaseIdentifier oldIdentifier = makePropertyIdentifier(oldToNew.getKey()); + DatabaseIdentifier newIdentifier = makePropertyIdentifier(oldToNew.getValue()); + if (!oldIdentifier.equals(newIdentifier)) + { + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" RENAME COLUMN ").appendIdentifier(oldIdentifier).append(" TO ").appendIdentifier(newIdentifier); + statements.add(f); + } + } + + // TODO: This loop should not guess the name of the old indices; instead, it should look them up. + // TableChange.setIndexedColumns() could set _indexRenames providing the name, and then this code uses that info. + // Or maybe schemaTableInfo.getAllIndices() and then use Index.isSameIndex() to find names. Issue 53838. + for (Map.Entry oldToNew : change.getIndexRenames().entrySet()) + { + PropertyStorageSpec.Index oldIndex = oldToNew.getKey(); + PropertyStorageSpec.Index newIndex = oldToNew.getValue(); + String oldName = nameIndex(change.getTableName(), oldIndex.columnNames); // TODO: Look up name + String newName = nameIndex(change.getTableName(), newIndex.columnNames); + if (!oldName.equals(newName)) + { + SQLFragment f = new SQLFragment("ALTER INDEX "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(oldName); + f.append(" RENAME TO ").appendIdentifier(newName); + statements.add(f); + } + } + + return statements; + } + + private SQLFragment getDropColumnsStatement(TableChange change) + { + List sqlParts = new ArrayList<>(); + for (PropertyStorageSpec prop : change.getColumns()) + { + SQLFragment sql = new SQLFragment("DROP COLUMN "); + if (prop.getExactName()) + { + sql.append(quoteIdentifier(prop.getName())); + } + else + { + sql.appendIdentifier(makePropertyIdentifier(prop.getName())); + } + sqlParts.add(sql); + } + + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ").append(sqlParts, ", "); + return f; + } + + // TODO if there are cases where user-defined columns need indices, this method will need to support + // creating indices like getCreateTableStatement does. + private List getAddColumnsStatements(TableChange change) + { + List statements = new ArrayList<>(); + String pkColumn = null; + Constraint constraint = null; + + List columnSpecs = new ArrayList<>(); + for (PropertyStorageSpec prop : change.getColumns()) + { + columnSpecs.add(getSqlColumnSpec(prop)); + if (prop.isPrimaryKey()) + { + assert null == pkColumn : "no more than one primary key defined"; + pkColumn = prop.getName(); + constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); + } + } + + SQLFragment alter = new SQLFragment("ALTER TABLE "); + alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + alter.append(" "); + String sep = ""; + for (SQLFragment col : columnSpecs) + { + alter.append(sep); + alter.append("ADD COLUMN "); + alter.append(col); + sep = ", "; + } + statements.add(alter); + if (null != pkColumn) + { + SQLFragment addPk = new SQLFragment("ALTER TABLE "); + addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) + .append(" ").append(constraint.getType().toString()).append(" (") + .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); + statements.add(addPk); + } + + return statements; + } + + private List getDropConstraintsStatement(TableChange change) + { + return change.getConstraints().stream().map(constraint -> { + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" DROP CONSTRAINT ").appendIdentifier(constraint.getName()); + return f; + }).collect(Collectors.toList()); + } + + private List getAddConstraintsStatement(TableChange change) + { + List statements = new ArrayList<>(); + Collection constraints = change.getConstraints(); + + if (null!=constraints && !constraints.isEmpty()) + { + statements = constraints.stream().map(constraint -> { + List columns = new ArrayList<>(); + for (String col : constraint.getColumns()) + { + columns.add(new SQLFragment().appendIdentifier(col)); + } + + SQLFragment f = new SQLFragment(); + f.append("DO $$\nBEGIN\nIF NOT EXISTS\n(SELECT 1 FROM information_schema.constraint_column_usage\nWHERE table_name = ") + .appendStringLiteral(change.getSchemaName() + "." + change.getTableName(), this) + .append(" and constraint_name = ") + .appendStringLiteral(constraint.getName(), this) + .append(") THEN\nALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()).append(" ") + .append(constraint.getType().toString()).append(" (") + .append(columns, ",") + .append(")").appendEOS().append("\nEND IF)").appendEOS().append("\nEND$$").appendEOS(); + return f; + }).collect(Collectors.toList()); + } + + return statements; + } + + private List getCreateTableStatements(TableChange change) + { + List statements = new ArrayList<>(); + List createTableSqlParts = new ArrayList<>(); + String pkColumn = null; + for (PropertyStorageSpec prop : change.getColumns()) + { + createTableSqlParts.add(getSqlColumnSpec(prop)); + if (prop.isPrimaryKey()) + { + assert null == pkColumn : "no more than one primary key defined"; + pkColumn = prop.getName(); + } + } + + for (PropertyStorageSpec.ForeignKey foreignKey : change.getForeignKeys()) + { + DbSchema schema = DbSchema.get(foreignKey.getSchemaName(), DbSchemaType.Module); + TableInfo tableInfo = foreignKey.isProvisioned() ? + foreignKey.getTableInfoProvisioned() : + schema.getTable(foreignKey.getTableName()); + String constraintName = "fk_" + foreignKey.getColumnName() + "_" + change.getTableName() + "_" + tableInfo.getName(); + SQLFragment fkFrag = new SQLFragment("CONSTRAINT "); + fkFrag.appendIdentifier(constraintName) + .append(" FOREIGN KEY (") + .appendIdentifier(makePropertyIdentifier(foreignKey.getColumnName())) + .append(") REFERENCES ") + .appendIdentifier(tableInfo.getSchema().getName()).append(".").appendIdentifier(tableInfo.getName()) + .append(" (") + .appendIdentifier(makePropertyIdentifier(foreignKey.getForeignColumnName())) + .append(")"); + createTableSqlParts.add(fkFrag); + } + + SQLFragment create = new SQLFragment("CREATE TABLE "); + create.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + create.append(" (").append(createTableSqlParts, ", ").append(")"); + statements.add(create); + if (null != pkColumn) + { + // Making this just for consistent naming + Constraint constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); + + SQLFragment addPk = new SQLFragment("ALTER TABLE "); + addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) + .append(" ").append(constraint.getType().toString()).append(" (") + .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); + statements.add(addPk); + } + + statements.addAll(getCreateIndexStatements(change)); + statements.addAll(getAddConstraintsStatement(change)); + return statements; + } + + private List getCreateIndexStatements(TableChange change) + { + List statements = new ArrayList<>(); + for (PropertyStorageSpec.Index index : change.getIndexedColumns()) + { + String newIndexName = nameIndex(change.getTableName(), index.columnNames); + SQLFragment f = new SQLFragment("CREATE "); + if (index.isUnique) + f.append("UNIQUE "); + f.append("INDEX ").appendIdentifier(newIndexName).append(" ON "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ("); + String separator = ""; + for (String columnName : index.columnNames) + { + f.append(separator).appendIdentifier(makePropertyIdentifier(columnName)); + separator = ", "; + } + f.append(")"); + f.appendEOS(); + statements.add(f); + + if (index.isClustered) + { + SQLFragment c = new SQLFragment(); + c.append(PropertyStorageSpec.CLUSTER_TYPE.CLUSTER.toString()).append(" "); + c.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + c.append(" USING ").appendIdentifier(newIndexName); + statements.add(c); + } + } + return statements; + } + + @Override + public String nameIndex(String tableName, String[] indexedColumns) + { + return AliasManager.makeLegalName(tableName + '_' + StringUtils.join(indexedColumns, "_"), this); + } + + private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop) + { + return getSqlColumnSpec(prop, prop.getName()); + } + + private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop, String columnName) + { + SQLFragment colSpec = new SQLFragment(); + colSpec.appendIdentifier(makePropertyIdentifier(columnName)).append(" "); + colSpec.append(getSqlTypeName(prop)); + + // Apply size and precision to varchar and Decimal types + if (prop.getJdbcType() == JdbcType.VARCHAR && prop.getSize() != -1 && prop.getSize() <= SqlDialect.MAX_VARCHAR_SIZE) + { + colSpec.append("(").append(prop.getSize().toString()).append(")"); + } + else if (prop.getJdbcType() == JdbcType.DECIMAL) + { + colSpec.append(DEFAULT_DECIMAL_SCALE_PRECISION); + } + + if (prop.isPrimaryKey() || !prop.isNullable()) + colSpec.append(" NOT NULL"); + + if (null != prop.getDefaultValue()) + { + if (prop.getJdbcType() == JdbcType.BOOLEAN) + { + colSpec.append(" DEFAULT "); + colSpec.append((Boolean)prop.getDefaultValue() ? getBooleanTRUE() : getBooleanFALSE()); + } + else if (prop.getJdbcType() == JdbcType.VARCHAR) + { + colSpec.append(" DEFAULT "); + colSpec.append(getStringHandler().quoteStringLiteral(prop.getDefaultValue().toString())); + } + else + { + throw new IllegalArgumentException("Default value on type " + prop.getJdbcType().name() + " is not supported."); + } + } + return colSpec; + } + + @Override + public void purgeTempSchema(Map createdTableNames) + { + try + { + trackTempTables(createdTableNames); + } + catch (SQLException e) + { + LOG.warn("error cleaning up temp schema", e); + } + + DbSchema coreSchema = CoreSchema.getInstance().getSchema(); + SqlExecutor executor = new SqlExecutor(coreSchema); + + //rs = conn.getMetaData().getFunctions(dbName, tempSchemaName, "%"); + + new SqlSelector(coreSchema, "SELECT proname AS SPECIFIC_NAME, CAST(proargtypes AS VARCHAR) FROM pg_proc WHERE pronamespace=(select oid from pg_namespace where nspname = ?)", DbSchema.getTemp().getName()).forEach( + new ForEachBlock<>() + { + private Map _types = null; + + @Override + public void exec(ResultSet rs) throws SQLException + { + if (null == _types) + { + _types = new HashMap<>(); + new SqlSelector(coreSchema, "SELECT CAST(oid AS VARCHAR) as oid, typname, (select nspname from pg_namespace where oid = typnamespace) as nspname FROM pg_type").forEach(type -> + _types.put(type.getString(1), quoteIdentifier(type.getString(3)) + "." + quoteIdentifier(type.getString(2)))); + } + + String name = rs.getString(1); + String[] oids = StringUtils.split(rs.getString(2), ' '); + SQLFragment drop = new SQLFragment("DROP FUNCTION temp.").append(name); + drop.append("("); + String comma = ""; + for (String oid : oids) + { + drop.append(comma).append(_types.get(oid)); + comma = ","; + } + drop.append(")"); + + try + { + executor.execute(drop); + } + catch (BadSqlGrammarException x) + { + LOG.warn("could not clean up postgres function : temp." + name, x); + } + } + }); + + // TODO delete types in temp schema as well! search for "CREATE TYPE" in StatementUtils.java + } // // ARRAY and SET syntax