Join core.Documents rows to parent details

announcements/src/org/labkey/announcements/model/AnnouncementType.java

@@ -16,7 +16,6 @@
 package org.labkey.announcements.model;
 
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 import org.labkey.api.announcements.CommSchema;
 import org.labkey.api.attachments.AttachmentParentType;
 import org.labkey.api.data.SQLFragment;
@@ -41,8 +40,8 @@ public static AttachmentParentType get()
     }
 
     @Override
-    public @Nullable SQLFragment getSelectParentEntityIdsSql()
+    public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
     {
-        return new SQLFragment("SELECT EntityId FROM ").append(CommSchema.getInstance().getTableInfoAnnouncements(), "ann");
+        return new SQLFragment("SELECT EntityId, Title AS Description FROM ").append(CommSchema.getInstance().getTableInfoAnnouncements(), "ann");
     }
 }

api/src/org/labkey/api/attachments/AttachmentParentType.java

@@ -16,7 +16,7 @@
 package org.labkey.api.attachments;
 
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.CoreSchema;
 import org.labkey.api.data.SQLFragment;
 
 /**
@@ -25,7 +25,9 @@
  */
 public interface AttachmentParentType
 {
-    SQLFragment NO_ENTITY_IDS = new SQLFragment("SELECT NULL AS EntityId WHERE 1 = 0");
+    SQLFragment NO_ROWS = new SQLFragment("SELECT NULL AS EntityId, NULL AS Description WHERE 1 = 0");
+    SQLFragment PARENT_CONTAINER_SQL = new SQLFragment("SELECT EntityId, COALESCE(Name, '<Root>') AS Description FROM ")
+        .append(CoreSchema.getInstance().getTableInfoContainers());
 
     AttachmentParentType UNKNOWN = new AttachmentParentType()
     {
@@ -37,9 +39,9 @@ public String getUniqueName()
         }
 
         @Override
-        public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn)
+        public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
         {
-            sql.append("0 = 1");
+            return NO_ROWS;
         }
     };
 
@@ -56,20 +58,27 @@ public void addWhereSql(SQLFragment sql, String parentColumn, String documentNam
     default void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn)
     {
         SQLFragment selectSql = getSelectParentEntityIdsSql();
-        if (selectSql == null)
-            throw new IllegalStateException("Must override either addWhereSql() or getSelectParentEntityIdsSql()");
         sql.append(parentColumn).append(" IN (").append(selectSql).append(")");
     }
 
     /**
-     * Return a SQLFragment that selects all the EntityIds that might be attachment parents from the table(s) that
-     * provide attachments of this type, without involving the core.Documents table. For example,
-     * {@code SELECT EntityId FROM comm.Announcements}. Return null if this is not-yet-implemented or inappropriate.
-     * For example, some attachments' parents are container IDs. If the method determines that no parents exist, then
-     * return a valid query that selects no rows, for example, {@code NO_ENTITY_IDS}.
+     * Return a SQLFragment that selects just the EntityId of rows that might be attachment parents from the table(s)
+     * that provide attachments of this type, without involving the core.Documents table.
      */
-    default @Nullable SQLFragment getSelectParentEntityIdsSql()
+    default @NotNull SQLFragment getSelectParentEntityIdsSql()
     {
-        return null;
+        SQLFragment selectSql = getSelectEntityIdAndDescriptionSql();
+
+        // The returned SQL is always used inside a subselect, so the alias doesn't have to be unique
+        return new SQLFragment("SELECT EntityId FROM (").append(selectSql).append(") x");
     }
+
+    /**
+     * Return a SQLFragment that selects the EntityId and an appropriate Description of all rows that might be
+     * attachment parents from the table(s) that provide attachment parents of this type, without involving the
+     * core.Documents table. For example, {@code SELECT EntityId, Title AS Description FROM comm.Announcements}.
+     * If the method determines that no parents exist, then return a valid query that selects no rows, for example,
+     * {@code NO_ROWS}.
+     */
+    @NotNull SQLFragment getSelectEntityIdAndDescriptionSql();
 }

api/src/org/labkey/api/attachments/LookAndFeelResourceType.java

@@ -49,4 +49,10 @@ public void addWhereSql(SQLFragment sql, String parentColumn, String documentNam
         sql.append(documentNameColumn).append(" LIKE '" + AttachmentCache.LOGO_FILE_NAME_PREFIX + "%' OR ");
         sql.append(documentNameColumn).append(" LIKE '" + AttachmentCache.MOBILE_LOGO_FILE_NAME_PREFIX + "%')");
     }
+
+    @Override
+    public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
+    {
+        return PARENT_CONTAINER_SQL;
+    }
 }

api/src/org/labkey/api/collections/LabKeyCollectors.java

@@ -6,6 +6,7 @@
 import org.json.JSONArray;
 import org.junit.Assert;
 import org.junit.Test;
+import org.labkey.api.data.SQLFragment;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.HtmlStringBuilder;
 
@@ -14,6 +15,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -222,6 +224,18 @@ public static Collector<HtmlString, HtmlStringBuilder, HtmlString> joining(HtmlS
         );
     }
 
+    /**
+     * Returns a {@link Collector} that joins {@link SQLFragment}s into a single {@link SQLFragment} separated by delimiter
+     */
+    public static Collector<SQLFragment, List<SQLFragment>, SQLFragment> joining(SQLFragment delimiter) {
+        return Collector.of(
+            LinkedList::new,
+            List::add,
+            (list1, list2) -> {list1.addAll(list2); return list1;},
+            (list) -> SQLFragment.join(list, delimiter)
+        );
+    }
+
     public static class TestCase extends Assert
     {
         @Test

api/src/org/labkey/api/data/SQLFragment.java

@@ -47,7 +47,6 @@
 import java.util.TreeSet;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS;
 
@@ -1339,25 +1338,23 @@ public int hashCode()
      * concatenation using the provided separator. The parameters are combined to form the new parameter list.
      *
      * @param fragments SQLFragments to join together
-     * @param separator Separator to use on the SQL portion
+     * @param separator Separator to use
      * @return A new SQLFragment that joins all the SQLFragments
      */
-    public static SQLFragment join(Iterable<SQLFragment> fragments, String separator)
+    public static SQLFragment join(Iterable<SQLFragment> fragments, SQLFragment separator)
     {
-        if (separator.contains("?"))
-            throw new IllegalStateException("separator must not include a parameter marker");
+        SQLFragment join = new SQLFragment();
+        boolean first = true;
 
-        // Join all the SQL statements
-        String sql = StreamSupport.stream(fragments.spliterator(), false)
-            .map(SQLFragment::getSQL)
-            .collect(Collectors.joining(separator));
-
-        // Collect all the parameters to a single list
-        List<?> params = StreamSupport.stream(fragments.spliterator(), false)
-            .map(SQLFragment::getParams)
-            .flatMap(Collection::stream)
-            .collect(Collectors.toList());
+        for (SQLFragment fragment : fragments)
+        {
+            if (first)
+                first = false;
+            else
+                join.append(separator);
+            join.append(fragment);
+        }
 
-        return new SQLFragment(sql, params);
+        return join;
     }
 }

api/src/org/labkey/api/exp/Lsid.java

@@ -25,8 +25,10 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.labkey.api.data.Builder;
+import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.dialect.SqlDialect;
 import org.labkey.api.settings.AppProps;
+import org.labkey.api.util.GUID;
 import org.labkey.api.util.Pair;
 
 import java.net.URI;
@@ -36,8 +38,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
-import static org.apache.commons.lang3.StringUtils.repeat;
-
 /**
  * Life-sciences identifier (LSID). A structured URI to describe things like samples, data files, assay runs, and protocols.
  * User: migra
@@ -122,7 +122,6 @@ private Lsid(String src, String authority, String namespace, String objectId, St
      *
      * To spend less time parsing, maybe cached parsed Lsid in ExpObjectImpl? (not that straightforward)
      */
-
     @Nullable
     private static String[] parseLsid(@Nullable String s)
     {
@@ -138,28 +137,32 @@ private static String[] parseLsid(@Nullable String s)
         return new String[] {parts[2], parts[3], parts[4], parts.length < 6 ? null : parts[5]};
     }
 
-
-    // Keep in sync with LSID_REGEX (above)
-    public static Pair<String, String> getSqlExpressionToExtractObjectId(String lsidExpression, SqlDialect dialect)
+    // Keep in sync with LSID_REGEX (above). Note: AttachmentServiceImpl.TestCase.testLsidGuidExtraction tests this.
+    public static Pair<SQLFragment, SQLFragment> getSqlExpressionToExtractObjectId(SQLFragment lsidExpression, SqlDialect dialect)
     {
+        String objectId = GUID.SQL_LIKE_GUID_PATTERN;
+
         if (dialect.isPostgreSQL())
         {
-            // PostgreSQL SUBSTRING supports simple regular expressions. This captures all the text from the third
-            // colon to the end of the string (or to the fourth colon, if present).
-            String expression = "SUBSTRING(" + lsidExpression + " FROM '%urn:lsid:%:#\"%#\":?%' FOR '#')";
-            String where = lsidExpression + " SIMILAR TO '%urn:lsid:%:[0-9a-f\\-]{36}:?%'";
+            // PostgreSQL SUBSTRING supports simple regular expressions. This captures all the text from the fourth
+            // colon to the end of the string (or to the fifth colon, if present).
+            SQLFragment expression = new SQLFragment("SUBSTRING(")
+                .append(lsidExpression)
+                .append(" FROM '%urn:lsid:%:%:#\"[0-9a-f\\-]{36}#\":?%' FOR '#')");
+            SQLFragment where = new SQLFragment(lsidExpression).append(" SIMILAR TO '%urn:lsid:%:%:[0-9a-f\\-]{36}:?%'");
 
             return new Pair<>(expression, where);
         }
 
         if (dialect.isSqlServer())
         {
-            // SQL Server doesn't support regular expressions; this uses an unwieldy pattern to extract the objectid
-            String d = "[0-9a-f]"; // pattern for a single digit
-            String objectId = repeat(d, 8) + "-" + repeat(d, 4) + "-" + repeat(d, 4) + "-" + repeat(d, 4) + "-" + repeat(d, 12);
-
-            String expression = "SUBSTRING(" + lsidExpression + ", PATINDEX('%:" + objectId + "%', " + lsidExpression + ") + 1, 36)";
-            String where = lsidExpression + " LIKE '%urn:lsid:%:" + objectId + "%'";
+            // SQL Server doesn't support regular expressions
+            SQLFragment expression = new SQLFragment("SUBSTRING(")
+                .append(lsidExpression)
+                .append(", PATINDEX('%:" + objectId + "%', ")
+                .append(lsidExpression)
+                .append(") + 1, 36)");
+            SQLFragment where = new SQLFragment(lsidExpression).append(" LIKE '%urn:lsid:%:%:" + objectId + "%'");
 
             return new Pair<>(expression, where);
         }

api/src/org/labkey/api/exp/api/ExpProtocolAttachmentType.java

@@ -16,7 +16,6 @@
 package org.labkey.api.exp.api;
 
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 import org.labkey.api.attachments.AttachmentParentType;
 import org.labkey.api.data.SQLFragment;
 
@@ -40,8 +39,8 @@ private ExpProtocolAttachmentType()
     }
 
     @Override
-    public @Nullable SQLFragment getSelectParentEntityIdsSql()
+    public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
     {
-        return new SQLFragment("SELECT EntityId FROM ").append(ExperimentService.get().getTinfoProtocol(), "ep");
+        return new SQLFragment("SELECT EntityId, Name AS Description FROM ").append(ExperimentService.get().getTinfoProtocol(), "ep");
     }
 }

api/src/org/labkey/api/exp/api/ExpRunAttachmentType.java

@@ -16,7 +16,6 @@
 package org.labkey.api.exp.api;
 
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 import org.labkey.api.attachments.AttachmentParentType;
 import org.labkey.api.data.SQLFragment;
 
@@ -40,8 +39,8 @@ private ExpRunAttachmentType()
     }
 
     @Override
-    public @Nullable SQLFragment getSelectParentEntityIdsSql()
+    public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
     {
-        return new SQLFragment("SELECT EntityId FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "er");
+        return new SQLFragment("SELECT EntityId, Name AS Description FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "er");
     }
 }

api/src/org/labkey/api/exp/property/PropertyService.java

@@ -83,6 +83,18 @@ static void setInstance(PropertyService impl)
 
     Stream<? extends Domain> getDomainsStream(Container container, User user, Set<String> domainKinds, @Nullable Set<String> domainNames, boolean includeProjectAndShared);
 
+    /**
+     * Get a stream of all containers that have a domain with a DomainURI that starts with this server's LSID authority
+     * followed by any of the provided namespace prefixes. Not typically needed, but it can be useful for maintenance
+     * or reporting purposes. On servers with many containers, filtering the containers before enumerating domains can
+     * be much faster than enumerating domains in every container. Callers are responsible for passing in the namespace
+     * prefixes because DomainKinds don't provide a standard getter for this; the namespace prefix is often (but not
+     * always) getKindName(). Note that it's possible that containers that don't have your desired domain may be
+     * returned since different kinds' prefixes can overlap and this method doesn't use getPriority() to resolve
+     * domains.
+     */
+    Stream<Container> getContainersWithDomains(@NotNull Set<String> domainNamespacePrefixes);
+
     /** Creates an in-memory Domain. It is not automatically saved to the database */
     @NotNull
     Domain createDomain(Container container, String typeURI, String name);

api/src/org/labkey/api/files/FileSystemAttachmentType.java

@@ -16,7 +16,6 @@
 package org.labkey.api.files;
 
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 import org.labkey.api.attachments.AttachmentParentType;
 import org.labkey.api.data.CoreSchema;
 import org.labkey.api.data.SQLFragment;
@@ -41,8 +40,8 @@ private FileSystemAttachmentType()
     }
 
     @Override
-    public @Nullable SQLFragment getSelectParentEntityIdsSql()
+    public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
     {
-        return new SQLFragment("SELECT EntityId FROM ").append(CoreSchema.getInstance().getMappedDirectories(), "md");
+        return new SQLFragment("SELECT EntityId, Name AS Description FROM ").append(CoreSchema.getInstance().getMappedDirectories(), "md");
     }
 }

api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java

@@ -26,7 +26,6 @@
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.SchemaKey;
 import org.labkey.api.query.TableSorter;
-import org.labkey.api.util.ConfigurationException;
 import org.labkey.api.util.GUID;
 import org.labkey.api.util.StringUtilsLabKey;
 import org.labkey.api.util.logging.LogHelper;
@@ -249,20 +248,14 @@ public Collection<AttachmentParentType> copyAttachments(DatabaseMigrationConfigu
         // requires querying and re-filtering the source tables instead.
         Collection<AttachmentParentType> ret = new LinkedList<>();
 
+        // TODO: Select ParentType as well to avoid duplication
         getAttachmentTypes().forEach(type -> {
             SQLFragment sql = type.getSelectParentEntityIdsSql();
-            if (sql != null)
-            {
-                Collection<String> entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class);
-                SQLFragment selectParents = new SQLFragment("Parent");
-                // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator
-                sourceSchema.getSqlDialect().appendInClauseSqlWithCustomInClauseGenerator(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope()));
-                ret.addAll(copyAttachments(configuration, new SQLClause(selectParents), type));
-            }
-            else
-            {
-                throw new ConfigurationException("AttachmentType \"" + type.getUniqueName() + "\" is not configured to find parent EntityIds!");
-            }
+            Collection<String> entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class);
+            SQLFragment selectParents = new SQLFragment("Parent");
+            // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator
+            sourceSchema.getSqlDialect().appendInClauseSqlWithCustomInClauseGenerator(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope()));
+            ret.addAll(copyAttachments(configuration, new SQLClause(selectParents), type));
         });
 
         return ret;

api/src/org/labkey/api/query/QueryView.java

@@ -143,7 +143,7 @@
 
 /**
  * View that generates the majority of standard data grids/tables in the LabKey Server UI.
- * The backing query is lazily invoked when it comes times to render the QueryView.
+ * The backing query is lazily invoked when it comes time to render the QueryView.
  */
 public class QueryView extends WebPartView<Object> implements ContainerUser
 {

api/src/org/labkey/api/reports/report/ReportType.java

@@ -19,6 +19,7 @@
 import org.labkey.api.attachments.AttachmentParentType;
 import org.labkey.api.data.CoreSchema;
 import org.labkey.api.data.SQLFragment;
+import org.labkey.api.data.TableInfo;
 
 public class ReportType implements AttachmentParentType
 {
@@ -40,8 +41,12 @@ private ReportType()
     }
 
     @Override
-    public @NotNull SQLFragment getSelectParentEntityIdsSql()
+    public @NotNull SQLFragment getSelectEntityIdAndDescriptionSql()
     {
-        return new SQLFragment("SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoReport(), "reports");
+        TableInfo table = CoreSchema.getInstance().getTableInfoReport();
+        return new SQLFragment("SELECT EntityId, ")
+            .append(table.getSqlDialect().concatenate("ReportKey", "':'", "CAST(RowId AS VARCHAR)"))
+            .append(" AS Description FROM ")
+            .append(table, "reports");
     }
 }