Issue 695: Add feature to allow unreferenced files to be deleted from apps

api/src/org/labkey/api/exp/query/ExpDataTable.java

@@ -35,6 +35,7 @@ enum Column
         SourceProtocolApplication,
         SourceApplicationInput,
         DataFileUrl,
+        ReferenceCount,
         Run,
         RunApplication,
         RunApplicationOutput,

api/src/org/labkey/api/exp/query/ExpSchema.java

@@ -47,6 +47,7 @@
 import org.labkey.api.security.User;
 import org.labkey.api.security.permissions.InsertPermission;
 import org.labkey.api.security.permissions.ReadPermission;
+import org.labkey.api.settings.AppProps;
 import org.labkey.api.util.StringExpression;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.ViewContext;
@@ -70,6 +71,7 @@ public class ExpSchema extends AbstractExpSchema
     public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType";
     public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType";
     public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits";
+    public static final String SAMPLE_FILES_TABLE = "UnreferencedSampleFiles";
 
     public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME);
     public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name());
@@ -220,6 +222,20 @@ public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFil
                 return expSchema.setupTable(result);
             }
         },
+        UnreferencedSampleFiles
+        {
+             @Override
+             public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf)
+             {
+                 return new ExpUnreferencedSampleFilesTable(expSchema, cf);
+             }
+
+            @Override
+            public boolean includeTable()
+            {
+                return AppProps.getInstance().isOptionalFeatureEnabled(SAMPLE_FILES_TABLE);
+            }
+        },
         SampleStatus
         {
             @Override

api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java

@@ -0,0 +1,114 @@
+package org.labkey.api.exp.query;
+
+import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.data.BaseColumnInfo;
+import org.labkey.api.data.ContainerFilter;
+import org.labkey.api.data.CoreSchema;
+import org.labkey.api.data.JdbcType;
+import org.labkey.api.data.SQLFragment;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.VirtualTable;
+import org.labkey.api.exp.api.ExperimentService;
+import org.labkey.api.files.FileContentService;
+import org.labkey.api.query.FilteredTable;
+import org.labkey.api.query.UserIdQueryForeignKey;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.query.column.BuiltInColumnTypes;
+
+public class ExpUnreferencedSampleFilesTable extends FilteredTable<ExpSchema>
+{
+    public ExpUnreferencedSampleFilesTable(@NotNull ExpSchema schema, ContainerFilter cf)
+    {
+        super(createVirtualTable(schema), schema, cf);
+        setDescription("Contains all sample files that are not referenced by any domain fields.");
+        wrapAllColumns(true);
+    }
+
+    private static TableInfo createVirtualTable(@NotNull ExpSchema schema)
+    {
+        return new ExpUnreferencedSampleFilesTable.FileUnionTable(schema);
+    }
+
+    private static class FileUnionTable extends VirtualTable<ExpSchema>
+    {
+        private final SQLFragment _query;
+
+        public FileUnionTable(@NotNull ExpSchema schema)
+        {
+            super(CoreSchema.getInstance().getSchema(), ExpSchema.SAMPLE_FILES_TABLE, schema);
+
+            FileContentService svc = FileContentService.get();
+
+            _query = new SQLFragment();
+            if (svc == null)
+                return;
+
+            SQLFragment listQuery = svc.listSampleFilesQuery(schema.getUser());
+            if (StringUtils.isEmpty(listQuery))
+                return;
+
+            TableInfo expDataTable = ExperimentService.get().getTinfoData();
+            TableInfo materialTable = ExperimentService.get().getTinfoMaterial();
+
+            _query.appendComment("<SampleFileListTableInfo>", getSchema().getSqlDialect());
+
+            SQLFragment sampleFileSql = new SQLFragment("SELECT m.Container, if.FilePathShort \n")
+                    .append("FROM (")
+                    .append(svc.listSampleFilesQuery(schema.getUser()))
+                    .append(") AS if \n")
+                    .append("JOIN ")
+                    .append(materialTable, "m")
+                    .append(" ON if.SourceKey = m.RowId");
+
+            SQLFragment unreferencedFileSql = new SQLFragment("SELECT ed.rowId, ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ")
+                    .append(expDataTable, "ed")
+                    .append(" LEFT JOIN (")
+                    .append(sampleFileSql)
+                    .append(" ) sf\n")
+                    .append(" ON ed.name = sf.FilePathShort AND ed.container = sf.container\n")
+                    .append(" WHERE ed.datafileurl LIKE ")
+                    .appendValue("%@files/sampletype/%")
+                    .append(" AND sf.FilePathShort IS NULL");
+
+            _query.append(unreferencedFileSql);
+
+            _query.appendComment("</SampleFileListTableInfo>", getSchema().getSqlDialect());
+
+            var rowIdCol = new BaseColumnInfo("RowId", this, JdbcType.INTEGER);
+            rowIdCol.setHidden(true);
+            rowIdCol.setKeyField(true);
+            addColumn(rowIdCol);
+
+            var fileNameCol = new BaseColumnInfo("FileName", this, JdbcType.VARCHAR);
+            addColumn(fileNameCol);
+
+            if (schema.getUser().hasApplicationAdminPermission())
+            {
+                var filePathCol = new BaseColumnInfo("DataFileUrl", this, JdbcType.VARCHAR);
+                filePathCol.setHidden(true);
+                addColumn(filePathCol);
+            }
+
+            var containerCol = new BaseColumnInfo("Container", this, JdbcType.VARCHAR);
+            containerCol.setConceptURI(BuiltInColumnTypes.CONTAINERID_CONCEPT_URI);
+            addColumn(containerCol);
+
+            var createdCol = new BaseColumnInfo("Created", this, JdbcType.DATE);
+            addColumn(createdCol);
+
+            var createdByCol = new BaseColumnInfo("CreatedBy", this, JdbcType.INTEGER);
+            createdByCol.setFk(new UserIdQueryForeignKey(getUserSchema(), true));
+            addColumn(createdByCol);
+        }
+
+        @NotNull
+        @Override
+        public SQLFragment getFromSQL()
+        {
+            return _query;
+        }
+
+    }
+
+}

api/src/org/labkey/api/files/FileContentService.java

@@ -325,6 +325,8 @@ default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @N
      */
     SQLFragment listFilesQuery(@NotNull User currentUser);
 
+    SQLFragment listSampleFilesQuery(@NotNull User currentUser);
+
     void setWebfilesEnabled(boolean enabled, User user);
 
     /**

api/src/org/labkey/api/files/FileListener.java

@@ -94,4 +94,9 @@ default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable C
      * </ul>
      */
     SQLFragment listFilesQuery();
+
+    @Nullable default SQLFragment listSampleFilesQuery()
+    {
+        return null;
+    }
 }

api/src/org/labkey/api/files/TableUpdaterFileListener.java

@@ -353,10 +353,10 @@ public Collection<File> listFiles(@Nullable Container container)
     @Override
     public SQLFragment listFilesQuery()
     {
-        return listFilesQuery(false, null);
+        return listFilesQuery(false, null, false);
     }
 
-    public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
+    public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath, boolean extractName)
     {
         SQLFragment selectFrag = new SQLFragment();
         selectFrag.append("SELECT\n");
@@ -395,6 +395,16 @@ else if (_table.getColumn("Folder") != null)
 
         selectFrag.append("  ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n");
 
+        if (extractName)
+        {
+            SqlDialect dialect = _table.getSchema().getSqlDialect();
+            SQLFragment fileNameFrag = new SQLFragment();
+            fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", ");
+            fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", ");
+            fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");;
+            selectFrag.append("  ").append(fileNameFrag).append(" AS FilePathShort,\n");
+        }
+
         if (_keyColumn != null)
             selectFrag.append("  ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n");
         else
@@ -408,6 +418,8 @@ else if (_table.getColumn("Folder") != null)
 
         if (StringUtils.isEmpty(filePath))
             selectFrag.append(" IS NOT NULL\n");
+        else if (filePath instanceof SQLFragment)
+            selectFrag.append(" = ").append(filePath).append("\n");
         else
             selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n");
 

experiment/src/org/labkey/experiment/ExperimentModule.java

@@ -18,6 +18,7 @@
 import org.apache.commons.lang3.math.NumberUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.json.JSONObject;
 import org.labkey.api.admin.FolderSerializationRegistry;
 import org.labkey.api.assay.AssayProvider;
 import org.labkey.api.assay.AssayService;
@@ -112,6 +113,7 @@
 import org.labkey.api.vocabulary.security.DesignVocabularyPermission;
 import org.labkey.api.webdav.WebdavResource;
 import org.labkey.api.webdav.WebdavService;
+import org.labkey.api.writer.ContainerUser;
 import org.labkey.experiment.api.DataClassDomainKind;
 import org.labkey.experiment.api.ExpDataClassImpl;
 import org.labkey.experiment.api.ExpDataClassTableImpl;
@@ -181,6 +183,7 @@
 import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI;
 import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI;
 import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME;
+import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE;
 
 public class ExperimentModule extends SpringModule
 {
@@ -266,6 +269,9 @@ protected void init()
         }
         else
         {
+            OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files",
+                    "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false);
+
             OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression",
                 "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true);
         }
@@ -1117,4 +1123,12 @@ public Collection<String> getProvisionedSchemaNames()
     {
         return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME);
     }
+
+    @Override
+    public JSONObject getPageContextJson(ContainerUser context)
+    {
+        JSONObject json = super.getPageContextJson(context);
+        json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE));
+        return json;
+    }
 }

experiment/src/org/labkey/experiment/FileLinkFileListener.java

@@ -275,7 +275,7 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified)
         return listFilesQuery(skipCreatedModified, null);
     }
 
-    public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
+    public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath)
     {
         final SQLFragment frag = new SQLFragment();
 
@@ -298,8 +298,11 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
         frag.append("WHERE\n");
         if (StringUtils.isEmpty(filePath))
             frag.append("  op.StringValue IS NOT NULL AND\n");
+        else if (filePath instanceof SQLFragment)
+            frag.append("  op.StringValue = ").append(filePath).append(" AND\n");
         else
             frag.append("  op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n");
+
         frag.append("  o.ObjectId = op.ObjectId AND\n");
         frag.append("  PropertyId IN (\n");
         frag.append("    SELECT PropertyId\n");
@@ -311,7 +314,26 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
             SQLFragment containerFrag = new SQLFragment("?", containerId);
             TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag);
             frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n");
-            frag.append(updater.listFilesQuery(skipCreatedModified, filePath));
+            frag.append(updater.listFilesQuery(skipCreatedModified, filePath, false));
+        });
+
+        return frag;
+    }
+
+    @Override
+    public SQLFragment listSampleFilesQuery()
+    {
+        final SQLFragment frag = new SQLFragment();
+
+        hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> {
+            if (PROVISIONED_SCHEMA_NAME.equalsIgnoreCase(schema.getName()))
+            {
+                SQLFragment containerFrag = new SQLFragment("?", containerId);
+                TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag);
+                if (!frag.isEmpty())
+                    frag.append("UNION").append("").append("\n");
+                frag.append(updater.listFilesQuery(true, null, true));
+            }
         });
 
         return frag;