Allow table fields to reference process queries, but only for single record questions

Model/src/main/java/org/gusdb/wdk/core/api/JsonKeys.java

@@ -91,6 +91,7 @@ public class JsonKeys {
   public static final String IS_AVAILABLE = "isAvailable";
   public static final String NEW_BUILD = "newBuild";
   public static final String REVISE_BUILD = "reviseBuild";
+  public static final String SINGLE_RECORD_ONLY = "supportsSingleRecordOnly";
 
   // patch command keys
   public static final String DELETE = "delete";

Model/src/main/java/org/gusdb/wdk/model/WdkModelException.java

@@ -1,5 +1,12 @@
 package org.gusdb.wdk.model;
 
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import org.gusdb.fgputil.functional.FunctionalInterfaces.ConsumerWithException;
+import org.gusdb.fgputil.functional.FunctionalInterfaces.Procedure;
+import org.gusdb.fgputil.functional.FunctionalInterfaces.SupplierWithException;
+
 /**
  * This exception should be thrown out if the cause is not related to user's
  * input. For example, the cause of the exception can be a mistake in the model
@@ -43,6 +50,64 @@ public static WdkModelException translateFrom(Exception e, String newMessage) {
     return (t instanceof WdkModelException ? (WdkModelException)t : new WdkModelException(newMessage, t));
   }
 
+  /**
+   * Meant to be used with the Supplier version of unwrap().  This function will
+   * take a SupplierWithException, catch the exception and wrap it in a Runtime
+   * Exception, returning a Supplier appropriate for "naked" lambdas.
+   *
+   * @param <T> type returned by the returned supplier
+   * @param supplier supplier with exception
+   * @return supplier without exception (wrapped in runtime exception)
+   */
+  public static <T> T wrap(SupplierWithException<T> supplier) {
+    try {
+      return supplier.get();
+    }
+    catch (Exception e) {
+      throw (e instanceof RuntimeException) ? (RuntimeException)e : new WdkRuntimeException(e);
+    }
+  }
+
+  /**
+   * Meant to be used with wrap().  This function takes a non-throwing Supplier,
+   * calls it, catches any RuntimeExceptions, and throws a WdkModelException
+   * that is either 1) the underlying cause, if the cause is a WdkModelException,
+   * or 2) a new WdkModelException wrapping the exception.
+   *
+   * @param <T> type returned by the supplier
+   * @param supplier a supplier
+   * @return the supplied value
+   * @throws WdkModelException if an exception occurs
+   */
+  public static <T> T unwrap(Supplier<T> supplier) throws WdkModelException {
+    try {
+      return supplier.get();
+    }
+    catch(Exception e) {
+      return unwrap(e);
+    }
+  }
+
+  public static <T> Consumer<T> wrap(ConsumerWithException<T> consumer) {
+    return val -> {
+      try {
+        consumer.accept(val);
+      }
+      catch (Exception e) {
+        throw (e instanceof RuntimeException) ? (RuntimeException)e : new WdkRuntimeException(e);
+      }
+    };
+  }
+
+  public static <T> void unwrap(Procedure p) throws WdkModelException {
+    try {
+      p.perform();
+    }
+    catch (Exception e) {
+      unwrap(e);
+    }
+  }
+
   public static <T> T unwrap(Exception e) throws WdkModelException {
     throw translateFrom(e, e.getMessage());
   }

Model/src/main/java/org/gusdb/wdk/model/analysis/ExternalAnalyzer.java

@@ -26,6 +26,7 @@
 import org.gusdb.wdk.model.WdkModelException;
 import org.gusdb.wdk.model.WdkUserException;
 import org.gusdb.wdk.model.answer.AnswerValue;
+import org.gusdb.wdk.model.question.Question;
 import org.gusdb.wdk.model.record.Field;
 import org.gusdb.wdk.model.record.RecordClass;
 import org.gusdb.wdk.model.record.TableField;
@@ -150,7 +151,7 @@ public ExecutionStatus runAnalysis(AnswerValue answerValue, StatusLogger log)
 
     // if hasHeader and display value map requested, dump
     if (hasHeader && determineBooleanProperty(DUMP_HEADER_DISPLAY_MAP_PROP_KEY, DUMP_HEADER_DISPLAY_MAP_BY_DEFAULT)) {
-      dumpHeaderDisplayMap(answerValue.getQuestion().getRecordClass(), storageDir);
+      dumpHeaderDisplayMap(answerValue.getQuestion(), storageDir);
     }
 
     try {
@@ -197,7 +198,8 @@ protected static void writeContentToFile(String storageDir, String fileName, Str
     }
   }
 
-  private void dumpHeaderDisplayMap(RecordClass recordClass, String storageDir) throws WdkModelException {
+  private void dumpHeaderDisplayMap(Question question, String storageDir) throws WdkModelException {
+    RecordClass recordClass = question.getRecordClass();
     List<String> attributeNames = getConfiguredFields(EXTRACTED_ATTRIBS_PROP_KEY);
     List<String> tableNames = getConfiguredFields(EXTRACTED_TABLES_PROP_KEY);
     File mappingOutFile = Paths.get(storageDir, HEADER_MAPPING_FILE_NAME).toFile();
@@ -212,7 +214,7 @@ private void dumpHeaderDisplayMap(RecordClass recordClass, String storageDir) th
         writeField(out, attr.get(), "");
       }
       for (String tableName : tableNames) {
-        TableField table = recordClass.getTableFieldMap().get(tableName);
+        TableField table = question.getTableFieldMap().get(tableName);
         if (table == null) {
           LOG.warn("Table '" + tableName + "', specified in analysis plugin, is not valid for record class '" + recordClass.getFullName() + "'.");
           continue;

Model/src/main/java/org/gusdb/wdk/model/answer/AnswerValue.java

@@ -368,14 +368,18 @@ public String getAnswerTableSql(Query tableQuery)
 
   public String getTableFieldResultSql(TableField tableField) throws WdkModelException {
     // has to get a clean copy of the attribute query, without pk params appended
-    Query tableQuery = tableField.getUnwrappedQuery();
+    Query tableQuery = tableField.getQuery().leftOrElseThrow(() ->
+        new WdkModelException("Table field " + tableField.getFullName() +
+            " does not reference a SqlQuery, required for this function."))
+        .getUnwrappedQuery();
 
     // get and run the paged table query sql
     LOG.debug("AnswerValue: getTableFieldResultSql(): going to getPagedTableSql()");
 
     return getAnswerTableSql(tableQuery);
   }
 
+  // NOTE: this method is overridden in SingleRecordAnswerValue to support tables with process queries; can assume SQL queries here
   public ResultList getTableFieldResultList(TableField tableField) throws WdkModelException {
     return getTableFieldResultList(tableField, getTableFieldResultSql(tableField));
   }
@@ -388,7 +392,7 @@ public ResultList getTableFieldResultList(TableField tableField, String customSq
     ResultSet resultSet = null;
     try {
       LOG.debug("AnswerValue: getTableFieldResultList(): returning SQL for TableField '" + tableField.getName() + "': \n" + customSql);
-      resultSet = SqlUtils.executeQuery(dataSource, customSql, tableField.getUnwrappedQuery().getFullName() + "_table");
+      resultSet = SqlUtils.executeQuery(dataSource, customSql, tableField.getQueryFullName() + "_table");
     }
     catch (SQLException e) {
       throw new WdkModelException(e);

Model/src/main/java/org/gusdb/wdk/model/answer/TableFieldProcessQueryResult.java

@@ -0,0 +1,44 @@
+package org.gusdb.wdk.model.answer;
+
+import java.util.Map;
+
+import org.gusdb.wdk.model.WdkModelException;
+import org.gusdb.wdk.model.answer.single.SingleRecordQuestionParam;
+import org.gusdb.wdk.model.dbms.ArrayResultList;
+import org.gusdb.wdk.model.query.ProcessQuery;
+import org.gusdb.wdk.model.query.ProcessQueryInstance;
+import org.gusdb.wdk.model.query.spec.QueryInstanceSpec;
+import org.gusdb.wdk.model.record.TableField;
+import org.gusdb.wdk.model.user.StepContainer;
+import org.gusdb.wdk.model.user.User;
+import org.json.JSONArray;
+
+public class TableFieldProcessQueryResult {
+
+  public static ArrayResultList getResultList(User user, TableField tableField, Map<String, Object> _pkMap) throws WdkModelException {
+
+    if (tableField.hasSqlQuery()) {
+      throw new WdkModelException("Table field " + tableField.getFullName() + " does not reference a ProcessQuery, required for this function.");
+    }
+
+    // create param map
+    Map<String,String> params = Map.of(
+
+        // add primary key param
+        SingleRecordQuestionParam.PRIMARY_KEY_PARAM_NAME,
+        new JSONArray(_pkMap.values()).toString(),
+
+        // add table name param
+        TableField.TABLE_NAME_PARAM_NAME,
+        tableField.getName()
+    );
+
+    // create process query instance to fetch results
+    ProcessQueryInstance queryInstance = (ProcessQueryInstance)ProcessQuery.makeQueryInstance(
+      QueryInstanceSpec.builder().putAll(params)
+        .buildRunnable(user, tableField.getQuery().rightOrElseThrow(WdkModelException::new), StepContainer.emptyContainer()));
+
+    return queryInstance.getUncachedResults();
+  }
+
+}

Model/src/main/java/org/gusdb/wdk/model/answer/factory/DynamicRecordInstanceList.java

@@ -212,28 +212,30 @@ public void integrateTableQuery(TableField tableField) throws WdkModelException,
 
     // make table values
     PrimaryKeyDefinition pkDef = _answerValue.getQuestion().getRecordClass().getPrimaryKeyDefinition();
-    Query tableQuery = tableField.getWrappedQuery();
 
     while (resultList.next()) {
       PrimaryKeyValue primaryKey = new PrimaryKeyValue(pkDef, resultList);
       DynamicRecordInstance record = get(primaryKey);
 
       if (record == null) {
-        StringBuffer error = new StringBuffer();
-        error.append("Paged table query [" + tableQuery.getFullName());
+        StringBuilder error = new StringBuilder();
+        error.append("Paged table query [" + tableField.getQueryFullName());
         error.append("] returned rows that doesn't match the paged ");
         error.append("records. (");
         error.append(primaryKey.getValuesAsString());
-        error.append(").\nPaged table SQL:\n" + _answerValue.getAnswerTableSql(tableQuery));
-        error.append("\n" + "Paged ID SQL:\n" + _answerValue.getPagedIdSql());
+        if (tableField.hasSqlQuery()) {
+          Query tableQuery = tableField.getQuery().getLeft().getWrappedQuery();
+          error.append(").\nPaged table SQL:\n" + _answerValue.getAnswerTableSql(tableQuery));
+          error.append("\n" + "Paged ID SQL:\n" + _answerValue.getPagedIdSql());
+        }
         throw new WdkModelException(error.toString());
       }
 
       TableValue tableValue = record.getTableValue(tableField.getName());
       // initialize a row in table value
       tableValue.initializeRow(resultList);
     }
-    LOG.debug("Table query [" + tableQuery + "] integrated.");
+    LOG.debug("Table query [" + tableField.getQueryFullName() + "] integrated.");
   }
 
 }

Model/src/main/java/org/gusdb/wdk/model/answer/single/SingleRecordAnswerValue.java

@@ -17,11 +17,14 @@
 import org.gusdb.wdk.model.WdkUserException;
 import org.gusdb.wdk.model.answer.AnswerValue;
 import org.gusdb.wdk.model.answer.ResultSizeFactory;
+import org.gusdb.wdk.model.answer.TableFieldProcessQueryResult;
 import org.gusdb.wdk.model.answer.spec.AnswerSpec;
+import org.gusdb.wdk.model.dbms.ResultList;
 import org.gusdb.wdk.model.record.DynamicRecordInstance;
 import org.gusdb.wdk.model.record.PrimaryKeyIterator;
 import org.gusdb.wdk.model.record.RecordClass;
 import org.gusdb.wdk.model.record.RecordInstance;
+import org.gusdb.wdk.model.record.TableField;
 
 public class SingleRecordAnswerValue extends AnswerValue {
 
@@ -151,4 +154,14 @@ public boolean cacheInitiallyExistedForSpec() throws WdkModelException {
     // does not use WDK cache
     return false;
   }
+
+  @Override
+  public ResultList getTableFieldResultList(TableField tableField) throws WdkModelException {
+    if (tableField.hasSqlQuery()) {
+      return super.getTableFieldResultList(tableField);
+    }
+    else {
+      return TableFieldProcessQueryResult.getResultList(_requestingUser, tableField, _pkMap);
+    }
+  }
 }

Model/src/main/java/org/gusdb/wdk/model/answer/single/SingleRecordQuestion.java

@@ -10,6 +10,7 @@
 import org.gusdb.wdk.model.question.DynamicAttributeSet;
 import org.gusdb.wdk.model.question.Question;
 import org.gusdb.wdk.model.record.RecordClass;
+import org.gusdb.wdk.model.record.TableField;
 
 public class SingleRecordQuestion extends Question {
 
@@ -67,4 +68,9 @@ public boolean isBoolean() {
   public SingleRecordQuestionParam getParam() {
     return _param;
   }
+
+  @Override
+  public Map<String, TableField> getTableFieldMap() {
+    return getRecordClass().getTableFieldMap(true);
+  }
 }

Model/src/main/java/org/gusdb/wdk/model/answer/stream/FileBasedRecordStream.java

@@ -400,7 +400,7 @@ private static Map<Path, TwoTuple<TableField,List<String>>> assembleTableFiles(A
    */
   private static TwoTuple<Path,List<String>> writeTableFile(AnswerValue answerValue, Path tempDir, TableField table) throws WdkModelException {
     Timer t = new Timer();
-    LOG.debug("writeTableFile(): Starting table: " + table.getName() + "(query: " + table.getWrappedQuery().getName() + ")");
+    LOG.debug("writeTableFile(): Starting table: " + table.getName() + "(query: " + table.getQueryFullName() + ")");
 
     // Appending table designation to query name for file to more easily distinguish
     // these files from those supporting attribute queries and to avoid name collisions.
@@ -413,9 +413,9 @@ private static TwoTuple<Path,List<String>> writeTableFile(AnswerValue answerValu
       resultList = answerValue.getTableFieldResultList(table);
 
       // Transfer the result list content to the CSV file provided.
-      LOG.debug("writeTableFile(): Starting iteration over result list for query " + table.getWrappedQuery().getName() + ": " + t.getElapsedString());
+      LOG.debug("writeTableFile(): Starting iteration over result list for query " + table.getQueryFullName() + ": " + t.getElapsedString());
       assembleCsvFile(filePath, columnNames, resultList);
-      LOG.debug("writeTableFile(): Finished iteration over result list for query " + table.getWrappedQuery().getName() + ": " + t.getElapsedString());
+      LOG.debug("writeTableFile(): Finished iteration over result list for query " + table.getQueryFullName() + ": " + t.getElapsedString());
 
       // open file permissions and return the path to the temporary CSV file
       filePath.toFile().setWritable(true, false);

Model/src/main/java/org/gusdb/wdk/model/dbms/ArrayResultList.java

@@ -29,8 +29,11 @@ public class ArrayResultList implements ResultList, WsfResponseListener {
    * @throws WdkModelException
    *           if the result has fewer columns than the column definition
    */
-  public ArrayResultList(Map<String, Integer> columns) throws WdkModelException {
-    _columns = new LinkedHashMap<String, Integer>(columns);
+  public ArrayResultList(List<String> columns) throws WdkModelException {
+    _columns = new LinkedHashMap<String, Integer>();
+    for (int i = 0; i < columns.size(); i++) {
+      _columns.put(columns.get(i), i);
+    }
     _rows = new ArrayList<>();
     _attachments = new LinkedHashMap<>();
     _rowIndex = -1;

Model/src/main/java/org/gusdb/wdk/model/query/ProcessQueryInstance.java

@@ -11,6 +11,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.log4j.Logger;
 import org.gusdb.fgputil.db.SqlUtils;
@@ -21,6 +22,7 @@
 import org.gusdb.wdk.model.Utilities;
 import org.gusdb.wdk.model.WdkDelayedResultException;
 import org.gusdb.wdk.model.WdkModelException;
+import org.gusdb.wdk.model.dbms.ArrayResultList;
 import org.gusdb.wdk.model.dbms.CacheFactory;
 import org.gusdb.wdk.model.dbms.ResultList;
 import org.gusdb.wdk.model.query.spec.QueryInstanceSpec;
@@ -223,6 +225,13 @@ private String buildCacheInsertSql(
     return sql.append(")").toString();
   }
 
+  public ArrayResultList getUncachedResults() throws WdkModelException {
+    List<String> columns = Arrays.stream(_query.getColumns()).map(Column::getName).collect(Collectors.toList());
+    ArrayResultList resultsListener = new ArrayResultList(columns);
+    invokeWsf(resultsListener);
+    return resultsListener;
+  }
+
   private void invokeWsf(WsfResponseListener listener) throws WdkModelException {
     long start = System.currentTimeMillis();
 

Model/src/main/java/org/gusdb/wdk/model/question/Question.java

@@ -305,7 +305,7 @@ public void addDynamicAttributeSet(DynamicAttributeSet dynamicAttributes) {
   public Map<String, Field> getFields() {
     Map<String, Field> fields = new LinkedHashMap<>();
     Map<String, AttributeField> attributes = getAttributeFieldMap();
-    Map<String, TableField> tables = _recordClass.getTableFieldMap();
+    Map<String, TableField> tables = getTableFieldMap();
 
     fields.putAll(attributes);
     fields.putAll(tables);
@@ -1134,4 +1134,8 @@ public String getNameForLogging() {
     return getFullName();
   }
 
+  public Map<String, TableField> getTableFieldMap() {
+    return getRecordClass().getTableFieldMap(false);
+  }
+
 }

Model/src/main/java/org/gusdb/wdk/model/record/DynamicRecordInstance.java

@@ -60,7 +60,7 @@ public DynamicRecordInstance(User user, Question question, DynamicRecordInstance
 
   @Override
   protected Collection<TableField> getAvailableTableFields() {
-    return _recordClass.getTableFieldMap().values();
+    return _recordClass.getTableFieldMap(true).values();
   }
 
   private void fillColumnAttributeValues(Query attributeQuery)