Migrate Parquet support to Mason package

dep/checkArrow.chpl

@@ -1,33 +1,6 @@
-use IO;
-use CTypes;
-
-require "../src/parquet/WriteParquet.h";
-require "../src/WriteParquet.o";
-require "../src/parquet/ReadParquet.h";
-require "../src/ReadParquet.o";
-require "../src/parquet/UtilParquet.h";
-require "../src/UtilParquet.o";
-
-proc getVersionInfo() {
-  extern proc c_getVersionInfo(): c_ptrConst(c_char);
-  extern proc strlen(str): c_int;
-  extern proc c_free_string(ptr);
-  var cVersionString = c_getVersionInfo();
-  defer {
-    c_free_string(cVersionString: c_ptr(void));
-  }
-  var ret: string;
-  try {
-    ret = string.createCopyingBuffer(cVersionString,
-                              strlen(cVersionString));
-  } catch e {
-    ret = "Error converting Arrow version message to Chapel string";
-  }
-  return ret;
-}
+use Parquet;
 
 proc main() {
-  var ArrowVersion = getVersionInfo();
-  writeln("Found Arrow version: ", ArrowVersion);
+  writeln("Found Arrow version: ", getVersionInfo());
   return 0;
 }

make/Arkouda.mk

@@ -46,15 +46,16 @@ endif
 
 SERVER_CONFIG_SCRIPT=$(ARKOUDA_SOURCE_DIR)/parseServerConfig.py
 # This is the main compilation statement section
-$(ARKOUDA_MAIN_MODULE): check-deps register-commands $(ARROW_UTIL_O) $(ARROW_READ_O) $(ARROW_WRITE_O) $(ARKOUDA_SOURCES) $(ARKOUDA_MAKEFILES)
+$(ARKOUDA_MAIN_MODULE): check-deps register-commands $(ARKOUDA_SOURCES) $(ARKOUDA_MAKEFILES)
 	$(eval MOD_GEN_OUT=$(shell python3 $(SERVER_CONFIG_SCRIPT) $(ARKOUDA_CONFIG_FILE) $(ARKOUDA_SOURCE_DIR)))
 
-	$(CHPL) $(CHPL_DEBUG_FLAGS) $(CHPL_USER_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(CHPL_COMPAT_FLAGS) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) $(ARKOUDA_SERVER_USER_MODULES) $(MOD_GEN_OUT) $(ARKOUDA_RW_DEFAULT_FLAG) $(ARKOUDA_KEYPART_FLAG) $(ARKOUDA_REGISTRY_DIR)/Commands.chpl -I$(ARKOUDA_SOURCE_DIR)/parquet -o $@
+	PARQUET_PKG_FLAGS="$$($(PARQUET_PACKAGE_FLAGS_CMD))" && \
+	$(CHPL) $(CHPL_DEBUG_FLAGS) $(CHPL_USER_FLAGS) $(PRINT_PASSES_FLAGS) $(REGEX_MAX_CAPTURES_FLAG) $(OPTIONAL_SERVER_FLAGS) $(CHPL_FLAGS_WITH_VERSION) $(CHPL_COMPAT_FLAGS) $(ARKOUDA_MAIN_SOURCE) $(ARKOUDA_COMPAT_MODULES) $(ARKOUDA_SERVER_USER_MODULES) $(MOD_GEN_OUT) $(ARKOUDA_RW_DEFAULT_FLAG) $(ARKOUDA_KEYPART_FLAG) $(ARKOUDA_REGISTRY_DIR)/Commands.chpl $$PARQUET_PKG_FLAGS -o $@
 
 CLEAN_TARGETS += arkouda-clean
 .PHONY: arkouda-clean
 arkouda-clean:
-	$(RM) $(ARKOUDA_MAIN_MODULE) $(ARKOUDA_MAIN_MODULE)_real $(ARROW_UTIL_O) $(ARROW_READ_O) $(ARROW_WRITE_O)
+	$(RM) $(ARKOUDA_MAIN_MODULE) $(ARKOUDA_MAIN_MODULE)_real
 
 .PHONY: tags
 tags:

make/Prologue.mk

@@ -5,10 +5,10 @@ include make/prologue/core.mk
 include make/prologue/chapel.mk
 include make/prologue/paths.mk
 include make/prologue/config.mk
-include make/prologue/arrow_shims.mk
 
 # Dependency meta-targets and per-dependency rules
 include make/deps/common.mk
+include make/deps/parquet.mk
 include make/deps/zmq.mk
 include make/deps/hdf5.mk
 include make/deps/arrow.mk

make/deps/parquet.mk

@@ -0,0 +1,12 @@
+# External Chapel `Parquet` Mason package integration.
+#
+# get_parquet_package.sh clones/builds the package and prints the chpl arguments
+# (C++ prerequisite headers/objects, include paths, Arrow link flags) plus the
+# module source path. We use this instead of `mason modules`, which does not
+# work reliably with Chapel 2.4. Override PARQUET_INSTALL_DIR to relocate the
+# clone; set ARKOUDA_PARQUET_SRC_DIR to reuse an existing checkout.
+PARQUET_PACKAGE_SCRIPT := $(ARKOUDA_PROJECT_DIR)/scripts/get_parquet_package.sh
+PARQUET_INSTALL_DIR ?= $(DEP_BUILD_DIR)/Parquet
+
+PARQUET_PACKAGE_FLAGS_CMD = env ARKOUDA_CHPL_HOME="$(ARKOUDA_CHPL_HOME)" \
+	"$(PARQUET_PACKAGE_SCRIPT)" "$(PARQUET_INSTALL_DIR)"

make/prologue/checks.mk

@@ -35,15 +35,14 @@ check-re2: $(RE2_CHECK)
 	@rm -f $(DEP_INSTALL_DIR)/$@ $(DEP_INSTALL_DIR)/$@_real
 
 ARROW_CHECK = $(DEP_INSTALL_DIR)/checkArrow.chpl
-check-arrow: $(ARROW_CHECK) $(ARROW_UTIL_O) $(ARROW_READ_O) $(ARROW_WRITE_O)
+check-arrow: $(ARROW_CHECK)
 	@echo "Checking for Arrow"
-	$(MAKE) compile-arrow-cpp --no-print-directory
-	@$(CHPL) $(CHPL_FLAGS) $(ARKOUDA_COMPAT_MODULES) $< \
-		$(ARROW_M) -M $(ARKOUDA_SOURCE_DIR) \
-		-I $(ARKOUDA_SOURCE_DIR)/parquet \
+	@PARQUET_PKG_FLAGS="$$($(PARQUET_PACKAGE_FLAGS_CMD))" && \
+		$(CHPL) $(CHPL_FLAGS) $(ARKOUDA_COMPAT_MODULES) $< \
+		$$PARQUET_PKG_FLAGS \
 		-o $(DEP_INSTALL_DIR)/$@ && \
-		([ $$? -eq 0 ] && echo "Success compiling program") || \
-		echo "\nERROR: Please ensure that dependencies have been installed correctly (see -> https://github.com/Bears-R-Us/arkouda/blob/main/pydoc/setup/BUILD.md)\n"
+		echo "Success compiling program" || \
+		{ echo "\nERROR: Please ensure that dependencies have been installed correctly (see -> https://github.com/Bears-R-Us/arkouda/blob/main/pydoc/setup/BUILD.md)\n"; exit 1; }
 	$(DEP_INSTALL_DIR)/$@ -nl 1
 	@rm -f $(DEP_INSTALL_DIR)/$@ $(DEP_INSTALL_DIR)/$@_real
 

make/prologue/paths.mk

@@ -9,10 +9,20 @@ ifneq ("$(wildcard $(1)/lib64)","")
 CHPL_FLAGS += -L$(1)/lib64 --ldflags="-Wl,-rpath,$(1)/lib64"
 endif
 
+ifneq ("$(wildcard $(1)/lib64/pkgconfig)","")
+PKG_CONFIG_PATH := $(1)/lib64/pkgconfig$$(if $$(PKG_CONFIG_PATH),:$$(PKG_CONFIG_PATH))
+endif
+
 # Add lib if present
 ifneq ("$(wildcard $(1)/lib)","")
 CHPL_FLAGS += -L$(1)/lib --ldflags="-Wl,-rpath,$(1)/lib"
 endif
+
+ifneq ("$(wildcard $(1)/lib/pkgconfig)","")
+PKG_CONFIG_PATH := $(1)/lib/pkgconfig$$(if $$(PKG_CONFIG_PATH),:$$(PKG_CONFIG_PATH))
+endif
+
+export PKG_CONFIG_PATH
 endef
 # Usage: $(eval $(call add-path,/home/user/anaconda3/envs/arkouda))
 #                               ^ no space after comma

pydoc/setup/BUILD.md

@@ -110,6 +110,32 @@ conda install boost-cpp snappy thrift-cpp re2 utf8proc
 pip install boost snappy thrift re2 utf8proc
 ```
 
+#### Chapel Parquet Package
+
+The server build downloads and compiles the Chapel `Parquet` package into
+`dep/build/Parquet`. Arrow and Parquet C++ libraries must still be available
+through `pkg-config` as described above.
+
+You can optionally build against an existing package checkout or select a different
+repository reference. You can also download the `Parquet` source ahead of time
+using `wget https://github.com/chapel-lang/Parquet/archive/refs/heads/main.tar.gz`,
+untar it, and then point `ARKOUDA_PARQUET_SRC_DIR` to the untarred location before running `make`.
+
+```bash
+# Use an existing checkout without cloning it into dep/build.
+ARKOUDA_PARQUET_SRC_DIR=/path/to/Parquet make
+
+# Select a branch or tag when creating a new checkout.
+ARKOUDA_PARQUET_REF=my-branch make
+
+# Relocate Arkouda's downloaded checkout.
+make PARQUET_INSTALL_DIR=/path/to/build/Parquet
+```
+
+Delete the downloaded checkout before changing `ARKOUDA_PARQUET_REF`; an
+existing checkout is reused as-is. `ARKOUDA_PARQUET_REPO` can override the
+default package repository.
+
 #### Distributable Package
 
 Alternatively you can build a distributable package:

scripts/get_parquet_package.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+#
+# Fetch and build the external Chapel `Parquet` Mason package, then print the
+# `chpl` arguments (C++ prerequisite headers/objects, include paths, Arrow link
+# flags) and the Parquet module source path needed to compile it into the
+# Arkouda server.
+#
+# This is used by the Makefile in place of `mason modules`, which does not work
+# reliably with Chapel 2.4.
+#
+# Contract: ALL human-readable status is written to stderr. The ONLY thing
+# written to stdout is a single line of `chpl` arguments, so a Make recipe can
+# capture it with shell command substitution.
+#
+# Usage:
+#   get_parquet_package.sh <install-dir>
+#
+# Environment overrides:
+#   ARKOUDA_PARQUET_REPO      git URL to clone (default: chapel-lang/Parquet)
+#   ARKOUDA_PARQUET_REF       branch or tag to check out (default: repo HEAD)
+#   ARKOUDA_PARQUET_SRC_DIR   use an existing checkout instead of cloning
+#   ARKOUDA_CHPL_HOME         used to select Chapel's C++ compiler for prereqs
+
+set -euo pipefail
+
+log() { echo "get_parquet_package: $*" >&2; }
+
+PARQUET_REPO="${ARKOUDA_PARQUET_REPO:-https://github.com/chapel-lang/Parquet}"
+PARQUET_REF="${ARKOUDA_PARQUET_REF:-}"
+
+INSTALL_DIR="${1:-${ARKOUDA_PARQUET_INSTALL_DIR:-}}"
+if [[ -z "${INSTALL_DIR}" ]]; then
+  log "ERROR: no install directory provided (pass it as the first argument)"
+  exit 1
+fi
+
+# Allow pointing at an existing checkout (e.g. a Mason clone) to skip cloning.
+PARQUET_SRC="${ARKOUDA_PARQUET_SRC_DIR:-${INSTALL_DIR}}"
+
+if [[ -f "${PARQUET_SRC}/Mason.toml" ]]; then
+  log "Using existing Parquet checkout at ${PARQUET_SRC}"
+else
+  log "Cloning ${PARQUET_REPO}${PARQUET_REF:+@${PARQUET_REF}} into ${PARQUET_SRC}"
+  mkdir -p "$(dirname "${PARQUET_SRC}")"
+  git clone --depth 1 ${PARQUET_REF:+--branch "${PARQUET_REF}"} \
+      "${PARQUET_REPO}" "${PARQUET_SRC}" >&2
+fi
+
+# Resolve to an absolute path so the emitted flags work from any CWD.
+PARQUET_ROOT="$(cd "${PARQUET_SRC}" && pwd -P)"
+PREREQ_DIR="${PARQUET_ROOT}/prereqs/cpp"
+PARQUET_MODULE="${PARQUET_ROOT}/src/Parquet.chpl"
+
+if [[ ! -d "${PREREQ_DIR}" ]]; then
+  log "ERROR: expected C++ prerequisites at ${PREREQ_DIR}, but they are missing"
+  exit 1
+fi
+
+if [[ ! -f "${PARQUET_MODULE}" ]]; then
+  log "ERROR: expected Chapel module at ${PARQUET_MODULE}, but it is missing"
+  exit 1
+fi
+
+log "Building C++ prerequisites in ${PREREQ_DIR}"
+make -s -C "${PREREQ_DIR}" CHPL_HOME="${ARKOUDA_CHPL_HOME:-}" >&2
+
+# Gather the chpl flags the package needs.
+FLAGS="$(make -s -C "${PREREQ_DIR}" CHPL_HOME="${ARKOUDA_CHPL_HOME}" printchplflags)"
+if [[ -z "${FLAGS}" ]]; then
+  log "ERROR: the Parquet prerequisite build returned no Chapel flags"
+  exit 1
+fi
+
+printf '%s %s\n' "${FLAGS}" "${PARQUET_MODULE}"

src/CheckpointMsg.chpl

@@ -386,7 +386,7 @@ module CheckpointMsg {
         return;
     }
 
-    if hasPrimitiveElements(entry) {
+    if hasPrimitiveElements(entry) && entry.etype != uint(8) {
         saveSymEntryPrimitive(entry, name, path, mdName, mdWriter);
 
     } else if entry.etype == bigint {