Repository navigation
Conversation
|
|
9e429c7 to
eea3482
Compare
|
Here are the suggested changes for Assumption: PMC member running diff --git a/dev/release/07-flightsqlodbc-upload.sh b/dev/release/07-flightsqlodbc-upload.sh
index c823bc84ae..bb924cc785 100755
--- a/dev/release/07-flightsqlodbc-upload.sh
+++ b/dev/release/07-flightsqlodbc-upload.sh
@@ -53,6 +53,7 @@ fi
. "${SOURCE_DIR}/utils-env.sh"
+# TODO: Add checks for required macos signer storepass
if [ -z "${ESIGNER_STOREPASS:-}" ]; then
echo "ERROR: ESIGNER_STOREPASS is not set" >&2
exit 1
@@ -73,12 +74,17 @@ dll_signed="arrow_flight_sql_odbc.dll"
: "${GITHUB_REPOSITORY:=apache/arrow}"
+# In PHASE_BUILD_MSI step, both unsigned MSI and PKG will be built
: "${PHASE_DEFAULT:=1}"
: "${PHASE_SIGN_DLL:=${PHASE_DEFAULT}}"
+: "${PHASE_SIGN_DYLIB:=${PHASE_DEFAULT}}"
: "${PHASE_BUILD_MSI:=${PHASE_DEFAULT}}"
: "${PHASE_SIGN_MSI:=${PHASE_DEFAULT}}"
+: "${PHASE_SIGN_PKG:=${PHASE_DEFAULT}}"
-if [ "${PHASE_SIGN_DLL}" -eq 0 ] && [ "${PHASE_BUILD_MSI}" -eq 0 ] && [ "${PHASE_SIGN_MSI}" -eq 0 ]; then
+if [ "${PHASE_SIGN_DLL}" -eq 0 ] && [ "${PHASE_SIGN_DYLIB}" -eq 0 ] &&
+ [ "${PHASE_BUILD_MSI}" -eq 0 ] && [ "${PHASE_SIGN_MSI}" -eq 0 ] &&
+ [ "${PHASE_SIGN_PKG}" -eq 0 ]; then
echo "No phases specified. Exiting."
exit 1
fi
@@ -102,7 +108,7 @@ if [ -e "${tmp_dir}" ]; then
fi
if [ "${PHASE_SIGN_DLL}" -gt 0 ]; then
- echo "[1/8] Downloading ${dll_unsigned} from release..."
+ echo "[1/15] Downloading ${dll_unsigned} from release..."
gh release download "${tag}" \
--repo "${GITHUB_REPOSITORY}" \
--pattern "${dll_unsigned}" \
@@ -112,7 +118,7 @@ if [ "${PHASE_SIGN_DLL}" -gt 0 ]; then
exit 1
fi
- echo "[2/8] Signing ${dll_unsigned}..."
+ echo "[2/15] Signing ${dll_unsigned}..."
echo "NOTE: Running jsign. You may be prompted for your OTP PIN..."
jsign --storetype ESIGNER \
--alias d97c5110-c66a-4c0c-ac0c-1cd6af812ee6 \
@@ -128,27 +134,66 @@ if [ "${PHASE_SIGN_DLL}" -gt 0 ]; then
exit 1
fi
- echo "[3/8] Uploading signed DLL to GitHub Release..."
+ echo "[3/15] Uploading signed DLL to GitHub Release..."
gh release upload "${tag}" \
--repo "${GITHUB_REPOSITORY}" \
--clobber \
"${tmp_dir}/${dll_signed}"
fi
+# The signed DYLIB must be on the release before PHASE_BUILD_MSI: the workflow
+# downloads it (libarrow_flight_sql_odbc.*.dylib) to build the universal PKG.
+if [ "${PHASE_SIGN_DYLIB}" -gt 0 ]; then
+ echo "[4/15] Downloading unsigned universal DYLIB from release..."
+ gh release download "${tag}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --pattern "libarrow_flight_sql_odbc_unsigned.*.dylib" \
+ --dir "${tmp_dir}"
+ dylib_unsigned="$(ls "${tmp_dir}"/libarrow_flight_sql_odbc_unsigned.*.dylib)"
+ dylib_unsigned_name="$(basename "${dylib_unsigned}")"
+ dylib_signed="${tmp_dir}/${dylib_unsigned_name/_unsigned/}"
+ if is_dylib_signed "${dylib_unsigned}"; then
+ echo "ERROR: ${dylib_unsigned_name} is already signed" >&2
+ exit 1
+ fi
+
+ echo "[5/15] Signing ${dylib_unsigned_name}..."
+ # Rename before signing: codesign derives the signature identifier from the
+ # file name, so signing it as *_unsigned.* would leak that into the identifier.
+ mv "${dylib_unsigned}" "${dylib_signed}"
+ # TODO: replace example `codesign` command with ASF approved way to sign DYLIB
+ codesign --force \
+ --timestamp \
+ --options runtime \
+ --sign "${APPLE_CODESIGN_IDENTITY}" \
+ "${dylib_signed}"
+ if ! is_dylib_signed "${dylib_signed}"; then
+ echo "ERROR: $(basename "${dylib_signed}") is not signed" >&2
+ exit 1
+ fi
+
+ echo "[6/15] Uploading signed DYLIB to GitHub Release..."
+ gh release upload "${tag}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --clobber \
+ "${dylib_signed}"
+fi
+
+# This also builds the universal macOS PKG, using the signed DYLIB above.
if [ "${PHASE_BUILD_MSI}" -gt 0 ]; then
- echo "[4/8] Triggering odbc_release_step in package_odbc.yml workflow..."
+ echo "[7/15] Triggering odbc_release_step in package_odbc.yml workflow..."
gh workflow run package_odbc.yml \
--repo "${GITHUB_REPOSITORY}" \
--ref "${tag}" \
--field odbc_release_step=true
- echo "[5/8] Waiting for workflow to complete. This can take a very long time..."
+ echo "[8/15] Waiting for workflow to complete. This can take a very long time..."
REPOSITORY="${GITHUB_REPOSITORY}" \
"${SOURCE_DIR}/utils-watch-gh-workflow.sh" "${tag}" package_odbc.yml
fi
if [ "${PHASE_SIGN_MSI}" -gt 0 ]; then
- echo "[6/8] Downloading unsigned MSI..."
+ echo "[9/15] Downloading unsigned MSI..."
gh release download "${tag}" \
--repo "${GITHUB_REPOSITORY}" \
--pattern "Apache-Arrow-Flight-SQL-ODBC-${version}-win64.msi" \
@@ -159,7 +204,7 @@ if [ "${PHASE_SIGN_MSI}" -gt 0 ]; then
exit 1
fi
- echo "[7/8] Signing MSI..."
+ echo "[10/15] Signing MSI..."
echo "NOTE: Running jsign. You may be prompted for your OTP PIN..."
jsign --storetype ESIGNER \
--alias d97c5110-c66a-4c0c-ac0c-1cd6af812ee6 \
@@ -174,9 +219,53 @@ if [ "${PHASE_SIGN_MSI}" -gt 0 ]; then
exit 1
fi
- echo "[8/8] Uploading signed MSI to GitHub Release..."
+ echo "[11/15] Uploading signed MSI to GitHub Release..."
gh release upload "${tag}" \
--repo "${GITHUB_REPOSITORY}" \
--clobber \
"${msi}"
fi
+
+if [ "${PHASE_SIGN_PKG}" -gt 0 ]; then
+ echo "[12/15] Downloading unsigned PKG..."
+ gh release download "${tag}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --pattern "ArrowFlightSQLODBC-*_unsigned.pkg" \
+ --dir "${tmp_dir}"
+ pkg_unsigned="$(ls "${tmp_dir}"/ArrowFlightSQLODBC-*_unsigned.pkg)"
+ pkg_name="$(basename "${pkg_unsigned}" _unsigned.pkg)"
+ pkg_signed="${tmp_dir}/${pkg_name}.pkg"
+ if is_pkg_signed "${pkg_unsigned}"; then
+ echo "ERROR: PKG is already signed" >&2
+ exit 1
+ fi
+
+ echo "[13/15] Signing PKG..."
+ # TODO: replace example `productsign` command with ASF approved way to sign PKG
+ productsign \
+ --timestamp \
+ --sign "${APPLE_INSTALLER_IDENTITY}" \
+ "${pkg_unsigned}" \
+ "${pkg_signed}"
+ if ! is_pkg_signed "${pkg_signed}"; then
+ echo "ERROR: PKG is not signed" >&2
+ exit 1
+ fi
+
+ echo "[14/15] Notarizing and stapling PKG. This can take a while..."
+ # TODO: replace example `xcrun` command with ASF approved way to notarize PKG
+ xcrun notarytool submit "${pkg_signed}" \
+ --keychain-profile "${APPLE_NOTARY_PROFILE}" \
+ --wait
+ # Stapling fails if there is no notarization ticket, so a rejected submission
+ # can't slip through even if notarytool exits successfully.
+ xcrun stapler staple "${pkg_signed}"
+ xcrun stapler validate "${pkg_signed}"
+ spctl --assess --type install --verbose "${pkg_signed}"
+
+ echo "[15/15] Uploading signed PKG to GitHub Release..."
+ gh release upload "${tag}" \
+ --repo "${GITHUB_REPOSITORY}" \
+ --clobber \
+ "${pkg_signed}"
+fi
|
amoeba
left a comment
There was a problem hiding this comment.
Thanks, left a few comments.
Can you check the file permissions final universal pkg contents and the installed files? Looking at the code here I think it's possible some of the files could end up owned by the GHA runner user.
| # skip the process if MSI/PKG artifacts with signed | ||
| # ODBC libraries are already generated |
There was a problem hiding this comment.
This addition of this job seems fine. Just curious, this hasn't been an issue yet right?
There was a problem hiding this comment.
yes, I don't think there's an existing GitHub issue for this
edited msg
There was a problem hiding this comment.
Sorry, just meant to ask whether we ran into a scenario yet where this was a problem. I'll resolve this conversation since it sounds like it hasn't.
There was a problem hiding this comment.
I am reopening this as I think @amoeba has a point here. I think AI tends to be overly defensive on lots of places. In general those defenses are worth taking a look at but in this case I think adding more is detrimental for the release.
In this specific case it will work as is but if in the future we make changes to other workflows for example changing the tag or the name of the assets this is a place where we have to update "stuff". The problem is that we might forget to do so and this would require a new RC as it will only fail mid release. I'd rather remove this step entirely as if this fails it would still fail the following steps nonetheless.
In my opinion this only adds maintenance burden without adding too much value.
There was a problem hiding this comment.
That makes sense, let's remove it.
There was a problem hiding this comment.
@raulcd you have a good point, I have removed this check. Since future changes to release workflows are expected, I think the costs outweigh the gain here.
Just a side note, I actually asked AI to add this check, the reason is that I wanted the CI to skip the rebuild is the MSI or PKG is already generated. For example, with the check implemented, if a workflow run fails for macOS but succeeds for Windows, re-running the workflow will only try to generate the missing macOS .pkg installer.
if this fails it would still fail the following steps nonetheless.
Good point. So re-running won't really help in this sense
| # skip the process if MSI/PKG artifacts with signed | ||
| # ODBC libraries are already generated |
There was a problem hiding this comment.
yes, I don't think there's an existing GitHub issue for this
edited msg
| # skip the process if MSI/PKG artifacts with signed | ||
| # ODBC libraries are already generated |
There was a problem hiding this comment.
I am reopening this as I think @amoeba has a point here. I think AI tends to be overly defensive on lots of places. In general those defenses are worth taking a look at but in this case I think adding more is detrimental for the release.
In this specific case it will work as is but if in the future we make changes to other workflows for example changing the tag or the name of the assets this is a place where we have to update "stuff". The problem is that we might forget to do so and this would require a new RC as it will only fail mid release. I'd rather remove this step entirely as if this fails it would still fail the following steps nonetheless.
In my opinion this only adds maintenance burden without adding too much value.
| # skip the process if MSI/PKG artifacts with signed | ||
| # ODBC libraries are already generated |
There was a problem hiding this comment.
@raulcd you have a good point, I have removed this check. Since future changes to release workflows are expected, I think the costs outweigh the gain here.
Just a side note, I actually asked AI to add this check, the reason is that I wanted the CI to skip the rebuild is the MSI or PKG is already generated. For example, with the check implemented, if a workflow run fails for macOS but succeeds for Windows, re-running the workflow will only try to generate the missing macOS .pkg installer.
if this fails it would still fail the following steps nonetheless.
Good point. So re-running won't really help in this sense
2f4d307 to
e26b604
Compare
amoeba
left a comment
There was a problem hiding this comment.
Thanks for the changes @alinaliBQ. This looks good now but I'll give @raulcd a chance to re-review.
I think the 07 script will need a few tweaks to work with the changes we're about to merge here but we can work on that in another PR.
|
Sounds good Bryce. I assume an Arrow PMC member would need to work on the 07 script, since the changes can only be tested with PMC access. My suggestions are in #55321 (comment) comment. |
The fork CI results are successful. PR ready for another round of review |
e26b604 to
0aaeeae
Compare
| gh release download ${GITHUB_REF_NAME} \ | ||
| --repo ${{ github.repository }} \ | ||
| --pattern 'ArrowFlightSQLODBC-*_ARM64_base.pkg' \ | ||
| --pattern 'libarrow_flight_sql_odbc.*.dylib' \ | ||
| --clobber |
There was a problem hiding this comment.
I am not sure I understand the whole workflow.
We build AMD64 and ARM64, upload to the job. Build universal, upload to the job. Then upload everything to the GitHub release. The sign and upload (this is expected to be done locally) with the gpg signature?
At this point the artifacts are signed, right?
And we seem to be generating a new artifact repackaging the universal signed artifacts that won't be signed with this job. Doesn't this require signing too?
Can't we just sign the whole universal generated previously and avoid this step of repac
There was a problem hiding this comment.
Hi @raulcd, I'm happy to explain here. The workflow is follows the same flow as the ODBC Windows msi installer which is sign ODBC library .dll first, then sign the newly built .msi installer which contains the signed ODBC library file. Difference is that at step where we prepare for unsigned installer, we do repackaging for macOS, and we build from scratch for Windows.
The sign and upload (this is expected to be done locally) with the gpg signature?
At this point the artifacts are signed, right?
Only the universal ODBC .dylib file is signed at 1st stage.
The file 'ArrowFlightSQLODBC-*_ARM64_base.pkg' is left unsigned, because the signature will be cleared during repackaging. So if 'ArrowFlightSQLODBC-*_ARM64_base.pkg' is signed at this stage it won't be helpful to us.
... generating a new artifact ... Doesn't this require signing too?
Yes it does. The new repackaged artifact will be an unsigned universal .pkg installer that contains the signed universal ODBC .dylib file (from 1st stage). Then the universal .pkg file should be signed in 2nd stage. This way we get a signed universal .pkg containing a signed ODBC .dylib file.
Another different design route is, the workflow generates and uploads the universal .pkg installer with unsigned universal ODBC .dylib file to Git Release in one go, and then let the signing script dev/release/07-flightsqlodbc-upload.sh handle the rest steps:
- Extract the unsigned ODBC .dylib file from .pkg file to sign,
- do the repackaging to put the signed .dylib file back to universal .pkg,
- sign the universal .pkg.
But this will make the signing script too complicated I think.. It's probably better to let CI handle the repackaging.
There was a problem hiding this comment.
Can't we just sign the universal .pkg even if the internal .dylib file is not signed?
I am trying to find a way so we can sign a single time (as we do today) otherwise we are adding more and more steps to our release which is already quite convoluted.
I am wondering what is the difference of behaviour as an end user if we sign once. In the end, we are providing the packaged universal to the users, and we are signing to tell them "this is official and is from us" why does the internal .dylib have to be signed individually that's where I don't understand it.
There was a problem hiding this comment.
Good question. The reason is that the internal .dylib file needs to be signed in order for the .pkg to be notarized. Apple's Notarization page says:
Enable code-signing for all of the executables you distribute, and ensure that executables have valid code signatures, as described in Ensure a valid code signature.
The .dylib file fall under the executables category. So if the .dylib file isn't signed, my understanding is that Apple's notarization process will detect it, which will cause notarization to fail on the .pkg.
By the way, currently there are 2 signing steps for Windows ODBC MSI:
first we sign the ODBC .dll library file,
arrow/dev/release/07-flightsqlodbc-upload.sh
Lines 115 to 117 in 5580366
and then we sign the ODBC Windows MSI installer. The plan is for macOS to follow the same pattern (and notarization is also needed for macOS).
arrow/dev/release/07-flightsqlodbc-upload.sh
Lines 162 to 164 in 5580366
There was a problem hiding this comment.
Why notarization will be needed: A user who downloads the signed but not notarized installer in a browser gets the “Apple cannot check it for malicious software” warning, and has to override it (right-click, Open, or approve it in Privacy & Security).
There was a problem hiding this comment.
I can do those in a follow-up PR. The follow-up tasks to this PR are to update the release script and the release guide (which can be done in one PR).
There was a problem hiding this comment.
Thanks for opening the issue. Let's work on it as a follow-up task.
In-progress fix CI logic Revert "Increase nugettimeout to 30 mins" This reverts commit 62b8d41. Increase nugettimeout to 30 mins Upload gRPC can take long for vcpkg Clean up IN-PROGRESS add check Address Raul and Bryce comments - removed `Check runner architecture` - remove `odbc-release-check` Fix sudo issue Empty commit to retrigger CI to fix https://github.com/apache/arrow/actions/runs/41538385140/job/125596157426 ODBC debug AMD64 macOS 26-intel error Re-use macOS ODBC workflow .github/actions/odbc-mac/action.yml mirrors .github/actions/odbc-windows/action.yml Address Bryce's comments Co-Authored-By: Bryce Mecum <brycemecum@gmail.com>
0aaeeae to
315ce7b
Compare
There was a problem hiding this comment.
Could you add this file to Dependabot targets?
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 1acdf1e9ca..d4f702e8c0 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -18,7 +18,9 @@
version: 2
updates:
- package-ecosystem: "github-actions"
- directory: "/"
+ directories:
+ - "/"
+ - "/.github/actions/*"
schedule:
interval: "weekly"
commit-message:| - name: Build universal macOS installer | ||
| run: | | ||
| set -euo pipefail | ||
|
|
||
| pkg_path="$(ls ArrowFlightSQLODBC-*_ARM64_base.pkg)" | ||
| pkg_name="$(basename "${pkg_path}" _ARM64_base.pkg)" | ||
|
|
||
| # Use the ARM64 .pkg and swap in the signed universal ODBC dylib. | ||
| # Use sudo for every step that touches the expanded payload so the | ||
| # root-owned files in the PKG keep their ownership instead of being | ||
| # re-stamped with the runner user's uid/gid. | ||
| sudo pkgutil --expand-full "${pkg_path}" expanded | ||
|
|
||
| # The unversioned name libarrow_flight_sql_odbc.dylib is a symlink | ||
| # to the versioned dylib that the install name | ||
| # (@rpath/libarrow_flight_sql_odbc.<SOVERSION>.dylib) resolves to. | ||
| # Replace the real file so the symlinks stay intact. | ||
| symlink_path="$(find expanded -name libarrow_flight_sql_odbc.dylib)" | ||
| lib_dir="$(dirname "${symlink_path}")" | ||
| dylib_path="$(realpath "${symlink_path}")" | ||
| signed_dylib="$(ls libarrow_flight_sql_odbc.*.dylib)" | ||
| if [ "${signed_dylib}" != "$(basename "${dylib_path}")" ]; then | ||
| echo "Signed DYLIB ${signed_dylib} doesn't match PKG DYLIB $(basename "${dylib_path}")" | ||
| exit 1 | ||
| fi | ||
| echo "Replacing ${dylib_path} (via ${symlink_path}) with ${signed_dylib}" | ||
| # cp onto an existing file keeps its owner and mode. | ||
| sudo cp "${signed_dylib}" "${dylib_path}" | ||
|
|
||
| # Verify every dylib (including the symlinks) is universal before re-packaging. | ||
| for lib in "${lib_dir}"/libarrow_flight_sql_odbc*.dylib; do | ||
| dylib_archs="$(lipo -archs "${lib}")" | ||
| echo "${lib}: ${dylib_archs}" | ||
| for arch in arm64 x86_64; do | ||
| if ! grep -qw "${arch}" <<< "${dylib_archs}"; then | ||
| echo "${lib} is missing '${arch}' (got: ${dylib_archs})" | ||
| exit 1 | ||
| fi | ||
| done | ||
| done | ||
|
|
||
| # --flatten-full regenerates each component's Bom and PackageInfo | ||
| # installKBytes, but leaves Distribution with the ARM64 base PKG's | ||
| # sizes. Copy the regenerated sizes into Distribution, then | ||
| # re-flatten without -full so the payloads are reused as-is. | ||
| sudo pkgutil --flatten-full expanded flattened.pkg | ||
| sudo pkgutil --expand flattened.pkg shallow | ||
| for component in shallow/*.pkg; do | ||
| component_name="$(basename "${component}")" | ||
| kbytes="$(sudo sed -n 's/.*installKBytes="\([0-9]*\)".*/\1/p' "${component}/PackageInfo")" | ||
| sudo sed -i '' \ | ||
| "/>#${component_name}<\/pkg-ref>/ s/installKBytes=\"[0-9]*\"/installKBytes=\"${kbytes}\"/" \ | ||
| shallow/Distribution | ||
| done | ||
| mkdir universal | ||
| sudo pkgutil --flatten shallow "universal/${pkg_name}.pkg" | ||
| sudo chown "$(id -u):$(id -g)" "universal/${pkg_name}.pkg" |
There was a problem hiding this comment.
It seems that this step is similar to
arrow/.github/workflows/cpp_extra.yml
Lines 701 to 737 in 21156bc
Can we extract common code to ci/scripts/ and reuse it?
| fetch-depth: 0 | ||
| submodules: recursive |
There was a problem hiding this comment.
It seems that we don't need them.
| fetch-depth: 0 | |
| submodules: recursive |
| gh release download ${GITHUB_REF_NAME} \ | ||
| --repo ${{ github.repository }} \ | ||
| --pattern 'ArrowFlightSQLODBC-*_ARM64_base.pkg' \ | ||
| --pattern 'libarrow_flight_sql_odbc.*.dylib' \ | ||
| --clobber |
There was a problem hiding this comment.
Thanks for opening the issue. Let's work on it as a follow-up task.
Rationale for this change
GH-51694
Also see #49411 (comment) for context
What changes are included in this PR?
Upload unsigned macOS DYLIB and PKG to draft releases.
Adds a check to skip upload process for universal PKG/MSI artifacts if they already exist.
To release macOS and Windows ODBC after this PR:
dev/release/07-flightsqlodbc-upload.shwould be able to sign both Windows and macOS artifacts on the same machine.Generate unsigned DYLIB, macOS base PKG, and DLL with CI
Sign DYLIB and DLL and upload to GitHub release
Generate unsigned universal PKG (using base PKG) and MSI with CI
Sign universal PKG and MSI and upload to GitHub release
Are these changes tested?
Tested in fork
Are there any user-facing changes?
N/A
Was AI used for this PR?
In accordance to the AI generation guidelines, please disclose below whether and how AI was used in this PR.
PR code and description written by:
Reviewed before submission by: