"mapped file has no cdhash, completely unsigned?" when cdhash exists

Question

GeorgeHall OP

Created 6d

Replies 3

Boosts 0

Participants 2

Hi,

I have created a conda python environment which I have packaged into a .tar.gz (using conda-pack) and which runs correctly when extracted (in this example, it only contains the scipy package). However, when I sign the necessary files within the environment (i.e. the binaries, the dylibs, the .so files), attempting to load scipy.sparse now fails with the error "mapped file has no cdhash, completely unsigned" about one of the .so files. Furthermore, I believe that this file does in fact have a cdhash.

The signing process represented by my example below has been working for about a year, and I am unsure why it has suddenly stopped working. I am on a 2020 MacBook Pro with an i7 processor and running Sequoia 15.1.1.

Here is a minimal example showing the creating of the conda environment, codesigning, and the error message. Many thanks in advance!

 # Create and activate conda env
> conda create -y -n mwe_env python=3.10
> conda activate mwe_env
# Verify scipy not initially installed
(mwe_env) > python

 Python 3.10.16 (main, Dec 11 2024, 10:24:41) [Clang 14.0.6 ] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import scipy
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ModuleNotFoundError: No module named 'scipy'

 (mwe_env) > pip install scipy
Collecting scipy
  Downloading scipy-1.15.2-cp310-cp310-macosx_14_0_x86_64.whl.metadata (61 kB)
Collecting numpy<2.5,>=1.23.5 (from scipy)
  Downloading numpy-2.2.4-cp310-cp310-macosx_14_0_x86_64.whl.metadata (62 kB)
Downloading scipy-1.15.2-cp310-cp310-macosx_14_0_x86_64.whl (25.1 MB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 25.1/25.1 MB 17.3 MB/s eta 0:00:00
Downloading numpy-2.2.4-cp310-cp310-macosx_14_0_x86_64.whl (7.0 MB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.0/7.0 MB 16.4 MB/s eta 0:00:00
Installing collected packages: numpy, scipy
Successfully installed numpy-2.2.4 scipy-1.15.2
 
(mwe_env) > python

 Python 3.10.16 (main, Dec 11 2024, 10:24:41) [Clang 14.0.6 ] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import scipy.sparse
>>> # success!

 # Package conda env
(mwe_env) > conda-pack --output mwe_env.tar.gz --name mwe_env
Collecting packages...
Packing environment at '/path/to/my/conda/envs/mwe_env' to 'mwe_env.tar.gz'
[########################################] | 100% Completed |  7.8s
 
(mwe_env) > conda deactivate
> mkdir mwe_dir && cd mwe_dir
> tar -xzvf ../mwe_env.tar.gz
> source bin/activate
 
(mwe_dir) > python

 Python 3.10.16 (main, Dec 11 2024, 10:24:41) [Clang 14.0.6 ] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import scipy.sparse
>>> # success!

 # Sign the binaries and .dylibs and .so files
(mwe_dir) > find bin -type f | xargs -n1 xcrun codesign -f -o runtime --timestamp --sign "Developer ID Application: MY_TEAM_ID"
(mwe_dir) > find . -name "*.dylib" -o -name "*.so" -type f | xargs -n1 xcrun codesign -f -o runtime --timestamp --sign "Developer ID Application: MY_TEAM_ID"
# the second command prints many lines saying it is "replacing existing signature"
 
(mwe_dir) > python

 Python 3.10.16 (main, Dec 11 2024, 10:24:41) [Clang 14.0.6 ] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import scipy.sparse
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/__init__.py", line 315, in <module>
    from . import csgraph
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/csgraph/__init__.py", line 187, in <module>
    from ._laplacian import laplacian
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/csgraph/_laplacian.py", line 7, in <module>
    from scipy.sparse.linalg import LinearOperator
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/__init__.py", line 134, in <module>
    from ._eigen import *
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/__init__.py", line 9, in <module>
    from .arpack import *
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/__init__.py", line 20, in <module>
    from .arpack import *
  File "/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/arpack.py", line 50, in <module>
    from . import _arpack
ImportError: dlopen(/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so, 0x0002): tried: '/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so' (code signature in <5DD8FC01-7360-3DB9-8273-C8A45ABB19A9> '/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so' not valid for use in process: mapped file has no cdhash, completely unsigned? Code has to be at least ad-hoc signed.), '/System/Volumes/Preboot/Cryptexes/OS/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so' (no such file), '/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so' (code signature in <5DD8FC01-7360-3DB9-8273-C8A45ABB19A9> '/path/to/mwe_dir/conda_env/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so' not valid for use in process: mapped file has no cdhash, completely unsigned? Code has to be at least ad-hoc signed.)

 # But:
(mwe_dir) > xcrun codesign -dvvv /path/to/mwe_dir/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so
Executable=/path/to/mwe_dir/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so
Identifier=_arpack.cpython-310-darwin
Format=Mach-O thin (x86_64)
CodeDirectory v=20400 size=4318 flags=0x10000(runtime) hashes=129+2 location=embedded
Library validation warning=OS X SDK version before 10.9 does not support Library Validation
Hash type=sha256 size=32
CandidateCDHash sha256=816731ecd1ad01b38555cbfef8c000628696d0ca
CandidateCDHashFull sha256=816731ecd1ad01b38555cbfef8c000628696d0ca53376aebf6fae28d8c02f519
Hash choices=sha256
CMSDigest=816731ecd1ad01b38555cbfef8c000628696d0ca53376aebf6fae28d8c02f519
CMSDigestType=2
CDHash=816731ecd1ad01b38555cbfef8c000628696d0ca
Signature size=9000
Authority=Developer ID Application: MY_TEAM_ID
Authority=Developer ID Certification Authority
Authority=Apple Root CA
Timestamp=2 Apr 2025 at 16:24:52
Info.plist=not bound
TeamIdentifier=MY_TEAM_ID
Sealed Resources=none
Internal requirements count=1 size=188

Answered by DTS Engineer in 832927022

Buried deep in the logs you posted there’s this snippet:

Library validation warning=OS X SDK version before 10.9 does not support Library Validation

Modern Mac software must be built with the macOS 10.9 SDK or later, because that’s required to pass notarisation. Without that, the system won’t load your code signature, and hence this complaining about a cdhash not being present even though it clearly is.

For more on this issue, Notarisation and the macOS 10.9 SDK. Oh, and I just updated it to make it easier to find based on the errors you saw.

Share and Enjoy
—
Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"

Boost

Answer 1

DTS Engineer OP

Apple

6d

Accepted Answer

Buried deep in the logs you posted there’s this snippet:

Library validation warning=OS X SDK version before 10.9 does not support Library Validation

Modern Mac software must be built with the macOS 10.9 SDK or later, because that’s required to pass notarisation. Without that, the system won’t load your code signature, and hence this complaining about a cdhash not being present even though it clearly is.

For more on this issue, Notarisation and the macOS 10.9 SDK. Oh, and I just updated it to make it easier to find based on the errors you saw.

Share and Enjoy
—
Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"

0

Answer 2

GeorgeHall OP

6d

Hi Quinn( @DTS Engineer),

Thank you for the quick response and for the (updated) post! I had wondered about this warning, but I decided it was unimportant since it is also present on versions of my software which work. For instance, if I check the same file in the conda environment contained in a working version of the app on the same machine, it also displays this warning.

I should also note that, on the M2 MacBook Pro (MacOS 14.1) I'm working with today, my MWE does not reproduce the error (i.e. scipy.sparse can be imported without error following signing), and it also has this warning.

 > codesign -dvvv /path/to/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-311-darwin.so
 
Executable=/path/to/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so
Identifier=_arpack.cpython-310-darwin
Format=Mach-O thin (arm64)
CodeDirectory v=20400 size=3806 flags=0x10000(runtime) hashes=113+2 location=embedded
Library validation warning=OS X SDK version before 10.9 does not support Library Validation
Hash type=sha256 size=32
CandidateCDHash sha256=e887c0fd9372bf3c78b8446609a8d3f9eddee120
CandidateCDHashFull sha256=e887c0fd9372bf3c78b8446609a8d3f9eddee1205a8729a33ac117d2763e5909
Hash choices=sha256
CMSDigest=e887c0fd9372bf3c78b8446609a8d3f9eddee1205a8729a33ac117d2763e5909
CMSDigestType=2
CDHash=e887c0fd9372bf3c78b8446609a8d3f9eddee120
Signature size=9000
Authority=Developer ID Application: MY_TEAM_ID
Authority=Developer ID Certification Authority
Authority=Apple Root CA
Timestamp=3 Apr 2025 at 12:09:22
Info.plist=not bound
TeamIdentifier=MY_TEAM_ID
Sealed Resources=none
Internal requirements count=1 size=188

When I use vtool -show to check the associated SDK of /path/to/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-311-darwin.so for the intel machine's conda env, it says n/a:

 /path/to/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-311-darwin.so:
Load command 8
      cmd LC_BUILD_VERSION
  cmdsize 32
 platform MACOS
    minos 14.0
      sdk n/a
   ntools 1
     tool LD
  version 1022.1
Load command 9
      cmd LC_SOURCE_VERSION
  cmdsize 16
  version 0.0

Am I misunderstanding how to further investigate this issue? Or do you think it could be some other issue since this warning also present in some signed versions of the software which work correctly? This bug currently appears on intel but not on apple-silicon, where I use the same build process on both machines.

Many thanks, George

0

Answer 3

DTS Engineer OP

Apple

5d

There are many factors in play here, some of which are quite subtle:

The macOS 10.9 SDK requirement only comes into play if you have the hardened runtime enabled.
You have the hardened runtime enabled on these libraries, but that’s irrelevant. macOS enforces protections at the process boundary, so what matters is the hardened runtime on the main executable.
There are entitlements to opt out of the various security protections enabled by the hardend runtime and, again, it’s the entitlements on the main executable that matter.
Our trusted execution policies vary by platform. For example, on Intel Macs we still allow unsigned code O-:

I see a couple of paths to follow here:

You can invest a bunch o’ timing learning all our trusted execution rules to fully understand what’s going on here.
Or your can build your code against a modern SDK and see if that helps.

I recommend starting with the second path because:

vtool lets you prototype it without changing your build environment [1].
It’s likely that the end result of the first path is that you’ll have to walk the second path anyway.
And that’s definitely true if you want to distribute your code, because the notary service enforces this requirement separately.

Share and Enjoy
—
Quinn “The Eskimo!” @ Developer Technical Support @ Apple
let myEmail = "eskimo" + "1" + "@" + "apple.com"

[1] If you do this, beware of the gotcha described in Updating Mac Software. So, to run vtool in a situation like this:

Make a copy of the file.
Munge it with vtool.
Re-sign it.
Move that copy into place, replacing the original file.

0

	# Create and activate conda env
	> conda create -y -n mwe_env python=3.10
	> conda activate mwe_env

	# Verify scipy not initially installed
	(mwe_env) > python

	Python 3.10.16 (main, Dec 11 2024, 10:24:41) [Clang 14.0.6 ] on darwin
	Type "help", "copyright", "credits" or "license" for more information.
	>>> import scipy
	Traceback (most recent call last):
	File "<stdin>", line 1, in <module>
	ModuleNotFoundError: No module named 'scipy'

	(mwe_env) > pip install scipy
	Collecting scipy
	Downloading scipy-1.15.2-cp310-cp310-macosx_14_0_x86_64.whl.metadata (61 kB)
	Collecting numpy<2.5,>=1.23.5 (from scipy)
	Downloading numpy-2.2.4-cp310-cp310-macosx_14_0_x86_64.whl.metadata (62 kB)
	Downloading scipy-1.15.2-cp310-cp310-macosx_14_0_x86_64.whl (25.1 MB)
	━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 25.1/25.1 MB 17.3 MB/s eta 0:00:00
	Downloading numpy-2.2.4-cp310-cp310-macosx_14_0_x86_64.whl (7.0 MB)
	━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.0/7.0 MB 16.4 MB/s eta 0:00:00
	Installing collected packages: numpy, scipy
	Successfully installed numpy-2.2.4 scipy-1.15.2

	(mwe_env) > python

	Python 3.10.16 (main, Dec 11 2024, 10:24:41) [Clang 14.0.6 ] on darwin
	Type "help", "copyright", "credits" or "license" for more information.
	>>> import scipy.sparse
	>>> # success!

	# Package conda env
	(mwe_env) > conda-pack --output mwe_env.tar.gz --name mwe_env
	Collecting packages...
	Packing environment at '/path/to/my/conda/envs/mwe_env' to 'mwe_env.tar.gz'
	[########################################] \| 100% Completed \| 7.8s

	(mwe_env) > conda deactivate

	> mkdir mwe_dir && cd mwe_dir
	> tar -xzvf ../mwe_env.tar.gz
	> source bin/activate

	(mwe_dir) > python

	# Sign the binaries and .dylibs and .so files
	(mwe_dir) > find bin -type f \| xargs -n1 xcrun codesign -f -o runtime --timestamp --sign "Developer ID Application: MY_TEAM_ID"
	(mwe_dir) > find . -name ".dylib" -o -name ".so" -type f \| xargs -n1 xcrun codesign -f -o runtime --timestamp --sign "Developer ID Application: MY_TEAM_ID"
	# the second command prints many lines saying it is "replacing existing signature"

	(mwe_dir) > python

	# But:
	(mwe_dir) > xcrun codesign -dvvv /path/to/mwe_dir/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so
	Executable=/path/to/mwe_dir/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so
	Identifier=_arpack.cpython-310-darwin
	Format=Mach-O thin (x86_64)
	CodeDirectory v=20400 size=4318 flags=0x10000(runtime) hashes=129+2 location=embedded
	Library validation warning=OS X SDK version before 10.9 does not support Library Validation
	Hash type=sha256 size=32
	CandidateCDHash sha256=816731ecd1ad01b38555cbfef8c000628696d0ca
	CandidateCDHashFull sha256=816731ecd1ad01b38555cbfef8c000628696d0ca53376aebf6fae28d8c02f519
	Hash choices=sha256
	CMSDigest=816731ecd1ad01b38555cbfef8c000628696d0ca53376aebf6fae28d8c02f519
	CMSDigestType=2
	CDHash=816731ecd1ad01b38555cbfef8c000628696d0ca
	Signature size=9000
	Authority=Developer ID Application: MY_TEAM_ID
	Authority=Developer ID Certification Authority
	Authority=Apple Root CA
	Timestamp=2 Apr 2025 at 16:24:52
	Info.plist=not bound
	TeamIdentifier=MY_TEAM_ID
	Sealed Resources=none
	Internal requirements count=1 size=188

	> codesign -dvvv /path/to/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-311-darwin.so

	Executable=/path/to/lib/python3.10/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-310-darwin.so
	Identifier=_arpack.cpython-310-darwin
	Format=Mach-O thin (arm64)
	CodeDirectory v=20400 size=3806 flags=0x10000(runtime) hashes=113+2 location=embedded
	Library validation warning=OS X SDK version before 10.9 does not support Library Validation
	Hash type=sha256 size=32
	CandidateCDHash sha256=e887c0fd9372bf3c78b8446609a8d3f9eddee120
	CandidateCDHashFull sha256=e887c0fd9372bf3c78b8446609a8d3f9eddee1205a8729a33ac117d2763e5909
	Hash choices=sha256
	CMSDigest=e887c0fd9372bf3c78b8446609a8d3f9eddee1205a8729a33ac117d2763e5909
	CMSDigestType=2
	CDHash=e887c0fd9372bf3c78b8446609a8d3f9eddee120
	Signature size=9000
	Authority=Developer ID Application: MY_TEAM_ID
	Authority=Developer ID Certification Authority
	Authority=Apple Root CA
	Timestamp=3 Apr 2025 at 12:09:22
	Info.plist=not bound
	TeamIdentifier=MY_TEAM_ID
	Sealed Resources=none
	Internal requirements count=1 size=188

	/path/to/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/arpack/_arpack.cpython-311-darwin.so:
	Load command 8
	cmd LC_BUILD_VERSION
	cmdsize 32
	platform MACOS
	minos 14.0
	sdk n/a
	ntools 1
	tool LD
	version 1022.1
	Load command 9
	cmd LC_SOURCE_VERSION
	cmdsize 16
	version 0.0