I tested with the following config:
# Test 1: IPv4 + tunnel mode IPSec + IPComp
spdadd 172.16.0.9 172.16.0.11 any -P out ipsec ipcomp/tunnel/10.9.8.9-10.9.8.11/default;
add 10.9.8.9 10.9.8.11 ipcomp 15701 -m tunnel -C deflate;
add 10.9.8.11 10.9.8.9 ipcomp 24501 -m tunnel -C deflate;
# tcpdump -ni em0 proto ipcomp or proto ipencap
# ping -c1 172.16.0.11
17:02:52.410553 IP 10.9.8.9 > 10.9.8.11: IP 172.16.0.9 > 172.16.0.11: ICMP echo request, id 46650, seq 0, length 64 (ipip-proto-4)
17:02:52.411037 IP 10.9.8.11 > 10.9.8.9: IP 172.16.0.11 > 172.16.0.9: ICMP echo reply, id 46650, seq 0, length 64 (ipip-proto-4)
# ping -c1 -s1000 172.16.0.11
17:03:38.828548 IP 10.9.8.9 > 10.9.8.11: IPComp(cpi=0x3d55)
17:03:38.829143 IP 10.9.8.11 > 10.9.8.9: IPComp(cpi=0x5fb5)
# Test 2: IPv4 + tunnel mode IPComp + transport mode ESP
spdadd 172.16.0.9 172.16.0.11 any -P out ipsec ipcomp/tunnel/10.9.8.9-10.9.8.11/default esp/transport//default;
add 10.9.8.9 10.9.8.11 ipcomp 15701 -m tunnel -C deflate;
add 10.9.8.11 10.9.8.9 ipcomp 24501 -m tunnel -C deflate;
add 10.9.8.9 10.9.8.11 esp 15701 -m transport -E rijndael-cbc "1111111111111111";
add 10.9.8.11 10.9.8.9 esp 24501 -m transport -E rijndael-cbc "1111111111111111";
# tcpdump -ni em0 proto ipcomp or proto ipencap or esp
# ping -c1 172.16.0.11
17:09:07.549512 IP 10.9.8.9 > 10.9.8.11: ESP(spi=0x00003d55,seq=0x1), length 120
17:09:07.558344 IP 10.9.8.11 > 10.9.8.9: ESP(spi=0x00005fb5,seq=0x1), length 120
# dtrace -n 'fbt::ipcomp4_nonexp_encapcheck:return {printf("%d", arg1);}'
dtrace: description 'fbt::ipcomp4_nonexp_encapcheck:return ' matched 1 probe
CPU ID FUNCTION:NAME
1 18454 ipcomp4_nonexp_encapcheck:return 64
# ping -c1 -s1000 172.16.0.11
17:09:46.326608 IP 10.9.8.9 > 10.9.8.11: ESP(spi=0x00003d55,seq=0x2), length 360
17:09:46.327316 IP 10.9.8.11 > 10.9.8.9: ESP(spi=0x00005fb5,seq=0x2), length 360