Skip to content

Commit 78cd14c

Browse files
cconloncconlon
committed
F-5637: validate RSA JNI buffer sizes against DirectByteBuffer capacity
1 parent c8d6372 commit 78cd14c

2 files changed

Lines changed: 166 additions & 0 deletions

File tree

‎native/com_wolfssl_WolfCryptRSA.c‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,18 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doSign
7272
}
7373

7474
/* get output buffer size */
75+
if ((*jenv)->GetArrayLength(jenv, outSz) < 1) {
76+
return -1;
77+
}
7578
(*jenv)->GetIntArrayRegion(jenv, outSz, 0, 1, (jint*)&tmpOut);
7679

80+
/* Reject sizes larger than their backing direct buffers */
81+
if ((inSz > (*jenv)->GetDirectBufferCapacity(jenv, in)) ||
82+
(keySz > (*jenv)->GetDirectBufferCapacity(jenv, keyDer)) ||
83+
((jlong)tmpOut > (*jenv)->GetDirectBufferCapacity(jenv, out))) {
84+
return -1;
85+
}
86+
7787
ret = wc_InitRng(&rng);
7888
if (ret != 0) {
7989
printf("wc_InitRng failed, ret = %d\n", ret);
@@ -151,6 +161,13 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doVerify
151161
return -1;
152162
}
153163

164+
/* Reject sizes larger than their backing direct buffers */
165+
if ((sigSz > (*jenv)->GetDirectBufferCapacity(jenv, sig)) ||
166+
(keySz > (*jenv)->GetDirectBufferCapacity(jenv, keyDer)) ||
167+
(outSz > (*jenv)->GetDirectBufferCapacity(jenv, out))) {
168+
return -1;
169+
}
170+
154171
wc_InitRsaKey(&myKey, NULL);
155172
idx = 0;
156173

@@ -211,8 +228,18 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doEnc
211228
}
212229

213230
/* get output buffer size */
231+
if ((*jenv)->GetArrayLength(jenv, outSz) < 1) {
232+
return -1;
233+
}
214234
(*jenv)->GetIntArrayRegion(jenv, outSz, 0, 1, (jint*)&tmpOut);
215235

236+
/* Reject sizes larger than their backing direct buffers */
237+
if ((inSz > (*jenv)->GetDirectBufferCapacity(jenv, in)) ||
238+
(keySz > (*jenv)->GetDirectBufferCapacity(jenv, keyDer)) ||
239+
((jlong)tmpOut > (*jenv)->GetDirectBufferCapacity(jenv, out))) {
240+
return -1;
241+
}
242+
216243
ret = wc_InitRng(&rng);
217244
if (ret != 0) {
218245
printf("wc_InitRng failed, ret = %d\n", ret);
@@ -299,8 +326,18 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doPssSign
299326
}
300327

301328
/* get output buffer size */
329+
if ((*jenv)->GetArrayLength(jenv, outSz) < 1) {
330+
return -1;
331+
}
302332
(*jenv)->GetIntArrayRegion(jenv, outSz, 0, 1, (jint*)&tmpOut);
303333

334+
/* Reject sizes larger than their backing direct buffers */
335+
if ((inSz > (*jenv)->GetDirectBufferCapacity(jenv, in)) ||
336+
(keySz > (*jenv)->GetDirectBufferCapacity(jenv, keyDer)) ||
337+
((jlong)tmpOut > (*jenv)->GetDirectBufferCapacity(jenv, out))) {
338+
return -1;
339+
}
340+
304341
ret = wc_InitRng(&rng);
305342
if (ret != 0) {
306343
printf("wc_InitRng failed, ret = %d\n", ret);
@@ -394,6 +431,13 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doPssVerify
394431
return -1;
395432
}
396433

434+
/* Reject sizes larger than their backing direct buffers */
435+
if ((sigSz > (*jenv)->GetDirectBufferCapacity(jenv, sig)) ||
436+
(keySz > (*jenv)->GetDirectBufferCapacity(jenv, keyDer)) ||
437+
(outSz > (*jenv)->GetDirectBufferCapacity(jenv, out))) {
438+
return -1;
439+
}
440+
397441
ret = wc_InitRsaKey(&myKey, NULL);
398442
if (ret != 0) {
399443
printf("wc_InitRsaKey failed, ret = %d\n", ret);
@@ -472,6 +516,13 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfCryptRSA_doDec
472516
return -1;
473517
}
474518

519+
/* Reject sizes larger than their backing direct buffers */
520+
if ((inSz > (*jenv)->GetDirectBufferCapacity(jenv, in)) ||
521+
(keySz > (*jenv)->GetDirectBufferCapacity(jenv, keyDer)) ||
522+
(outSz > (*jenv)->GetDirectBufferCapacity(jenv, out))) {
523+
return -1;
524+
}
525+
475526
wc_InitRsaKey(&myKey, NULL);
476527
idx = 0;
477528

‎src/test/com/wolfssl/test/WolfCryptRSATest.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,17 @@
2121

2222
package com.wolfssl.test;
2323

24+
import org.junit.Assume;
2425
import org.junit.BeforeClass;
2526
import org.junit.Rule;
2627
import org.junit.Test;
2728
import org.junit.rules.TestRule;
2829
import static org.junit.Assert.assertNotNull;
30+
import static org.junit.Assert.assertEquals;
2931

32+
import java.nio.ByteBuffer;
33+
34+
import com.wolfssl.WolfSSL;
3035
import com.wolfssl.WolfSSLException;
3136
import com.wolfssl.WolfCryptRSA;
3237

@@ -44,4 +49,114 @@ public static void beforeClass() {
4449
public void testRSANew() throws WolfSSLException {
4550
assertNotNull(new WolfCryptRSA());
4651
}
52+
53+
/* A size larger than its backing direct buffer must be rejected */
54+
@Test
55+
public void testDoSignRejectsOversizedSz() {
56+
WolfCryptRSA rsa = new WolfCryptRSA();
57+
ByteBuffer in = ByteBuffer.allocateDirect(64);
58+
ByteBuffer out = ByteBuffer.allocateDirect(256);
59+
ByteBuffer key = ByteBuffer.allocateDirect(128);
60+
61+
assertEquals(-1, rsa.doSign(in, 65, out, new int[]{256}, key, 128));
62+
assertEquals(-1, rsa.doSign(in, 64, out, new int[]{257}, key, 128));
63+
assertEquals(-1,
64+
rsa.doSign(in, 64, out, new int[]{256}, key, 0x100000000L + 16));
65+
}
66+
67+
@Test
68+
public void testDoEncRejectsOversizedSz() {
69+
WolfCryptRSA rsa = new WolfCryptRSA();
70+
ByteBuffer in = ByteBuffer.allocateDirect(64);
71+
ByteBuffer out = ByteBuffer.allocateDirect(256);
72+
ByteBuffer key = ByteBuffer.allocateDirect(128);
73+
74+
assertEquals(-1, rsa.doEnc(in, 65, out, new int[]{256}, key, 128));
75+
assertEquals(-1, rsa.doEnc(in, 64, out, new int[]{257}, key, 128));
76+
assertEquals(-1,
77+
rsa.doEnc(in, 64, out, new int[]{256}, key, 0x100000000L + 16));
78+
}
79+
80+
@Test
81+
public void testDoVerifyRejectsOversizedSz() {
82+
WolfCryptRSA rsa = new WolfCryptRSA();
83+
ByteBuffer sig = ByteBuffer.allocateDirect(64);
84+
ByteBuffer out = ByteBuffer.allocateDirect(256);
85+
ByteBuffer key = ByteBuffer.allocateDirect(128);
86+
87+
assertEquals(-1, rsa.doVerify(sig, 65, out, 256, key, 128));
88+
assertEquals(-1, rsa.doVerify(sig, 64, out, 257, key, 128));
89+
assertEquals(-1,
90+
rsa.doVerify(sig, 64, out, 256, key, 0x100000000L + 16));
91+
}
92+
93+
@Test
94+
public void testDoDecRejectsOversizedSz() {
95+
WolfCryptRSA rsa = new WolfCryptRSA();
96+
ByteBuffer in = ByteBuffer.allocateDirect(64);
97+
ByteBuffer out = ByteBuffer.allocateDirect(256);
98+
ByteBuffer key = ByteBuffer.allocateDirect(128);
99+
100+
assertEquals(-1, rsa.doDec(in, 65, out, 256, key, 128));
101+
assertEquals(-1, rsa.doDec(in, 64, out, 257, key, 128));
102+
assertEquals(-1,
103+
rsa.doDec(in, 64, out, 256, key, 0x100000000L + 16));
104+
}
105+
106+
/* SHA-256 hash OID sum, which wolfSSL encodes two ways depending on the
107+
* build: current default, or the legacy WOLFSSL_OLD_OID_SUM value
108+
* (see wolfSSL oid_sum.h). */
109+
private static final int SHA256_OID = 0x7cb37afb;
110+
private static final int SHA256_OID_OLD = 414;
111+
112+
/* Return a SHA-256 hash OID wc_OidGetHash accepts, or -1 if none
113+
* (PSS not compiled, or an OID-sum scheme we do not know). */
114+
private static int findPssHashOid(WolfCryptRSA rsa) {
115+
ByteBuffer b = ByteBuffer.allocateDirect(64);
116+
int[] candidates = { SHA256_OID, SHA256_OID_OLD };
117+
for (int oid : candidates) {
118+
int ret = rsa.doPssVerify(b, 64, b, 64, oid, 0, b, 64);
119+
if (ret != -1 && ret != WolfSSL.NOT_COMPILED_IN) {
120+
return oid;
121+
}
122+
}
123+
return -1;
124+
}
125+
126+
@Test
127+
public void testDoPssSignRejectsOversizedSz() {
128+
WolfCryptRSA rsa = new WolfCryptRSA();
129+
int oid = findPssHashOid(rsa);
130+
Assume.assumeTrue("PSS unavailable or unknown hash OID scheme",
131+
oid != -1);
132+
133+
ByteBuffer in = ByteBuffer.allocateDirect(64);
134+
ByteBuffer out = ByteBuffer.allocateDirect(256);
135+
ByteBuffer key = ByteBuffer.allocateDirect(128);
136+
137+
assertEquals(-1,
138+
rsa.doPssSign(in, 65, out, new int[]{256}, oid, 0, key, 128));
139+
assertEquals(-1,
140+
rsa.doPssSign(in, 64, out, new int[]{257}, oid, 0, key, 128));
141+
assertEquals(-1,
142+
rsa.doPssSign(in, 64, out, new int[]{256}, oid, 0, key,
143+
0x100000000L + 16));
144+
}
145+
146+
@Test
147+
public void testDoPssVerifyRejectsOversizedSz() {
148+
WolfCryptRSA rsa = new WolfCryptRSA();
149+
int oid = findPssHashOid(rsa);
150+
Assume.assumeTrue("PSS unavailable or unknown hash OID scheme",
151+
oid != -1);
152+
153+
ByteBuffer sig = ByteBuffer.allocateDirect(64);
154+
ByteBuffer out = ByteBuffer.allocateDirect(256);
155+
ByteBuffer key = ByteBuffer.allocateDirect(128);
156+
157+
assertEquals(-1, rsa.doPssVerify(sig, 65, out, 256, oid, 0, key, 128));
158+
assertEquals(-1, rsa.doPssVerify(sig, 64, out, 257, oid, 0, key, 128));
159+
assertEquals(-1,
160+
rsa.doPssVerify(sig, 64, out, 256, oid, 0, key, 0x100000000L + 16));
161+
}
47162
}

0 commit comments

Comments
 (0)