fix XFWRITE called with negative size

Author: miyazakh

src/sign-verify/clu_sign.c

@@ -729,6 +729,16 @@ int wolfCLU_sign_data_dilithium (byte* data, char* out, word32 dataSz, char* pri
     else {
         XFILE outFile;
         outFile = XFOPEN(out, "wb");
+        if (outFile == NULL) {
+            wolfCLU_LogError("Failed to open output file %s", out);
+            XFREE(outBuf, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+            wc_FreeRng(&rng);
+            wc_dilithium_free(key);
+        #ifdef WOLFSSL_SMALL_STACK
+            XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+        #endif
+            return BAD_FUNC_ARG;
+        }
         XFWRITE(outBuf, 1, outBufSz, outFile);
         XFCLOSE(outFile);
     }

src/sign-verify/clu_verify.c

@@ -438,14 +438,16 @@ int wolfCLU_verify_signature_rsa(byte* sig, char* out, int sigSz, char* keyPath,
         }
 
         /* write the output to the specified file */
-        XFILE s = XFOPEN(out, "wb");
-        if (s == NULL) {
-            wolfCLU_LogError("Unable to open file %s", out);
-            ret = BAD_FUNC_ARG;
-        }
-        else {
-            XFWRITE(outBuf, 1, ret, s);
-            XFCLOSE(s);
+        if (ret > 0) {
+            XFILE s = XFOPEN(out, "wb");
+            if (s == NULL) {
+                wolfCLU_LogError("Unable to open file %s", out);
+                ret = BAD_FUNC_ARG;
+            }
+            else {
+                XFWRITE(outBuf, 1, ret, s);
+                XFCLOSE(s);
+            }
         }
     }
 
@@ -954,7 +956,7 @@ int wolfCLU_verify_signature_xmss(byte* sig, int sigSz,
         for (int i = 0; i < XMSS_OID_LEN; i++) {
             oid = (oid << 8) | keyBuf[i];
         }
-        
+
         switch (oid) {
             case WC_XMSS_OID_SHA2_10_256:
                 XMEMCPY(paramStr, "XMSS-SHA2_10_256\0", paramLen);
@@ -1109,7 +1111,7 @@ int wolfCLU_verify_signature_xmssmt(byte* sig, int sigSz,
         for (int i = 0; i < XMSS_OID_LEN; i++) {
             oid = (oid << 8) | keyBuf[i];
         }
-        
+
         switch (oid) {
             case WC_XMSSMT_OID_SHA2_20_2_256:
                 XMEMCPY(paramStr, "XMSSMT-SHA2_20/2_256\0\0", paramLen);

tests/genkey_sign_ver/genkey-sign-ver-test.sh

@@ -176,6 +176,16 @@ DERPEMRAW="der"
 VERIFYOUTNAME="rsa-sigout"
 gen_key_sign_ver_test ${ALGORITHM} ${KEYFILENAME} ${SIGOUTNAME} ${DERPEMRAW} ${VERIFYOUTNAME}
 
+# A verify with invalid signature must fail gracefully.
+./wolfssl -rsa -verify -inkey rsakey.pub -inform der \
+          -sigfile sign-this.txt -in sign-this.txt \
+          -out rsa_badverify_out.txt -pubin
+RESULT=$?
+[ $RESULT -eq 0 ] && \
+    printf '%s\n' "RSA verify with invalid sig should have failed" && exit 99
+[ -f rsa_badverify_out.txt ] && \
+    printf '%s\n' "RSA verify with invalid sig: output file must not be created" && exit 99
+
 # Regression test: -exponent value must not overwrite -size (was stored in
 # sizeArg instead of expArg, corrupting the key size).
 ./wolfssl -genkey rsa -size 2048 -exponent 65537 -out rsakey_exp \
@@ -227,6 +237,14 @@ for level in 2 3 5
 do
     gen_key_sign_ver_test ${ALGORITHM} ${KEYFILENAME} ${SIGOUTNAME} ${DERPEMRAW} ${level}
 done
+
+# Dilithium sign to an unwritable path must fail gracefully
+./wolfssl -genkey dilithium -level 2 -out mldsakey -outform der -output keypair
+./wolfssl -dilithium -sign -inkey mldsakey.priv -inform der \
+    -in sign-this.txt -out /nonexistent_dir/mldsa_bad.sig
+RESULT=$?
+[ $RESULT -eq 0 ] && \
+    printf '%s\n' "dilithium sign to invalid path should have failed" && exit 99
 fi
 
 # Check if xmss is availabe