Attempted to read or write protected memory. (#13)

* Refactor interop marshaling and buffer handling in certificate validation

Author: Kyrmasch

NKalkan/InteropDefinitions.cs

@@ -29,7 +29,7 @@ namespace NKalkan;
 internal delegate KalkanError KC_SignData(string? alias, int flags, byte[] inData, int inDataLength, string? inSign, int inSignLength, StringBuilder? outSign, ref int outSignoutSignLength);
 
 [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-internal delegate KalkanError KC_VerifyData(string? alias, int flags, byte[] inData, int inDataLength, string inoutSign, int inoutSignLength, 
+internal delegate KalkanError KC_VerifyData(string? alias, int flags, byte[] inData, int inDataLength, string inoutSign, int inoutSignLength,
     StringBuilder? outData, ref int outDataLength, StringBuilder? outVerifyInfo, ref int outVerifyInfoLength, int inCertID, StringBuilder? outCert, ref int outCertLength);
 
 [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
@@ -42,7 +42,7 @@ internal delegate KalkanError KC_VerifyData(string? alias, int flags, byte[] inD
 internal delegate KalkanError KC_X509CertificateGetInfo(string certificateData, int certificateDataLength, int propertyId, [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder certificatePropertyData, ref int certificatePropertyDataLength);
 
 [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-internal delegate KalkanError KC_X509ValidateCertificate(string certificateData, int certificateDataLength, int validType, [MarshalAs(UnmanagedType.LPUTF8Str)] string validPath, long checkTime, [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder? outputInformation, ref int outputInformationLength, int flag, [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder? ocsPResponse, ref int ocsPResponseLength);
+internal delegate KalkanError KC_X509ValidateCertificate([MarshalAs(UnmanagedType.LPUTF8Str)] string certificateData, int certificateDataLength, int validType, [MarshalAs(UnmanagedType.LPUTF8Str)] string validPath, long checkTime, [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 6)] byte[]? outputInformation, ref int outputInformationLength, int flag, [Out, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 9)] byte[]? ocsPResponse, ref int ocsPResponseLength);
 
 [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
 internal delegate KalkanError KC_SignWSSE(string? alias, int flags, byte[] inData, int inDataLength, byte[] outSign, ref int outSignoutSignLength, string? signNodeId);

NKalkan/KalkanApi.cs

@@ -168,28 +168,24 @@ public void ValidateCertificate(string certificate, KalkanValidationType validat
             throw new ArgumentNullException(nameof(certificate));
         }
 
-        int certificateLength = certificate.Length;
-        int outputInformationLength = 0;
-        int ospResponseLength = 0;
-        
-        int flag = (checkCertificateTime ? 0 : KalkanConstants.KC_NOCHECKCERTTIME) + (getOscpResponse ? KalkanConstants.KC_GET_OCSP_RESPONSE : 0);
+        int certificateLength = Encoding.UTF8.GetByteCount(certificate);
+        int outputInformationLength = 64 * 1024;
+        int ospResponseLength = getOscpResponse ? 128 * 1024 : 0;
 
-        var errorCode = StKCFunctionsType.X509ValidateCertificate(certificate, certificateLength, (int)validationType, validPath, checkTime: 0, null, ref outputInformationLength, flag, null, ref ospResponseLength);
-        if (errorCode != KalkanError.BUFFER_TOO_SMALL)
-        {
-            ThrowIfError(errorCode);
-        }
+        int flag = (checkCertificateTime ? 0 : KalkanConstants.KC_NOCHECKCERTTIME) + (getOscpResponse ? KalkanConstants.KC_GET_OCSP_RESPONSE : 0);
 
-        // Fix: BUFFER_TOO_SMALL
-        ospResponseLength = ospResponseLength + 200;
-        outputInformationLength = outputInformationLength + 200;
+        byte[] outputInformationBuf = new byte[outputInformationLength];
+        byte[]? ospResponseBuf = getOscpResponse ? new byte[ospResponseLength] : null;
 
-        StringBuilder ospResponseBuilder = new StringBuilder(ospResponseLength);
-        StringBuilder outputInformationBuilder = new StringBuilder(outputInformationLength);
-        errorCode = StKCFunctionsType.X509ValidateCertificate(certificate, certificateLength, (int)validationType, validPath, checkTime: 0, outputInformationBuilder, ref outputInformationLength, flag, ospResponseBuilder, ref ospResponseLength);
+        var errorCode = StKCFunctionsType.X509ValidateCertificate(certificate, certificateLength, (int)validationType, validPath, checkTime: 0, outputInformation: outputInformationBuf, ref outputInformationLength, flag, ocsPResponse: ospResponseBuf, ref ospResponseLength);
         ThrowIfError(errorCode);
-        outputInformation = outputInformationBuilder.ToString();
-        ospResponse = ospResponseBuilder.ToString();
+
+        outputInformation = Encoding.UTF8.GetString(outputInformationBuf, 0, outputInformationLength);
+
+        if (getOscpResponse && ospResponseBuf != null && ospResponseLength > 0)
+            ospResponse = Convert.ToBase64String(ospResponseBuf, 0, ospResponseLength);
+        else
+            ospResponse = string.Empty;
     }
 
     public string SignXml(string content, KalkanSignFlags flags = 0, string? certificateAlias = null, string? signNodeId = null, string? parentSignNode = null, string parentNameSpace = "")
@@ -319,7 +315,7 @@ public string SignWsse(string content, string? signNodeId, KalkanSignFlags flags
         ThrowIfError(errorCode);
         return Encoding.UTF8.GetString(signedPayload, 0, signedPayloadLength).TrimEnd('\0');
     }
-    
+
     /// <summary>
     /// Sing envelope with custom xml attributes 
     /// </summary>
@@ -530,7 +526,7 @@ private static void ThrowIfError(KalkanError errorCode)
 
         throw new InvalidOperationException(err.ToString());
     }
-    
+
     private static KalkanSignFlags SignFlags(KalkanSignType signType, KalkanInputFormat inputFormat,
         KalkanOutputFormat outputFormat)
     {