1#!/usr/bin/env python3 2 3"""Analyze the test outcomes from a full CI run. 4 5This script can also run on outcomes from a partial run, but the results are 6less likely to be useful. 7""" 8 9import re 10import typing 11 12import scripts_path # pylint: disable=unused-import 13from mbedtls_framework import outcome_analysis 14 15 16class CoverageTask(outcome_analysis.CoverageTask): 17 """Justify test cases that are never executed.""" 18 19 @staticmethod 20 def _has_word_re(words: typing.Iterable[str], 21 exclude: typing.Optional[str] = None) -> typing.Pattern: 22 """Construct a regex that matches if any of the words appears. 23 24 The occurrence must start and end at a word boundary. 25 26 If exclude is specified, strings containing a match for that 27 regular expression will not match the returned pattern. 28 """ 29 exclude_clause = r'' 30 if exclude: 31 exclude_clause = r'(?!.*' + exclude + ')' 32 return re.compile(exclude_clause + 33 r'.*\b(?:' + r'|'.join(words) + r')\b.*', 34 re.DOTALL) 35 36 IGNORED_TESTS = { 37 'ssl-opt': [ 38 # We don't run ssl-opt.sh with Valgrind on the CI because 39 # it's extremely slow. We don't intend to change this. 40 'DTLS client reconnect from same port: reconnect, nbio, valgrind', 41 # We don't have IPv6 in our CI environment. 42 # https://github.com/Mbed-TLS/mbedtls-test/issues/176 43 'DTLS cookie: enabled, IPv6', 44 # Disabled due to OpenSSL bug. 45 # https://github.com/openssl/openssl/issues/18887 46 'DTLS fragmenting: 3d, openssl client, DTLS 1.2', 47 # We don't run ssl-opt.sh with Valgrind on the CI because 48 # it's extremely slow. We don't intend to change this. 49 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)', 50 # TLS doesn't use restartable ECDH yet. 51 # https://github.com/Mbed-TLS/mbedtls/issues/7294 52 re.compile(r'EC restart:.*no USE_PSA.*'), 53 ], 54 'test_suite_config.mbedtls_boolean': [ 55 # Missing coverage of test configurations. 56 # https://github.com/Mbed-TLS/mbedtls/issues/9585 57 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY', 58 # Missing coverage of test configurations. 59 # https://github.com/Mbed-TLS/mbedtls/issues/9585 60 'Config: !MBEDTLS_SSL_DTLS_HELLO_VERIFY', 61 # We don't run test_suite_config when we test this. 62 # https://github.com/Mbed-TLS/mbedtls/issues/9586 63 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED', 64 ], 65 'test_suite_config.crypto_combinations': [ 66 # New thing in crypto. Not intended to be tested separately 67 # in mbedtls. 68 # https://github.com/Mbed-TLS/mbedtls/issues/10300 69 'Config: entropy: NV seed only', 70 ], 71 'test_suite_config.psa_boolean': [ 72 # We don't test with HMAC disabled. 73 # https://github.com/Mbed-TLS/mbedtls/issues/9591 74 'Config: !PSA_WANT_ALG_HMAC', 75 # The DERIVE key type is always enabled. 76 'Config: !PSA_WANT_KEY_TYPE_DERIVE', 77 # More granularity of key pair type enablement macros 78 # than we care to test. 79 # https://github.com/Mbed-TLS/mbedtls/issues/9590 80 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT', 81 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE', 82 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT', 83 # More granularity of key pair type enablement macros 84 # than we care to test. 85 # https://github.com/Mbed-TLS/mbedtls/issues/9590 86 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT', 87 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT', 88 # We don't test with HMAC disabled. 89 # https://github.com/Mbed-TLS/mbedtls/issues/9591 90 'Config: !PSA_WANT_KEY_TYPE_HMAC', 91 # The PASSWORD key type is always enabled. 92 'Config: !PSA_WANT_KEY_TYPE_PASSWORD', 93 # The PASSWORD_HASH key type is always enabled. 94 'Config: !PSA_WANT_KEY_TYPE_PASSWORD_HASH', 95 # The RAW_DATA key type is always enabled. 96 'Config: !PSA_WANT_KEY_TYPE_RAW_DATA', 97 # More granularity of key pair type enablement macros 98 # than we care to test. 99 # https://github.com/Mbed-TLS/mbedtls/issues/9590 100 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT', 101 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT', 102 # Algorithm declared but not supported. 103 'Config: PSA_WANT_ALG_CBC_MAC', 104 # Algorithm declared but not supported. 105 'Config: PSA_WANT_ALG_XTS', 106 # More granularity of key pair type enablement macros 107 # than we care to test. 108 # https://github.com/Mbed-TLS/mbedtls/issues/9590 109 'Config: PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE', 110 'Config: PSA_WANT_KEY_TYPE_ECC_KEY_PAIR', 111 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR', 112 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE', 113 # https://github.com/Mbed-TLS/mbedtls/issues/9583 114 'Config: !MBEDTLS_ECP_NIST_OPTIM', 115 # We never test without the PSA client code. Should we? 116 # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/112 117 'Config: !MBEDTLS_PSA_CRYPTO_CLIENT', 118 # We only test multithreading with pthreads. 119 # https://github.com/Mbed-TLS/mbedtls/issues/9584 120 'Config: !MBEDTLS_THREADING_PTHREAD', 121 # Built but not tested. 122 # https://github.com/Mbed-TLS/mbedtls/issues/9587 123 'Config: MBEDTLS_AES_USE_HARDWARE_ONLY', 124 # Untested platform-specific optimizations. 125 # https://github.com/Mbed-TLS/mbedtls/issues/9588 126 'Config: MBEDTLS_HAVE_SSE2', 127 # Obsolete configuration options, to be replaced by 128 # PSA entropy drivers. 129 # https://github.com/Mbed-TLS/mbedtls/issues/8150 130 'Config: MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES', 131 # Untested aspect of the platform interface. 132 # https://github.com/Mbed-TLS/mbedtls/issues/9589 133 'Config: MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', 134 # In a client-server build, test_suite_config runs in the 135 # client configuration, so it will never report 136 # MBEDTLS_PSA_CRYPTO_SPM as enabled. That's ok. 137 'Config: MBEDTLS_PSA_CRYPTO_SPM', 138 # We don't test on armv8 yet. 139 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT', 140 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY', 141 'Config: MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY', 142 'Config: MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY', 143 # We don't run test_suite_config when we test this. 144 # https://github.com/Mbed-TLS/mbedtls/issues/9586 145 'Config: MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND', 146 ], 147 'test_suite_config.psa_combinations': [ 148 # We don't test this unusual, but sensible configuration. 149 # https://github.com/Mbed-TLS/mbedtls/issues/9592 150 'Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA', 151 ], 152 'test_suite_pkcs12': [ 153 # We never test with CBC/PKCS5/PKCS12 enabled but 154 # PKCS7 padding disabled. 155 # https://github.com/Mbed-TLS/mbedtls/issues/9580 156 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)', 157 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)', 158 ], 159 'test_suite_pkcs5': [ 160 # We never test with CBC/PKCS5/PKCS12 enabled but 161 # PKCS7 padding disabled. 162 # https://github.com/Mbed-TLS/mbedtls/issues/9580 163 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)', 164 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)', 165 'PBES2 Encrypt, pad=8 (PKCS7 padding disabled)', 166 ], 167 'test_suite_psa_crypto': [ 168 # We don't test this unusual, but sensible configuration. 169 # https://github.com/Mbed-TLS/mbedtls/issues/9592 170 re.compile(r'.*ECDSA.*only deterministic supported'), 171 ], 172 'test_suite_psa_crypto_metadata': [ 173 # Algorithms declared but not supported. 174 # https://github.com/Mbed-TLS/mbedtls/issues/9579 175 'Asymmetric signature: Ed25519ph', 176 'Asymmetric signature: Ed448ph', 177 'Asymmetric signature: pure EdDSA', 178 'Cipher: XTS', 179 'MAC: CBC_MAC-3DES', 180 'MAC: CBC_MAC-AES-128', 181 'MAC: CBC_MAC-AES-192', 182 'MAC: CBC_MAC-AES-256', 183 ], 184 'test_suite_psa_crypto_not_supported.generated': [ 185 # We never test with DH key support disabled but support 186 # for a DH group enabled. The dependencies of these test 187 # cases don't really make sense. 188 # https://github.com/Mbed-TLS/mbedtls/issues/9574 189 re.compile(r'PSA \w+ DH_.*type not supported'), 190 # We only test partial support for DH with the 2048-bit group 191 # enabled and the other groups disabled. 192 # https://github.com/Mbed-TLS/mbedtls/issues/9575 193 'PSA generate DH_KEY_PAIR(RFC7919) 2048-bit group not supported', 194 'PSA import DH_KEY_PAIR(RFC7919) 2048-bit group not supported', 195 'PSA import DH_PUBLIC_KEY(RFC7919) 2048-bit group not supported', 196 ], 197 'test_suite_psa_crypto_op_fail.generated': [ 198 # We don't test this unusual, but sensible configuration. 199 # https://github.com/Mbed-TLS/mbedtls/issues/9592 200 re.compile(r'.*: !ECDSA but DETERMINISTIC_ECDSA with ECC_.*'), 201 # We never test with the HMAC algorithm enabled but the HMAC 202 # key type disabled. Those dependencies don't really make sense. 203 # https://github.com/Mbed-TLS/mbedtls/issues/9573 204 re.compile(r'.* !HMAC with HMAC'), 205 # We don't test with ECDH disabled but the key type enabled. 206 # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/161 207 re.compile(r'PSA key_agreement.* !ECDH with ECC_KEY_PAIR\(.*'), 208 # We don't test with FFDH disabled but the key type enabled. 209 # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/160 210 re.compile(r'PSA key_agreement.* !FFDH with DH_KEY_PAIR\(.*'), 211 ], 212 'test_suite_psa_crypto_op_fail.misc': [ 213 # We don't test this unusual, but sensible configuration. 214 # https://github.com/Mbed-TLS/mbedtls/issues/9592 215 'PSA sign DETERMINISTIC_ECDSA(SHA_256): !ECDSA but DETERMINISTIC_ECDSA with ECC_KEY_PAIR(SECP_R1)', #pylint: disable=line-too-long 216 ], 217 'tls13-misc': [ 218 # Disabled due to OpenSSL bug. 219 # https://github.com/openssl/openssl/issues/10714 220 'TLS 1.3 O->m: resumption', 221 # Disabled due to OpenSSL command line limitation. 222 # https://github.com/Mbed-TLS/mbedtls/issues/9582 223 'TLS 1.3 m->O: resumption with early data', 224 ], 225 } 226 227 228# The names that we give to classes derived from DriverVSReference do not 229# follow the usual naming convention, because it's more readable to use 230# underscores and parts of the configuration names. Also, these classes 231# are just there to specify some data, so they don't need repetitive 232# documentation. 233#pylint: disable=invalid-name,missing-class-docstring 234 235class DriverVSReference_hash(outcome_analysis.DriverVSReference): 236 REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa' 237 DRIVER = 'test_psa_crypto_config_accel_hash_use_psa' 238 IGNORED_SUITES = [ 239 'shax', 'mdx', # the software implementations that are being excluded 240 'md.psa', # purposefully depends on whether drivers are present 241 'psa_crypto_low_hash.generated', # testing the builtins 242 ] 243 IGNORED_TESTS = { 244 'test_suite_config': [ 245 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), 246 ], 247 'test_suite_platform': [ 248 # Incompatible with sanitizers (e.g. ASan). If the driver 249 # component uses a sanitizer but the reference component 250 # doesn't, we have a PASS vs SKIP mismatch. 251 'Check mbedtls_calloc overallocation', 252 ], 253 } 254 255class DriverVSReference_hmac(outcome_analysis.DriverVSReference): 256 REFERENCE = 'test_psa_crypto_config_reference_hmac' 257 DRIVER = 'test_psa_crypto_config_accel_hmac' 258 IGNORED_SUITES = [ 259 # These suites require legacy hash support, which is disabled 260 # in the accelerated component. 261 'shax', 'mdx', 262 # This suite tests builtins directly, but these are missing 263 # in the accelerated case. 264 'psa_crypto_low_hash.generated', 265 ] 266 IGNORED_TESTS = { 267 'test_suite_config': [ 268 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), 269 re.compile(r'.*\bMBEDTLS_MD_C\b') 270 ], 271 'test_suite_md': [ 272 # Builtin HMAC is not supported in the accelerate component. 273 re.compile('.*HMAC.*'), 274 # Following tests make use of functions which are not available 275 # when MD_C is disabled, as it happens in the accelerated 276 # test component. 277 re.compile('generic .* Hash file .*'), 278 'MD list', 279 ], 280 'test_suite_md.psa': [ 281 # "legacy only" tests require hash algorithms to be NOT 282 # accelerated, but this of course false for the accelerated 283 # test component. 284 re.compile('PSA dispatch .* legacy only'), 285 ], 286 'test_suite_platform': [ 287 # Incompatible with sanitizers (e.g. ASan). If the driver 288 # component uses a sanitizer but the reference component 289 # doesn't, we have a PASS vs SKIP mismatch. 290 'Check mbedtls_calloc overallocation', 291 ], 292 } 293 294class DriverVSReference_cipher_aead_cmac(outcome_analysis.DriverVSReference): 295 REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac' 296 DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac' 297 # Modules replaced by drivers. 298 IGNORED_SUITES = [ 299 # low-level (block/stream) cipher modules 300 'aes', 'aria', 'camellia', 'des', 'chacha20', 301 # AEAD modes, CMAC and POLY1305 302 'ccm', 'chachapoly', 'cmac', 'gcm', 'poly1305', 303 # The Cipher abstraction layer 304 'cipher', 305 ] 306 IGNORED_TESTS = { 307 'test_suite_config': [ 308 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'), 309 re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM|POLY1305)_.*'), 310 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), 311 re.compile(r'.*\bMBEDTLS_CIPHER_.*'), 312 ], 313 # PEM decryption is not supported so far. 314 # The rest of PEM (write, unencrypted read) works though. 315 'test_suite_pem': [ 316 re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), 317 ], 318 'test_suite_platform': [ 319 # Incompatible with sanitizers (e.g. ASan). If the driver 320 # component uses a sanitizer but the reference component 321 # doesn't, we have a PASS vs SKIP mismatch. 322 'Check mbedtls_calloc overallocation', 323 ], 324 # Following tests depend on AES_C/DES_C but are not about 325 # them really, just need to know some error code is there. 326 'test_suite_error': [ 327 'Low and high error', 328 'Single low error' 329 ], 330 # The en/decryption part of PKCS#12 is not supported so far. 331 # The rest of PKCS#12 (key derivation) works though. 332 'test_suite_pkcs12': [ 333 re.compile(r'PBE Encrypt, .*'), 334 re.compile(r'PBE Decrypt, .*'), 335 ], 336 # The en/decryption part of PKCS#5 is not supported so far. 337 # The rest of PKCS#5 (PBKDF2) works though. 338 'test_suite_pkcs5': [ 339 re.compile(r'PBES2 Encrypt, .*'), 340 re.compile(r'PBES2 Decrypt .*'), 341 ], 342 # Encrypted keys are not supported so far. 343 # pylint: disable=line-too-long 344 'test_suite_pkparse': [ 345 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', 346 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', 347 re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), 348 ], 349 # Encrypted keys are not supported so far. 350 'ssl-opt': [ 351 'TLS: password protected server key', 352 'TLS: password protected client key', 353 'TLS: password protected server key, two certificates', 354 ], 355 } 356 357class DriverVSReference_ecp_light_only(outcome_analysis.DriverVSReference): 358 REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only' 359 DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only' 360 IGNORED_SUITES = [ 361 # Modules replaced by drivers 362 'ecdsa', 'ecdh', 'ecjpake', 363 # Unit tests for the built-in implementation 364 'psa_crypto_ecp', 365 ] 366 IGNORED_TESTS = { 367 'test_suite_config': [ 368 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 369 ], 370 'test_suite_platform': [ 371 # Incompatible with sanitizers (e.g. ASan). If the driver 372 # component uses a sanitizer but the reference component 373 # doesn't, we have a PASS vs SKIP mismatch. 374 'Check mbedtls_calloc overallocation', 375 ], 376 # This test wants a legacy function that takes f_rng, p_rng 377 # arguments, and uses legacy ECDSA for that. The test is 378 # really about the wrapper around the PSA RNG, not ECDSA. 379 'test_suite_random': [ 380 'PSA classic wrapper: ECDSA signature (SECP256R1)', 381 ], 382 # In the accelerated test ECP_C is not set (only ECP_LIGHT is) 383 # so we must ignore disparities in the tests for which ECP_C 384 # is required. 385 'test_suite_ecp': [ 386 re.compile(r'ECP check public-private .*'), 387 re.compile(r'ECP calculate public: .*'), 388 re.compile(r'ECP gen keypair .*'), 389 re.compile(r'ECP point muladd .*'), 390 re.compile(r'ECP point multiplication .*'), 391 re.compile(r'ECP test vectors .*'), 392 ], 393 } 394 395class DriverVSReference_no_ecp_at_all(outcome_analysis.DriverVSReference): 396 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all' 397 DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all' 398 IGNORED_SUITES = [ 399 # Modules replaced by drivers 400 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 401 # Unit tests for the built-in implementation 402 'psa_crypto_ecp', 403 ] 404 IGNORED_TESTS = { 405 'test_suite_config': [ 406 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 407 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 408 ], 409 'test_suite_platform': [ 410 # Incompatible with sanitizers (e.g. ASan). If the driver 411 # component uses a sanitizer but the reference component 412 # doesn't, we have a PASS vs SKIP mismatch. 413 'Check mbedtls_calloc overallocation', 414 ], 415 # See ecp_light_only 416 'test_suite_random': [ 417 'PSA classic wrapper: ECDSA signature (SECP256R1)', 418 ], 419 'test_suite_pkparse': [ 420 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED 421 # is automatically enabled in build_info.h (backward compatibility) 422 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a 423 # consequence compressed points are supported in the reference 424 # component but not in the accelerated one, so they should be skipped 425 # while checking driver's coverage. 426 re.compile(r'Parse EC Key .*compressed\)'), 427 re.compile(r'Parse Public EC Key .*compressed\)'), 428 ], 429 } 430 431class DriverVSReference_ecc_no_bignum(outcome_analysis.DriverVSReference): 432 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum' 433 DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum' 434 IGNORED_SUITES = [ 435 # Modules replaced by drivers 436 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 437 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 438 'bignum.generated', 'bignum.misc', 439 # Unit tests for the built-in implementation 440 'psa_crypto_ecp', 441 ] 442 IGNORED_TESTS = { 443 'test_suite_config': [ 444 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 445 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 446 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 447 ], 448 'test_suite_platform': [ 449 # Incompatible with sanitizers (e.g. ASan). If the driver 450 # component uses a sanitizer but the reference component 451 # doesn't, we have a PASS vs SKIP mismatch. 452 'Check mbedtls_calloc overallocation', 453 ], 454 # See ecp_light_only 455 'test_suite_random': [ 456 'PSA classic wrapper: ECDSA signature (SECP256R1)', 457 ], 458 # See no_ecp_at_all 459 'test_suite_pkparse': [ 460 re.compile(r'Parse EC Key .*compressed\)'), 461 re.compile(r'Parse Public EC Key .*compressed\)'), 462 ], 463 'test_suite_asn1parse': [ 464 'INTEGER too large for mpi', 465 ], 466 'test_suite_asn1write': [ 467 re.compile(r'ASN.1 Write mpi.*'), 468 ], 469 'test_suite_debug': [ 470 re.compile(r'Debug print mbedtls_mpi.*'), 471 ], 472 } 473 474class DriverVSReference_ecc_ffdh_no_bignum(outcome_analysis.DriverVSReference): 475 REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum' 476 DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum' 477 IGNORED_SUITES = [ 478 # Modules replaced by drivers 479 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 480 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 481 'bignum.generated', 'bignum.misc', 482 # Unit tests for the built-in implementation 483 'psa_crypto_ecp', 484 ] 485 IGNORED_TESTS = { 486 'test_suite_config': [ 487 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 488 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 489 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 490 ], 491 'test_suite_platform': [ 492 # Incompatible with sanitizers (e.g. ASan). If the driver 493 # component uses a sanitizer but the reference component 494 # doesn't, we have a PASS vs SKIP mismatch. 495 'Check mbedtls_calloc overallocation', 496 ], 497 # See ecp_light_only 498 'test_suite_random': [ 499 'PSA classic wrapper: ECDSA signature (SECP256R1)', 500 ], 501 # See no_ecp_at_all 502 'test_suite_pkparse': [ 503 re.compile(r'Parse EC Key .*compressed\)'), 504 re.compile(r'Parse Public EC Key .*compressed\)'), 505 ], 506 'test_suite_asn1parse': [ 507 'INTEGER too large for mpi', 508 ], 509 'test_suite_asn1write': [ 510 re.compile(r'ASN.1 Write mpi.*'), 511 ], 512 'test_suite_debug': [ 513 re.compile(r'Debug print mbedtls_mpi.*'), 514 ], 515 } 516 517class DriverVSReference_ffdh_alg(outcome_analysis.DriverVSReference): 518 REFERENCE = 'test_psa_crypto_config_reference_ffdh' 519 DRIVER = 'test_psa_crypto_config_accel_ffdh' 520 IGNORED_TESTS = { 521 'test_suite_platform': [ 522 # Incompatible with sanitizers (e.g. ASan). If the driver 523 # component uses a sanitizer but the reference component 524 # doesn't, we have a PASS vs SKIP mismatch. 525 'Check mbedtls_calloc overallocation', 526 ], 527 } 528 529class DriverVSReference_tfm_config(outcome_analysis.DriverVSReference): 530 REFERENCE = 'test_tfm_config_no_p256m' 531 DRIVER = 'test_tfm_config_p256m_driver_accel_ec' 532 IGNORED_SUITES = [ 533 # Modules replaced by drivers 534 'asn1parse', 'asn1write', 535 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 536 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 537 'bignum.generated', 'bignum.misc', 538 # Unit tests for the built-in implementation 539 'psa_crypto_ecp', 540 ] 541 IGNORED_TESTS = { 542 'test_suite_config': [ 543 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 544 re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'), 545 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'), 546 re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*') 547 ], 548 'test_suite_config.crypto_combinations': [ 549 'Config: ECC: Weierstrass curves only', 550 ], 551 'test_suite_platform': [ 552 # Incompatible with sanitizers (e.g. ASan). If the driver 553 # component uses a sanitizer but the reference component 554 # doesn't, we have a PASS vs SKIP mismatch. 555 'Check mbedtls_calloc overallocation', 556 ], 557 # See ecp_light_only 558 'test_suite_random': [ 559 'PSA classic wrapper: ECDSA signature (SECP256R1)', 560 ], 561 } 562 563class DriverVSReference_rsa(outcome_analysis.DriverVSReference): 564 REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto' 565 DRIVER = 'test_psa_crypto_config_accel_rsa_crypto' 566 IGNORED_SUITES = [ 567 # Modules replaced by drivers. 568 'rsa', 'pkcs1_v15', 'pkcs1_v21', 569 # We temporarily don't care about PK stuff. 570 'pk', 'pkwrite', 'pkparse' 571 ] 572 IGNORED_TESTS = { 573 'test_suite_bignum.misc': [ 574 re.compile(r'.*\bmbedtls_mpi_is_prime.*'), 575 re.compile(r'.*\bmbedtls_mpi_gen_prime.*'), 576 ], 577 'test_suite_config': [ 578 re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'), 579 re.compile(r'.*\bMBEDTLS_GENPRIME\b.*') 580 ], 581 'test_suite_platform': [ 582 # Incompatible with sanitizers (e.g. ASan). If the driver 583 # component uses a sanitizer but the reference component 584 # doesn't, we have a PASS vs SKIP mismatch. 585 'Check mbedtls_calloc overallocation', 586 ], 587 # Following tests depend on RSA_C but are not about 588 # them really, just need to know some error code is there. 589 'test_suite_error': [ 590 'Low and high error', 591 'Single high error' 592 ], 593 # Constant time operations only used for PKCS1_V15 594 'test_suite_constant_time': [ 595 re.compile(r'mbedtls_ct_zeroize_if .*'), 596 re.compile(r'mbedtls_ct_memmove_left .*') 597 ], 598 'test_suite_psa_crypto': [ 599 # We don't support generate_key_custom entry points 600 # in drivers yet. 601 re.compile(r'PSA generate key custom: RSA, e=.*'), 602 re.compile(r'PSA generate key ext: RSA, e=.*'), 603 ], 604 } 605 606class DriverVSReference_block_cipher_dispatch(outcome_analysis.DriverVSReference): 607 REFERENCE = 'test_full_block_cipher_legacy_dispatch' 608 DRIVER = 'test_full_block_cipher_psa_dispatch' 609 IGNORED_SUITES = [ 610 # Skipped in the accelerated component 611 'aes', 'aria', 'camellia', 612 # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in 613 # order for the cipher module (actually cipher_wrapper) to work 614 # properly. However these symbols are disabled in the accelerated 615 # component so we ignore them. 616 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', 617 'cipher.camellia', 618 ] 619 IGNORED_TESTS = { 620 'test_suite_config': [ 621 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'), 622 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), 623 ], 624 'test_suite_cmac': [ 625 # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, 626 # but these are not available in the accelerated component. 627 'CMAC null arguments', 628 re.compile('CMAC.* (AES|ARIA|Camellia).*'), 629 ], 630 'test_suite_cipher.padding': [ 631 # Following tests require AES_C/CAMELLIA_C to be enabled, 632 # but these are not available in the accelerated component. 633 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), 634 ], 635 'test_suite_pkcs5': [ 636 # The AES part of PKCS#5 PBES2 is not yet supported. 637 # The rest of PKCS#5 (PBKDF2) works, though. 638 re.compile(r'PBES2 .* AES-.*') 639 ], 640 'test_suite_pkparse': [ 641 # PEM (called by pkparse) requires AES_C in order to decrypt 642 # the key, but this is not available in the accelerated 643 # component. 644 re.compile('Parse RSA Key.*(password|AES-).*'), 645 ], 646 'test_suite_pem': [ 647 # Following tests require AES_C, but this is diabled in the 648 # accelerated component. 649 re.compile('PEM read .*AES.*'), 650 'PEM read (unknown encryption algorithm)', 651 ], 652 'test_suite_error': [ 653 # Following tests depend on AES_C but are not about them 654 # really, just need to know some error code is there. 655 'Single low error', 656 'Low and high error', 657 ], 658 'test_suite_platform': [ 659 # Incompatible with sanitizers (e.g. ASan). If the driver 660 # component uses a sanitizer but the reference component 661 # doesn't, we have a PASS vs SKIP mismatch. 662 'Check mbedtls_calloc overallocation', 663 ], 664 } 665 666#pylint: enable=invalid-name,missing-class-docstring 667 668 669# List of tasks with a function that can handle this task and additional arguments if required 670KNOWN_TASKS = { 671 'analyze_coverage': CoverageTask, 672 'analyze_driver_vs_reference_hash': DriverVSReference_hash, 673 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac, 674 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac, 675 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only, 676 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all, 677 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum, 678 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum, 679 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg, 680 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config, 681 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa, 682 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch, 683} 684 685if __name__ == '__main__': 686 outcome_analysis.main(KNOWN_TASKS) 687