00001
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038
00039
00040
00041
00042
00043
00044
00045
00046
00047
00048
00049
00050
00051
00052
00053
00054
00055
00056
00057
00058
00059
00060
00061
00062
00063 #ifndef MBEDTLS_BN_MUL_H
00064 #define MBEDTLS_BN_MUL_H
00065
00066 #if !defined(MBEDTLS_CONFIG_FILE)
00067 #include "config.h"
00068 #else
00069 #include MBEDTLS_CONFIG_FILE
00070 #endif
00071
00072 #include "bignum.h"
00073
00074 #if defined(MBEDTLS_HAVE_ASM)
00075
00076 #ifndef asm
00077 #define asm __asm
00078 #endif
00079
00080
00081 #if defined(__GNUC__) && \
00082 ( !defined(__ARMCC_VERSION) || __ARMCC_VERSION >= 6000000 )
00083
00084
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096 #if defined(__GNUC__) && __GNUC__ < 5 && defined(__PIC__)
00097 #define MULADDC_CANNOT_USE_EBX
00098 #endif
00099
00100
00101
00102
00103
00104
00105
00106 #if defined(__i386__) && defined(__OPTIMIZE__) && !defined(MULADDC_CANNOT_USE_EBX)
00107
00108 #define MULADDC_INIT \
00109 asm( \
00110 "movl %%ebx, %0 \n\t" \
00111 "movl %5, %%esi \n\t" \
00112 "movl %6, %%edi \n\t" \
00113 "movl %7, %%ecx \n\t" \
00114 "movl %8, %%ebx \n\t"
00115
00116 #define MULADDC_CORE \
00117 "lodsl \n\t" \
00118 "mull %%ebx \n\t" \
00119 "addl %%ecx, %%eax \n\t" \
00120 "adcl $0, %%edx \n\t" \
00121 "addl (%%edi), %%eax \n\t" \
00122 "adcl $0, %%edx \n\t" \
00123 "movl %%edx, %%ecx \n\t" \
00124 "stosl \n\t"
00125
00126 #if defined(MBEDTLS_HAVE_SSE2)
00127
00128 #define MULADDC_HUIT \
00129 "movd %%ecx, %%mm1 \n\t" \
00130 "movd %%ebx, %%mm0 \n\t" \
00131 "movd (%%edi), %%mm3 \n\t" \
00132 "paddq %%mm3, %%mm1 \n\t" \
00133 "movd (%%esi), %%mm2 \n\t" \
00134 "pmuludq %%mm0, %%mm2 \n\t" \
00135 "movd 4(%%esi), %%mm4 \n\t" \
00136 "pmuludq %%mm0, %%mm4 \n\t" \
00137 "movd 8(%%esi), %%mm6 \n\t" \
00138 "pmuludq %%mm0, %%mm6 \n\t" \
00139 "movd 12(%%esi), %%mm7 \n\t" \
00140 "pmuludq %%mm0, %%mm7 \n\t" \
00141 "paddq %%mm2, %%mm1 \n\t" \
00142 "movd 4(%%edi), %%mm3 \n\t" \
00143 "paddq %%mm4, %%mm3 \n\t" \
00144 "movd 8(%%edi), %%mm5 \n\t" \
00145 "paddq %%mm6, %%mm5 \n\t" \
00146 "movd 12(%%edi), %%mm4 \n\t" \
00147 "paddq %%mm4, %%mm7 \n\t" \
00148 "movd %%mm1, (%%edi) \n\t" \
00149 "movd 16(%%esi), %%mm2 \n\t" \
00150 "pmuludq %%mm0, %%mm2 \n\t" \
00151 "psrlq $32, %%mm1 \n\t" \
00152 "movd 20(%%esi), %%mm4 \n\t" \
00153 "pmuludq %%mm0, %%mm4 \n\t" \
00154 "paddq %%mm3, %%mm1 \n\t" \
00155 "movd 24(%%esi), %%mm6 \n\t" \
00156 "pmuludq %%mm0, %%mm6 \n\t" \
00157 "movd %%mm1, 4(%%edi) \n\t" \
00158 "psrlq $32, %%mm1 \n\t" \
00159 "movd 28(%%esi), %%mm3 \n\t" \
00160 "pmuludq %%mm0, %%mm3 \n\t" \
00161 "paddq %%mm5, %%mm1 \n\t" \
00162 "movd 16(%%edi), %%mm5 \n\t" \
00163 "paddq %%mm5, %%mm2 \n\t" \
00164 "movd %%mm1, 8(%%edi) \n\t" \
00165 "psrlq $32, %%mm1 \n\t" \
00166 "paddq %%mm7, %%mm1 \n\t" \
00167 "movd 20(%%edi), %%mm5 \n\t" \
00168 "paddq %%mm5, %%mm4 \n\t" \
00169 "movd %%mm1, 12(%%edi) \n\t" \
00170 "psrlq $32, %%mm1 \n\t" \
00171 "paddq %%mm2, %%mm1 \n\t" \
00172 "movd 24(%%edi), %%mm5 \n\t" \
00173 "paddq %%mm5, %%mm6 \n\t" \
00174 "movd %%mm1, 16(%%edi) \n\t" \
00175 "psrlq $32, %%mm1 \n\t" \
00176 "paddq %%mm4, %%mm1 \n\t" \
00177 "movd 28(%%edi), %%mm5 \n\t" \
00178 "paddq %%mm5, %%mm3 \n\t" \
00179 "movd %%mm1, 20(%%edi) \n\t" \
00180 "psrlq $32, %%mm1 \n\t" \
00181 "paddq %%mm6, %%mm1 \n\t" \
00182 "movd %%mm1, 24(%%edi) \n\t" \
00183 "psrlq $32, %%mm1 \n\t" \
00184 "paddq %%mm3, %%mm1 \n\t" \
00185 "movd %%mm1, 28(%%edi) \n\t" \
00186 "addl $32, %%edi \n\t" \
00187 "addl $32, %%esi \n\t" \
00188 "psrlq $32, %%mm1 \n\t" \
00189 "movd %%mm1, %%ecx \n\t"
00190
00191 #define MULADDC_STOP \
00192 "emms \n\t" \
00193 "movl %4, %%ebx \n\t" \
00194 "movl %%ecx, %1 \n\t" \
00195 "movl %%edi, %2 \n\t" \
00196 "movl %%esi, %3 \n\t" \
00197 : "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
00198 : "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
00199 : "eax", "ebx", "ecx", "edx", "esi", "edi" \
00200 );
00201
00202 #else
00203
00204 #define MULADDC_STOP \
00205 "movl %4, %%ebx \n\t" \
00206 "movl %%ecx, %1 \n\t" \
00207 "movl %%edi, %2 \n\t" \
00208 "movl %%esi, %3 \n\t" \
00209 : "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
00210 : "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
00211 : "eax", "ebx", "ecx", "edx", "esi", "edi" \
00212 );
00213 #endif
00214 #endif
00215
00216 #if defined(__amd64__) || defined (__x86_64__)
00217
00218 #define MULADDC_INIT \
00219 asm( \
00220 "xorq %%r8, %%r8\n"
00221
00222 #define MULADDC_CORE \
00223 "movq (%%rsi), %%rax\n" \
00224 "mulq %%rbx\n" \
00225 "addq $8, %%rsi\n" \
00226 "addq %%rcx, %%rax\n" \
00227 "movq %%r8, %%rcx\n" \
00228 "adcq $0, %%rdx\n" \
00229 "nop \n" \
00230 "addq %%rax, (%%rdi)\n" \
00231 "adcq %%rdx, %%rcx\n" \
00232 "addq $8, %%rdi\n"
00233
00234 #define MULADDC_STOP \
00235 : "+c" (c), "+D" (d), "+S" (s) \
00236 : "b" (b) \
00237 : "rax", "rdx", "r8" \
00238 );
00239
00240 #endif
00241
00242 #if defined(__mc68020__) || defined(__mcpu32__)
00243
00244 #define MULADDC_INIT \
00245 asm( \
00246 "movl %3, %%a2 \n\t" \
00247 "movl %4, %%a3 \n\t" \
00248 "movl %5, %%d3 \n\t" \
00249 "movl %6, %%d2 \n\t" \
00250 "moveq #0, %%d0 \n\t"
00251
00252 #define MULADDC_CORE \
00253 "movel %%a2@+, %%d1 \n\t" \
00254 "mulul %%d2, %%d4:%%d1 \n\t" \
00255 "addl %%d3, %%d1 \n\t" \
00256 "addxl %%d0, %%d4 \n\t" \
00257 "moveq #0, %%d3 \n\t" \
00258 "addl %%d1, %%a3@+ \n\t" \
00259 "addxl %%d4, %%d3 \n\t"
00260
00261 #define MULADDC_STOP \
00262 "movl %%d3, %0 \n\t" \
00263 "movl %%a3, %1 \n\t" \
00264 "movl %%a2, %2 \n\t" \
00265 : "=m" (c), "=m" (d), "=m" (s) \
00266 : "m" (s), "m" (d), "m" (c), "m" (b) \
00267 : "d0", "d1", "d2", "d3", "d4", "a2", "a3" \
00268 );
00269
00270 #define MULADDC_HUIT \
00271 "movel %%a2@+, %%d1 \n\t" \
00272 "mulul %%d2, %%d4:%%d1 \n\t" \
00273 "addxl %%d3, %%d1 \n\t" \
00274 "addxl %%d0, %%d4 \n\t" \
00275 "addl %%d1, %%a3@+ \n\t" \
00276 "movel %%a2@+, %%d1 \n\t" \
00277 "mulul %%d2, %%d3:%%d1 \n\t" \
00278 "addxl %%d4, %%d1 \n\t" \
00279 "addxl %%d0, %%d3 \n\t" \
00280 "addl %%d1, %%a3@+ \n\t" \
00281 "movel %%a2@+, %%d1 \n\t" \
00282 "mulul %%d2, %%d4:%%d1 \n\t" \
00283 "addxl %%d3, %%d1 \n\t" \
00284 "addxl %%d0, %%d4 \n\t" \
00285 "addl %%d1, %%a3@+ \n\t" \
00286 "movel %%a2@+, %%d1 \n\t" \
00287 "mulul %%d2, %%d3:%%d1 \n\t" \
00288 "addxl %%d4, %%d1 \n\t" \
00289 "addxl %%d0, %%d3 \n\t" \
00290 "addl %%d1, %%a3@+ \n\t" \
00291 "movel %%a2@+, %%d1 \n\t" \
00292 "mulul %%d2, %%d4:%%d1 \n\t" \
00293 "addxl %%d3, %%d1 \n\t" \
00294 "addxl %%d0, %%d4 \n\t" \
00295 "addl %%d1, %%a3@+ \n\t" \
00296 "movel %%a2@+, %%d1 \n\t" \
00297 "mulul %%d2, %%d3:%%d1 \n\t" \
00298 "addxl %%d4, %%d1 \n\t" \
00299 "addxl %%d0, %%d3 \n\t" \
00300 "addl %%d1, %%a3@+ \n\t" \
00301 "movel %%a2@+, %%d1 \n\t" \
00302 "mulul %%d2, %%d4:%%d1 \n\t" \
00303 "addxl %%d3, %%d1 \n\t" \
00304 "addxl %%d0, %%d4 \n\t" \
00305 "addl %%d1, %%a3@+ \n\t" \
00306 "movel %%a2@+, %%d1 \n\t" \
00307 "mulul %%d2, %%d3:%%d1 \n\t" \
00308 "addxl %%d4, %%d1 \n\t" \
00309 "addxl %%d0, %%d3 \n\t" \
00310 "addl %%d1, %%a3@+ \n\t" \
00311 "addxl %%d0, %%d3 \n\t"
00312
00313 #endif
00314
00315 #if defined(__powerpc64__) || defined(__ppc64__)
00316
00317 #if defined(__MACH__) && defined(__APPLE__)
00318
00319 #define MULADDC_INIT \
00320 asm( \
00321 "ld r3, %3 \n\t" \
00322 "ld r4, %4 \n\t" \
00323 "ld r5, %5 \n\t" \
00324 "ld r6, %6 \n\t" \
00325 "addi r3, r3, -8 \n\t" \
00326 "addi r4, r4, -8 \n\t" \
00327 "addic r5, r5, 0 \n\t"
00328
00329 #define MULADDC_CORE \
00330 "ldu r7, 8(r3) \n\t" \
00331 "mulld r8, r7, r6 \n\t" \
00332 "mulhdu r9, r7, r6 \n\t" \
00333 "adde r8, r8, r5 \n\t" \
00334 "ld r7, 8(r4) \n\t" \
00335 "addze r5, r9 \n\t" \
00336 "addc r8, r8, r7 \n\t" \
00337 "stdu r8, 8(r4) \n\t"
00338
00339 #define MULADDC_STOP \
00340 "addze r5, r5 \n\t" \
00341 "addi r4, r4, 8 \n\t" \
00342 "addi r3, r3, 8 \n\t" \
00343 "std r5, %0 \n\t" \
00344 "std r4, %1 \n\t" \
00345 "std r3, %2 \n\t" \
00346 : "=m" (c), "=m" (d), "=m" (s) \
00347 : "m" (s), "m" (d), "m" (c), "m" (b) \
00348 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00349 );
00350
00351
00352 #else
00353
00354 #define MULADDC_INIT \
00355 asm( \
00356 "ld %%r3, %3 \n\t" \
00357 "ld %%r4, %4 \n\t" \
00358 "ld %%r5, %5 \n\t" \
00359 "ld %%r6, %6 \n\t" \
00360 "addi %%r3, %%r3, -8 \n\t" \
00361 "addi %%r4, %%r4, -8 \n\t" \
00362 "addic %%r5, %%r5, 0 \n\t"
00363
00364 #define MULADDC_CORE \
00365 "ldu %%r7, 8(%%r3) \n\t" \
00366 "mulld %%r8, %%r7, %%r6 \n\t" \
00367 "mulhdu %%r9, %%r7, %%r6 \n\t" \
00368 "adde %%r8, %%r8, %%r5 \n\t" \
00369 "ld %%r7, 8(%%r4) \n\t" \
00370 "addze %%r5, %%r9 \n\t" \
00371 "addc %%r8, %%r8, %%r7 \n\t" \
00372 "stdu %%r8, 8(%%r4) \n\t"
00373
00374 #define MULADDC_STOP \
00375 "addze %%r5, %%r5 \n\t" \
00376 "addi %%r4, %%r4, 8 \n\t" \
00377 "addi %%r3, %%r3, 8 \n\t" \
00378 "std %%r5, %0 \n\t" \
00379 "std %%r4, %1 \n\t" \
00380 "std %%r3, %2 \n\t" \
00381 : "=m" (c), "=m" (d), "=m" (s) \
00382 : "m" (s), "m" (d), "m" (c), "m" (b) \
00383 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00384 );
00385
00386 #endif
00387
00388 #elif defined(__powerpc__) || defined(__ppc__)
00389
00390 #if defined(__MACH__) && defined(__APPLE__)
00391
00392 #define MULADDC_INIT \
00393 asm( \
00394 "lwz r3, %3 \n\t" \
00395 "lwz r4, %4 \n\t" \
00396 "lwz r5, %5 \n\t" \
00397 "lwz r6, %6 \n\t" \
00398 "addi r3, r3, -4 \n\t" \
00399 "addi r4, r4, -4 \n\t" \
00400 "addic r5, r5, 0 \n\t"
00401
00402 #define MULADDC_CORE \
00403 "lwzu r7, 4(r3) \n\t" \
00404 "mullw r8, r7, r6 \n\t" \
00405 "mulhwu r9, r7, r6 \n\t" \
00406 "adde r8, r8, r5 \n\t" \
00407 "lwz r7, 4(r4) \n\t" \
00408 "addze r5, r9 \n\t" \
00409 "addc r8, r8, r7 \n\t" \
00410 "stwu r8, 4(r4) \n\t"
00411
00412 #define MULADDC_STOP \
00413 "addze r5, r5 \n\t" \
00414 "addi r4, r4, 4 \n\t" \
00415 "addi r3, r3, 4 \n\t" \
00416 "stw r5, %0 \n\t" \
00417 "stw r4, %1 \n\t" \
00418 "stw r3, %2 \n\t" \
00419 : "=m" (c), "=m" (d), "=m" (s) \
00420 : "m" (s), "m" (d), "m" (c), "m" (b) \
00421 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00422 );
00423
00424 #else
00425
00426 #define MULADDC_INIT \
00427 asm( \
00428 "lwz %%r3, %3 \n\t" \
00429 "lwz %%r4, %4 \n\t" \
00430 "lwz %%r5, %5 \n\t" \
00431 "lwz %%r6, %6 \n\t" \
00432 "addi %%r3, %%r3, -4 \n\t" \
00433 "addi %%r4, %%r4, -4 \n\t" \
00434 "addic %%r5, %%r5, 0 \n\t"
00435
00436 #define MULADDC_CORE \
00437 "lwzu %%r7, 4(%%r3) \n\t" \
00438 "mullw %%r8, %%r7, %%r6 \n\t" \
00439 "mulhwu %%r9, %%r7, %%r6 \n\t" \
00440 "adde %%r8, %%r8, %%r5 \n\t" \
00441 "lwz %%r7, 4(%%r4) \n\t" \
00442 "addze %%r5, %%r9 \n\t" \
00443 "addc %%r8, %%r8, %%r7 \n\t" \
00444 "stwu %%r8, 4(%%r4) \n\t"
00445
00446 #define MULADDC_STOP \
00447 "addze %%r5, %%r5 \n\t" \
00448 "addi %%r4, %%r4, 4 \n\t" \
00449 "addi %%r3, %%r3, 4 \n\t" \
00450 "stw %%r5, %0 \n\t" \
00451 "stw %%r4, %1 \n\t" \
00452 "stw %%r3, %2 \n\t" \
00453 : "=m" (c), "=m" (d), "=m" (s) \
00454 : "m" (s), "m" (d), "m" (c), "m" (b) \
00455 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00456 );
00457
00458 #endif
00459
00460 #endif
00461
00462
00463
00464
00465
00466 #if 0 && defined(__sparc__)
00467 #if defined(__sparc64__)
00468
00469 #define MULADDC_INIT \
00470 asm( \
00471 "ldx %3, %%o0 \n\t" \
00472 "ldx %4, %%o1 \n\t" \
00473 "ld %5, %%o2 \n\t" \
00474 "ld %6, %%o3 \n\t"
00475
00476 #define MULADDC_CORE \
00477 "ld [%%o0], %%o4 \n\t" \
00478 "inc 4, %%o0 \n\t" \
00479 "ld [%%o1], %%o5 \n\t" \
00480 "umul %%o3, %%o4, %%o4 \n\t" \
00481 "addcc %%o4, %%o2, %%o4 \n\t" \
00482 "rd %%y, %%g1 \n\t" \
00483 "addx %%g1, 0, %%g1 \n\t" \
00484 "addcc %%o4, %%o5, %%o4 \n\t" \
00485 "st %%o4, [%%o1] \n\t" \
00486 "addx %%g1, 0, %%o2 \n\t" \
00487 "inc 4, %%o1 \n\t"
00488
00489 #define MULADDC_STOP \
00490 "st %%o2, %0 \n\t" \
00491 "stx %%o1, %1 \n\t" \
00492 "stx %%o0, %2 \n\t" \
00493 : "=m" (c), "=m" (d), "=m" (s) \
00494 : "m" (s), "m" (d), "m" (c), "m" (b) \
00495 : "g1", "o0", "o1", "o2", "o3", "o4", \
00496 "o5" \
00497 );
00498
00499 #else
00500
00501 #define MULADDC_INIT \
00502 asm( \
00503 "ld %3, %%o0 \n\t" \
00504 "ld %4, %%o1 \n\t" \
00505 "ld %5, %%o2 \n\t" \
00506 "ld %6, %%o3 \n\t"
00507
00508 #define MULADDC_CORE \
00509 "ld [%%o0], %%o4 \n\t" \
00510 "inc 4, %%o0 \n\t" \
00511 "ld [%%o1], %%o5 \n\t" \
00512 "umul %%o3, %%o4, %%o4 \n\t" \
00513 "addcc %%o4, %%o2, %%o4 \n\t" \
00514 "rd %%y, %%g1 \n\t" \
00515 "addx %%g1, 0, %%g1 \n\t" \
00516 "addcc %%o4, %%o5, %%o4 \n\t" \
00517 "st %%o4, [%%o1] \n\t" \
00518 "addx %%g1, 0, %%o2 \n\t" \
00519 "inc 4, %%o1 \n\t"
00520
00521 #define MULADDC_STOP \
00522 "st %%o2, %0 \n\t" \
00523 "st %%o1, %1 \n\t" \
00524 "st %%o0, %2 \n\t" \
00525 : "=m" (c), "=m" (d), "=m" (s) \
00526 : "m" (s), "m" (d), "m" (c), "m" (b) \
00527 : "g1", "o0", "o1", "o2", "o3", "o4", \
00528 "o5" \
00529 );
00530
00531 #endif
00532 #endif
00533
00534 #if defined(__microblaze__) || defined(microblaze)
00535
00536 #define MULADDC_INIT \
00537 asm( \
00538 "lwi r3, %3 \n\t" \
00539 "lwi r4, %4 \n\t" \
00540 "lwi r5, %5 \n\t" \
00541 "lwi r6, %6 \n\t" \
00542 "andi r7, r6, 0xffff \n\t" \
00543 "bsrli r6, r6, 16 \n\t"
00544
00545 #define MULADDC_CORE \
00546 "lhui r8, r3, 0 \n\t" \
00547 "addi r3, r3, 2 \n\t" \
00548 "lhui r9, r3, 0 \n\t" \
00549 "addi r3, r3, 2 \n\t" \
00550 "mul r10, r9, r6 \n\t" \
00551 "mul r11, r8, r7 \n\t" \
00552 "mul r12, r9, r7 \n\t" \
00553 "mul r13, r8, r6 \n\t" \
00554 "bsrli r8, r10, 16 \n\t" \
00555 "bsrli r9, r11, 16 \n\t" \
00556 "add r13, r13, r8 \n\t" \
00557 "add r13, r13, r9 \n\t" \
00558 "bslli r10, r10, 16 \n\t" \
00559 "bslli r11, r11, 16 \n\t" \
00560 "add r12, r12, r10 \n\t" \
00561 "addc r13, r13, r0 \n\t" \
00562 "add r12, r12, r11 \n\t" \
00563 "addc r13, r13, r0 \n\t" \
00564 "lwi r10, r4, 0 \n\t" \
00565 "add r12, r12, r10 \n\t" \
00566 "addc r13, r13, r0 \n\t" \
00567 "add r12, r12, r5 \n\t" \
00568 "addc r5, r13, r0 \n\t" \
00569 "swi r12, r4, 0 \n\t" \
00570 "addi r4, r4, 4 \n\t"
00571
00572 #define MULADDC_STOP \
00573 "swi r5, %0 \n\t" \
00574 "swi r4, %1 \n\t" \
00575 "swi r3, %2 \n\t" \
00576 : "=m" (c), "=m" (d), "=m" (s) \
00577 : "m" (s), "m" (d), "m" (c), "m" (b) \
00578 : "r3", "r4", "r5", "r6", "r7", "r8", \
00579 "r9", "r10", "r11", "r12", "r13" \
00580 );
00581
00582 #endif
00583
00584 #if defined(__tricore__)
00585
00586 #define MULADDC_INIT \
00587 asm( \
00588 "ld.a %%a2, %3 \n\t" \
00589 "ld.a %%a3, %4 \n\t" \
00590 "ld.w %%d4, %5 \n\t" \
00591 "ld.w %%d1, %6 \n\t" \
00592 "xor %%d5, %%d5 \n\t"
00593
00594 #define MULADDC_CORE \
00595 "ld.w %%d0, [%%a2+] \n\t" \
00596 "madd.u %%e2, %%e4, %%d0, %%d1 \n\t" \
00597 "ld.w %%d0, [%%a3] \n\t" \
00598 "addx %%d2, %%d2, %%d0 \n\t" \
00599 "addc %%d3, %%d3, 0 \n\t" \
00600 "mov %%d4, %%d3 \n\t" \
00601 "st.w [%%a3+], %%d2 \n\t"
00602
00603 #define MULADDC_STOP \
00604 "st.w %0, %%d4 \n\t" \
00605 "st.a %1, %%a3 \n\t" \
00606 "st.a %2, %%a2 \n\t" \
00607 : "=m" (c), "=m" (d), "=m" (s) \
00608 : "m" (s), "m" (d), "m" (c), "m" (b) \
00609 : "d0", "d1", "e2", "d4", "a2", "a3" \
00610 );
00611
00612 #endif
00613
00614
00615
00616
00617
00618
00619
00620
00621
00622
00623
00624
00625
00626 #if defined(__GNUC__) && !defined(__OPTIMIZE__)
00627 #define MULADDC_CANNOT_USE_R7
00628 #endif
00629
00630 #if defined(__arm__) && !defined(MULADDC_CANNOT_USE_R7)
00631
00632 #if defined(__thumb__) && !defined(__thumb2__)
00633
00634 #define MULADDC_INIT \
00635 asm( \
00636 "ldr r0, %3 \n\t" \
00637 "ldr r1, %4 \n\t" \
00638 "ldr r2, %5 \n\t" \
00639 "ldr r3, %6 \n\t" \
00640 "lsr r7, r3, #16 \n\t" \
00641 "mov r9, r7 \n\t" \
00642 "lsl r7, r3, #16 \n\t" \
00643 "lsr r7, r7, #16 \n\t" \
00644 "mov r8, r7 \n\t"
00645
00646 #define MULADDC_CORE \
00647 "ldmia r0!, {r6} \n\t" \
00648 "lsr r7, r6, #16 \n\t" \
00649 "lsl r6, r6, #16 \n\t" \
00650 "lsr r6, r6, #16 \n\t" \
00651 "mov r4, r8 \n\t" \
00652 "mul r4, r6 \n\t" \
00653 "mov r3, r9 \n\t" \
00654 "mul r6, r3 \n\t" \
00655 "mov r5, r9 \n\t" \
00656 "mul r5, r7 \n\t" \
00657 "mov r3, r8 \n\t" \
00658 "mul r7, r3 \n\t" \
00659 "lsr r3, r6, #16 \n\t" \
00660 "add r5, r5, r3 \n\t" \
00661 "lsr r3, r7, #16 \n\t" \
00662 "add r5, r5, r3 \n\t" \
00663 "add r4, r4, r2 \n\t" \
00664 "mov r2, #0 \n\t" \
00665 "adc r5, r2 \n\t" \
00666 "lsl r3, r6, #16 \n\t" \
00667 "add r4, r4, r3 \n\t" \
00668 "adc r5, r2 \n\t" \
00669 "lsl r3, r7, #16 \n\t" \
00670 "add r4, r4, r3 \n\t" \
00671 "adc r5, r2 \n\t" \
00672 "ldr r3, [r1] \n\t" \
00673 "add r4, r4, r3 \n\t" \
00674 "adc r2, r5 \n\t" \
00675 "stmia r1!, {r4} \n\t"
00676
00677 #define MULADDC_STOP \
00678 "str r2, %0 \n\t" \
00679 "str r1, %1 \n\t" \
00680 "str r0, %2 \n\t" \
00681 : "=m" (c), "=m" (d), "=m" (s) \
00682 : "m" (s), "m" (d), "m" (c), "m" (b) \
00683 : "r0", "r1", "r2", "r3", "r4", "r5", \
00684 "r6", "r7", "r8", "r9", "cc" \
00685 );
00686
00687 #else
00688
00689 #define MULADDC_INIT \
00690 asm( \
00691 "ldr r0, %3 \n\t" \
00692 "ldr r1, %4 \n\t" \
00693 "ldr r2, %5 \n\t" \
00694 "ldr r3, %6 \n\t"
00695
00696 #define MULADDC_CORE \
00697 "ldr r4, [r0], #4 \n\t" \
00698 "mov r5, #0 \n\t" \
00699 "ldr r6, [r1] \n\t" \
00700 "umlal r2, r5, r3, r4 \n\t" \
00701 "adds r7, r6, r2 \n\t" \
00702 "adc r2, r5, #0 \n\t" \
00703 "str r7, [r1], #4 \n\t"
00704
00705 #define MULADDC_STOP \
00706 "str r2, %0 \n\t" \
00707 "str r1, %1 \n\t" \
00708 "str r0, %2 \n\t" \
00709 : "=m" (c), "=m" (d), "=m" (s) \
00710 : "m" (s), "m" (d), "m" (c), "m" (b) \
00711 : "r0", "r1", "r2", "r3", "r4", "r5", \
00712 "r6", "r7", "cc" \
00713 );
00714
00715 #endif
00716
00717 #endif
00718
00719 #if defined(__alpha__)
00720
00721 #define MULADDC_INIT \
00722 asm( \
00723 "ldq $1, %3 \n\t" \
00724 "ldq $2, %4 \n\t" \
00725 "ldq $3, %5 \n\t" \
00726 "ldq $4, %6 \n\t"
00727
00728 #define MULADDC_CORE \
00729 "ldq $6, 0($1) \n\t" \
00730 "addq $1, 8, $1 \n\t" \
00731 "mulq $6, $4, $7 \n\t" \
00732 "umulh $6, $4, $6 \n\t" \
00733 "addq $7, $3, $7 \n\t" \
00734 "cmpult $7, $3, $3 \n\t" \
00735 "ldq $5, 0($2) \n\t" \
00736 "addq $7, $5, $7 \n\t" \
00737 "cmpult $7, $5, $5 \n\t" \
00738 "stq $7, 0($2) \n\t" \
00739 "addq $2, 8, $2 \n\t" \
00740 "addq $6, $3, $3 \n\t" \
00741 "addq $5, $3, $3 \n\t"
00742
00743 #define MULADDC_STOP \
00744 "stq $3, %0 \n\t" \
00745 "stq $2, %1 \n\t" \
00746 "stq $1, %2 \n\t" \
00747 : "=m" (c), "=m" (d), "=m" (s) \
00748 : "m" (s), "m" (d), "m" (c), "m" (b) \
00749 : "$1", "$2", "$3", "$4", "$5", "$6", "$7" \
00750 );
00751 #endif
00752
00753 #if defined(__mips__) && !defined(__mips64)
00754
00755 #define MULADDC_INIT \
00756 asm( \
00757 "lw $10, %3 \n\t" \
00758 "lw $11, %4 \n\t" \
00759 "lw $12, %5 \n\t" \
00760 "lw $13, %6 \n\t"
00761
00762 #define MULADDC_CORE \
00763 "lw $14, 0($10) \n\t" \
00764 "multu $13, $14 \n\t" \
00765 "addi $10, $10, 4 \n\t" \
00766 "mflo $14 \n\t" \
00767 "mfhi $9 \n\t" \
00768 "addu $14, $12, $14 \n\t" \
00769 "lw $15, 0($11) \n\t" \
00770 "sltu $12, $14, $12 \n\t" \
00771 "addu $15, $14, $15 \n\t" \
00772 "sltu $14, $15, $14 \n\t" \
00773 "addu $12, $12, $9 \n\t" \
00774 "sw $15, 0($11) \n\t" \
00775 "addu $12, $12, $14 \n\t" \
00776 "addi $11, $11, 4 \n\t"
00777
00778 #define MULADDC_STOP \
00779 "sw $12, %0 \n\t" \
00780 "sw $11, %1 \n\t" \
00781 "sw $10, %2 \n\t" \
00782 : "=m" (c), "=m" (d), "=m" (s) \
00783 : "m" (s), "m" (d), "m" (c), "m" (b) \
00784 : "$9", "$10", "$11", "$12", "$13", "$14", "$15", "lo", "hi" \
00785 );
00786
00787 #endif
00788 #endif
00789
00790 #if (defined(_MSC_VER) && defined(_M_IX86)) || defined(__WATCOMC__)
00791
00792 #define MULADDC_INIT \
00793 __asm mov esi, s \
00794 __asm mov edi, d \
00795 __asm mov ecx, c \
00796 __asm mov ebx, b
00797
00798 #define MULADDC_CORE \
00799 __asm lodsd \
00800 __asm mul ebx \
00801 __asm add eax, ecx \
00802 __asm adc edx, 0 \
00803 __asm add eax, [edi] \
00804 __asm adc edx, 0 \
00805 __asm mov ecx, edx \
00806 __asm stosd
00807
00808 #if defined(MBEDTLS_HAVE_SSE2)
00809
00810 #define EMIT __asm _emit
00811
00812 #define MULADDC_HUIT \
00813 EMIT 0x0F EMIT 0x6E EMIT 0xC9 \
00814 EMIT 0x0F EMIT 0x6E EMIT 0xC3 \
00815 EMIT 0x0F EMIT 0x6E EMIT 0x1F \
00816 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00817 EMIT 0x0F EMIT 0x6E EMIT 0x16 \
00818 EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
00819 EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x04 \
00820 EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
00821 EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x08 \
00822 EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
00823 EMIT 0x0F EMIT 0x6E EMIT 0x7E EMIT 0x0C \
00824 EMIT 0x0F EMIT 0xF4 EMIT 0xF8 \
00825 EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
00826 EMIT 0x0F EMIT 0x6E EMIT 0x5F EMIT 0x04 \
00827 EMIT 0x0F EMIT 0xD4 EMIT 0xDC \
00828 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x08 \
00829 EMIT 0x0F EMIT 0xD4 EMIT 0xEE \
00830 EMIT 0x0F EMIT 0x6E EMIT 0x67 EMIT 0x0C \
00831 EMIT 0x0F EMIT 0xD4 EMIT 0xFC \
00832 EMIT 0x0F EMIT 0x7E EMIT 0x0F \
00833 EMIT 0x0F EMIT 0x6E EMIT 0x56 EMIT 0x10 \
00834 EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
00835 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00836 EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x14 \
00837 EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
00838 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00839 EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x18 \
00840 EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
00841 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x04 \
00842 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00843 EMIT 0x0F EMIT 0x6E EMIT 0x5E EMIT 0x1C \
00844 EMIT 0x0F EMIT 0xF4 EMIT 0xD8 \
00845 EMIT 0x0F EMIT 0xD4 EMIT 0xCD \
00846 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x10 \
00847 EMIT 0x0F EMIT 0xD4 EMIT 0xD5 \
00848 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x08 \
00849 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00850 EMIT 0x0F EMIT 0xD4 EMIT 0xCF \
00851 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x14 \
00852 EMIT 0x0F EMIT 0xD4 EMIT 0xE5 \
00853 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x0C \
00854 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00855 EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
00856 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x18 \
00857 EMIT 0x0F EMIT 0xD4 EMIT 0xF5 \
00858 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x10 \
00859 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00860 EMIT 0x0F EMIT 0xD4 EMIT 0xCC \
00861 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x1C \
00862 EMIT 0x0F EMIT 0xD4 EMIT 0xDD \
00863 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x14 \
00864 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00865 EMIT 0x0F EMIT 0xD4 EMIT 0xCE \
00866 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x18 \
00867 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00868 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00869 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x1C \
00870 EMIT 0x83 EMIT 0xC7 EMIT 0x20 \
00871 EMIT 0x83 EMIT 0xC6 EMIT 0x20 \
00872 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00873 EMIT 0x0F EMIT 0x7E EMIT 0xC9
00874
00875 #define MULADDC_STOP \
00876 EMIT 0x0F EMIT 0x77 \
00877 __asm mov c, ecx \
00878 __asm mov d, edi \
00879 __asm mov s, esi \
00880
00881 #else
00882
00883 #define MULADDC_STOP \
00884 __asm mov c, ecx \
00885 __asm mov d, edi \
00886 __asm mov s, esi \
00887
00888 #endif
00889 #endif
00890
00891 #endif
00892
00893 #if !defined(MULADDC_CORE)
00894 #if defined(MBEDTLS_HAVE_UDBL)
00895
00896 #define MULADDC_INIT \
00897 { \
00898 mbedtls_t_udbl r; \
00899 mbedtls_mpi_uint r0, r1;
00900
00901 #define MULADDC_CORE \
00902 r = *(s++) * (mbedtls_t_udbl) b; \
00903 r0 = (mbedtls_mpi_uint) r; \
00904 r1 = (mbedtls_mpi_uint)( r >> biL ); \
00905 r0 += c; r1 += (r0 < c); \
00906 r0 += *d; r1 += (r0 < *d); \
00907 c = r1; *(d++) = r0;
00908
00909 #define MULADDC_STOP \
00910 }
00911
00912 #else
00913 #define MULADDC_INIT \
00914 { \
00915 mbedtls_mpi_uint s0, s1, b0, b1; \
00916 mbedtls_mpi_uint r0, r1, rx, ry; \
00917 b0 = ( b << biH ) >> biH; \
00918 b1 = ( b >> biH );
00919
00920 #define MULADDC_CORE \
00921 s0 = ( *s << biH ) >> biH; \
00922 s1 = ( *s >> biH ); s++; \
00923 rx = s0 * b1; r0 = s0 * b0; \
00924 ry = s1 * b0; r1 = s1 * b1; \
00925 r1 += ( rx >> biH ); \
00926 r1 += ( ry >> biH ); \
00927 rx <<= biH; ry <<= biH; \
00928 r0 += rx; r1 += (r0 < rx); \
00929 r0 += ry; r1 += (r0 < ry); \
00930 r0 += c; r1 += (r0 < c); \
00931 r0 += *d; r1 += (r0 < *d); \
00932 c = r1; *(d++) = r0;
00933
00934 #define MULADDC_STOP \
00935 }
00936
00937 #endif
00938 #endif
00939
00940 #endif