|
12 | 12 | * BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations |
13 | 13 | * BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations |
14 | 14 | * BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations |
15 | | - * BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned |
16 | | - read and overlapping memcpy; this reduces decompression speed by 5% |
17 | 15 | * BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs |
| 16 | + * BROTLI_BUILD_NO_UNALIGNED_READ_FAST forces off the fast-unaligned-read |
| 17 | + optimizations (mainly for testing purposes) |
18 | 18 | * BROTLI_DEBUG dumps file name and line number when decoder detects stream |
19 | 19 | or memory error |
20 | 20 | * BROTLI_ENABLE_LOG enables asserts and dumps various state information |
| 21 | + * BROTLI_ENABLE_DUMP overrides default "dump" behaviour |
21 | 22 | */ |
22 | 23 |
|
23 | 24 | #ifndef BROTLI_COMMON_PLATFORM_H_ |
|
40 | 41 | #define BROTLI_X_BIG_ENDIAN BIG_ENDIAN |
41 | 42 | #endif |
42 | 43 |
|
43 | | -#if BROTLI_MSVC_VERSION_CHECK(12, 0, 0) |
| 44 | +#if BROTLI_MSVC_VERSION_CHECK(18, 0, 0) |
44 | 45 | #include <intrin.h> |
45 | 46 | #endif |
46 | 47 |
|
@@ -156,24 +157,6 @@ To apply compiler hint, enclose the branching condition into macros, like this: |
156 | 157 | #define BROTLI_NOINLINE |
157 | 158 | #endif |
158 | 159 |
|
159 | | -/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */ |
160 | | -#if !defined(BROTLI_INTERNAL) |
161 | | -#if defined(_WIN32) || defined(__CYGWIN__) |
162 | | -#define BROTLI_INTERNAL |
163 | | -#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \ |
164 | | - BROTLI_TI_VERSION_CHECK(8, 0, 0) || \ |
165 | | - BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \ |
166 | | - BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \ |
167 | | - BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \ |
168 | | - BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \ |
169 | | - (BROTLI_TI_VERSION_CHECK(7, 3, 0) && \ |
170 | | - defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__)) |
171 | | -#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden"))) |
172 | | -#else |
173 | | -#define BROTLI_INTERNAL |
174 | | -#endif |
175 | | -#endif |
176 | | - |
177 | 160 | /* <<< <<< <<< end of hedley macros. */ |
178 | 161 |
|
179 | 162 | #if BROTLI_GNUC_HAS_ATTRIBUTE(unused, 2, 7, 0) || \ |
@@ -226,15 +209,24 @@ To apply compiler hint, enclose the branching condition into macros, like this: |
226 | 209 | #define BROTLI_TARGET_RISCV64 |
227 | 210 | #endif |
228 | 211 |
|
| 212 | +#if defined(__loongarch_lp64) |
| 213 | +#define BROTLI_TARGET_LOONGARCH64 |
| 214 | +#endif |
| 215 | + |
| 216 | +#if defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8_64) || \ |
| 217 | + defined(BROTLI_TARGET_POWERPC64) || defined(BROTLI_TARGET_RISCV64) || \ |
| 218 | + defined(BROTLI_TARGET_LOONGARCH64) |
| 219 | +#define BROTLI_TARGET_64_BITS 1 |
| 220 | +#else |
| 221 | +#define BROTLI_TARGET_64_BITS 0 |
| 222 | +#endif |
| 223 | + |
229 | 224 | #if defined(BROTLI_BUILD_64_BIT) |
230 | 225 | #define BROTLI_64_BITS 1 |
231 | 226 | #elif defined(BROTLI_BUILD_32_BIT) |
232 | 227 | #define BROTLI_64_BITS 0 |
233 | | -#elif defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8_64) || \ |
234 | | - defined(BROTLI_TARGET_POWERPC64) || defined(BROTLI_TARGET_RISCV64) |
235 | | -#define BROTLI_64_BITS 1 |
236 | 228 | #else |
237 | | -#define BROTLI_64_BITS 0 |
| 229 | +#define BROTLI_64_BITS BROTLI_TARGET_64_BITS |
238 | 230 | #endif |
239 | 231 |
|
240 | 232 | #if (BROTLI_64_BITS) |
@@ -278,18 +270,19 @@ To apply compiler hint, enclose the branching condition into macros, like this: |
278 | 270 | #undef BROTLI_X_BIG_ENDIAN |
279 | 271 | #endif |
280 | 272 |
|
281 | | -#if defined(BROTLI_BUILD_PORTABLE) |
282 | | -#define BROTLI_ALIGNED_READ (!!1) |
283 | | -#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \ |
| 273 | +#if defined(BROTLI_BUILD_NO_UNALIGNED_READ_FAST) |
| 274 | +#define BROTLI_UNALIGNED_READ_FAST (!!0) |
| 275 | +#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \ |
284 | 276 | defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY) || \ |
285 | | - defined(BROTLI_TARGET_RISCV64) |
286 | | -/* Allow unaligned read only for white-listed CPUs. */ |
287 | | -#define BROTLI_ALIGNED_READ (!!0) |
| 277 | + defined(BROTLI_TARGET_RISCV64) || defined(BROTLI_TARGET_LOONGARCH64) |
| 278 | +/* These targets are known to generate efficient code for unaligned reads |
| 279 | + * (e.g. a single instruction, not multiple 1-byte loads, shifted and or'd |
| 280 | + * together). */ |
| 281 | +#define BROTLI_UNALIGNED_READ_FAST (!!1) |
288 | 282 | #else |
289 | | -#define BROTLI_ALIGNED_READ (!!1) |
| 283 | +#define BROTLI_UNALIGNED_READ_FAST (!!0) |
290 | 284 | #endif |
291 | 285 |
|
292 | | -#if BROTLI_ALIGNED_READ |
293 | 286 | /* Portable unaligned memory access: read / write values via memcpy. */ |
294 | 287 | static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) { |
295 | 288 | uint16_t t; |
@@ -309,75 +302,6 @@ static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) { |
309 | 302 | static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) { |
310 | 303 | memcpy(p, &v, sizeof v); |
311 | 304 | } |
312 | | -#else /* BROTLI_ALIGNED_READ */ |
313 | | -/* Unaligned memory access is allowed: just cast pointer to requested type. */ |
314 | | -#if BROTLI_SANITIZED |
315 | | -/* Consider we have an unaligned load/store of 4 bytes from address 0x...05. |
316 | | - AddressSanitizer will treat it as a 3-byte access to the range 05:07 and |
317 | | - will miss a bug if 08 is the first unaddressable byte. |
318 | | - ThreadSanitizer will also treat this as a 3-byte access to 05:07 and will |
319 | | - miss a race between this access and some other accesses to 08. |
320 | | - MemorySanitizer will correctly propagate the shadow on unaligned stores |
321 | | - and correctly report bugs on unaligned loads, but it may not properly |
322 | | - update and report the origin of the uninitialized memory. |
323 | | - For all three tools, replacing an unaligned access with a tool-specific |
324 | | - callback solves the problem. */ |
325 | | -#if defined(__cplusplus) |
326 | | -extern "C" { |
327 | | -#endif /* __cplusplus */ |
328 | | - uint16_t __sanitizer_unaligned_load16(const void* p); |
329 | | - uint32_t __sanitizer_unaligned_load32(const void* p); |
330 | | - uint64_t __sanitizer_unaligned_load64(const void* p); |
331 | | - void __sanitizer_unaligned_store64(void* p, uint64_t v); |
332 | | -#if defined(__cplusplus) |
333 | | -} /* extern "C" */ |
334 | | -#endif /* __cplusplus */ |
335 | | -#define BrotliUnalignedRead16 __sanitizer_unaligned_load16 |
336 | | -#define BrotliUnalignedRead32 __sanitizer_unaligned_load32 |
337 | | -#define BrotliUnalignedRead64 __sanitizer_unaligned_load64 |
338 | | -#define BrotliUnalignedWrite64 __sanitizer_unaligned_store64 |
339 | | -#else /* BROTLI_SANITIZED */ |
340 | | -static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) { |
341 | | - return *(const uint16_t*)p; |
342 | | -} |
343 | | -static BROTLI_INLINE uint32_t BrotliUnalignedRead32(const void* p) { |
344 | | - return *(const uint32_t*)p; |
345 | | -} |
346 | | -#if (BROTLI_64_BITS) |
347 | | -static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) { |
348 | | - return *(const uint64_t*)p; |
349 | | -} |
350 | | -static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) { |
351 | | - *(uint64_t*)p = v; |
352 | | -} |
353 | | -#else /* BROTLI_64_BITS */ |
354 | | -/* Avoid emitting LDRD / STRD, which require properly aligned address. */ |
355 | | -/* If __attribute__(aligned) is available, use that. Otherwise, memcpy. */ |
356 | | - |
357 | | -#if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) |
358 | | -typedef BROTLI_ALIGNED(1) uint64_t brotli_unaligned_uint64_t; |
359 | | - |
360 | | -static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) { |
361 | | - return (uint64_t) ((const brotli_unaligned_uint64_t*) p)[0]; |
362 | | -} |
363 | | -static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) { |
364 | | - brotli_unaligned_uint64_t* dwords = (brotli_unaligned_uint64_t*) p; |
365 | | - dwords[0] = (brotli_unaligned_uint64_t) v; |
366 | | -} |
367 | | -#else /* BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) */ |
368 | | -static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) { |
369 | | - uint64_t v; |
370 | | - memcpy(&v, p, sizeof(uint64_t)); |
371 | | - return v; |
372 | | -} |
373 | | - |
374 | | -static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) { |
375 | | - memcpy(p, &v, sizeof(uint64_t)); |
376 | | -} |
377 | | -#endif /* BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) */ |
378 | | -#endif /* BROTLI_64_BITS */ |
379 | | -#endif /* BROTLI_SANITIZED */ |
380 | | -#endif /* BROTLI_ALIGNED_READ */ |
381 | 305 |
|
382 | 306 | #if BROTLI_LITTLE_ENDIAN |
383 | 307 | /* Straight endianness. Just read / write values. */ |
@@ -453,6 +377,16 @@ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) { |
453 | 377 | } |
454 | 378 | #endif /* BROTLI_LITTLE_ENDIAN */ |
455 | 379 |
|
| 380 | +static BROTLI_INLINE void* BROTLI_UNALIGNED_LOAD_PTR(const void* p) { |
| 381 | + void* v; |
| 382 | + memcpy(&v, p, sizeof(void*)); |
| 383 | + return v; |
| 384 | +} |
| 385 | + |
| 386 | +static BROTLI_INLINE void BROTLI_UNALIGNED_STORE_PTR(void* p, const void* v) { |
| 387 | + memcpy(p, &v, sizeof(void*)); |
| 388 | +} |
| 389 | + |
456 | 390 | /* BROTLI_IS_CONSTANT macros returns true for compile-time constants. */ |
457 | 391 | #if BROTLI_GNUC_HAS_BUILTIN(__builtin_constant_p, 3, 0, 1) || \ |
458 | 392 | BROTLI_INTEL_VERSION_CHECK(16, 0, 0) |
@@ -474,22 +408,34 @@ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) { |
474 | 408 | #endif |
475 | 409 |
|
476 | 410 | #if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG) |
| 411 | +#define BROTLI_ENABLE_DUMP_DEFAULT 1 |
477 | 412 | #define BROTLI_DCHECK(x) assert(x) |
| 413 | +#else |
| 414 | +#define BROTLI_ENABLE_DUMP_DEFAULT 0 |
| 415 | +#define BROTLI_DCHECK(x) |
| 416 | +#endif |
| 417 | + |
| 418 | +#if !defined(BROTLI_ENABLE_DUMP) |
| 419 | +#define BROTLI_ENABLE_DUMP BROTLI_ENABLE_DUMP_DEFAULT |
| 420 | +#endif |
| 421 | + |
| 422 | +#if BROTLI_ENABLE_DUMP |
478 | 423 | static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) { |
479 | 424 | fprintf(stderr, "%s:%d (%s)\n", f, l, fn); |
480 | 425 | fflush(stderr); |
481 | 426 | } |
482 | 427 | #define BROTLI_DUMP() BrotliDump(__FILE__, __LINE__, __FUNCTION__) |
483 | 428 | #else |
484 | | -#define BROTLI_DCHECK(x) |
485 | 429 | #define BROTLI_DUMP() (void)(0) |
486 | 430 | #endif |
487 | 431 |
|
488 | | -/* TODO: add appropriate icc/sunpro/arm/ibm/ti checks. */ |
| 432 | +/* BrotliRBit assumes brotli_reg_t fits native CPU register type. */ |
| 433 | +#if (BROTLI_64_BITS == BROTLI_TARGET_64_BITS) |
| 434 | +/* TODO(eustas): add appropriate icc/sunpro/arm/ibm/ti checks. */ |
489 | 435 | #if (BROTLI_GNUC_VERSION_CHECK(3, 0, 0) || defined(__llvm__)) && \ |
490 | 436 | !defined(BROTLI_BUILD_NO_RBIT) |
491 | 437 | #if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY) |
492 | | -/* TODO: detect ARMv6T2 and enable this code for it. */ |
| 438 | +/* TODO(eustas): detect ARMv6T2 and enable this code for it. */ |
493 | 439 | static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) { |
494 | 440 | brotli_reg_t output; |
495 | 441 | __asm__("rbit %0, %1\n" : "=r"(output) : "r"(input)); |
@@ -498,15 +444,14 @@ static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) { |
498 | 444 | #define BROTLI_RBIT(x) BrotliRBit(x) |
499 | 445 | #endif /* armv7 / armv8 */ |
500 | 446 | #endif /* gcc || clang */ |
| 447 | +#endif /* brotli_reg_t is native */ |
501 | 448 | #if !defined(BROTLI_RBIT) |
502 | 449 | static BROTLI_INLINE void BrotliRBit(void) { /* Should break build if used. */ } |
503 | 450 | #endif /* BROTLI_RBIT */ |
504 | 451 |
|
505 | | -#define BROTLI_REPEAT(N, X) { \ |
506 | | - if ((N & 1) != 0) {X;} \ |
507 | | - if ((N & 2) != 0) {X; X;} \ |
508 | | - if ((N & 4) != 0) {X; X; X; X;} \ |
509 | | -} |
| 452 | +#define BROTLI_REPEAT_4(X) {X; X; X; X;} |
| 453 | +#define BROTLI_REPEAT_5(X) {X; X; X; X; X;} |
| 454 | +#define BROTLI_REPEAT_6(X) {X; X; X; X; X; X;} |
510 | 455 |
|
511 | 456 | #define BROTLI_UNUSED(X) (void)(X) |
512 | 457 |
|
@@ -529,7 +474,7 @@ BROTLI_MIN_MAX(size_t) BROTLI_MIN_MAX(uint32_t) BROTLI_MIN_MAX(uint8_t) |
529 | 474 | #if BROTLI_GNUC_HAS_BUILTIN(__builtin_ctzll, 3, 4, 0) || \ |
530 | 475 | BROTLI_INTEL_VERSION_CHECK(16, 0, 0) |
531 | 476 | #define BROTLI_TZCNT64 __builtin_ctzll |
532 | | -#elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0) |
| 477 | +#elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0) |
533 | 478 | #if defined(BROTLI_TARGET_X64) |
534 | 479 | #define BROTLI_TZCNT64 _tzcnt_u64 |
535 | 480 | #else /* BROTLI_TARGET_X64 */ |
@@ -546,7 +491,7 @@ static BROTLI_INLINE uint32_t BrotliBsf64Msvc(uint64_t x) { |
546 | 491 | #if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \ |
547 | 492 | BROTLI_INTEL_VERSION_CHECK(16, 0, 0) |
548 | 493 | #define BROTLI_BSR32(x) (31u ^ (uint32_t)__builtin_clz(x)) |
549 | | -#elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0) |
| 494 | +#elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0) |
550 | 495 | static BROTLI_INLINE uint32_t BrotliBsr32Msvc(uint32_t x) { |
551 | 496 | unsigned long msb; |
552 | 497 | _BitScanReverse(&msb, x); |
@@ -571,6 +516,8 @@ BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) { |
571 | 516 | BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD32LE); |
572 | 517 | BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD64LE); |
573 | 518 | BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE64LE); |
| 519 | + BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD_PTR); |
| 520 | + BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE_PTR); |
574 | 521 | BROTLI_UNUSED(&BrotliRBit); |
575 | 522 | BROTLI_UNUSED(&brotli_min_double); |
576 | 523 | BROTLI_UNUSED(&brotli_max_double); |
@@ -586,7 +533,7 @@ BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) { |
586 | 533 | BROTLI_UNUSED(&brotli_max_uint8_t); |
587 | 534 | BROTLI_UNUSED(&BrotliDefaultAllocFunc); |
588 | 535 | BROTLI_UNUSED(&BrotliDefaultFreeFunc); |
589 | | -#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG) |
| 536 | +#if BROTLI_ENABLE_DUMP |
590 | 537 | BROTLI_UNUSED(&BrotliDump); |
591 | 538 | #endif |
592 | 539 | } |
|