2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 1) /* Copyright (C) 2016 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 2) *
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 3) * This file is provided under a dual BSD/GPLv2 license.
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 4) *
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 5) * SipHash: a fast short-input PRF
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 6) * https://131002.net/siphash/
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 7) *
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 8) * This implementation is specifically for SipHash2-4 for a secure PRF
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 9) * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 10) * hashtables.
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 11) */
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 12)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 13) #include <linux/siphash.h>
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 14) #include <asm/unaligned.h>
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 15)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 16) #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 17) #include <linux/dcache.h>
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 18) #include <asm/word-at-a-time.h>
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 19) #endif
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 20)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 21) #define SIPROUND \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 22) do { \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 23) v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 24) v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 25) v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 26) v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 27) } while (0)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 28)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 29) #define PREAMBLE(len) \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 30) u64 v0 = 0x736f6d6570736575ULL; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 31) u64 v1 = 0x646f72616e646f6dULL; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 32) u64 v2 = 0x6c7967656e657261ULL; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 33) u64 v3 = 0x7465646279746573ULL; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 34) u64 b = ((u64)(len)) << 56; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 35) v3 ^= key->key[1]; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 36) v2 ^= key->key[0]; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 37) v1 ^= key->key[1]; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 38) v0 ^= key->key[0];
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 39)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 40) #define POSTAMBLE \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 41) v3 ^= b; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 42) SIPROUND; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 43) SIPROUND; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 44) v0 ^= b; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 45) v2 ^= 0xff; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 46) SIPROUND; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 47) SIPROUND; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 48) SIPROUND; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 49) SIPROUND; \
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 50) return (v0 ^ v1) ^ (v2 ^ v3);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 51)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 52) u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 53) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 54) const u8 *end = data + len - (len % sizeof(u64));
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 55) const u8 left = len & (sizeof(u64) - 1);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 56) u64 m;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 57) PREAMBLE(len)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 58) for (; data != end; data += sizeof(u64)) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 59) m = le64_to_cpup(data);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 60) v3 ^= m;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 61) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 62) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 63) v0 ^= m;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 64) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 65) #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 66) if (left)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 67) b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 68) bytemask_from_count(left)));
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 69) #else
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 70) switch (left) {
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 71) case 7: b |= ((u64)end[6]) << 48; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 72) case 6: b |= ((u64)end[5]) << 40; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 73) case 5: b |= ((u64)end[4]) << 32; fallthrough;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 74) case 4: b |= le32_to_cpup(data); break;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 75) case 3: b |= ((u64)end[2]) << 16; fallthrough;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 76) case 2: b |= le16_to_cpup(data); break;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 77) case 1: b |= end[0];
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 78) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 79) #endif
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 80) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 81) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 82) EXPORT_SYMBOL(__siphash_aligned);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 83)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 84) #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 85) u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 86) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 87) const u8 *end = data + len - (len % sizeof(u64));
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 88) const u8 left = len & (sizeof(u64) - 1);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 89) u64 m;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 90) PREAMBLE(len)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 91) for (; data != end; data += sizeof(u64)) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 92) m = get_unaligned_le64(data);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 93) v3 ^= m;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 94) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 95) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 96) v0 ^= m;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 97) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 98) #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 99) if (left)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 100) b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 101) bytemask_from_count(left)));
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 102) #else
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 103) switch (left) {
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 104) case 7: b |= ((u64)end[6]) << 48; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 105) case 6: b |= ((u64)end[5]) << 40; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 106) case 5: b |= ((u64)end[4]) << 32; fallthrough;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 107) case 4: b |= get_unaligned_le32(end); break;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 108) case 3: b |= ((u64)end[2]) << 16; fallthrough;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 109) case 2: b |= get_unaligned_le16(end); break;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 110) case 1: b |= end[0];
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 111) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 112) #endif
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 113) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 114) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 115) EXPORT_SYMBOL(__siphash_unaligned);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 116) #endif
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 117)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 118) /**
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 119) * siphash_1u64 - compute 64-bit siphash PRF value of a u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 120) * @first: first u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 121) * @key: the siphash key
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 122) */
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 123) u64 siphash_1u64(const u64 first, const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 124) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 125) PREAMBLE(8)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 126) v3 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 127) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 128) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 129) v0 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 130) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 131) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 132) EXPORT_SYMBOL(siphash_1u64);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 133)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 134) /**
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 135) * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 136) * @first: first u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 137) * @second: second u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 138) * @key: the siphash key
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 139) */
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 140) u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 141) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 142) PREAMBLE(16)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 143) v3 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 144) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 145) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 146) v0 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 147) v3 ^= second;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 148) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 149) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 150) v0 ^= second;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 151) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 152) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 153) EXPORT_SYMBOL(siphash_2u64);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 154)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 155) /**
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 156) * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 157) * @first: first u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 158) * @second: second u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 159) * @third: third u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 160) * @key: the siphash key
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 161) */
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 162) u64 siphash_3u64(const u64 first, const u64 second, const u64 third,
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 163) const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 164) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 165) PREAMBLE(24)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 166) v3 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 167) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 168) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 169) v0 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 170) v3 ^= second;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 171) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 172) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 173) v0 ^= second;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 174) v3 ^= third;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 175) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 176) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 177) v0 ^= third;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 178) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 179) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 180) EXPORT_SYMBOL(siphash_3u64);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 181)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 182) /**
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 183) * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 184) * @first: first u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 185) * @second: second u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 186) * @third: third u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 187) * @forth: forth u64
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 188) * @key: the siphash key
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 189) */
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 190) u64 siphash_4u64(const u64 first, const u64 second, const u64 third,
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 191) const u64 forth, const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 192) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 193) PREAMBLE(32)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 194) v3 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 195) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 196) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 197) v0 ^= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 198) v3 ^= second;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 199) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 200) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 201) v0 ^= second;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 202) v3 ^= third;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 203) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 204) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 205) v0 ^= third;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 206) v3 ^= forth;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 207) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 208) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 209) v0 ^= forth;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 210) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 211) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 212) EXPORT_SYMBOL(siphash_4u64);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 213)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 214) u64 siphash_1u32(const u32 first, const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 215) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 216) PREAMBLE(4)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 217) b |= first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 218) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 219) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 220) EXPORT_SYMBOL(siphash_1u32);
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 221)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 222) u64 siphash_3u32(const u32 first, const u32 second, const u32 third,
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 223) const siphash_key_t *key)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 224) {
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 225) u64 combined = (u64)second << 32 | first;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 226) PREAMBLE(12)
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 227) v3 ^= combined;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 228) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 229) SIPROUND;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 230) v0 ^= combined;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 231) b |= third;
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 232) POSTAMBLE
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 233) }
2c956a60778cb (Jason A. Donenfeld 2017-01-08 13:54:00 +0100 234) EXPORT_SYMBOL(siphash_3u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 235)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 236) #if BITS_PER_LONG == 64
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 237) /* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 238) * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3.
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 239) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 240)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 241) #define HSIPROUND SIPROUND
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 242) #define HPREAMBLE(len) PREAMBLE(len)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 243) #define HPOSTAMBLE \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 244) v3 ^= b; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 245) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 246) v0 ^= b; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 247) v2 ^= 0xff; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 248) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 249) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 250) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 251) return (v0 ^ v1) ^ (v2 ^ v3);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 252)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 253) u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 254) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 255) const u8 *end = data + len - (len % sizeof(u64));
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 256) const u8 left = len & (sizeof(u64) - 1);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 257) u64 m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 258) HPREAMBLE(len)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 259) for (; data != end; data += sizeof(u64)) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 260) m = le64_to_cpup(data);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 261) v3 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 262) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 263) v0 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 264) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 265) #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 266) if (left)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 267) b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 268) bytemask_from_count(left)));
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 269) #else
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 270) switch (left) {
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 271) case 7: b |= ((u64)end[6]) << 48; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 272) case 6: b |= ((u64)end[5]) << 40; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 273) case 5: b |= ((u64)end[4]) << 32; fallthrough;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 274) case 4: b |= le32_to_cpup(data); break;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 275) case 3: b |= ((u64)end[2]) << 16; fallthrough;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 276) case 2: b |= le16_to_cpup(data); break;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 277) case 1: b |= end[0];
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 278) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 279) #endif
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 280) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 281) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 282) EXPORT_SYMBOL(__hsiphash_aligned);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 283)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 284) #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 285) u32 __hsiphash_unaligned(const void *data, size_t len,
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 286) const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 287) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 288) const u8 *end = data + len - (len % sizeof(u64));
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 289) const u8 left = len & (sizeof(u64) - 1);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 290) u64 m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 291) HPREAMBLE(len)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 292) for (; data != end; data += sizeof(u64)) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 293) m = get_unaligned_le64(data);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 294) v3 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 295) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 296) v0 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 297) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 298) #if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 299) if (left)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 300) b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 301) bytemask_from_count(left)));
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 302) #else
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 303) switch (left) {
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 304) case 7: b |= ((u64)end[6]) << 48; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 305) case 6: b |= ((u64)end[5]) << 40; fallthrough;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 306) case 5: b |= ((u64)end[4]) << 32; fallthrough;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 307) case 4: b |= get_unaligned_le32(end); break;
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 308) case 3: b |= ((u64)end[2]) << 16; fallthrough;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 309) case 2: b |= get_unaligned_le16(end); break;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 310) case 1: b |= end[0];
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 311) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 312) #endif
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 313) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 314) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 315) EXPORT_SYMBOL(__hsiphash_unaligned);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 316) #endif
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 317)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 318) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 319) * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 320) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 321) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 322) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 323) u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 324) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 325) HPREAMBLE(4)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 326) b |= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 327) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 328) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 329) EXPORT_SYMBOL(hsiphash_1u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 330)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 331) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 332) * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 333) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 334) * @second: second u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 335) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 336) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 337) u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 338) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 339) u64 combined = (u64)second << 32 | first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 340) HPREAMBLE(8)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 341) v3 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 342) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 343) v0 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 344) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 345) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 346) EXPORT_SYMBOL(hsiphash_2u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 347)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 348) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 349) * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 350) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 351) * @second: second u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 352) * @third: third u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 353) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 354) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 355) u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 356) const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 357) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 358) u64 combined = (u64)second << 32 | first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 359) HPREAMBLE(12)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 360) v3 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 361) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 362) v0 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 363) b |= third;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 364) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 365) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 366) EXPORT_SYMBOL(hsiphash_3u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 367)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 368) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 369) * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 370) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 371) * @second: second u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 372) * @third: third u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 373) * @forth: forth u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 374) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 375) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 376) u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 377) const u32 forth, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 378) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 379) u64 combined = (u64)second << 32 | first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 380) HPREAMBLE(16)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 381) v3 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 382) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 383) v0 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 384) combined = (u64)forth << 32 | third;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 385) v3 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 386) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 387) v0 ^= combined;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 388) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 389) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 390) EXPORT_SYMBOL(hsiphash_4u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 391) #else
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 392) #define HSIPROUND \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 393) do { \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 394) v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 395) v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 396) v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 397) v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 398) } while (0)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 399)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 400) #define HPREAMBLE(len) \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 401) u32 v0 = 0; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 402) u32 v1 = 0; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 403) u32 v2 = 0x6c796765U; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 404) u32 v3 = 0x74656462U; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 405) u32 b = ((u32)(len)) << 24; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 406) v3 ^= key->key[1]; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 407) v2 ^= key->key[0]; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 408) v1 ^= key->key[1]; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 409) v0 ^= key->key[0];
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 410)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 411) #define HPOSTAMBLE \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 412) v3 ^= b; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 413) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 414) v0 ^= b; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 415) v2 ^= 0xff; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 416) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 417) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 418) HSIPROUND; \
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 419) return v1 ^ v3;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 420)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 421) u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 422) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 423) const u8 *end = data + len - (len % sizeof(u32));
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 424) const u8 left = len & (sizeof(u32) - 1);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 425) u32 m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 426) HPREAMBLE(len)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 427) for (; data != end; data += sizeof(u32)) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 428) m = le32_to_cpup(data);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 429) v3 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 430) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 431) v0 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 432) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 433) switch (left) {
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 434) case 3: b |= ((u32)end[2]) << 16; fallthrough;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 435) case 2: b |= le16_to_cpup(data); break;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 436) case 1: b |= end[0];
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 437) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 438) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 439) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 440) EXPORT_SYMBOL(__hsiphash_aligned);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 441)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 442) #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 443) u32 __hsiphash_unaligned(const void *data, size_t len,
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 444) const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 445) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 446) const u8 *end = data + len - (len % sizeof(u32));
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 447) const u8 left = len & (sizeof(u32) - 1);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 448) u32 m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 449) HPREAMBLE(len)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 450) for (; data != end; data += sizeof(u32)) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 451) m = get_unaligned_le32(data);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 452) v3 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 453) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 454) v0 ^= m;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 455) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 456) switch (left) {
4c1ca831adb10 (Nick Desaulniers 2020-11-15 20:35:31 -0800 457) case 3: b |= ((u32)end[2]) << 16; fallthrough;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 458) case 2: b |= get_unaligned_le16(end); break;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 459) case 1: b |= end[0];
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 460) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 461) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 462) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 463) EXPORT_SYMBOL(__hsiphash_unaligned);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 464) #endif
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 465)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 466) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 467) * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 468) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 469) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 470) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 471) u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 472) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 473) HPREAMBLE(4)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 474) v3 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 475) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 476) v0 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 477) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 478) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 479) EXPORT_SYMBOL(hsiphash_1u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 480)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 481) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 482) * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 483) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 484) * @second: second u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 485) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 486) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 487) u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 488) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 489) HPREAMBLE(8)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 490) v3 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 491) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 492) v0 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 493) v3 ^= second;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 494) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 495) v0 ^= second;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 496) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 497) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 498) EXPORT_SYMBOL(hsiphash_2u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 499)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 500) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 501) * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 502) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 503) * @second: second u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 504) * @third: third u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 505) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 506) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 507) u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 508) const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 509) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 510) HPREAMBLE(12)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 511) v3 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 512) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 513) v0 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 514) v3 ^= second;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 515) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 516) v0 ^= second;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 517) v3 ^= third;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 518) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 519) v0 ^= third;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 520) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 521) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 522) EXPORT_SYMBOL(hsiphash_3u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 523)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 524) /**
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 525) * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 526) * @first: first u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 527) * @second: second u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 528) * @third: third u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 529) * @forth: forth u32
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 530) * @key: the hsiphash key
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 531) */
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 532) u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 533) const u32 forth, const hsiphash_key_t *key)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 534) {
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 535) HPREAMBLE(16)
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 536) v3 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 537) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 538) v0 ^= first;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 539) v3 ^= second;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 540) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 541) v0 ^= second;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 542) v3 ^= third;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 543) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 544) v0 ^= third;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 545) v3 ^= forth;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 546) HSIPROUND;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 547) v0 ^= forth;
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 548) HPOSTAMBLE
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 549) }
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 550) EXPORT_SYMBOL(hsiphash_4u32);
1ae2324f732c9 (Jason A. Donenfeld 2017-01-08 13:54:01 +0100 551) #endif
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
33 Branches
57 Tags