From e1dbd6cfdd86b513d4bb4c0a6afd7464d7f949d9 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Thu, 29 May 2025 09:53:11 +0100 Subject: [PATCH 1/3] chore(deps): Update module github.com/cloudquery/plugin-sdk/v4 to v4.83.0 (#2178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/cloudquery/plugin-sdk/v4](https://redirect.github.com/cloudquery/plugin-sdk) | require | minor | `v4.82.2` -> `v4.83.0` | --- ### Release Notes
cloudquery/plugin-sdk (github.com/cloudquery/plugin-sdk/v4) ### [`v4.83.0`](https://redirect.github.com/cloudquery/plugin-sdk/releases/tag/v4.83.0) [Compare Source](https://redirect.github.com/cloudquery/plugin-sdk/compare/v4.82.2...v4.83.0) ##### Features - Switch state grpc client to NewClient rather than DialContext ([#​2176](https://redirect.github.com/cloudquery/plugin-sdk/issues/2176)) ([9356d9d](https://redirect.github.com/cloudquery/plugin-sdk/commit/9356d9d14f89d3c1ea58848ae3e53d671f5b4c8f)) ##### Bug Fixes - **deps:** Update dependency go to v1.24.3 ([#​2041](https://redirect.github.com/cloudquery/plugin-sdk/issues/2041)) ([c438d69](https://redirect.github.com/cloudquery/plugin-sdk/commit/c438d690057cb2b8fb4944a5108b0c9bd5bfe294))
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://redirect.github.com/renovatebot/renovate). --- examples/simple_plugin/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index e21c75898f..07931e7976 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -4,7 +4,7 @@ go 1.24.3 require ( github.com/apache/arrow-go/v18 v18.3.0 - github.com/cloudquery/plugin-sdk/v4 v4.82.2 + github.com/cloudquery/plugin-sdk/v4 v4.83.0 github.com/rs/zerolog v1.34.0 ) From 5a34e3522179831f991dd8b4b59844bc1c918c1b Mon Sep 17 00:00:00 2001 From: Marcel <144216124+maaarcelino@users.noreply.github.com> Date: Fri, 30 May 2025 12:34:27 +0200 Subject: [PATCH 2/3] feat: Make SDK FIPS-compliant by using internal SHA1 module (#2179) The Go standard library has removed support for using SHA1 when in FIPS mode in https://github.com/golang/go/commit/54693a81fd605a9c1abbee83da072c61e38d3ebf. However the [NIST FIPS spec](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-131Ar1.pdf) allows for using SHA1 in certain cases: > SHA-1 for non-digital signature applications: > For all other hash function applications, the use of SHA-1 is **acceptable**. The other > applications include HMAC, Key Derivation Functions (KDFs), Random Bit Generation, and > hash-only applications (e.g., hashing passwords and using SHA-1 to compute a checksum, > such as the approved integrity technique specified in Section 4.6.1 of [FIPS 140]). The above removal from the standard lib is too restrictive, as we should be able to use this algo in our SDK for checksums. I've borrowed the implementation of packages `sha1` and `byteorder` and added them as internal modules. --- internal/sha1/byteorder.go | 54 ++++++ internal/sha1/sha1.go | 242 ++++++++++++++++++++++++ internal/sha1/sha1_test.go | 285 +++++++++++++++++++++++++++++ internal/sha1/sha1block.go | 79 ++++++++ internal/sha1/sha1block_generic.go | 5 + schema/arrow.go | 8 +- schema/arrow_test.go | 15 +- schema/resource.go | 4 +- schema/testdata.go | 2 +- 9 files changed, 688 insertions(+), 6 deletions(-) create mode 100644 internal/sha1/byteorder.go create mode 100644 internal/sha1/sha1.go create mode 100644 internal/sha1/sha1_test.go create mode 100644 internal/sha1/sha1block.go create mode 100644 internal/sha1/sha1block_generic.go diff --git a/internal/sha1/byteorder.go b/internal/sha1/byteorder.go new file mode 100644 index 0000000000..deeedf6889 --- /dev/null +++ b/internal/sha1/byteorder.go @@ -0,0 +1,54 @@ +package sha1 + +func BEUint32(b []byte) uint32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func BEPutUint32(b []byte, v uint32) { + _ = b[3] // early bounds check to guarantee safety of writes below + b[0] = byte(v >> 24) + b[1] = byte(v >> 16) + b[2] = byte(v >> 8) + b[3] = byte(v) +} + +func BEAppendUint32(b []byte, v uint32) []byte { + return append(b, + byte(v>>24), + byte(v>>16), + byte(v>>8), + byte(v), + ) +} + +func BEUint64(b []byte) uint64 { + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} + +func BEPutUint64(b []byte, v uint64) { + _ = b[7] // early bounds check to guarantee safety of writes below + b[0] = byte(v >> 56) + b[1] = byte(v >> 48) + b[2] = byte(v >> 40) + b[3] = byte(v >> 32) + b[4] = byte(v >> 24) + b[5] = byte(v >> 16) + b[6] = byte(v >> 8) + b[7] = byte(v) +} + +func BEAppendUint64(b []byte, v uint64) []byte { + return append(b, + byte(v>>56), + byte(v>>48), + byte(v>>40), + byte(v>>32), + byte(v>>24), + byte(v>>16), + byte(v>>8), + byte(v), + ) +} diff --git a/internal/sha1/sha1.go b/internal/sha1/sha1.go new file mode 100644 index 0000000000..c391710157 --- /dev/null +++ b/internal/sha1/sha1.go @@ -0,0 +1,242 @@ +//nolint:revive,errcheck,staticcheck +package sha1 + +import ( + "errors" + "hash" +) + +// The size of a SHA-1 checksum in bytes. +const Size = 20 + +// The blocksize of SHA-1 in bytes. +const BlockSize = 64 + +const ( + chunk = 64 + init0 = 0x67452301 + init1 = 0xEFCDAB89 + init2 = 0x98BADCFE + init3 = 0x10325476 + init4 = 0xC3D2E1F0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [5]uint32 + x [chunk]byte + nx int + len uint64 +} + +const ( + magic = "sha\x01" + marshaledSize = len(magic) + 5*4 + chunk + 8 +) + +func (d *digest) MarshalBinary() ([]byte, error) { + return d.AppendBinary(make([]byte, 0, marshaledSize)) +} + +func (d *digest) AppendBinary(b []byte) ([]byte, error) { + b = append(b, magic...) + b = BEAppendUint32(b, d.h[0]) + b = BEAppendUint32(b, d.h[1]) + b = BEAppendUint32(b, d.h[2]) + b = BEAppendUint32(b, d.h[3]) + b = BEAppendUint32(b, d.h[4]) + b = append(b, d.x[:d.nx]...) + b = append(b, make([]byte, len(d.x)-d.nx)...) + b = BEAppendUint64(b, d.len) + return b, nil +} + +func (d *digest) UnmarshalBinary(b []byte) error { + if len(b) < len(magic) || string(b[:len(magic)]) != magic { + return errors.New("crypto/sha1: invalid hash state identifier") + } + if len(b) != marshaledSize { + return errors.New("crypto/sha1: invalid hash state size") + } + b = b[len(magic):] + b, d.h[0] = consumeUint32(b) + b, d.h[1] = consumeUint32(b) + b, d.h[2] = consumeUint32(b) + b, d.h[3] = consumeUint32(b) + b, d.h[4] = consumeUint32(b) + b = b[copy(d.x[:], b):] + b, d.len = consumeUint64(b) + d.nx = int(d.len % chunk) + return nil +} + +func consumeUint64(b []byte) ([]byte, uint64) { + return b[8:], BEUint64(b) +} + +func consumeUint32(b []byte) ([]byte, uint32) { + return b[4:], BEUint32(b) +} + +func (d *digest) Reset() { + d.h[0] = init0 + d.h[1] = init1 + d.h[2] = init2 + d.h[3] = init3 + d.h[4] = init4 + d.nx = 0 + d.len = 0 +} + +// New returns a new [hash.Hash] computing the SHA1 checksum. The Hash +// also implements [encoding.BinaryMarshaler], [encoding.BinaryAppender] and +// [encoding.BinaryUnmarshaler] to marshal and unmarshal the internal +// state of the hash. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { return Size } + +func (d *digest) BlockSize() int { return BlockSize } + +func (d *digest) Write(p []byte) (nn int, err error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := copy(d.x[d.nx:], p) + d.nx += n + if d.nx == chunk { + block(d, d.x[:]) + d.nx = 0 + } + p = p[n:] + } + if len(p) >= chunk { + n := len(p) &^ (chunk - 1) + block(d, p[:n]) + p = p[n:] + } + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d *digest) Sum(in []byte) []byte { + // Make a copy of d so that caller can keep writing and summing. + d0 := *d + hash := d0.checkSum() + return append(in, hash[:]...) +} + +func (d *digest) checkSum() [Size]byte { + len := d.len + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + var tmp [64 + 8]byte // padding + length buffer + tmp[0] = 0x80 + var t uint64 + if len%64 < 56 { + t = 56 - len%64 + } else { + t = 64 + 56 - len%64 + } + + // Length in bits. + len <<= 3 + padlen := tmp[:t+8] + BEPutUint64(padlen[t:], len) + d.Write(padlen) + + if d.nx != 0 { + panic("d.nx != 0") + } + + var digest [Size]byte + + BEPutUint32(digest[0:], d.h[0]) + BEPutUint32(digest[4:], d.h[1]) + BEPutUint32(digest[8:], d.h[2]) + BEPutUint32(digest[12:], d.h[3]) + BEPutUint32(digest[16:], d.h[4]) + + return digest +} + +// ConstantTimeSum computes the same result of [Sum] but in constant time +func (d *digest) ConstantTimeSum(in []byte) []byte { + d0 := *d + hash := d0.constSum() + return append(in, hash[:]...) +} + +func (d *digest) constSum() [Size]byte { + var length [8]byte + l := d.len << 3 + for i := uint(0); i < 8; i++ { + length[i] = byte(l >> (56 - 8*i)) + } + + nx := byte(d.nx) + t := nx - 56 // if nx < 56 then the MSB of t is one + mask1b := byte(int8(t) >> 7) // mask1b is 0xFF iff one block is enough + + separator := byte(0x80) // gets reset to 0x00 once used + for i := byte(0); i < chunk; i++ { + mask := byte(int8(i-nx) >> 7) // 0x00 after the end of data + + // if we reached the end of the data, replace with 0x80 or 0x00 + d.x[i] = (^mask & separator) | (mask & d.x[i]) + + // zero the separator once used + separator &= mask + + if i >= 56 { + // we might have to write the length here if all fit in one block + d.x[i] |= mask1b & length[i-56] + } + } + + // compress, and only keep the digest if all fit in one block + block(d, d.x[:]) + + var digest [Size]byte + for i, s := range d.h { + digest[i*4] = mask1b & byte(s>>24) + digest[i*4+1] = mask1b & byte(s>>16) + digest[i*4+2] = mask1b & byte(s>>8) + digest[i*4+3] = mask1b & byte(s) + } + + for i := byte(0); i < chunk; i++ { + // second block, it's always past the end of data, might start with 0x80 + if i < 56 { + d.x[i] = separator + separator = 0 + } else { + d.x[i] = length[i-56] + } + } + + // compress, and only keep the digest if we actually needed the second block + block(d, d.x[:]) + + for i, s := range d.h { + digest[i*4] |= ^mask1b & byte(s>>24) + digest[i*4+1] |= ^mask1b & byte(s>>16) + digest[i*4+2] |= ^mask1b & byte(s>>8) + digest[i*4+3] |= ^mask1b & byte(s) + } + + return digest +} + +// Sum returns the SHA-1 checksum of the data. +func Sum(data []byte) [Size]byte { + var d digest + d.Reset() + d.Write(data) + return d.checkSum() +} diff --git a/internal/sha1/sha1_test.go b/internal/sha1/sha1_test.go new file mode 100644 index 0000000000..1fde1ee4f1 --- /dev/null +++ b/internal/sha1/sha1_test.go @@ -0,0 +1,285 @@ +//nolint:revive,errcheck +package sha1 + +import ( + "bytes" + "crypto/rand" + "encoding" + "fmt" + "github.com/stretchr/testify/require" + "hash" + "io" + "testing" +) + +type sha1Test struct { + out string + in string + halfState string // marshaled hash state after first half of in written, used by TestGoldenMarshal +} + +var golden = []sha1Test{ + {"76245dbf96f661bd221046197ab8b9f063f11bad", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n", "sha\x01\v\xa0)I\xdeq(8h\x9ev\xe5\x88[\xf8\x81\x17\xba4Daaaaaaaaaaaaaaaaaaaaaa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x96"}, + {"da39a3ee5e6b4b0d3255bfef95601890afd80709", "", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01"}, + {"a9993e364706816aba3e25717850c26c9cd0d89d", "abc", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01"}, + {"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0ab\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02"}, + {"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0ab\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02"}, + {"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0abc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03"}, + {"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0abc\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03"}, + {"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0abcd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04"}, + {"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0abcd\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04"}, + {"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0abcde\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05"}, + {"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0Discard medicine mor\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14"}, + {"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0He who has a shady past know\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c"}, + {"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0I wouldn't marry him \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15"}, + {"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0Free! Free!/A trip/to Mars/f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c"}, + {"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered. -Tom Stoppard", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0The days of the digital watch\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d"}, + {"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0Nepal premier\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\r"}, + {"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0For every action there is an equa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00!"}, + {"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0His money is twice tainted: \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c"}, + {"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0There is no reason for any individual to hav\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,"}, + {"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0It's a tiny change to the code and no\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00%"}, + {"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size: a.out: bad magic", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0size: a.out\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\f"}, + {"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail. -Mark Horton", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0The major problem is wit\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18"}, + {"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world. CCFestoon", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0Give me a rock, paper and scissors a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$"}, + {"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0If the enemy is within \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17"}, + {"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0It's well we cannot hear the scream\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#"}, + {"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway.", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0You remind me of a TV show, but th\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\""}, + {"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0C is as portable\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10"}, + {"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0Even if I could be Shakespeare, I think I sh\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,"}, + {"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule", "sha\x01x}\xf4\r\xeb\xf2\x10\x87\xe8[\xb2JA$D\xb7\u063ax8em\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00B"}, + {"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++? -Paul Glick", "sha\x01gE#\x01\xef\u036b\x89\x98\xba\xdc\xfe\x102Tv\xc3\xd2\xe1\xf0How can you write a big syst\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + s := fmt.Sprintf("%x", Sum([]byte(g.in))) + if s != g.out { + t.Fatalf("Sum function: sha1(%s) = %s want %s", g.in, s, g.out) + } + c := New() + for j := 0; j < 4; j++ { + var sum []byte + switch j { + case 0, 1: + _, err := io.WriteString(c, g.in) + require.NoError(t, err) + sum = c.Sum(nil) + case 2: + _, err := io.WriteString(c, g.in[:len(g.in)/2]) + require.NoError(t, err) + c.Sum(nil) + _, err = io.WriteString(c, g.in[len(g.in)/2:]) + require.NoError(t, err) + sum = c.Sum(nil) + case 3: + _, err := io.WriteString(c, g.in[:len(g.in)/2]) + require.NoError(t, err) + c.(*digest).ConstantTimeSum(nil) + _, err = io.WriteString(c, g.in[len(g.in)/2:]) + require.NoError(t, err) + sum = c.(*digest).ConstantTimeSum(nil) + } + s := fmt.Sprintf("%x", sum) + if s != g.out { + t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} + +func TestGoldenMarshal(t *testing.T) { + h := New() + h2 := New() + for _, g := range golden { + h.Reset() + h2.Reset() + + _, err := io.WriteString(h, g.in[:len(g.in)/2]) + require.NoError(t, err) + + state, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + t.Errorf("could not marshal: %v", err) + continue + } + + stateAppend, err := h.(encoding.BinaryAppender).AppendBinary(make([]byte, 4, 32)) + if err != nil { + t.Errorf("could not marshal: %v", err) + continue + } + stateAppend = stateAppend[4:] + + if string(state) != g.halfState { + t.Errorf("sha1(%q) state = %+q, want %+q", g.in, state, g.halfState) + continue + } + + if string(stateAppend) != g.halfState { + t.Errorf("sha1(%q) stateAppend = %+q, want %+q", g.in, stateAppend, g.halfState) + continue + } + + if err := h2.(encoding.BinaryUnmarshaler).UnmarshalBinary(state); err != nil { + t.Errorf("could not unmarshal: %v", err) + continue + } + + _, err = io.WriteString(h, g.in[len(g.in)/2:]) + require.NoError(t, err) + + _, err = io.WriteString(h2, g.in[len(g.in)/2:]) + require.NoError(t, err) + + if actual, actual2 := h.Sum(nil), h2.Sum(nil); !bytes.Equal(actual, actual2) { + t.Errorf("sha1(%q) = 0x%x != marshaled 0x%x", g.in, actual, actual2) + } + } +} + +func TestSize(t *testing.T) { + c := New() + if got := c.Size(); got != Size { + t.Errorf("Size = %d; want %d", got, Size) + } +} + +func TestBlockSize(t *testing.T) { + c := New() + if got := c.BlockSize(); got != BlockSize { + t.Errorf("BlockSize = %d; want %d", got, BlockSize) + } +} + +// Tests that blockGeneric (pure Go) and block (in assembly for some architectures) match. +func TestBlockGeneric(t *testing.T) { + for i := 1; i < 30; i++ { // arbitrary factor + gen, asm := New().(*digest), New().(*digest) + buf := make([]byte, BlockSize*i) + rand.Read(buf) + blockGeneric(gen, buf) + block(asm, buf) + if *gen != *asm { + t.Errorf("For %#v block and blockGeneric resulted in different states", buf) + } + } +} + +// Tests for unmarshaling hashes that have hashed a large amount of data +// The initial hash generation is omitted from the test, because it takes a long time. +// The test contains some already-generated states, and their expected sums +// Tests a problem that is outlined in GitHub issue #29543 +// The problem is triggered when an amount of data has been hashed for which +// the data length has a 1 in the 32nd bit. When casted to int, this changes +// the sign of the value, and causes the modulus operation to return a +// different result. +type unmarshalTest struct { + state string + sum string +} + +var largeUnmarshalTests = []unmarshalTest{ + // Data length: 7_102_415_735 + { + state: "sha\x01\x13\xbc\xfe\x83\x8c\xbd\xdfP\x1f\xd8ڿ<\x9eji8t\xe1\xa5@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuv\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\xa7VCw", + sum: "bc6245c9959cc33e1c2592e5c9ea9b5d0431246c", + }, + // Data length: 6_565_544_823 + { + state: "sha\x01m;\x16\xa6R\xbe@\xa9nĈ\xf9S\x03\x00B\xc2\xdcv\xcf@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuv\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x87VCw", + sum: "8f2d1c0e4271768f35feb918bfe21ea1387a2072", + }, +} + +func safeSum(h hash.Hash) (sum []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("sum panic: %v", r) + } + }() + + return h.Sum(nil), nil +} + +func TestLargeHashes(t *testing.T) { + for i, test := range largeUnmarshalTests { + + h := New() + if err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary([]byte(test.state)); err != nil { + t.Errorf("test %d could not unmarshal: %v", i, err) + continue + } + + sum, err := safeSum(h) + if err != nil { + t.Errorf("test %d could not sum: %v", i, err) + continue + } + + if fmt.Sprintf("%x", sum) != test.sum { + t.Errorf("test %d sum mismatch: expect %s got %x", i, test.sum, sum) + } + } +} + +func TestAllocations(t *testing.T) { + in := []byte("hello, world!") + out := make([]byte, 0, Size) + h := New() + n := int(testing.AllocsPerRun(10, func() { + h.Reset() + h.Write(in) + out = h.Sum(out[:0]) + })) + if n > 0 { + t.Errorf("allocs = %d, want 0", n) + } +} + +// func TestSHA1Hash(t *testing.T) { +// cryptotest.TestHash(t, New) +// } + +var bench = New() +var buf = make([]byte, 8192) + +func benchmarkSize(b *testing.B, size int) { + sum := make([]byte, bench.Size()) + b.Run("New", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(size)) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:size]) + bench.Sum(sum[:0]) + } + }) + b.Run("Sum", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(size)) + for i := 0; i < b.N; i++ { + Sum(buf[:size]) + } + }) +} + +func BenchmarkHash8Bytes(b *testing.B) { + benchmarkSize(b, 8) +} + +func BenchmarkHash320Bytes(b *testing.B) { + benchmarkSize(b, 320) +} + +func BenchmarkHash1K(b *testing.B) { + benchmarkSize(b, 1024) +} + +func BenchmarkHash8K(b *testing.B) { + benchmarkSize(b, 8192) +} diff --git a/internal/sha1/sha1block.go b/internal/sha1/sha1block.go new file mode 100644 index 0000000000..89ec719100 --- /dev/null +++ b/internal/sha1/sha1block.go @@ -0,0 +1,79 @@ +package sha1 + +import ( + "math/bits" +) + +const ( + _K0 = 0x5A827999 + _K1 = 0x6ED9EBA1 + _K2 = 0x8F1BBCDC + _K3 = 0xCA62C1D6 +) + +// blockGeneric is a portable, pure Go version of the SHA-1 block step. +// It's used by sha1block_generic.go and tests. +func blockGeneric(dig *digest, p []byte) { + var w [16]uint32 + + h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] + for len(p) >= chunk { + // Can interlace the computation of w with the + // rounds below if needed for speed. + for i := 0; i < 16; i++ { + j := i * 4 + w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3]) + } + + a, b, c, d, e := h0, h1, h2, h3, h4 + + // Each of the four 20-iteration rounds + // differs only in the computation of f and + // the choice of K (_K0, _K1, etc). + i := 0 + for ; i < 16; i++ { + f := b&c | (^b)&d + t := bits.RotateLeft32(a, 5) + f + e + w[i&0xf] + _K0 + a, b, c, d, e = t, a, bits.RotateLeft32(b, 30), c, d + } + for ; i < 20; i++ { + tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf] + w[i&0xf] = bits.RotateLeft32(tmp, 1) + + f := b&c | (^b)&d + t := bits.RotateLeft32(a, 5) + f + e + w[i&0xf] + _K0 + a, b, c, d, e = t, a, bits.RotateLeft32(b, 30), c, d + } + for ; i < 40; i++ { + tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf] + w[i&0xf] = bits.RotateLeft32(tmp, 1) + f := b ^ c ^ d + t := bits.RotateLeft32(a, 5) + f + e + w[i&0xf] + _K1 + a, b, c, d, e = t, a, bits.RotateLeft32(b, 30), c, d + } + for ; i < 60; i++ { + tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf] + w[i&0xf] = bits.RotateLeft32(tmp, 1) + f := ((b | c) & d) | (b & c) + t := bits.RotateLeft32(a, 5) + f + e + w[i&0xf] + _K2 + a, b, c, d, e = t, a, bits.RotateLeft32(b, 30), c, d + } + for ; i < 80; i++ { + tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf] + w[i&0xf] = bits.RotateLeft32(tmp, 1) + f := b ^ c ^ d + t := bits.RotateLeft32(a, 5) + f + e + w[i&0xf] + _K3 + a, b, c, d, e = t, a, bits.RotateLeft32(b, 30), c, d + } + + h0 += a + h1 += b + h2 += c + h3 += d + h4 += e + + p = p[chunk:] + } + + dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4 +} diff --git a/internal/sha1/sha1block_generic.go b/internal/sha1/sha1block_generic.go new file mode 100644 index 0000000000..a4b74f99d9 --- /dev/null +++ b/internal/sha1/sha1block_generic.go @@ -0,0 +1,5 @@ +package sha1 + +func block(dig *digest, p []byte) { + blockGeneric(dig, p) +} diff --git a/schema/arrow.go b/schema/arrow.go index 0560426695..81b8a39c3c 100644 --- a/schema/arrow.go +++ b/schema/arrow.go @@ -1,12 +1,12 @@ package schema import ( - "crypto/sha1" "time" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/cloudquery/plugin-sdk/v4/internal/sha1" "github.com/cloudquery/plugin-sdk/v4/types" "github.com/google/uuid" ) @@ -63,11 +63,15 @@ func hashRecord(record arrow.Record) arrow.Array { _, _ = rowHash.Write([]byte(value)) } // This part ensures that we conform to the UUID spec - hashArray.Append(uuid.NewSHA1(uuid.NameSpaceURL, rowHash.Sum(nil))) + hashArray.Append(newUUID(uuid.NameSpaceURL, rowHash.Sum(nil))) } return hashArray.NewArray() } +func newUUID(space uuid.UUID, data []byte) uuid.UUID { + return uuid.NewHash(sha1.New(), space, data, 5) +} + func nullUUIDsForRecord(numRows int) arrow.Array { uuidArray := types.NewUUIDBuilder(memory.DefaultAllocator) uuidArray.AppendNulls(numRows) diff --git a/schema/arrow_test.go b/schema/arrow_test.go index 25bbcd7ebc..5744e4a8f8 100644 --- a/schema/arrow_test.go +++ b/schema/arrow_test.go @@ -11,6 +11,7 @@ import ( "github.com/cloudquery/plugin-sdk/v4/types" "github.com/google/uuid" "github.com/samber/lo" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -80,7 +81,7 @@ func buildTestRecord(withClientIDValue string) arrow.Record { } case *types.UUIDBuilder: for i := range testValuesCount { - b.Append(uuid.NewSHA1(uuid.NameSpaceURL, []byte(fmt.Sprintf("test%d", i)))) + b.Append(newUUID(uuid.NameSpaceURL, []byte(fmt.Sprintf("test%d", i)))) } } } @@ -162,3 +163,15 @@ func TestAddInternalColumnsToRecord(t *testing.T) { }) } } + +func TestCQIDHashingConsistency(t *testing.T) { + record := buildTestRecord("") + got, err := AddInternalColumnsToRecord(record, "") + require.NoError(t, err) + cqIDFields := got.Schema().FieldIndices(CqIDColumn.Name) + require.Len(t, cqIDFields, 1) + // we are now using an internal version of the official SHA1 module + // this test ensures that our implementation produces the same hash value as the official one + // d8f3b1de-8c63-5a0e-a1aa-19e9b5311c24 is the hash value from the official implementation + assert.Equal(t, "d8f3b1de-8c63-5a0e-a1aa-19e9b5311c24", got.Column(cqIDFields[0]).ValueStr(0)) +} diff --git a/schema/resource.go b/schema/resource.go index 0e7106271e..9d6da66a80 100644 --- a/schema/resource.go +++ b/schema/resource.go @@ -83,7 +83,7 @@ func (r *Resource) CalculateCQID(deterministicCQID bool) error { // if `PrimaryKeyComponent` is set, we calculate the CQID based on those components pkComponents := r.Table.PrimaryKeyComponents() if len(pkComponents) > 0 { - return r.storeCQID(uuid.NewSHA1(uuid.UUID{}, calculateCqIDValue(r, pkComponents).Sum(nil))) + return r.storeCQID(newUUID(uuid.UUID{}, calculateCqIDValue(r, pkComponents).Sum(nil))) } // If deterministicCQID is false, we generate a random CQID @@ -96,7 +96,7 @@ func (r *Resource) CalculateCQID(deterministicCQID bool) error { return r.storeCQID(uuid.New()) } - return r.storeCQID(uuid.NewSHA1(uuid.UUID{}, calculateCqIDValue(r, names).Sum(nil))) + return r.storeCQID(newUUID(uuid.UUID{}, calculateCqIDValue(r, names).Sum(nil))) } func calculateCqIDValue(r *Resource, cols []string) hash.Hash { diff --git a/schema/testdata.go b/schema/testdata.go index 71c8831b4d..96446c01b8 100644 --- a/schema/testdata.go +++ b/schema/testdata.go @@ -294,7 +294,7 @@ func (tg TestDataGenerator) getExampleJSON(colName string, dataType arrow.DataTy // This will make UUIDs deterministic like all other types hash := sha256.New() hash.Write([]byte(fmt.Sprintf(`"AString%d"`, rnd.Intn(100000)))) - u := uuid.NewSHA1(uuid.UUID{}, hash.Sum(nil)) + u := newUUID(uuid.UUID{}, hash.Sum(nil)) return `"` + u.String() + `"` } if arrow.TypeEqual(dataType, types.ExtensionTypes.JSON) { From a759070f2d90fd18d15abc8e005ade32ff743879 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Fri, 30 May 2025 11:51:18 +0100 Subject: [PATCH 3/3] chore(main): Release v4.84.0 (#2180) :robot: I have created a release *beep* *boop* --- ## [4.84.0](https://github.com/cloudquery/plugin-sdk/compare/v4.83.0...v4.84.0) (2025-05-30) ### Features * Make SDK FIPS-compliant by using internal SHA1 module ([#2179](https://github.com/cloudquery/plugin-sdk/issues/2179)) ([5a34e35](https://github.com/cloudquery/plugin-sdk/commit/5a34e3522179831f991dd8b4b59844bc1c918c1b)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- .release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6eb0f130ec..063dfb8fd9 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "4.83.0" + ".": "4.84.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index d1007952e3..8c42422c27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.84.0](https://github.com/cloudquery/plugin-sdk/compare/v4.83.0...v4.84.0) (2025-05-30) + + +### Features + +* Make SDK FIPS-compliant by using internal SHA1 module ([#2179](https://github.com/cloudquery/plugin-sdk/issues/2179)) ([5a34e35](https://github.com/cloudquery/plugin-sdk/commit/5a34e3522179831f991dd8b4b59844bc1c918c1b)) + ## [4.83.0](https://github.com/cloudquery/plugin-sdk/compare/v4.82.2...v4.83.0) (2025-05-28)