// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. #include "meta-store/codec.h" #include #include #include #include "common/util.h" using namespace doris; int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } TEST(CodecTest, StringCodecTest) { using namespace doris::cloud; std::mt19937 gen(std::random_device("/dev/urandom")()); const int max_len = (2 << 16) + 10086; std::uniform_int_distribution rd_len(0, max_len); std::uniform_int_distribution rd_char(std::numeric_limits::min(), std::numeric_limits::max()); int ret = -1; // Correctness test { int case_count = 50; std::string str1; std::string str2; str1.reserve(max_len); str2.reserve(max_len); std::string b1; std::string b2; std::string d1; std::string d2; b1.reserve(1 + max_len * 2 + 2); b2.reserve(1 + max_len * 2 + 2); d1.reserve(max_len); d2.reserve(max_len); while (case_count--) { str1.clear(); str2.clear(); b1.clear(); b2.clear(); d1.clear(); d2.clear(); int len1 = rd_len(gen); int len2 = rd_len(gen); int zero_count1 = 0; int zero_count2 = 0; while (str1.size() < len1) { str1.push_back(rd_char(gen)); str1.back() == 0x00 ? ++zero_count1 : zero_count1 += 0; } while (str2.size() < len2) { str2.push_back(rd_char(gen)); str2.back() == 0x00 ? ++zero_count2 : zero_count2 += 0; } cloud::encode_bytes(str1, &b1); cloud::encode_bytes(str2, &b2); // clang-format off int sequence = std::memcmp(&str1[0], &str2[0], str1.size() > str2.size() ? str2.size() : str1.size()); int sequence_encoded = std::memcmp(&b1[0], &b2[0], b1.size() > b2.size() ? b2.size() : b1.size()); #define CASE_INFO "sequence=" << sequence << " sequence_encoded=" << sequence_encoded << " str1=" << hex(str1)<< " str2=" << hex(str2)<< " b1=" << hex(b1)<< " b2=" << hex(b2) << " len1=" << len1 << " len2=" << len2 if (len1 > 0 && len2 > 0) { EXPECT_TRUE((sequence * sequence_encoded > 0) || (sequence == 0 && sequence_encoded == 0)) << CASE_INFO; } else { // sequence is not 0 if one of len1,len2 is 0 if (len1 > len2) { EXPECT_GT(sequence_encoded, 0) << CASE_INFO; } else if (len1 < len2) { EXPECT_LT(sequence_encoded, 0) << CASE_INFO; } else { // len1 == len2 == 0 EXPECT_TRUE(len1 == 0 && len2 == 0 && sequence == 0 && sequence == sequence_encoded) << CASE_INFO; } } #undef CASE_INFO EXPECT_EQ(b1[0], cloud::EncodingTag::BYTES_TAG) << " str1=" << hex(str1)<< " str2=" << hex(str2)<< " b1=" << hex(b1)<< " b2=" << hex(b2); EXPECT_EQ(b2[0], cloud::EncodingTag::BYTES_TAG) << " str1=" << hex(str1)<< " str2=" << hex(str2)<< " b1=" << hex(b1)<< " b2=" << hex(b2); // Check encoded value size, marker + zero_escape + terminator EXPECT_EQ(b1.size(), (str1.size() + 1 + zero_count1 + 2)) << "zc1=" << zero_count1; EXPECT_EQ(b2.size(), (str2.size() + 1 + zero_count2 + 2)) << "zc2=" << zero_count2; // Decoding test b1 += "cloud is good"; b2 += "cloud will be better"; std::string_view b1_sv(b1); ret = cloud::decode_bytes(&b1_sv, &d1); EXPECT_EQ(ret, 0); EXPECT_EQ(d1, str1); std::string_view b2_sv(b2); ret = cloud::decode_bytes(&b2_sv, &d2); EXPECT_EQ(ret, 0); EXPECT_EQ(d2, str2); EXPECT_EQ(b1_sv, "cloud is good"); EXPECT_EQ(b2_sv, "cloud will be better"); // clang-format on } } // Boundary tests { std::vector strs; std::vector expected; int zeroes = 1 * 1024 * 1024; strs.emplace_back(zeroes, static_cast(0x00)); expected.push_back(""); expected.back().push_back(cloud::EncodingTag::BYTES_TAG); while (zeroes--) { expected.back().push_back(cloud::EncodingTag::BYTE_ESCAPE); expected.back().push_back(cloud::EncodingTag::ESCAPED_00); } expected.back().push_back(cloud::EncodingTag::BYTE_ESCAPE); expected.back().push_back(cloud::EncodingTag::BYTES_ENDING); ASSERT_TRUE(strs.size() == expected.size()); for (int i = 0; i < strs.size(); ++i) { std::string b1; std::string d1; std::string_view sv(strs[i]); cloud::encode_bytes(sv, &b1); ASSERT_TRUE(b1.size() == expected[i].size()); for (int j = 0; j < b1.size(); ++j) { ASSERT_TRUE(expected[i][j] == b1[j]); } std::string_view b1_sv(b1); ret = cloud::decode_bytes(&b1_sv, &d1); ASSERT_EQ(ret, 0); ASSERT_EQ(b1_sv.size(), 0); ASSERT_EQ(d1.size(), strs[i].size()); ASSERT_TRUE(d1 == strs[i]); } } // Other tests { std::string str1 = "This is"; std::string str2 = "tHIS IS"; // Append something strange str1.push_back(static_cast(0x00)); str1 += "an string"; str1.push_back(static_cast(0xff)); str2.push_back(static_cast(0x00)); str2 += "AN STRING"; str2.push_back(static_cast(0xff)); // Output byte array std::string b1; std::string b2; cloud::encode_bytes(str1, &b1); cloud::encode_bytes(str2, &b2); ASSERT_TRUE(std::memcmp(&b1[0], &b2[0], b1.size() > b2.size() ? b2.size() : b1.size()) < 0); std::string str11; std::string str22; std::string_view b1_sv(b1); std::string_view b2_sv(b2); cloud::decode_bytes(&b1_sv, &str11); cloud::decode_bytes(&b2_sv, &str22); ASSERT_TRUE(str1 == str11); ASSERT_TRUE(str2 == str22); } } TEST(CodecTest, Int64CodecTest) { using namespace doris::cloud; int ret = 0; // Basic test { std::string out1; cloud::encode_int64(10086, &out1); ASSERT_EQ(out1[0], cloud::EncodingTag::POSITIVE_FIXED_INT_TAG); std::cout << hex(out1) << std::endl; int64_t val1 = 10010; std::string_view in(out1); ret = cloud::decode_int64(&in, &val1); ASSERT_EQ(ret, 0); ASSERT_EQ(val1, 10086); std::string out2; cloud::encode_int64(-1001011, &out2); ASSERT_EQ(out2[0], cloud::EncodingTag::NEGATIVE_FIXED_INT_TAG); std::cout << hex(out2) << std::endl; int64_t val2 = 10086; in = out2; ret = cloud::decode_int64(&in, &val2); ASSERT_EQ(ret, 0); ASSERT_EQ(val2, -1001011); // Compare lexical order ASSERT_LT(out2, out1); } }