Make difficulty 128 bit instead of 64 bit

Based on Boolberry work by:
  jahrsg <jahr@jahr.me>
  cr.zoidberg <crypto.zoidberg@gmail.com>

tests/difficulty/CMakeLists.txt

@@ -45,3 +45,6 @@ set_property(TARGET difficulty-tests
 add_test(
   NAME    difficulty
   COMMAND difficulty-tests "${CMAKE_CURRENT_SOURCE_DIR}/data.txt")
+add_test(
+  NAME    wide_difficulty
+  COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/wide_difficulty.py" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_wide_data.py" "${CMAKE_CURRENT_BINARY_DIR}/difficulty-tests" "${CMAKE_CURRENT_BINARY_DIR}/wide_data.txt")

tests/difficulty/difficulty.cpp

@@ -43,16 +43,15 @@ using namespace std;
 
 #define DEFAULT_TEST_DIFFICULTY_TARGET        120
 
-int main(int argc, char *argv[]) {
-    if (argc != 2) {
-        cerr << "Wrong arguments" << endl;
-        return 1;
-    }
-    vector<uint64_t> timestamps, cumulative_difficulties;
-    fstream data(argv[1], fstream::in);
+static int test_wide_difficulty(const char *filename)
+{
+    std::vector<uint64_t> timestamps;
+    std::vector<cryptonote::difficulty_type> cumulative_difficulties;
+    fstream data(filename, fstream::in);
     data.exceptions(fstream::badbit);
     data.clear(data.rdstate());
-    uint64_t timestamp, difficulty, cumulative_difficulty = 0;
+    uint64_t timestamp;
+    cryptonote::difficulty_type difficulty, cumulative_difficulty = 0;
     size_t n = 0;
     while (data >> timestamp >> difficulty) {
         size_t begin, end;
@@ -63,11 +62,11 @@ int main(int argc, char *argv[]) {
             end = n - DIFFICULTY_LAG;
             begin = end - DIFFICULTY_WINDOW;
         }
-        uint64_t res = cryptonote::next_difficulty(
-            vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
-            vector<uint64_t>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
+        cryptonote::difficulty_type res = cryptonote::next_difficulty(
+            std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
+            std::vector<cryptonote::difficulty_type>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
         if (res != difficulty) {
-            cerr << "Wrong difficulty for block " << n << endl
+            cerr << "Wrong wide difficulty for block " << n << endl
                 << "Expected: " << difficulty << endl
                 << "Found: " << res << endl;
             return 1;
@@ -81,3 +80,60 @@ int main(int argc, char *argv[]) {
     }
     return 0;
 }
+
+int main(int argc, char *argv[]) {
+    if (argc < 2) {
+        cerr << "Wrong arguments" << endl;
+        return 1;
+    }
+    if (argc == 3 && strcmp(argv[1], "--wide") == 0)
+    {
+        return test_wide_difficulty(argv[2]);
+    }
+
+    vector<uint64_t> timestamps, cumulative_difficulties;
+    std::vector<cryptonote::difficulty_type> wide_cumulative_difficulties;
+    fstream data(argv[1], fstream::in);
+    data.exceptions(fstream::badbit);
+    data.clear(data.rdstate());
+    uint64_t timestamp;
+    uint64_t difficulty, cumulative_difficulty = 0;
+    cryptonote::difficulty_type wide_cumulative_difficulty = 0;
+    size_t n = 0;
+    while (data >> timestamp >> difficulty) {
+        size_t begin, end;
+        if (n < DIFFICULTY_WINDOW + DIFFICULTY_LAG) {
+            begin = 0;
+            end = min(n, (size_t) DIFFICULTY_WINDOW);
+        } else {
+            end = n - DIFFICULTY_LAG;
+            begin = end - DIFFICULTY_WINDOW;
+        }
+        uint64_t res = cryptonote::next_difficulty_64(
+            vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
+            std::vector<uint64_t>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
+        if (res != difficulty) {
+            cerr << "Wrong difficulty for block " << n << endl
+                << "Expected: " << difficulty << endl
+                << "Found: " << res << endl;
+            return 1;
+        }
+        cryptonote::difficulty_type wide_res = cryptonote::next_difficulty(
+            std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end),
+            std::vector<cryptonote::difficulty_type>(wide_cumulative_difficulties.begin() + begin, wide_cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET);
+        if (wide_res.convert_to<uint64_t>() != res) {
+            cerr << "Wrong wide difficulty for block " << n << endl
+                << "Expected: " << res << endl
+                << "Found: " << wide_res << endl;
+            return 1;
+        }
+        timestamps.push_back(timestamp);
+        cumulative_difficulties.push_back(cumulative_difficulty += difficulty);
+        wide_cumulative_difficulties.push_back(wide_cumulative_difficulty += difficulty);
+        ++n;
+    }
+    if (!data.eof()) {
+        data.clear(fstream::badbit);
+    }
+    return 0;
+}

tests/difficulty/gen_wide_data.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import random
+
+DIFFICULTY_TARGET = 120
+DIFFICULTY_WINDOW = 720
+DIFFICULTY_LAG = 15
+DIFFICULTY_CUT = 60
+
+def difficulty():
+    times = []
+    diffs = []
+    while True:
+        if len(times) <= 1:
+            diff = 1
+        else:
+            begin = max(len(times) - DIFFICULTY_WINDOW - DIFFICULTY_LAG, 0)
+            end = min(begin + DIFFICULTY_WINDOW, len(times))
+            length = end - begin
+            assert length >= 2
+            if length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT:
+                cut_begin = 0
+                cut_end = length
+            else:
+                excess = length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT)
+                cut_begin = (excess + 1) // 2
+                cut_end = length - excess // 2
+            assert cut_begin + 2 <= cut_end
+            wnd = times[begin:end]
+            wnd.sort()
+            dtime = wnd[cut_end - 1] - wnd[cut_begin]
+            dtime = max(dtime, 1)
+            ddiff = sum(diffs[begin + cut_begin + 1:begin + cut_end])
+            diff = (ddiff * DIFFICULTY_TARGET + dtime - 1) // dtime
+        times.append((yield diff))
+        diffs.append(diff)
+
+random.seed(1)
+time = 1000
+gen = difficulty()
+diff = next(gen)
+for i in range(100000):
+    power = 100 if i < 10000 else 100000000 if i < 500 else 1000000000000 if i < 1000 else 1000000000000000 if i < 2000 else 10000000000000000000 if i < 4000 else 1000000000000000000000000 
+    time += random.randint(-diff // power - 10, 3 * diff // power + 10)
+    print(time, diff)
+    diff = gen.send(time)

tests/difficulty/wide_difficulty.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import sys
+import subprocess
+
+python = sys.argv[1]
+py = sys.argv[2]
+c = sys.argv[3]
+data = sys.argv[4]
+
+first = python + " " + py + " > " + data
+second = [c, '--wide', data]
+
+try:
+  print('running: ', first)
+  subprocess.check_call(first, shell=True)
+  print('running: ', second)
+  subprocess.check_call(second)
+except:
+  sys.exit(1)
+

tests/hash-target.cpp

@@ -40,7 +40,7 @@ using cryptonote::check_hash;
 
 int main(int argc, char *argv[]) {
   crypto::hash h;
-  for (uint64_t diff = 1;; diff += 1 + (diff >> 8)) {
+  for (cryptonote::difficulty_type diff = 1;; diff += 1 + (diff >> 8)) {
     for (uint16_t b = 0; b < 256; b++) {
       memset(&h, b, sizeof(crypto::hash));
       if (check_hash(h, diff) != (b == 0 || diff <= 255 / b)) {
@@ -50,7 +50,7 @@ int main(int argc, char *argv[]) {
         memset(&h, 0, sizeof(crypto::hash));
         ((char *) &h)[31] = b;
         if (check_hash(h, diff) != (diff <= 255 / b)) {
-          return 1;
+          return 2;
         }
       }
     }
@@ -58,11 +58,11 @@ int main(int argc, char *argv[]) {
       uint64_t val = 0;
       for (int i = 31; i >= 0; i--) {
         val = val * 256 + 255;
-        ((char *) &h)[i] = static_cast<char>(val / diff);
-        val %= diff;
+        ((char *) &h)[i] = static_cast<char>(static_cast<uint64_t>(val / diff));
+        val %= diff.convert_to<uint64_t>();
       }
       if (check_hash(h, diff) != true) {
-        return 1;
+        return 3;
       }
       if (diff > 1) {
         for (int i = 0;; i++) {
@@ -74,7 +74,7 @@ int main(int argc, char *argv[]) {
           }
         }
         if (check_hash(h, diff) != false) {
-          return 1;
+          return 4;
         }
       }
     }

tests/performance_tests/CMakeLists.txt

@@ -31,6 +31,7 @@ set(performance_tests_sources
 
 set(performance_tests_headers
   check_tx_signature.h
+  check_hash.h
   cn_slow_hash.h
   construct_tx.h
   derive_public_key.h

tests/performance_tests/check_hash.h

@@ -0,0 +1,66 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "string_tools.h"
+#include "cryptonote_basic/difficulty.h"
+
+template<uint64_t hash_target_high, uint64_t hash_target_low, uint64_t difficulty_high, uint64_t difficulty_low>
+class test_check_hash
+{
+public:
+  static const size_t loop_count = 100000;
+
+  bool init()
+  {
+    cryptonote::difficulty_type hash_target = hash_target_high;
+    hash_target = (hash_target << 64) | hash_target_low;
+    difficulty = difficulty_high;
+    difficulty = (difficulty << 64) | difficulty_low;
+    boost::multiprecision::uint256_t hash_value =  std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target;
+    ((uint64_t*)&hash)[0] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    hash_value >>= 64;
+    ((uint64_t*)&hash)[1] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    hash_value >>= 64;
+    ((uint64_t*)&hash)[2] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    hash_value >>= 64;
+    ((uint64_t*)&hash)[3] = (hash_value << 64 >> 64).convert_to<uint64_t>();
+    return true;
+  }
+
+  bool test()
+  {
+    cryptonote::check_hash_128(hash, difficulty);
+    return true;
+  }
+
+private:
+  crypto::hash hash;
+  cryptonote::difficulty_type difficulty;
+};

tests/performance_tests/main.cpp

@@ -38,6 +38,7 @@
 // tests
 #include "construct_tx.h"
 #include "check_tx_signature.h"
+#include "check_hash.h"
 #include "cn_slow_hash.h"
 #include "derive_public_key.h"
 #include "derive_secret_key.h"
@@ -181,6 +182,14 @@ int main(int argc, char** argv)
   TEST_PERFORMANCE4(filter, p, test_check_tx_signature_aggregated_bulletproofs, 2, 2, 56, 16);
   TEST_PERFORMANCE4(filter, p, test_check_tx_signature_aggregated_bulletproofs, 10, 2, 56, 16);
 
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 1, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 0xffffffffffffffff, 0, 0xffffffffffffffff);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 0xffffffffffffffff, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 1, 0, 1, 0);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 1, 0, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0xffffffffffffffff, 0xffffffffffffffff, 0, 1);
+  TEST_PERFORMANCE4(filter, p, test_check_hash, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff);
+
   TEST_PERFORMANCE0(filter, p, test_is_out_to_acc);
   TEST_PERFORMANCE0(filter, p, test_is_out_to_acc_precomp);
   TEST_PERFORMANCE0(filter, p, test_generate_key_image_helper);

tests/unit_tests/CMakeLists.txt

@@ -43,6 +43,7 @@ set(unit_tests_sources
   crypto.cpp
   decompose_amount_into_digits.cpp
   device.cpp
+  difficulty.cpp
   dns_resolver.cpp
   epee_boosted_tcp_server.cpp
   epee_levin_protocol_handler_async.cpp

tests/unit_tests/difficulty.cpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "gtest/gtest.h"
+#include "cryptonote_basic/difficulty.h"
+
+static cryptonote::difficulty_type MKDIFF(uint64_t high, uint64_t low)
+{
+  cryptonote::difficulty_type d = high;
+  d = (d << 64) | low;
+  return d;
+}
+
+static crypto::hash MKHASH(uint64_t high, uint64_t low)
+{
+  cryptonote::difficulty_type hash_target = high;
+  hash_target = (hash_target << 64) | low;
+  boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target;
+  crypto::hash h;
+  ((uint64_t*)&h)[0] = hash_value.convert_to<uint64_t>();
+  hash_value >>= 64;
+  ((uint64_t*)&h)[1] = hash_value.convert_to<uint64_t>();
+  hash_value >>= 64;
+  ((uint64_t*)&h)[2] = hash_value.convert_to<uint64_t>();
+  hash_value >>= 64;
+  ((uint64_t*)&h)[3] = hash_value.convert_to<uint64_t>();
+  return h;
+}
+
+TEST(difficulty, check_hash)
+{
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(0, 1), MKDIFF(0, 1)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(0, 1), MKDIFF(0, 2)));
+
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(0, 0xffffffffffffffff), MKDIFF(0, 0xffffffffffffffff)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(0, 0xffffffffffffffff), MKDIFF(1, 0)));
+
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(1, 1), MKDIFF(1, 1)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(1, 1), MKDIFF(1, 2)));
+
+  ASSERT_TRUE(cryptonote::check_hash(MKHASH(0xffffffffffffffff, 1), MKDIFF(0xffffffffffffffff, 1)));
+  ASSERT_FALSE(cryptonote::check_hash(MKHASH(0xffffffffffffffff, 1), MKDIFF(0xffffffffffffffff, 2)));
+}

tests/unit_tests/serialization.cpp

@@ -1187,3 +1187,26 @@ TEST(Serialization, portability_signed_tx)
   ASSERT_TRUE(epee::string_tools::pod_to_hex(ki1) == "d54cbd435a8d636ad9b01b8d4f3eb13bd0cf1ce98eddf53ab1617f9b763e66c0");
   ASSERT_TRUE(epee::string_tools::pod_to_hex(ki2) == "6c3cd6af97c4070a7aef9b1344e7463e29c7cd245076fdb65da447a34da3ca76");
 }
+
+TEST(Serialization, difficulty_type)
+{
+  std::vector<cryptonote::difficulty_type> v_original;
+
+  for(int i = 0; i != 100; i++)
+  {
+    v_original.push_back(cryptonote::difficulty_type("117868131154734361989189100"));
+    if(v_original.size() > 1)
+      v_original.back() *= v_original[v_original.size()-2];
+  }
+
+  std::stringstream ss;
+  boost::archive::portable_binary_oarchive a(ss);
+  a << v_original;
+
+  std::vector<cryptonote::difficulty_type> v_unserialized;
+
+  boost::archive::portable_binary_iarchive a2(ss);
+  a2 >> v_unserialized;
+
+  ASSERT_EQ(v_original, v_unserialized);
+}