ArticMine's new block weight algorithm

This curbs runaway growth while still allowing substantial
spikes in block weight

Original specification from ArticMine:

here is the scaling proposal
Define: LongTermBlockWeight
Before fork:
LongTermBlockWeight = BlockWeight
At or after fork:
LongTermBlockWeight = min(BlockWeight, 1.4*LongTermEffectiveMedianBlockWeight)
Note: To avoid possible consensus issues over rounding the LongTermBlockWeight for a given block should be calculated to the nearest byte, and stored as a integer in the block itself. The stored LongTermBlockWeight is then used for future calculations of the LongTermEffectiveMedianBlockWeight and not recalculated each time.
Define:   LongTermEffectiveMedianBlockWeight
LongTermEffectiveMedianBlockWeight = max(300000, MedianOverPrevious100000Blocks(LongTermBlockWeight))
Change Definition of EffectiveMedianBlockWeight
From (current definition)
EffectiveMedianBlockWeight  = max(300000, MedianOverPrevious100Blocks(BlockWeight))
To (proposed definition)
EffectiveMedianBlockWeight  = min(max(300000, MedianOverPrevious100Blocks(BlockWeight)), 50*LongTermEffectiveMedianBlockWeight)
Notes:
1) There are no other changes to the existing penalty formula, median calculation, fees etc.
2) There is the requirement to store the LongTermBlockWeight of a block unencrypted in the block itself. This  is to avoid possible consensus issues over rounding and also to prevent the calculations from becoming unwieldy as we move away from the fork.
3) When the  EffectiveMedianBlockWeight cap is reached it is still possible to mine blocks up to 2x the EffectiveMedianBlockWeight by paying the corresponding penalty.

Note: the long term block weight is stored in the database, but not in the actual block itself,
since it requires recalculating anyway for verification.

tests/block_weight/CMakeLists.txt

@@ -0,0 +1,45 @@
+# Copyright (c) 2014-2018, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(block_weight_sources
+  block_weight.cpp)
+
+set(block_weight_headers)
+
+add_executable(block_weight
+  ${block_weight_sources}
+  ${block_weight_headers})
+target_link_libraries(block_weight
+  PRIVATE
+    cryptonote_core
+    blockchain_db
+    ${EXTRA_LIBRARIES})
+
+add_test(
+  NAME    block_weight
+  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight)

tests/block_weight/block_weight.cpp

@@ -0,0 +1,183 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#define IN_UNIT_TESTS
+
+#include <stdio.h>
+#include <math.h>
+#include "cryptonote_core/blockchain.h"
+#include "cryptonote_core/tx_pool.h"
+#include "cryptonote_core/cryptonote_core.h"
+#include "blockchain_db/testdb.h"
+
+#define LONG_TERM_BLOCK_WEIGHT_WINDOW 5000
+
+enum test_t
+{
+  test_max = 0,
+  test_lcg = 1,
+  test_min = 2,
+};
+
+namespace
+{
+
+class TestDB: public cryptonote::BaseTestDB
+{
+private:
+  struct block_t
+  {
+    size_t weight;
+    uint64_t long_term_weight;
+  };
+
+public:
+  TestDB() { m_open = true; }
+
+  virtual void add_block( const cryptonote::block& blk
+                        , size_t block_weight
+                        , uint64_t long_term_block_weight
+                        , const cryptonote::difficulty_type& cumulative_difficulty
+                        , const uint64_t& coins_generated
+                        , uint64_t num_rct_outs
+                        , const crypto::hash& blk_hash
+                        ) override {
+    blocks.push_back({block_weight, long_term_block_weight});
+  }
+  virtual uint64_t height() const override { return blocks.size(); }
+  virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; }
+  virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; }
+  virtual crypto::hash top_block_hash() const override {
+    uint64_t h = height();
+    crypto::hash top = crypto::null_hash;
+    if (h)
+      *(uint64_t*)&top = h - 1;
+    return top;
+  }
+  virtual void pop_block(cryptonote::block &blk, std::vector<cryptonote::transaction> &txs) override { blocks.pop_back(); }
+  virtual void set_hard_fork_version(uint64_t height, uint8_t version) override { if (height >= hf.size()) hf.resize(height + 1); hf[height] = version; }
+  virtual uint8_t get_hard_fork_version(uint64_t height) const override { if (height >= hf.size()) return 255; return hf[height]; }
+
+private:
+  std::vector<block_t> blocks;
+  std::vector<uint8_t> hf;
+};
+
+}
+
+#define PREFIX_WINDOW(hf_version,window) \
+  std::unique_ptr<cryptonote::Blockchain> bc; \
+  cryptonote::tx_memory_pool txpool(*bc); \
+  bc.reset(new cryptonote::Blockchain(txpool)); \
+  struct get_test_options { \
+    const std::pair<uint8_t, uint64_t> hard_forks[3]; \
+    const cryptonote::test_options test_options = { \
+      hard_forks, \
+      window, \
+    }; \
+    get_test_options(): hard_forks{std::make_pair(1, (uint64_t)0), std::make_pair((uint8_t)hf_version, (uint64_t)LONG_TERM_BLOCK_WEIGHT_WINDOW), std::make_pair((uint8_t)0, (uint64_t)0)} {} \
+  } opts; \
+  cryptonote::Blockchain *blockchain = bc.get(); \
+  bool r = blockchain->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0, NULL); \
+  if (!r) \
+  { \
+    fprintf(stderr, "Failed to init blockchain\n"); \
+    exit(1); \
+  }
+
+#define PREFIX(hf_version) PREFIX_WINDOW(hf_version, LONG_TERM_BLOCK_WEIGHT_WINDOW)
+
+static uint32_t lcg_seed = 0;
+
+static uint32_t lcg()
+{
+  lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff;
+  return lcg_seed;
+}
+
+static void test(test_t t, uint64_t blocks)
+{
+  PREFIX(10);
+
+  for (uint64_t h = 0; h < LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h)
+  {
+    cryptonote::block b;
+    b.major_version = 1;
+    b.minor_version = 1;
+    bc->get_db().add_block(b, 300000, 300000, bc->get_db().height(), bc->get_db().height(), {});
+    if (!bc->update_next_cumulative_weight_limit())
+    {
+      fprintf(stderr, "Failed to update cumulative weight limit 1\n");
+      exit(1);
+    }
+  }
+
+  for (uint64_t h = 0; h < blocks; ++h)
+  {
+    uint64_t w;
+    uint64_t effective_block_weight_median = bc->get_current_cumulative_block_weight_median();
+    switch (t)
+    {
+      case test_lcg:
+      {
+        uint32_t r = lcg();
+        int64_t wi = 90 + r % 500000 + 250000 + sin(h / 200.) * 350000;
+        w = wi < 90 ? 90 : wi;
+        break;
+      }
+      case test_max:
+        w = bc->get_current_cumulative_block_weight_limit();
+        break;
+      case test_min:
+        w = 90;
+        break;
+      default:
+        exit(1);
+    }
+    uint64_t ltw = bc->get_next_long_term_block_weight(w);
+    cryptonote::block b;
+    b.major_version = 10;
+    b.minor_version = 10;
+    bc->get_db().add_block(std::move(b), w, ltw, bc->get_db().height(), bc->get_db().height(), {});
+
+    if (!bc->update_next_cumulative_weight_limit())
+    {
+      fprintf(stderr, "Failed to update cumulative weight limit\n");
+      exit(1);
+    }
+    std::cout << "H " << h << ", BW " << w << ", EMBW " << effective_block_weight_median << ", LTBW " << ltw << std::endl;
+  }
+}
+
+int main()
+{
+  test(test_max, 2 * LONG_TERM_BLOCK_WEIGHT_WINDOW);
+  test(test_lcg, 9 * LONG_TERM_BLOCK_WEIGHT_WINDOW);
+  test(test_min, 1 * LONG_TERM_BLOCK_WEIGHT_WINDOW);
+  return 0;
+}

tests/block_weight/block_weight.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# Simulate a maximal block attack on the Monero network
+# This uses the scheme proposed by ArticMine
+# Written by Sarang Nother
+# Copyright (c) 2019 The Monero Project
+import sys
+import math
+
+MEDIAN_WINDOW_SMALL = 100 # number of recent blocks for median computation
+MEDIAN_WINDOW_BIG = 5000
+MULTIPLIER_SMALL = 1.4 # multipliers for determining weights
+MULTIPLIER_BIG = 50.0
+MEDIAN_THRESHOLD = 300*1000 # initial value for median (scaled kB -> B)
+lcg_seed = 0
+embw = MEDIAN_THRESHOLD
+ltembw = MEDIAN_THRESHOLD
+
+weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_SMALL # weights of recent blocks (B), with index -1 most recent
+lt_weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_BIG # long-term weights
+
+# Compute the median of a list
+def get_median(vec):
+    #temp = vec
+    temp = sorted(vec)
+    if len(temp) % 2 == 1:
+        return temp[len(temp)/2]
+    else:
+        return int((temp[len(temp)/2]+temp[len(temp)/2-1])/2)
+
+def LCG():
+  global lcg_seed
+  lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff
+  return lcg_seed
+
+def run(t, blocks):
+  global embw
+  global ltembw
+
+  weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_SMALL # weights of recent blocks (B), with index -1 most recent
+  lt_weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_BIG # long-term weights
+
+  for block in range(blocks):
+      # determine the long-term effective weight
+      ltmedian = get_median(lt_weights[-MEDIAN_WINDOW_BIG:])
+      ltembw = max(MEDIAN_THRESHOLD,ltmedian)
+
+      # determine the effective weight
+      stmedian = get_median(weights[-MEDIAN_WINDOW_SMALL:])
+      embw = min(max(MEDIAN_THRESHOLD,stmedian),int(MULTIPLIER_BIG*ltembw))
+
+      # drop the lowest values
+      weights = weights[1:]
+      lt_weights = lt_weights[1:]
+
+      # add a block of max weight
+      if t == 0:
+        max_weight = 2 * embw
+      elif t == 1:
+        r = LCG()
+        max_weight = int(90 + r % 500000 + 250000 + math.sin(block / 200.) * 350000)
+        if max_weight < 90: max_weight = 90
+      elif t == 2:
+        max_weight = 90
+      else:
+        sys.exit(1)
+      weights.append(max_weight)
+      lt_weights.append(min(max_weight,int(ltembw + int(ltembw * 2 / 5))))
+
+      #print "H %u, r %u, BW %u, EMBW %u, LTBW %u, LTEMBW %u, ltmedian %u" % (block, r, max_weight, embw, lt_weights[-1], ltembw, ltmedian)
+      print "H %u, BW %u, EMBW %u, LTBW %u" % (block, max_weight, embw, lt_weights[-1])
+
+run(0, 2 * MEDIAN_WINDOW_BIG)
+run(1, 9 * MEDIAN_WINDOW_BIG)
+run(2, 1 * MEDIAN_WINDOW_BIG)

tests/block_weight/compare.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+
+import sys
+import subprocess
+
+print 'running: ', sys.argv[1]
+S0 = subprocess.check_output(sys.argv[1], stderr=subprocess.STDOUT)
+print 'running: ', sys.argv[2]
+S1 = subprocess.check_output(sys.argv[2], stderr=subprocess.STDOUT)
+print 'comparing'
+if S0 != S1:
+  sys.exit(1)
+sys.exit(0)