Merge pull request ton-blockchain#1139 from ton-blockchain/stable_testnet

Merge recent updates

Author: EmelyanenkoK

.github/workflows/docker-ubuntu-branch-image.yml

@@ -0,0 +1,48 @@
+name: Docker Ubuntu 22.04 branch image
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore:
+      - master
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get tag as branch name
+        id: tag
+        run: |
+          echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64
+          push: true
+          context: ./
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}

Dockerfile

@@ -27,14 +27,18 @@ RUN apt-get update && \
     apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \
     rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /var/ton-work/db /var/ton-work/scripts
+RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/ /usr/lib/fift/
 
 COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/
 COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
 COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/
 COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/
 COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/
 COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/
+COPY --from=builder /ton/build/crypto/fift /usr/local/bin/
+COPY --from=builder /ton/build/crypto/func /usr/local/bin/
+COPY --from=builder /ton/crypto/smartcont/* /usr/share/ton/smartcont/
+COPY --from=builder /ton/crypto/fift/lib/* /usr/lib/fift/
 
 WORKDIR /var/ton-work/db
 COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/

adnl/adnl-peer.cpp

@@ -269,7 +269,11 @@ void AdnlPeerPairImpl::send_messages_in(std::vector<OutboundAdnlMessage> message
     size_t ptr = 0;
     bool first = true;
     do {
+      respond_with_nop_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0));
       bool try_reinit = try_reinit_at_ && try_reinit_at_.is_in_past();
+      if (try_reinit) {
+        try_reinit_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.5));
+      }
       bool via_channel = channel_ready_ && !try_reinit;
       size_t s = (via_channel ? channel_packet_header_max_size() : packet_header_max_size());
       if (first) {
@@ -504,12 +508,6 @@ void AdnlPeerPairImpl::create_channel(pubkeys::Ed25519 pub, td::uint32 date) {
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCreateChannel &message) {
   create_channel(message.key(), message.date());
-  if (respond_to_channel_create_after_.is_in_past()) {
-    respond_to_channel_create_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0));
-    std::vector<OutboundAdnlMessage> messages;
-    messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0);
-    send_messages(std::move(messages));
-  }
 }
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChannel &message) {
@@ -526,6 +524,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChan
 }
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCustom &message) {
+  respond_with_nop();
   td::actor::send_closure(local_actor_, &AdnlLocalId::deliver, peer_id_short_, message.data());
 }
 
@@ -538,6 +537,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageReinit &mes
 }
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &message) {
+  respond_with_nop();
   auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = message.query_id(),
                                        flags = static_cast<td::uint32>(0)](td::Result<td::BufferSlice> R) {
     if (R.is_error()) {
@@ -556,6 +556,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess
 }
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &message) {
+  respond_with_nop();
   auto Q = out_queries_.find(message.query_id());
 
   if (Q == out_queries_.end()) {
@@ -573,6 +574,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &mes
 }
 
 void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &message) {
+  respond_with_nop();
   auto size = message.total_size();
   if (size > huge_packet_max_size()) {
     VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size;
@@ -635,6 +637,14 @@ void AdnlPeerPairImpl::delete_query(AdnlQueryId id) {
   }
 }
 
+void AdnlPeerPairImpl::respond_with_nop() {
+  if (respond_with_nop_after_.is_in_past()) {
+    std::vector<OutboundAdnlMessage> messages;
+    messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0);
+    send_messages(std::move(messages));
+  }
+}
+
 void AdnlPeerPairImpl::reinit(td::int32 date) {
   if (reinit_date_ == 0) {
     reinit_date_ = date;

adnl/adnl-peer.hpp

@@ -122,6 +122,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
   }
 
  private:
+  void respond_with_nop();
   void reinit(td::int32 date);
   td::Result<std::pair<td::actor::ActorId<AdnlNetworkConnection>, bool>> get_conn(bool direct_only);
   void create_channel(pubkeys::Ed25519 pub, td::uint32 date);
@@ -214,7 +215,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
   pubkeys::Ed25519 channel_pub_;
   td::int32 channel_pk_date_;
   td::actor::ActorOwn<AdnlChannel> channel_;
-  td::Timestamp respond_to_channel_create_after_;
+  td::Timestamp respond_with_nop_after_;
 
   td::uint64 in_seqno_ = 0;
   td::uint64 out_seqno_ = 0;

assembly/wasm/fift-func-wasm-build-ubuntu.sh

@@ -85,7 +85,7 @@ cd ..
 if [ ! -f "openssl/openssl_em" ]; then
   cd openssl
   make clean
-  emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
+  emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer
   sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
   sed -i 's/-ldl//g' Makefile
   sed -i 's/-O3/-Os/g' Makefile

catchain/catchain-receiver-source.h

@@ -61,7 +61,7 @@ class CatChainReceiverSource {
   virtual td::BufferSlice fork_proof() const = 0;
   virtual bool fork_is_found() const = 0;
 
-  // One block can be sent to one node only a limited number of times to prevent DoS
+  // One block can be sent to one node in catchain.getDifference only a limited number of times to prevent DoS
   virtual bool allow_send_block(CatChainBlockHash hash) = 0;
 
   static td::Result<std::unique_ptr<CatChainReceiverSource>> create(CatChainReceiver *chain, PublicKey pub_key,

catchain/catchain-receiver.cpp

@@ -697,12 +697,8 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
   } else {
     CatChainReceiverSource *S = get_source_by_adnl_id(src);
     CHECK(S != nullptr);
-    if (S->allow_send_block(it->second->get_hash())) {
-      promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_blockResult>(it->second->export_tl()),
-                                            true, it->second->get_payload().as_slice()));
-    } else {
-      promise.set_error(td::Status::Error("block was requested too many times"));
-    }
+    promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_blockResult>(it->second->export_tl()),
+                                          true, it->second->get_payload().as_slice()));
   }
 }
 

catchain/catchain.h

@@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor {
   virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
                               td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size,
                               td::actor::ActorId<adnl::AdnlSenderInterface> via) = 0;
+  virtual void get_source_heights(td::Promise<std::vector<CatChainBlockHeight>> promise) = 0;
   virtual void destroy() = 0;
 
   static td::actor::ActorOwn<CatChain> create(std::unique_ptr<Callback> callback, const CatChainOptions &opts,

catchain/catchain.hpp

@@ -115,6 +115,15 @@ class CatChainImpl : public CatChain {
     td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name,
                             std::move(promise), timeout, std::move(query), max_answer_size, via);
   }
+  void get_source_heights(td::Promise<std::vector<CatChainBlockHeight>> promise) override {
+    std::vector<CatChainBlockHeight> heights(top_source_blocks_.size(), 0);
+    for (size_t i = 0; i < top_source_blocks_.size(); ++i) {
+      if (top_source_blocks_[i]) {
+        heights[i] = top_source_blocks_[i]->height();
+      }
+    }
+    promise.set_result(std::move(heights));
+  }
   void destroy() override;
   CatChainImpl(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
                td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,

crypto/vm/bls.cpp

@@ -93,12 +93,13 @@ bool aggregate_verify(const std::vector<std::pair<P1, td::BufferSlice>> &pubs_ms
       return false;
     }
     std::unique_ptr<blst::Pairing> pairing = std::make_unique<blst::Pairing>(true, DST);
+    blst::P2_Affine p2_zero;
     for (const auto &p : pubs_msgs) {
       blst::P1_Affine p1(p.first.data(), P1_SIZE);
       if (!p1.in_group() || p1.is_inf()) {
         return false;
       }
-      pairing->aggregate(&p1, nullptr, (const td::uint8 *)p.second.data(), p.second.size());
+      pairing->aggregate(&p1, &p2_zero, (const td::uint8 *)p.second.data(), p.second.size());
     }
     pairing->commit();
     blst::P2_Affine p2(sig.data(), P2_SIZE);

dht-server/dht-server.cpp

@@ -1231,15 +1231,19 @@ int main(int argc, char *argv[]) {
   });
   td::uint32 threads = 7;
   p.add_checked_option(
-      't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) {
+      't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) {
         td::int32 v;
         try {
-          v = std::stoi(fname.str());
+          v = std::stoi(arg.str());
         } catch (...) {
           return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number");
         }
-        if (v < 1 || v > 256) {
-          return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]");
+        if (v <= 0) {
+          return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0");
+        }
+        if (v > 127) {
+          LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127";
+          v = 127;
         }
         threads = v;
         return td::Status::OK();

dht/dht-in.hpp

@@ -179,6 +179,7 @@ class DhtMemberImpl : public DhtMember {
   void get_value(DhtKey key, td::Promise<DhtValue> result) override {
     get_value_in(key.compute_key_id(), std::move(result));
   }
+  void get_value_many(DhtKey key, std::function<void(DhtValue)> callback, td::Promise<td::Unit> promise) override;
 
   void alarm() override {
     alarm_timestamp() = td::Timestamp::in(1.0);

dht/dht-query.cpp

@@ -210,8 +210,11 @@ void DhtQueryFindValue::on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeI
                   send_get_nodes = true;
                   return;
                 }
-                promise_.set_value(std::move(value));
-                need_stop = true;
+                if (on_value_found(std::move(value))) {
+                  send_get_nodes = true;
+                } else {
+                  need_stop = true;
+                }
               },
               [&](ton_api::dht_valueNotFound &v) {
                 add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()});
@@ -244,7 +247,32 @@ void DhtQueryFindValue::on_result_nodes(td::Result<td::BufferSlice> R, adnl::Adn
 }
 
 void DhtQueryFindValue::finish(DhtNodesList list) {
-  promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found"));
+}
+
+bool DhtQueryFindValueSingle::on_value_found(DhtValue value) {
+  promise_.set_value(std::move(value));
+  found_ = true;
+  return false;
+}
+
+void DhtQueryFindValueSingle::tear_down() {
+  if (!found_) {
+    promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found"));
+  }
+}
+
+bool DhtQueryFindValueMany::on_value_found(DhtValue value) {
+  callback_(std::move(value));
+  found_ = true;
+  return true;
+}
+
+void DhtQueryFindValueMany::tear_down() {
+  if (found_) {
+    promise_.set_value(td::Unit());
+  } else {
+    promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found"));
+  }
 }
 
 DhtQueryStore::DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src,

dht/dht-query.hpp

@@ -126,16 +126,11 @@ class DhtQueryFindNodes : public DhtQuery {
 };
 
 class DhtQueryFindValue : public DhtQuery {
- private:
-  td::Promise<DhtValue> promise_;
-
  public:
   DhtQueryFindValue(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list,
                     td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only,
-                    td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
-                    td::Promise<DhtValue> promise)
-      : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl)
-      , promise_(std::move(promise)) {
+                    td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl)
+      : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) {
     add_nodes(std::move(list));
   }
   void send_one_query(adnl::AdnlNodeIdShort id) override;
@@ -146,6 +141,48 @@ class DhtQueryFindValue : public DhtQuery {
   std::string get_name() const override {
     return "find value";
   }
+
+  virtual bool on_value_found(DhtValue value) = 0;
+};
+
+class DhtQueryFindValueSingle : public DhtQueryFindValue {
+ public:
+  DhtQueryFindValueSingle(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list,
+                          td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only,
+                          td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
+                          td::Promise<DhtValue> promise)
+      : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node,
+                          adnl)
+      , promise_(std::move(promise)) {
+    add_nodes(std::move(list));
+  }
+  bool on_value_found(DhtValue value) override;
+  void tear_down() override;
+
+ private:
+  td::Promise<DhtValue> promise_;
+  bool found_ = false;
+};
+
+class DhtQueryFindValueMany : public DhtQueryFindValue {
+ public:
+  DhtQueryFindValueMany(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list,
+                         td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only,
+                         td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
+                         std::function<void(DhtValue)> callback, td::Promise<td::Unit> promise)
+      : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node,
+                          adnl)
+      , callback_(std::move(callback))
+      , promise_(std::move(promise)) {
+    add_nodes(std::move(list));
+  }
+  bool on_value_found(DhtValue value) override;
+  void tear_down() override;
+
+ private:
+  std::function<void(DhtValue)> callback_;
+  td::Promise<td::Unit> promise_;
+  bool found_ = false;
 };
 
 class DhtQueryStore : public td::actor::Actor {