Encointer contributions upstreaming (#174)

* [enclave] ! fix: init-shard if it does not exist * back up chain relay db before update in case of file corruption * clean up chain relay sync logging * Ws server refactor (#13) * changed ws_server completely. Requests from client are now handled in the worker main event loop to prevent race conditions with on state/chain_relay access. * [ws_server] remove unwrap and send instead "invalid_client_request" to client * updating block number in stf state * [enclave/chain_relay] store only hashes of the headers instead of the headers themselves * enclave: patch log and env_logger to mesalock * worker should panic if it can't write to shard * add public getters for unpermissioned statistics (#16) * don't request key provisioning form other worker. assume its there or generate new (dangerous!) * bump version to 0.6.11 like encointer reference release Co-authored-by: clangenb <37865735+clangenb@users.noreply.github.com> Co-authored-by: Marcel Frei <marcel.frei@electronix.ch> Co-authored-by: Christian Langenbacher <christian.langenbacher91@gmail.com>
integritee-network · Sep 9, 2020 · 19452c0 · 19452c0
1 parent ee2eb13
commit 19452c0
Show file tree

Hide file tree

Showing 24 changed files with 441 additions and 286 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -3852,7 +3852,7 @@ checksum = "d2a965994514ab35d3893e9260245f2947fd1981cdd4fffd2c6e6d1a9ce02e6a"
 
 [[package]]
 name = "substratee-client"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "base58",
  "blake2-rfc",
@@ -3884,7 +3884,7 @@ dependencies = [
 
 [[package]]
 name = "substratee-node-primitives"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "base58",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3932,7 +3932,7 @@ dependencies = [
 
 [[package]]
 name = "substratee-stf"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "base58",
  "clap",
@@ -3957,7 +3957,7 @@ dependencies = [
 
 [[package]]
 name = "substratee-worker"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "base58",
  "cid",
@@ -3997,7 +3997,7 @@ dependencies = [
 
 [[package]]
 name = "substratee-worker-api"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "hex 0.4.2",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",

diff --git a/client/Cargo.toml b/client/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "substratee-client"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 authors = ["Supercomputing Systems AG <info@scs.ch>"]
 edition = "2018"
 

diff --git a/client/src/main.rs b/client/src/main.rs
@@ -57,8 +57,7 @@ use substratee_node_runtime::{
     AccountId, Event, Hash, Signature,
 };
 use substratee_stf::{
-    cli::get_identifiers, ShardIdentifier, TrustedCallSigned, TrustedGetterSigned,
-    TrustedOperationSigned,
+    cli::get_identifiers, Getter, ShardIdentifier, TrustedCallSigned, TrustedOperation,
 };
 use substratee_worker_api::Api as WorkerApi;
 
@@ -435,20 +434,17 @@ fn get_worker_api(matches: &ArgMatches<'_>) -> WorkerApi {
     WorkerApi::new(url)
 }
 
-fn perform_trusted_operation(
-    matches: &ArgMatches<'_>,
-    top: &TrustedOperationSigned,
-) -> Option<Vec<u8>> {
+fn perform_trusted_operation(matches: &ArgMatches<'_>, top: &TrustedOperation) -> Option<Vec<u8>> {
     match top {
-        TrustedOperationSigned::call(call) => send_request(matches, call.clone()),
-        TrustedOperationSigned::get(getter) => get_state(matches, getter.clone()),
+        TrustedOperation::call(call) => send_request(matches, call.clone()),
+        TrustedOperation::get(getter) => get_state(matches, getter.clone()),
     }
 }
 
-fn get_state(matches: &ArgMatches<'_>, getter: TrustedGetterSigned) -> Option<Vec<u8>> {
+fn get_state(matches: &ArgMatches<'_>, getter: Getter) -> Option<Vec<u8>> {
     let worker_api = get_worker_api(matches);
     let (_mrenclave, shard) = get_identifiers(matches);
-    debug!("calling workerapi to get state value");
+    debug!("calling workerapi to get state value, {:?}", getter);
     let ret = worker_api
         .get_stf_state(getter, &shard)
         .expect("getting value failed");

diff --git a/enclave/Cargo.lock b/enclave/Cargo.lock
@@ -225,7 +225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "chain-relay"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "derive_more 0.99.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "finality-grandpa 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -469,7 +469,7 @@ dependencies = [
  "frame-metadata 11.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "frame-support-procedural 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)",
  "parity-scale-codec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "paste 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "sp-arithmetic 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -811,14 +811,6 @@ dependencies = [
  "sgx_tstd 1.1.2 (git+https://github.com/apache/teaclave-sgx-sdk.git?rev=v1.1.2)",
 ]
 
-[[package]]
-name = "log"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "memchr"
 version = "2.2.1"
@@ -1952,7 +1944,7 @@ dependencies = [
  "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "libsecp256k1 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)",
  "merlin 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-scale-codec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2252,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "substratee-node-primitives"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "parity-scale-codec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "primitive-types 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2262,7 +2254,7 @@ dependencies = [
 
 [[package]]
 name = "substratee-stf"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "derive_more 0.99.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.7.1 (git+https://github.com/mesalock-linux/env_logger-sgx)",
@@ -2281,12 +2273,12 @@ dependencies = [
 
 [[package]]
 name = "substratee-worker-enclave"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 dependencies = [
  "aes 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "base64 0.10.1 (git+https://github.com/mesalock-linux/rust-base64-sgx)",
  "bit-vec 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "chain-relay 0.6.5-sub2.0.0-alpha.7",
+ "chain-relay 0.6.11-sub2.0.0-alpha.7",
  "chrono 0.4.11 (git+https://github.com/mesalock-linux/chrono-sgx)",
  "cid 0.5.1 (git+https://github.com/whalelephant/rust-cid?branch=nstd)",
  "env_logger 0.7.1 (git+https://github.com/mesalock-linux/env_logger-sgx)",
@@ -2323,8 +2315,8 @@ dependencies = [
  "sp-runtime 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "sp-std 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "substrate-api-client 0.4.6-sub2.0.0-alpha.7 (git+https://github.com/scs/substrate-api-client?tag=v0.4.6-sub2.0.0-alpha.7)",
- "substratee-node-primitives 0.6.5-sub2.0.0-alpha.7",
- "substratee-stf 0.6.5-sub2.0.0-alpha.7",
+ "substratee-node-primitives 0.6.11-sub2.0.0-alpha.7",
+ "substratee-stf 0.6.11-sub2.0.0-alpha.7",
  "webpki 0.21.2 (git+https://github.com/mesalock-linux/webpki?branch=mesalock_sgx)",
  "webpki-roots 0.19.0 (git+https://github.com/mesalock-linux/webpki-roots?branch=mesalock_sgx)",
  "yasna 0.3.1 (git+https://github.com/mesalock-linux/yasna.rs-sgx?rev=sgx_1.1.2)",
@@ -2445,7 +2437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)",
  "smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -2671,7 +2663,6 @@ dependencies = [
 "checksum libc 0.2.70 (registry+https://github.com/rust-lang/crates.io-index)" = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f"
 "checksum libsecp256k1 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962"
 "checksum log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)" = "<none>"
-"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
 "checksum memchr 2.2.1 (git+https://github.com/mesalock-linux/rust-memchr-sgx)" = "<none>"
 "checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
 "checksum memory-db 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be512cb2ccb4ecbdca937fdd4a62ea5f09f8e7195466a85e4632b3d5bcce82e6"

diff --git a/enclave/Cargo.toml b/enclave/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "substratee-worker-enclave"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 authors = ["Supercomputing Systems AG <info@scs.ch>"]
 edition = "2018"
 
@@ -127,12 +127,18 @@ path = "../substratee-node-primitives"
 default-features = false
 features = ["sgx"]
 
-[patch.crates-io]
-sp-io = { git = "https://github.com/scs/sgx-runtime", default-features = false, features = ["disable_oom", "disable_panic_handler", "disable_allocator", "sgx"]}
-
 [dependencies.ipfs-unixfs]
 git = "https://github.com/whalelephant/rust-ipfs"
 branch = "w-nstd"
 default-features = false
 
+[patch.crates-io]
+sp-io = { git = "https://github.com/scs/sgx-runtime", default-features = false, features = ["disable_oom", "disable_panic_handler", "disable_allocator", "sgx"]}
+log = { git = "https://github.com/mesalock-linux/log-sgx", version = "0.4" }
+env_logger = { git = "https://github.com/mesalock-linux/env_logger-sgx", version = "0.7" } 
+
+
+
+
+
 
diff --git a/enclave/chain_relay/Cargo.toml b/enclave/chain_relay/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "chain-relay"
-version = "0.6.5-sub2.0.0-alpha.7"
+version = "0.6.11-sub2.0.0-alpha.7"
 authors = ["Supercomputing Systems AG <info@scs.ch>"]
 edition = "2018"
 

diff --git a/enclave/chain_relay/src/lib.rs b/enclave/chain_relay/src/lib.rs
@@ -116,7 +116,7 @@ impl LightValidation {
 
         if grandpa_proof.is_none() {
             relay.last_finalized_block_header = header.clone();
-            relay.unjustified_headers.push(header);
+            relay.unjustified_headers.push(header.hash());
             info!(
                 "Syncing finalized block without grandpa proof. Amount of unjustified headers: {}",
                 relay.unjustified_headers.len()
@@ -142,8 +142,8 @@ impl LightValidation {
         Self::schedule_validator_set_change(&mut relay, &header);
 
         // a valid grandpa proof proofs finalization of all previous unjustified blocks
-        relay.headers.append(&mut relay.unjustified_headers);
-        relay.headers.push(header);
+        relay.header_hashes.append(&mut relay.unjustified_headers);
+        relay.header_hashes.push(header.hash());
 
         if validator_set_id > relay.current_validator_set_id {
             relay.current_validator_set = validator_set;
@@ -241,7 +241,7 @@ impl LightValidation {
             .tracked_relays
             .get(&relay_id)
             .ok_or(Error::NoSuchRelayExists)?;
-        Ok(relay.headers[0].hash())
+        Ok(relay.header_hashes[0])
     }
 
     pub fn latest_header(&self, relay_id: RelayId) -> Result<Header, Error> {

diff --git a/enclave/chain_relay/src/state.rs b/enclave/chain_relay/src/state.rs
@@ -10,8 +10,8 @@ pub struct RelayState<Block: BlockT> {
     pub last_finalized_block_header: Block::Header,
     pub current_validator_set: AuthorityList,
     pub current_validator_set_id: SetId,
-    pub headers: Vec<Block::Header>,
-    pub unjustified_headers: Vec<Block::Header>, // Finalized headers without grandpa proof
+    pub header_hashes: Vec<Block::Hash>,
+    pub unjustified_headers: Vec<Block::Hash>, // Finalized headers without grandpa proof
     pub verify_tx_inclusion: Vec<OpaqueExtrinsic>, // Transactions sent by the relay
     pub scheduled_change: Option<ScheduledChangeAtBlock<Block::Header>>, // Scheduled Authorities change as indicated in the header's digest.
 }
@@ -28,7 +28,7 @@ impl<Block: BlockT> RelayState<Block> {
             last_finalized_block_header: block_header.clone(),
             current_validator_set: validator_set,
             current_validator_set_id: 0,
-            headers: vec![block_header],
+            header_hashes: vec![block_header.hash()],
             unjustified_headers: Vec::new(),
             verify_tx_inclusion: Vec::new(),
             scheduled_change: None,

diff --git a/enclave/src/io.rs b/enclave/src/io.rs
@@ -14,7 +14,7 @@
     limitations under the License.
 
 */
-use std::fs::File;
+use std::fs;
 use std::io::{Read, Write};
 use std::sgxfs::SgxFile;
 use std::string::String;
@@ -31,7 +31,7 @@ pub fn unseal(filepath: &str) -> SgxResult<Vec<u8>> {
 }
 
 pub fn read(filepath: &str) -> SgxResult<Vec<u8>> {
-    File::open(filepath)
+    fs::File::open(filepath)
         .map(_read)
         .sgx_error_with_log(&format!("[Enclave] File '{}' not found!", filepath))?
 }
@@ -46,7 +46,7 @@ fn _read<F: Read>(mut file: F) -> SgxResult<Vec<u8>> {
 
 pub fn read_to_string(filepath: &str) -> SgxResult<String> {
     let mut contents = String::new();
-    File::open(filepath)
+    fs::File::open(filepath)
         .map(|mut f| f.read_to_string(&mut contents))
         .sgx_error_with_log(&format!("[Enclave] Could not read '{}'", filepath))?
         .sgx_error_with_log(&format!("[Enclave] File '{}' not found!", filepath))?;
@@ -61,7 +61,7 @@ pub fn seal(bytes: &[u8], filepath: &str) -> SgxResult<sgx_status_t> {
 }
 
 pub fn write(bytes: &[u8], filepath: &str) -> SgxResult<sgx_status_t> {
-    File::create(filepath)
+    fs::File::create(filepath)
         .map(|f| _write(bytes, f))
         .sgx_error_with_log(&format!("[Enclave] Creating '{}' failed", filepath))?
 }
@@ -82,6 +82,7 @@ pub mod light_validation {
     use log::*;
     use sgx_types::{sgx_status_t, SgxResult};
     use sp_finality_grandpa::VersionedAuthorityList;
+    use std::fs;
     use std::sgxfs::SgxFile;
 
     pub fn unseal() -> SgxResult<LightValidation> {
@@ -90,6 +91,10 @@ pub mod light_validation {
     }
 
     pub fn seal(validator: LightValidation) -> SgxResult<sgx_status_t> {
+        debug!("backup chain relay state");
+        if fs::copy(CHAIN_RELAY_DB, format!("{}.1", CHAIN_RELAY_DB)).is_err() {
+            warn!("could not backup previous chain relay state");
+        };
         debug!("Seal Chain Relay State. Current state: {:?}", validator);
         super::seal(validator.encode().as_slice(), CHAIN_RELAY_DB)
     }

diff --git a/enclave/src/ipfs.rs b/enclave/src/ipfs.rs
@@ -44,16 +44,14 @@ impl IpfsContent {
                 cid_str, self.stats.blocks, self.stats.block_bytes
             );
             match self.cid.as_ref() {
-				Ok(initial_cid) => {
-					if last_cid.hash().eq(&initial_cid.hash()) {
-						Ok(())
-					} else {
-						Err(IpfsError::Verification)
-					}
-				},
-				Err(_) => {
-					Err(IpfsError::InputCidInvalid)
-				}
+                Ok(initial_cid) => {
+                    if last_cid.hash().eq(&initial_cid.hash()) {
+                        Ok(())
+                    } else {
+                        Err(IpfsError::Verification)
+                    }
+                }
+                Err(_) => Err(IpfsError::InputCidInvalid),
             }
         } else {
             Err(IpfsError::FinalCidMissing)