From 19452c00e8a7b62b9ed3236190f5545361177240 Mon Sep 17 00:00:00 2001 From: brenzi Date: Wed, 9 Sep 2020 20:34:44 +0200 Subject: [PATCH] Encointer contributions upstreaming (#174) * [enclave] ! fix: init-shard if it does not exist * back up chain relay db before update in case of file corruption * clean up chain relay sync logging * Ws server refactor (#13) * changed ws_server completely. Requests from client are now handled in the worker main event loop to prevent race conditions with on state/chain_relay access. * [ws_server] remove unwrap and send instead "invalid_client_request" to client * updating block number in stf state * [enclave/chain_relay] store only hashes of the headers instead of the headers themselves * enclave: patch log and env_logger to mesalock * worker should panic if it can't write to shard * add public getters for unpermissioned statistics (#16) * don't request key provisioning form other worker. assume its there or generate new (dangerous!) * bump version to 0.6.11 like encointer reference release Co-authored-by: clangenb <37865735+clangenb@users.noreply.github.com> Co-authored-by: Marcel Frei Co-authored-by: Christian Langenbacher --- Cargo.lock | 10 +-- client/Cargo.toml | 2 +- client/src/main.rs | 16 ++-- enclave/Cargo.lock | 29 +++---- enclave/Cargo.toml | 14 ++- enclave/chain_relay/Cargo.toml | 2 +- enclave/chain_relay/src/lib.rs | 8 +- enclave/chain_relay/src/state.rs | 6 +- enclave/src/io.rs | 13 ++- enclave/src/ipfs.rs | 18 ++-- enclave/src/lib.rs | 79 ++++++++++------- enclave/src/state.rs | 57 ++++++++++--- stf/Cargo.toml | 2 +- stf/src/cli.rs | 76 +++++++++-------- stf/src/lib.rs | 57 ++++++++++++- stf/src/sgx.rs | 117 +++++++++++++++++--------- substratee-node-primitives/Cargo.toml | 2 +- worker/Cargo.toml | 2 +- worker/src/main.rs | 83 +++++++++--------- worker/src/ws_server.rs | 80 ++++++++++++------ worker/worker-api/Cargo.toml | 2 +- worker/worker-api/src/client.rs | 9 +- worker/worker-api/src/lib.rs | 31 +++---- worker/worker-api/src/requests.rs | 12 ++- 24 files changed, 441 insertions(+), 286 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c71164f539..7fa67858c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3852,7 +3852,7 @@ checksum = "d2a965994514ab35d3893e9260245f2947fd1981cdd4fffd2c6e6d1a9ce02e6a" [[package]] name = "substratee-client" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "base58", "blake2-rfc", @@ -3884,7 +3884,7 @@ dependencies = [ [[package]] name = "substratee-node-primitives" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "base58", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "substratee-stf" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "base58", "clap", @@ -3957,7 +3957,7 @@ dependencies = [ [[package]] name = "substratee-worker" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "base58", "cid", @@ -3997,7 +3997,7 @@ dependencies = [ [[package]] name = "substratee-worker-api" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "hex 0.4.2", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/client/Cargo.toml b/client/Cargo.toml index 9ead5b19e2..a7dd77d2a2 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substratee-client" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["Supercomputing Systems AG "] edition = "2018" diff --git a/client/src/main.rs b/client/src/main.rs index 888d08600b..bcc26238c6 100644 --- a/client/src/main.rs +++ b/client/src/main.rs @@ -57,8 +57,7 @@ use substratee_node_runtime::{ AccountId, Event, Hash, Signature, }; use substratee_stf::{ - cli::get_identifiers, ShardIdentifier, TrustedCallSigned, TrustedGetterSigned, - TrustedOperationSigned, + cli::get_identifiers, Getter, ShardIdentifier, TrustedCallSigned, TrustedOperation, }; use substratee_worker_api::Api as WorkerApi; @@ -435,20 +434,17 @@ fn get_worker_api(matches: &ArgMatches<'_>) -> WorkerApi { WorkerApi::new(url) } -fn perform_trusted_operation( - matches: &ArgMatches<'_>, - top: &TrustedOperationSigned, -) -> Option> { +fn perform_trusted_operation(matches: &ArgMatches<'_>, top: &TrustedOperation) -> Option> { match top { - TrustedOperationSigned::call(call) => send_request(matches, call.clone()), - TrustedOperationSigned::get(getter) => get_state(matches, getter.clone()), + TrustedOperation::call(call) => send_request(matches, call.clone()), + TrustedOperation::get(getter) => get_state(matches, getter.clone()), } } -fn get_state(matches: &ArgMatches<'_>, getter: TrustedGetterSigned) -> Option> { +fn get_state(matches: &ArgMatches<'_>, getter: Getter) -> Option> { let worker_api = get_worker_api(matches); let (_mrenclave, shard) = get_identifiers(matches); - debug!("calling workerapi to get state value"); + debug!("calling workerapi to get state value, {:?}", getter); let ret = worker_api .get_stf_state(getter, &shard) .expect("getting value failed"); diff --git a/enclave/Cargo.lock b/enclave/Cargo.lock index 46f09c191a..78327d03ba 100644 --- a/enclave/Cargo.lock +++ b/enclave/Cargo.lock @@ -225,7 +225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "chain-relay" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "derive_more 0.99.5 (registry+https://github.com/rust-lang/crates.io-index)", "finality-grandpa 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -469,7 +469,7 @@ dependencies = [ "frame-metadata 11.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", "frame-support-procedural 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)", "parity-scale-codec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "paste 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "sp-arithmetic 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -811,14 +811,6 @@ dependencies = [ "sgx_tstd 1.1.2 (git+https://github.com/apache/teaclave-sgx-sdk.git?rev=v1.1.2)", ] -[[package]] -name = "log" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "memchr" version = "2.2.1" @@ -1952,7 +1944,7 @@ dependencies = [ "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "libsecp256k1 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)", "merlin 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2252,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "substratee-node-primitives" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "parity-scale-codec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "primitive-types 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2262,7 +2254,7 @@ dependencies = [ [[package]] name = "substratee-stf" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "derive_more 0.99.5 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.7.1 (git+https://github.com/mesalock-linux/env_logger-sgx)", @@ -2281,12 +2273,12 @@ dependencies = [ [[package]] name = "substratee-worker-enclave" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" dependencies = [ "aes 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.10.1 (git+https://github.com/mesalock-linux/rust-base64-sgx)", "bit-vec 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "chain-relay 0.6.5-sub2.0.0-alpha.7", + "chain-relay 0.6.11-sub2.0.0-alpha.7", "chrono 0.4.11 (git+https://github.com/mesalock-linux/chrono-sgx)", "cid 0.5.1 (git+https://github.com/whalelephant/rust-cid?branch=nstd)", "env_logger 0.7.1 (git+https://github.com/mesalock-linux/env_logger-sgx)", @@ -2323,8 +2315,8 @@ dependencies = [ "sp-runtime 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", "sp-std 2.0.0-alpha.7 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-api-client 0.4.6-sub2.0.0-alpha.7 (git+https://github.com/scs/substrate-api-client?tag=v0.4.6-sub2.0.0-alpha.7)", - "substratee-node-primitives 0.6.5-sub2.0.0-alpha.7", - "substratee-stf 0.6.5-sub2.0.0-alpha.7", + "substratee-node-primitives 0.6.11-sub2.0.0-alpha.7", + "substratee-stf 0.6.11-sub2.0.0-alpha.7", "webpki 0.21.2 (git+https://github.com/mesalock-linux/webpki?branch=mesalock_sgx)", "webpki-roots 0.19.0 (git+https://github.com/mesalock-linux/webpki-roots?branch=mesalock_sgx)", "yasna 0.3.1 (git+https://github.com/mesalock-linux/yasna.rs-sgx?rev=sgx_1.1.2)", @@ -2445,7 +2437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)", "smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2671,7 +2663,6 @@ dependencies = [ "checksum libc 0.2.70 (registry+https://github.com/rust-lang/crates.io-index)" = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f" "checksum libsecp256k1 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" "checksum log 0.4.8 (git+https://github.com/mesalock-linux/log-sgx)" = "" -"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum memchr 2.2.1 (git+https://github.com/mesalock-linux/rust-memchr-sgx)" = "" "checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum memory-db 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be512cb2ccb4ecbdca937fdd4a62ea5f09f8e7195466a85e4632b3d5bcce82e6" diff --git a/enclave/Cargo.toml b/enclave/Cargo.toml index be651c87eb..14dc9910e7 100644 --- a/enclave/Cargo.toml +++ b/enclave/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substratee-worker-enclave" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["Supercomputing Systems AG "] edition = "2018" @@ -127,12 +127,18 @@ path = "../substratee-node-primitives" default-features = false features = ["sgx"] -[patch.crates-io] -sp-io = { git = "https://github.com/scs/sgx-runtime", default-features = false, features = ["disable_oom", "disable_panic_handler", "disable_allocator", "sgx"]} - [dependencies.ipfs-unixfs] git = "https://github.com/whalelephant/rust-ipfs" branch = "w-nstd" default-features = false +[patch.crates-io] +sp-io = { git = "https://github.com/scs/sgx-runtime", default-features = false, features = ["disable_oom", "disable_panic_handler", "disable_allocator", "sgx"]} +log = { git = "https://github.com/mesalock-linux/log-sgx", version = "0.4" } +env_logger = { git = "https://github.com/mesalock-linux/env_logger-sgx", version = "0.7" } + + + + + diff --git a/enclave/chain_relay/Cargo.toml b/enclave/chain_relay/Cargo.toml index 33b3a5edb3..a66d2008a8 100644 --- a/enclave/chain_relay/Cargo.toml +++ b/enclave/chain_relay/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chain-relay" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["Supercomputing Systems AG "] edition = "2018" diff --git a/enclave/chain_relay/src/lib.rs b/enclave/chain_relay/src/lib.rs index 50a0c6a51e..20442ed32d 100644 --- a/enclave/chain_relay/src/lib.rs +++ b/enclave/chain_relay/src/lib.rs @@ -116,7 +116,7 @@ impl LightValidation { if grandpa_proof.is_none() { relay.last_finalized_block_header = header.clone(); - relay.unjustified_headers.push(header); + relay.unjustified_headers.push(header.hash()); info!( "Syncing finalized block without grandpa proof. Amount of unjustified headers: {}", relay.unjustified_headers.len() @@ -142,8 +142,8 @@ impl LightValidation { Self::schedule_validator_set_change(&mut relay, &header); // a valid grandpa proof proofs finalization of all previous unjustified blocks - relay.headers.append(&mut relay.unjustified_headers); - relay.headers.push(header); + relay.header_hashes.append(&mut relay.unjustified_headers); + relay.header_hashes.push(header.hash()); if validator_set_id > relay.current_validator_set_id { relay.current_validator_set = validator_set; @@ -241,7 +241,7 @@ impl LightValidation { .tracked_relays .get(&relay_id) .ok_or(Error::NoSuchRelayExists)?; - Ok(relay.headers[0].hash()) + Ok(relay.header_hashes[0]) } pub fn latest_header(&self, relay_id: RelayId) -> Result { diff --git a/enclave/chain_relay/src/state.rs b/enclave/chain_relay/src/state.rs index 229170f030..1b1d0530f0 100644 --- a/enclave/chain_relay/src/state.rs +++ b/enclave/chain_relay/src/state.rs @@ -10,8 +10,8 @@ pub struct RelayState { pub last_finalized_block_header: Block::Header, pub current_validator_set: AuthorityList, pub current_validator_set_id: SetId, - pub headers: Vec, - pub unjustified_headers: Vec, // Finalized headers without grandpa proof + pub header_hashes: Vec, + pub unjustified_headers: Vec, // Finalized headers without grandpa proof pub verify_tx_inclusion: Vec, // Transactions sent by the relay pub scheduled_change: Option>, // Scheduled Authorities change as indicated in the header's digest. } @@ -28,7 +28,7 @@ impl RelayState { last_finalized_block_header: block_header.clone(), current_validator_set: validator_set, current_validator_set_id: 0, - headers: vec![block_header], + header_hashes: vec![block_header.hash()], unjustified_headers: Vec::new(), verify_tx_inclusion: Vec::new(), scheduled_change: None, diff --git a/enclave/src/io.rs b/enclave/src/io.rs index 4ebafbd9fc..ee59d25507 100644 --- a/enclave/src/io.rs +++ b/enclave/src/io.rs @@ -14,7 +14,7 @@ limitations under the License. */ -use std::fs::File; +use std::fs; use std::io::{Read, Write}; use std::sgxfs::SgxFile; use std::string::String; @@ -31,7 +31,7 @@ pub fn unseal(filepath: &str) -> SgxResult> { } pub fn read(filepath: &str) -> SgxResult> { - File::open(filepath) + fs::File::open(filepath) .map(_read) .sgx_error_with_log(&format!("[Enclave] File '{}' not found!", filepath))? } @@ -46,7 +46,7 @@ fn _read(mut file: F) -> SgxResult> { pub fn read_to_string(filepath: &str) -> SgxResult { let mut contents = String::new(); - File::open(filepath) + fs::File::open(filepath) .map(|mut f| f.read_to_string(&mut contents)) .sgx_error_with_log(&format!("[Enclave] Could not read '{}'", filepath))? .sgx_error_with_log(&format!("[Enclave] File '{}' not found!", filepath))?; @@ -61,7 +61,7 @@ pub fn seal(bytes: &[u8], filepath: &str) -> SgxResult { } pub fn write(bytes: &[u8], filepath: &str) -> SgxResult { - File::create(filepath) + fs::File::create(filepath) .map(|f| _write(bytes, f)) .sgx_error_with_log(&format!("[Enclave] Creating '{}' failed", filepath))? } @@ -82,6 +82,7 @@ pub mod light_validation { use log::*; use sgx_types::{sgx_status_t, SgxResult}; use sp_finality_grandpa::VersionedAuthorityList; + use std::fs; use std::sgxfs::SgxFile; pub fn unseal() -> SgxResult { @@ -90,6 +91,10 @@ pub mod light_validation { } pub fn seal(validator: LightValidation) -> SgxResult { + debug!("backup chain relay state"); + if fs::copy(CHAIN_RELAY_DB, format!("{}.1", CHAIN_RELAY_DB)).is_err() { + warn!("could not backup previous chain relay state"); + }; debug!("Seal Chain Relay State. Current state: {:?}", validator); super::seal(validator.encode().as_slice(), CHAIN_RELAY_DB) } diff --git a/enclave/src/ipfs.rs b/enclave/src/ipfs.rs index 0df5758f0b..b217218557 100644 --- a/enclave/src/ipfs.rs +++ b/enclave/src/ipfs.rs @@ -44,16 +44,14 @@ impl IpfsContent { cid_str, self.stats.blocks, self.stats.block_bytes ); match self.cid.as_ref() { - Ok(initial_cid) => { - if last_cid.hash().eq(&initial_cid.hash()) { - Ok(()) - } else { - Err(IpfsError::Verification) - } - }, - Err(_) => { - Err(IpfsError::InputCidInvalid) - } + Ok(initial_cid) => { + if last_cid.hash().eq(&initial_cid.hash()) { + Ok(()) + } else { + Err(IpfsError::Verification) + } + } + Err(_) => Err(IpfsError::InputCidInvalid), } } else { Err(IpfsError::FinalCidMissing) diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index af854cf4d0..9e4b7ac47e 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -37,9 +37,6 @@ use sgx_types::{sgx_epid_group_id_t, sgx_status_t, sgx_target_info_t, size_t, Sg use substrate_api_client::{compose_extrinsic_offline, utils::storage_key}; use substratee_node_primitives::{CallWorkerFn, ShieldFundsFn}; -use substratee_stf::{ - AccountId, ShardIdentifier, Stf, TrustedCall, TrustedCallSigned, TrustedGetterSigned, -}; use codec::{Decode, Encode}; use sp_core::{crypto::Pair, hashing::blake2_256}; @@ -66,6 +63,7 @@ use sp_runtime::OpaqueExtrinsic; use sp_runtime::{generic::SignedBlock, traits::Header as HeaderT}; use substrate_api_client::extrinsic::xt_primitives::UncheckedExtrinsicV4; use substratee_stf::sgx::{shards_key_hash, storage_hashes_to_update_per_shard, OpaqueCall}; +use substratee_stf::{AccountId, Getter, ShardIdentifier, Stf, TrustedCall, TrustedCallSigned}; mod aes; mod attestation; @@ -113,6 +111,14 @@ pub unsafe extern "C" fn init() -> sgx_status_t { return status; } + // for debug purposes, list shards. no problem to panic if fails + let shards = state::list_shards().unwrap(); + debug!("found the following {} shards on disk:", shards.len()); + for s in shards { + debug!("{}", s.encode().to_base58()) + } + //shards.into_iter().map(|s| debug!("{}", s.encode().to_base58())); + sgx_status_t::SGX_SUCCESS } @@ -214,12 +220,14 @@ pub unsafe extern "C" fn get_state( let shard = ShardIdentifier::from_slice(slice::from_raw_parts(shard, shard_size as usize)); let mut trusted_op_slice = slice::from_raw_parts(trusted_op, trusted_op_size as usize); let value_slice = slice::from_raw_parts_mut(value, value_size as usize); - let tusted_getter_signed = TrustedGetterSigned::decode(&mut trusted_op_slice).unwrap(); + let getter = Getter::decode(&mut trusted_op_slice).unwrap(); - debug!("verifying signature of TrustedCallSigned"); - if let false = tusted_getter_signed.verify_signature() { - error!("bad signature"); - return sgx_status_t::SGX_ERROR_UNEXPECTED; + if let Getter::trusted(trusted_getter_signed) = getter.clone() { + debug!("verifying signature of TrustedGetterSigned"); + if let false = trusted_getter_signed.verify_signature() { + error!("bad signature"); + return sgx_status_t::SGX_ERROR_UNEXPECTED; + } } if !state::exists(&shard) { @@ -241,12 +249,13 @@ pub unsafe extern "C" fn get_state( let latest_header = validator.latest_header(validator.num_relays).unwrap(); + // FIXME: not sure we will ever need this as we are querying trusted state, not onchain state + // i.e. demurrage could be correctly applied with this, but the client could do that too. debug!("Update STF storage!"); - let requests: Vec = - Stf::get_storage_hashes_to_update_for_getter(&tusted_getter_signed) - .into_iter() - .map(|key| WorkerRequest::ChainStorage(key, Some(latest_header.hash()))) - .collect(); + let requests: Vec = Stf::get_storage_hashes_to_update_for_getter(&getter) + .into_iter() + .map(|key| WorkerRequest::ChainStorage(key, Some(latest_header.hash()))) + .collect(); if !requests.is_empty() { let responses: Vec>> = match worker_request(requests) { @@ -263,7 +272,7 @@ pub unsafe extern "C" fn get_state( } debug!("calling into STF to get state"); - let value_opt = Stf::get_state(&mut state, tusted_getter_signed.getter); + let value_opt = Stf::get_state(&mut state, getter); debug!("returning getter result"); write_slice_and_whitespace_pad(value_slice, value_opt.encode()); @@ -360,7 +369,8 @@ pub unsafe extern "C" fn sync_chain_relay( } if update_states(signed_block.block.header.clone()).is_err() { - error!("Error performing state updates upon block import") + error!("Error performing state updates upon block import"); + return sgx_status_t::SGX_ERROR_UNEXPECTED; } match scan_block_for_relevant_xt(&signed_block.block) { @@ -387,7 +397,6 @@ pub fn update_states(header: Header) -> SgxResult<()> { return Ok(()); } - // global requests they are the same for every shard let responses: Vec>> = worker_request(requests)?; let update_map = verify_worker_responses(responses, header.clone())?; // look for new shards an initialize them @@ -396,6 +405,7 @@ pub fn update_states(header: Header) -> SgxResult<()> { Some(shards) => { let shards: Vec = Decode::decode(&mut shards.as_slice()) .sgx_error_with_log("error decoding shards")?; + for s in shards { if !state::exists(&s) { info!("Initialized new shard that was found on chain: {:?}", s); @@ -414,6 +424,10 @@ pub fn update_states(header: Header) -> SgxResult<()> { let mut state = state::load(&s)?; Stf::update_storage(&mut state, &per_shard_update_map); Stf::update_storage(&mut state, &update_map); + + // block number is purged from the substrate state so it can't be read like other storage values + Stf::update_block_number(&mut state, header.number); + state::write(state, &s)?; } } @@ -425,7 +439,7 @@ pub fn update_states(header: Header) -> SgxResult<()> { /// Scans blocks for extrinsics that ask the enclave to execute some actions. pub fn scan_block_for_relevant_xt(block: &Block) -> SgxResult> { - debug!("Scanning blocks for relevant xt"); + debug!("Scanning block {} for relevant xt", block.header.number()); let mut calls = Vec::::new(); for xt_opaque in block.extrinsics.iter() { if let Ok(xt) = @@ -457,8 +471,8 @@ fn handle_shield_funds_xt( xt: UncheckedExtrinsicV4, ) -> SgxResult<()> { let (call, account_encrypted, amount, shard) = xt.function.clone(); - info!("Found ShieldFunds extrinsic in block: \nCall: {:?} \nAccount Encrypted {:?} \nAmount: {} \nShard: {:?}", - call, account_encrypted, amount, shard + info!("Found ShieldFunds extrinsic in block: \nCall: {:?} \nAccount Encrypted {:?} \nAmount: {} \nShard: {}", + call, account_encrypted, amount, shard.encode().to_base58(), ); let mut state = if state::exists(&shard) { @@ -478,25 +492,25 @@ fn handle_shield_funds_xt( &mut state, TrustedCallSigned::new( TrustedCall::balance_shield(account, amount), - 0, - Default::default(), + 0, //nonce + Default::default(), //don't care about signature here ), calls, ) { error!("Error performing Stf::execute. Error: {:?}", e); return Ok(()); } - let xt_call = [SUBSRATEE_REGISTRY_MODULE, CALL_CONFIRMED]; + let state_hash = state::write(state, &shard)?; + + let xt_call = [SUBSRATEE_REGISTRY_MODULE, CALL_CONFIRMED]; + let call_hash = blake2_256(&xt.encode()); + debug!("Call hash 0x{}", hex::encode_hex(&call_hash)); + calls.push(OpaqueCall( - ( - xt_call, - shard, - blake2_256(&xt.encode()), - state_hash.encode(), - ) - .encode(), + (xt_call, shard, call_hash, state_hash.encode()).encode(), )); + Ok(()) } @@ -534,7 +548,12 @@ fn handle_call_worker_xt( return Ok(()); } - let mut state = state::load(&shard)?; + let mut state = if state::exists(&shard) { + state::load(&shard)? + } else { + state::init_shard(&shard)?; + Stf::init_state() + }; debug!("Update STF storage!"); let requests = Stf::get_storage_hashes_to_update(&stf_call_signed) diff --git a/enclave/src/state.rs b/enclave/src/state.rs index 03ff6089c0..c5ed0cf433 100644 --- a/enclave/src/state.rs +++ b/enclave/src/state.rs @@ -16,7 +16,7 @@ */ use std::fs; -use std::io::Write; + use std::vec::Vec; use log::*; @@ -28,8 +28,8 @@ use crate::constants::{ENCRYPTED_STATE_FILE, SHARDS_PATH}; use crate::hex; use crate::io; use crate::utils::UnwrapOrSgxErrorUnexpected; -use base58::ToBase58; -use codec::Encode; +use base58::{FromBase58, ToBase58}; +use codec::{Decode, Encode}; use sgx_externalities::SgxExternalitiesTrait; use sp_core::H256; use std::path::Path; @@ -43,21 +43,24 @@ pub fn load(shard: &ShardIdentifier) -> SgxResult { shard.encode().to_base58(), ENCRYPTED_STATE_FILE ); - debug!("loading state from: {}", state_path); + trace!("loading state from: {}", state_path); let state_vec = read(&state_path)?; // state is now decrypted! let state: StfState = match state_vec.len() { 0 => { - debug!("state is empty. will initialize it."); + debug!("state at {} is empty. will initialize it.", state_path); Stf::init_state() } n => { - debug!("State loaded with size {}B, deserializing...", n); + debug!( + "State loaded from {} with size {}B, deserializing...", + state_path, n + ); StfState::decode(state_vec) } }; - debug!("state decoded successfully"); + trace!("state decoded successfully"); Ok(state) } @@ -68,7 +71,7 @@ pub fn write(state: StfState, shard: &ShardIdentifier) -> SgxResult { shard.encode().to_base58(), ENCRYPTED_STATE_FILE ); - debug!("writing state to: {}", state_path); + trace!("writing state to: {}", state_path); let cyphertext = encrypt(state.encode())?; @@ -77,7 +80,11 @@ pub fn write(state: StfState, shard: &ShardIdentifier) -> SgxResult { Err(status) => return Err(status), }; - debug!("new state hash=0x{}", hex::encode_hex(&state_hash)); + debug!( + "new encrypted state with hash=0x{} written to {}", + hex::encode_hex(&state_hash), + state_path + ); io::write(&cyphertext, &state_path)?; Ok(state_hash.into()) @@ -94,10 +101,7 @@ pub fn exists(shard: &ShardIdentifier) -> bool { } pub fn init_shard(shard: &ShardIdentifier) -> SgxResult<()> { - let path = format!("{}/{}", SHARDS_PATH, shard.encode().to_base58()); - fs::create_dir_all(path.clone()).sgx_error()?; - let mut file = fs::File::create(format!("{}/{}", path, ENCRYPTED_STATE_FILE)).sgx_error()?; - file.write_all(b"").sgx_error() + fs::create_dir_all(format!("{}/{}", SHARDS_PATH, shard.encode().to_base58())).sgx_error() } fn read(path: &str) -> SgxResult> { @@ -108,9 +112,18 @@ fn read(path: &str) -> SgxResult> { }, Err(e) => return Err(e), }; + let state_hash = match rsgx_sha256_slice(&bytes) { + Ok(h) => h, + Err(status) => return Err(status), + }; + debug!( + "read encrypted state with hash 0x{} from {}", + hex::encode_hex(&state_hash), + path + ); aes::de_or_encrypt(&mut bytes)?; - debug!("buffer decrypted = {:?}", bytes); + trace!("buffer decrypted = {:?}", bytes); Ok(bytes) } @@ -129,6 +142,22 @@ fn encrypt(mut state: Vec) -> SgxResult> { Ok(state) } +pub fn list_shards() -> SgxResult> { + let files = fs::read_dir(SHARDS_PATH).sgx_error()?; + let mut shards = Vec::new(); + for file in files { + let s = file + .sgx_error()? + .file_name() + .into_string() + .sgx_error()? + .from_base58() + .sgx_error()?; + shards.push(ShardIdentifier::decode(&mut s.as_slice()).sgx_error()?); + } + Ok(shards) +} + pub fn test_encrypted_state_io_works() { let path = "test_state_file.bin"; let plaintext = b"The quick brown fox jumps over the lazy dog."; diff --git a/stf/Cargo.toml b/stf/Cargo.toml index 9c64711c29..f422ee444c 100644 --- a/stf/Cargo.toml +++ b/stf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substratee-stf" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["Supercomputing Systems AG "] edition = "2018" diff --git a/stf/src/cli.rs b/stf/src/cli.rs index 90358fbea0..cfa0de6537 100644 --- a/stf/src/cli.rs +++ b/stf/src/cli.rs @@ -15,7 +15,7 @@ */ -use crate::{AccountId, ShardIdentifier, TrustedCall, TrustedGetter, TrustedOperationSigned}; +use crate::{AccountId, ShardIdentifier, TrustedCall, TrustedGetter, TrustedOperation}; use base58::{FromBase58, ToBase58}; use clap::{Arg, ArgMatches}; use clap_nested::{Command, Commander, MultiCommand}; @@ -30,7 +30,7 @@ use std::path::PathBuf; const KEYSTORE_PATH: &str = "my_trusted_keystore"; pub fn cmd<'a>( - perform_operation: &'a dyn Fn(&ArgMatches<'_>, &TrustedOperationSigned) -> Option>, + perform_operation: &'a dyn Fn(&ArgMatches<'_>, &TrustedOperation) -> Option>, ) -> MultiCommand<'a, str, str> { Commander::new() .options(|app| { @@ -138,23 +138,22 @@ pub fn cmd<'a>( info!("from ss58 is {}", from.public().to_ss58check()); info!("to ss58 is {}", to.to_ss58check()); - let (mrenclave, shard) = get_identifiers(matches); - - let tcall = TrustedCall::balance_transfer( - sr25519_core::Public::from(from.public()), - to, - amount, - ); - let nonce = 0; // FIXME: hard coded for now - let tscall = - tcall.sign(&sr25519_core::Pair::from(from), nonce, &mrenclave, &shard); println!( "send trusted call transfer from {} to {}: {}", - tscall.call.account(), + from.public(), to, amount ); - let _ = perform_operation(matches, &TrustedOperationSigned::call(tscall)); + let (mrenclave, shard) = get_identifiers(matches); + let nonce = 0; // FIXME: hard coded for now + let top: TrustedOperation = TrustedCall::balance_transfer( + sr25519_core::Public::from(from.public()), + to, + amount, + ) + .sign(&sr25519_core::Pair::from(from), nonce, &mrenclave, &shard) + .into(); + let _ = perform_operation(matches, &top); Ok(()) }), ) @@ -185,23 +184,25 @@ pub fn cmd<'a>( let signer = get_pair_from_str(matches, "//AliceIncognito"); info!("account ss58 is {}", who.public().to_ss58check()); + println!( + "send trusted call set-balance({}, {})", + who.public(), + amount + ); + let (mrenclave, shard) = get_identifiers(matches); - let tcall = TrustedCall::balance_set_balance( + let nonce = 0; // FIXME: hard coded for now + + let top: TrustedOperation = TrustedCall::balance_set_balance( sr25519_core::Public::from(signer.public()), sr25519_core::Public::from(who.public()), amount, amount, - ); - let nonce = 0; // FIXME: hard coded for now - let tscall = - tcall.sign(&sr25519_core::Pair::from(signer), nonce, &mrenclave, &shard); - println!( - "send trusted call set-balance({}, {})", - tscall.call.account(), - amount - ); - let _ = perform_operation(matches, &TrustedOperationSigned::call(tscall)); + ) + .sign(&sr25519_core::Pair::from(signer), nonce, &mrenclave, &shard) + .into(); + let _ = perform_operation(matches, &top); Ok(()) }), ) @@ -221,10 +222,11 @@ pub fn cmd<'a>( let arg_who = matches.value_of("accountid").unwrap(); println!("arg_who = {:?}", arg_who); let who = get_pair_from_str(matches, arg_who); - let tgetter = - TrustedGetter::free_balance(sr25519_core::Public::from(who.public())); - let tsgetter = tgetter.sign(&sr25519_core::Pair::from(who)); - let res = perform_operation(matches, &TrustedOperationSigned::get(tsgetter)); + let top: TrustedOperation = + TrustedGetter::free_balance(sr25519_core::Public::from(who.public())) + .sign(&sr25519_core::Pair::from(who)) + .into(); + let res = perform_operation(matches, &top); let bal = if let Some(v) = res { if let Ok(vd) = crate::Balance::decode(&mut v.as_slice()) { vd @@ -282,8 +284,6 @@ pub fn cmd<'a>( println!("from ss58 is {}", from.public().to_ss58check()); println!("to ss58 is {}", to.to_ss58check()); - let (mrenclave, shard) = get_identifiers(matches); - println!( "send trusted call unshield_funds from {} to {}: {}", from.public(), @@ -291,16 +291,18 @@ pub fn cmd<'a>( amount ); - let tcall = TrustedCall::balance_unshield( + let (mrenclave, shard) = get_identifiers(matches); + let nonce = 0; // FIXME: hard coded for now + + let top: TrustedOperation = TrustedCall::balance_unshield( sr25519_core::Public::from(from.public()), to, amount, shard, - ); - let nonce = 0; // FIXME: hard coded for now - let tscall = - tcall.sign(&sr25519_core::Pair::from(from), nonce, &mrenclave, &shard); - perform_operation(matches, &TrustedOperationSigned::call(tscall)); + ) + .sign(&sr25519_core::Pair::from(from), nonce, &mrenclave, &shard) + .into(); + let _ = perform_operation(matches, &top); Ok(()) }), ) diff --git a/stf/src/lib.rs b/stf/src/lib.rs index 219e6c943e..bea3fa5be6 100644 --- a/stf/src/lib.rs +++ b/stf/src/lib.rs @@ -58,9 +58,58 @@ pub type State = sp_io::SgxExternalities; #[derive(Encode, Decode, Clone)] #[allow(non_camel_case_types)] -pub enum TrustedOperationSigned { +pub enum TrustedOperation { call(TrustedCallSigned), - get(TrustedGetterSigned), + get(Getter), +} + +impl From for TrustedOperation { + fn from(item: TrustedCallSigned) -> Self { + TrustedOperation::call(item) + } +} + +impl From for TrustedOperation { + fn from(item: Getter) -> Self { + TrustedOperation::get(item) + } +} + +impl From for TrustedOperation { + fn from(item: TrustedGetterSigned) -> Self { + TrustedOperation::get(item.into()) + } +} + +impl From for TrustedOperation { + fn from(item: PublicGetter) -> Self { + TrustedOperation::get(item.into()) + } +} + +#[derive(Encode, Decode, Clone, Debug)] +#[allow(non_camel_case_types)] +pub enum Getter { + public(PublicGetter), + trusted(TrustedGetterSigned), +} + +impl From for Getter { + fn from(item: PublicGetter) -> Self { + Getter::public(item) + } +} + +impl From for Getter { + fn from(item: TrustedGetterSigned) -> Self { + Getter::trusted(item) + } +} + +#[derive(Encode, Decode, Clone, Debug)] +#[allow(non_camel_case_types)] +pub enum PublicGetter { + some_value, } #[derive(Encode, Decode, Clone, Debug)] @@ -126,7 +175,7 @@ impl TrustedGetter { } } -#[derive(Encode, Decode, Clone)] +#[derive(Encode, Decode, Clone, Debug)] pub struct TrustedGetterSigned { pub getter: TrustedGetter, pub signature: AnySignature, @@ -143,7 +192,7 @@ impl TrustedGetterSigned { } } -#[derive(Encode, Decode, Clone)] +#[derive(Encode, Decode, Clone, Debug)] pub struct TrustedCallSigned { pub call: TrustedCall, pub nonce: u32, diff --git a/stf/src/sgx.rs b/stf/src/sgx.rs index 6c8593039e..f2c7390e4c 100644 --- a/stf/src/sgx.rs +++ b/stf/src/sgx.rs @@ -6,14 +6,14 @@ use codec::{Decode, Encode}; use derive_more::Display; use log_sgx::*; use metadata::StorageHasher; -use sgx_runtime::{Balance, Runtime}; +use sgx_runtime::{Balance, BlockNumber, Runtime}; use sp_core::crypto::AccountId32; use sp_io::SgxExternalitiesTrait; use sp_runtime::traits::Dispatchable; use crate::{ - AccountId, ShardIdentifier, State, Stf, TrustedCall, TrustedCallSigned, TrustedGetter, - TrustedGetterSigned, SUBSRATEE_REGISTRY_MODULE, UNSHIELD, + AccountId, Getter, PublicGetter, ShardIdentifier, State, Stf, TrustedCall, TrustedCallSigned, + TrustedGetter, SUBSRATEE_REGISTRY_MODULE, UNSHIELD, }; use sp_core::blake2_256; @@ -80,6 +80,13 @@ impl Stf { }); } + pub fn update_block_number(ext: &mut State, number: BlockNumber) { + ext.execute_with(|| { + let key = storage_value_key("System", "Number"); + sp_io::storage::set(&key, &number.encode()); + }); + } + pub fn execute( ext: &mut State, call: TrustedCallSigned, @@ -88,23 +95,47 @@ impl Stf { ext.execute_with(|| match call.call { TrustedCall::balance_set_balance(root, who, free_balance, reserved_balance) => { Self::ensure_root(root)?; + debug!( + "balance_set_balance({:x?}, {}, {})", + who.encode(), + free_balance, + reserved_balance + ); sgx_runtime::BalancesCall::::set_balance( AccountId32::from(who), free_balance, reserved_balance, ) .dispatch(sgx_runtime::Origin::ROOT) - .map_err(|_| StfError::Dispatch)?; + .map_err(|_| StfError::Dispatch("balance_set_balance".to_string()))?; Ok(()) } TrustedCall::balance_transfer(from, to, value) => { let origin = sgx_runtime::Origin::signed(AccountId32::from(from)); + debug!( + "balance_transfer({:x?}, {:x?}, {})", + from.encode(), + to.encode(), + value + ); + if let Some(info) = get_account_info(&from) { + debug!("sender balance is {}", info.data.free); + } else { + debug!("sender balance is zero"); + } sgx_runtime::BalancesCall::::transfer(AccountId32::from(to), value) .dispatch(origin) - .map_err(|_| StfError::Dispatch)?; + .map_err(|_| StfError::Dispatch("balance_transfer".to_string()))?; Ok(()) } TrustedCall::balance_unshield(account_incognito, beneficiary, value, shard) => { + debug!( + "balance_unshield({:x?}, {:x?}, {}, {})", + account_incognito.encode(), + beneficiary.encode(), + value, + shard + ); Self::unshield_funds(account_incognito, value)?; calls.push(OpaqueCall( ( @@ -119,30 +150,38 @@ impl Stf { Ok(()) } TrustedCall::balance_shield(who, value) => { + debug!("balance_shield({:x?}, {})", who.encode(), value); Self::shield_funds(who, value)?; Ok(()) } }) } - pub fn get_state(ext: &mut State, getter: TrustedGetter) -> Option> { + pub fn get_state(ext: &mut State, getter: Getter) -> Option> { ext.execute_with(|| match getter { - TrustedGetter::free_balance(who) => { - if let Some(info) = get_account_info(&who) { - debug!("AccountInfo for {:?} is {:?}", who, info); - Some(info.data.free.encode()) - } else { - None + Getter::trusted(g) => match g.getter { + TrustedGetter::free_balance(who) => { + if let Some(info) = get_account_info(&who) { + debug!("AccountInfo for {:x?} is {:?}", who.encode(), info); + debug!("Account free balance is {}", info.data.free); + Some(info.data.free.encode()) + } else { + None + } } - } - TrustedGetter::reserved_balance(who) => { - if let Some(info) = get_account_info(&who) { - debug!("AccountInfo for {:?} is {:?}", who, info); - Some(info.data.reserved.encode()) - } else { - None + TrustedGetter::reserved_balance(who) => { + if let Some(info) = get_account_info(&who) { + debug!("AccountInfo for {:x?} is {:?}", who.encode(), info); + debug!("Account reserved balance is {}", info.data.reserved); + Some(info.data.reserved.encode()) + } else { + None + } } - } + }, + Getter::public(g) => match g { + PublicGetter::some_value => Some(42u32.encode()), + }, }) } @@ -162,10 +201,10 @@ impl Stf { account_info.data.reserved, ) .dispatch(sgx_runtime::Origin::ROOT) - .map_err(|_| StfError::Dispatch)?, + .map_err(|_| StfError::Dispatch("shield_funds".to_string()))?, None => sgx_runtime::BalancesCall::::set_balance(account.into(), amount, 0) .dispatch(sgx_runtime::Origin::ROOT) - .map_err(|_| StfError::Dispatch)?, + .map_err(|_| StfError::Dispatch("shield_funds::set_balance".to_string()))?, }; Ok(()) } @@ -183,7 +222,7 @@ impl Stf { account_info.data.reserved, ) .dispatch(sgx_runtime::Origin::ROOT) - .map_err(|_| StfError::Dispatch)?; + .map_err(|_| StfError::Dispatch("unshield_funds::set_balance".to_string()))?; Ok(()) } None => Err(StfError::InexistentAccount(account)), @@ -191,37 +230,29 @@ impl Stf { } pub fn get_storage_hashes_to_update(call: &TrustedCallSigned) -> Vec> { - let mut key_hashes = Vec::new(); + let key_hashes = Vec::new(); match call.call { - TrustedCall::balance_set_balance(account, _, _, _) => { - key_hashes.push(nonce_key_hash(&account)) // dummy, actually not necessary - } - TrustedCall::balance_transfer(account, _, _) => { - key_hashes.push(nonce_key_hash(&account)) // dummy, actually not necessary - } - TrustedCall::balance_unshield(account, _, _, _) => { - key_hashes.push(nonce_key_hash(&account)) - } + TrustedCall::balance_set_balance(_, _, _, _) => debug!("No storage updates needed..."), + TrustedCall::balance_transfer(_, _, _) => debug!("No storage updates needed..."), + TrustedCall::balance_unshield(_, _, _, _) => debug!("No storage updates needed..."), TrustedCall::balance_shield(_, _) => debug!("No storage updates needed..."), }; key_hashes } - pub fn get_storage_hashes_to_update_for_getter(getter: &TrustedGetterSigned) -> Vec> { + pub fn get_storage_hashes_to_update_for_getter(getter: &Getter) -> Vec> { info!( "No specific storage updates needed for getter. Returning those for on block: {:?}", - getter.getter + getter ); Self::storage_hashes_to_update_on_block() } pub fn storage_hashes_to_update_on_block() -> Vec> { - let mut key_hashes = Vec::new(); - - // get all shards that are currently registered - key_hashes.push(shards_key_hash()); - - key_hashes + // let key_hashes = Vec::new(); + // key_hashes.push(storage_value_key("dummy", "dummy")); + // key_hashes + Vec::new() } } @@ -230,6 +261,8 @@ pub fn storage_hashes_to_update_per_shard(_shard: &ShardIdentifier) -> Vec Vec { + // here you have to point to a storage value containing a Vec of ShardIdentifiers + // the enclave uses this to autosubscribe to no shards storage_value_key("EncointerCurrencies", "CurrencyIdentifiers") } @@ -320,7 +353,7 @@ pub enum StfError { #[display(fmt = "Insufficient privileges {:?}, are you sure you are root?", _0)] MissingPrivileges(AccountId), #[display(fmt = "Error dispatching runtime call")] - Dispatch, + Dispatch(String), #[display(fmt = "Not enough funds to perform operation")] MissingFunds, #[display(fmt = "Account does not exist {:?}", _0)] diff --git a/substratee-node-primitives/Cargo.toml b/substratee-node-primitives/Cargo.toml index 046f5601cb..d0cc9fc590 100644 --- a/substratee-node-primitives/Cargo.toml +++ b/substratee-node-primitives/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substratee-node-primitives" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["clangenbacher "] edition = "2018" diff --git a/worker/Cargo.toml b/worker/Cargo.toml index b475f5e96f..3b993c17d8 100644 --- a/worker/Cargo.toml +++ b/worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substratee-worker" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["Supercomputing Systems AG "] build = "build.rs" edition = "2018" diff --git a/worker/src/main.rs b/worker/src/main.rs index 28e5658ec7..1265f2bb34 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -52,8 +52,9 @@ use enclave::api::{ }; use enclave::tls_ra::{enclave_request_key_provisioning, enclave_run_key_provisioning_server}; use sp_finality_grandpa::{AuthorityList, VersionedAuthorityList, GRANDPA_AUTHORITIES_KEY}; -use substratee_node_primitives::calls::get_first_worker_that_is_not_equal_to_self; -use substratee_worker_api::Api as WorkerApi; +use std::time::Duration; +//use substratee_worker_api::requests::ClientRequest; +//use substratee_worker_api::Api as WorkerApi; use ws_server::start_ws_server; mod constants; @@ -62,6 +63,9 @@ mod ipfs; mod tests; mod ws_server; +/// how many blocks will be synced before storing the chain db to disk +const BLOCK_SYNC_BATCH_SIZE: u32 = 1000; + fn main() { // Setup logging env_logger::init(); @@ -75,7 +79,7 @@ fn main() { info!("Interacting with node on {}", n_url); *NODE_URL.lock().unwrap() = n_url; - let w_ip = matches.value_of("w-server").unwrap_or("127.0.0.1"); + let w_ip = matches.value_of("w-server").unwrap_or("ws://127.0.0.1"); let w_port = matches.value_of("w-port").unwrap_or("2000"); info!("Worker listening on {}:{}", w_ip, w_port); @@ -225,12 +229,12 @@ fn worker(w_ip: &str, w_port: &str, mu_ra_port: &str, shard: &ShardIdentifier) { let eid = enclave.geteid(); // ------------------------------------------------------------------------ // start the ws server to listen for worker requests + let (ws_sender, ws_receiver) = channel(); let w_url = format!("{}:{}", w_ip, w_port); - start_ws_server(eid, w_url.clone(), mu_ra_port.to_string()); + start_ws_server(w_url.clone(), ws_sender); // ------------------------------------------------------------------------ // let new workers call us for key provisioning - let eid = enclave.geteid(); let ra_url = format!("{}:{}", w_ip, mu_ra_port); thread::spawn(move || { enclave_run_key_provisioning_server( @@ -267,33 +271,29 @@ fn worker(w_ip: &str, w_port: &str, mu_ra_port: &str, shard: &ShardIdentifier) { let tx_hash = api.send_extrinsic(_xthex, XtStatus::InBlock).unwrap(); println!("[<] Extrinsic got finalized. Hash: {:?}\n", tx_hash); - // browse enclave registry - match get_first_worker_that_is_not_equal_to_self(&api, &tee_accountid) { - Some(w) => { - let _url = String::from_utf8_lossy(&w.url[..]).to_string(); - let _w_api = WorkerApi::new(_url.clone()); - let _url_split: Vec<_> = _url.split(':').collect(); - let mura_url = format!("{}:{}", _url_split[0], _w_api.get_mu_ra_port().unwrap()); - - info!("Requesting key provisioning from worker at {}", mura_url); - enclave_request_key_provisioning( - eid, - sgx_quote_sign_type_t::SGX_UNLINKABLE_SIGNATURE, - &mura_url, - ) - .unwrap(); - debug!("key provisioning successfully performed"); - } - None => { - info!("there are no other workers"); - } - } - - println!("*** [+] finished remote attestation\n"); + // // browse enclave registry + // match get_first_worker_that_is_not_equal_to_self(&api, &tee_accountid) { + // Some(w) => { + // let _url = String::from_utf8_lossy(&w.url[..]).to_string(); + // let _w_api = WorkerApi::new(_url.clone()); + // let _url_split: Vec<_> = _url.split(':').collect(); + // let mura_url = format!("{}:{}", _url_split[0], _w_api.get_mu_ra_port().unwrap()); + + // info!("Requesting key provisioning from worker at {}", mura_url); + // enclave_request_key_provisioning( + // eid, + // sgx_quote_sign_type_t::SGX_UNLINKABLE_SIGNATURE, + // &mura_url, + // ) + // .unwrap(); + // debug!("key provisioning successfully performed"); + // } + // None => { + // info!("there are no other workers"); + // } + // } - println!("*** Syncing chain relay\n\n"); let mut latest_head = init_chain_relay(eid, &api); - println!("*** [+] Finished syncing chain relay\n"); // ------------------------------------------------------------------------ // subscribe to events and react on firing @@ -317,14 +317,17 @@ fn worker(w_ip: &str, w_port: &str, mu_ra_port: &str, shard: &ShardIdentifier) { println!("[+] Subscribed to events. waiting..."); + let timeout = Duration::from_millis(10); loop { - let msg = receiver.recv().unwrap(); - if let Ok(events) = parse_events(msg.clone()) { - print_events(events, sender.clone()) - } else if let Ok(_header) = parse_header(msg.clone()) { - latest_head = sync_chain_relay(eid, &api, latest_head) - } else { - println!("[-] Unable to parse received message!") + if let Ok(msg) = receiver.recv_timeout(timeout) { + if let Ok(events) = parse_events(msg.clone()) { + print_events(events, sender.clone()) + } else if let Ok(_header) = parse_header(msg.clone()) { + latest_head = sync_chain_relay(eid, &api, latest_head) + } + } + if let Ok(req) = ws_receiver.recv_timeout(timeout) { + ws_server::handle_request(req, eid, mu_ra_port.to_string()).unwrap() } } } @@ -493,7 +496,7 @@ pub fn sync_chain_relay( .unwrap(); blocks_to_sync.push(head.clone()); - if head.block.header.number % 100 == 0 { + if head.block.header.number % BLOCK_SYNC_BATCH_SIZE == 0 { println!( "Remaining blocks to fetch until last synced header: {:?}", head.block.header.number - last_synced_head.number @@ -504,9 +507,9 @@ pub fn sync_chain_relay( let tee_accountid = enclave_account(eid); - // only feed 100 blocks at a time into the enclave to save enclave state regularly + // only feed BLOCK_SYNC_BATCH_SIZE blocks at a time into the enclave to save enclave state regularly let mut i = blocks_to_sync[0].block.header.number as usize; - for chunk in blocks_to_sync.chunks(100) { + for chunk in blocks_to_sync.chunks(BLOCK_SYNC_BATCH_SIZE as usize) { let tee_nonce = get_nonce(&api, &tee_accountid); let xts = enclave_sync_chain_relay(eid, chunk.to_vec(), tee_nonce).unwrap(); let extrinsics: Vec> = Decode::decode(&mut xts.as_slice()).unwrap(); diff --git a/worker/src/ws_server.rs b/worker/src/ws_server.rs index c2516ed569..879f6d7ca6 100644 --- a/worker/src/ws_server.rs +++ b/worker/src/ws_server.rs @@ -20,37 +20,50 @@ use std::thread; use sgx_types::*; -use codec::Encode; +use codec::{Decode, Encode}; use log::*; -use substratee_stf::ShardIdentifier; +use std::sync::mpsc::Sender as MpscSender; +use substratee_stf::{Getter, ShardIdentifier}; use substratee_worker_api::requests::*; use ws::{listen, CloseCode, Handler, Message, Result, Sender}; use crate::enclave::api::{enclave_query_state, enclave_shielding_key}; -pub fn start_ws_server(eid: sgx_enclave_id_t, addr: String, mu_ra_port: String) { +#[derive(Clone, Debug)] +pub struct WsServerRequest { + client: Sender, + request: ClientRequest, +} + +impl WsServerRequest { + pub fn new(client: Sender, request: ClientRequest) -> Self { + Self { client, request } + } +} + +pub fn start_ws_server(addr: String, worker: MpscSender) { // Server WebSocket handler struct Server { - out: Sender, - eid: sgx_enclave_id_t, - mu_ra_port: String, + client: Sender, + worker: MpscSender, } impl Handler for Server { fn on_message(&mut self, msg: Message) -> Result<()> { - info!(" [WS Server] Got message '{}'. ", msg); - - let msg_txt = msg.into_text().unwrap(); - let args: Vec<&str> = msg_txt.split("::").collect(); - - let answer = match args[0] { - MSG_GET_PUB_KEY_WORKER => get_worker_pub_key(self.eid), - MSG_GET_MU_RA_PORT => Message::text(self.mu_ra_port.clone()), - MSG_GET_STF_STATE => handle_get_stf_state_msg(self.eid, args[1], args[2]), - _ => Message::text("[WS Server]: unrecognized msg pattern"), - }; - - self.out.send(answer) + info!( + "[WS Server] Forwarding message to worker event loop: {:?}", + msg + ); + + match ClientRequest::decode(&mut msg.into_data().as_slice()) { + Ok(req) => { + self.worker + .send(WsServerRequest::new(self.client.clone(), req)) + .unwrap(); + } + Err(_) => self.client.send("Could not decode request").unwrap(), + } + Ok(()) } fn on_close(&mut self, code: CloseCode, reason: &str) { @@ -61,20 +74,31 @@ pub fn start_ws_server(eid: sgx_enclave_id_t, addr: String, mu_ra_port: String) info!("Starting WebSocket server on {}", addr); thread::spawn(move || { listen(addr, |out| Server { - out, - eid, - mu_ra_port: mu_ra_port.clone(), + client: out, + worker: worker.clone(), }) .unwrap() }); } -fn handle_get_stf_state_msg(eid: sgx_enclave_id_t, getter_str: &str, shard_str: &str) -> Message { - info!(" [WS Server] Query state"); - let getter_vec = hex::decode(getter_str).unwrap(); - let shard = ShardIdentifier::from_slice(&hex::decode(shard_str).unwrap()); +pub fn handle_request( + req: WsServerRequest, + eid: sgx_enclave_id_t, + mu_ra_port: String, +) -> Result<()> { + info!(" [WS Server] Got message '{:?}'. ", req); + let answer = match req.request { + ClientRequest::PubKeyWorker => get_pubkey(eid), + ClientRequest::MuRaPortWorker => Message::text(mu_ra_port), + ClientRequest::StfState(getter, shard) => get_stf_state(eid, getter, shard), + }; - let value = match enclave_query_state(eid, getter_vec, shard.encode()) { + req.client.send(answer) +} + +fn get_stf_state(eid: sgx_enclave_id_t, getter: Getter, shard: ShardIdentifier) -> Message { + info!(" [WS Server] Query state"); + let value = match enclave_query_state(eid, getter.encode(), shard.encode()) { Ok(val) => Some(val), Err(_) => { error!("query state failed"); @@ -86,7 +110,7 @@ fn handle_get_stf_state_msg(eid: sgx_enclave_id_t, getter_str: &str, shard_str: Message::text(hex::encode(value.encode())) } -fn get_worker_pub_key(eid: sgx_enclave_id_t) -> Message { +fn get_pubkey(eid: sgx_enclave_id_t) -> Message { let rsa_pubkey = enclave_shielding_key(eid).unwrap(); debug!(" [WS Server] RSA pubkey {:?}\n", rsa_pubkey); diff --git a/worker/worker-api/Cargo.toml b/worker/worker-api/Cargo.toml index 48195f0041..6a02d22d23 100644 --- a/worker/worker-api/Cargo.toml +++ b/worker/worker-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substratee-worker-api" -version = "0.6.5-sub2.0.0-alpha.7" +version = "0.6.11-sub2.0.0-alpha.7" authors = ["Supercomputing Systems AG "] edition = "2018" diff --git a/worker/worker-api/src/client.rs b/worker/worker-api/src/client.rs index be42f101ea..1b01c74ba3 100644 --- a/worker/worker-api/src/client.rs +++ b/worker/worker-api/src/client.rs @@ -17,20 +17,21 @@ use std::sync::mpsc::Sender as ThreadOut; +use crate::requests::ClientRequest; +use codec::Encode; use log::*; use ws::{CloseCode, Handler, Handshake, Message, Result, Sender}; pub struct WsClient { pub out: Sender, - pub request: String, + pub request: ClientRequest, pub result: ThreadOut, } impl Handler for WsClient { fn on_open(&mut self, _: Handshake) -> Result<()> { - info!("sending request: {}", self.request); - - match self.out.send(self.request.clone()) { + debug!("sending request: {:?}", self.request); + match self.out.send(Message::Binary(self.request.encode())) { Ok(_) => Ok(()), Err(e) => Err(e), } diff --git a/worker/worker-api/src/lib.rs b/worker/worker-api/src/lib.rs index 92ec04a4d0..08dc4f70e8 100644 --- a/worker/worker-api/src/lib.rs +++ b/worker/worker-api/src/lib.rs @@ -20,13 +20,13 @@ use std::thread; use sgx_crypto_helper::rsa3072::Rsa3072PubKey; -use codec::{Decode, Encode}; +use codec::Decode; use log::*; use ws::connect; use client::WsClient; use requests::*; -use substratee_stf::{ShardIdentifier, TrustedGetterSigned}; +use substratee_stf::{Getter, ShardIdentifier}; pub mod client; pub mod requests; @@ -37,16 +37,16 @@ pub struct Api { } impl Api { - pub fn new(url: String) -> Api { - Api { url } + pub fn new(url: String) -> Self { + Self { url } } pub fn get_mu_ra_port(&self) -> Result { - Self::get(&self, MSG_GET_MU_RA_PORT) + Self::get(&self, ClientRequest::MuRaPortWorker) } pub fn get_rsa_pubkey(&self) -> Result { - let keystr = Self::get(&self, MSG_GET_PUB_KEY_WORKER)?; + let keystr = Self::get(&self, ClientRequest::PubKeyWorker)?; let rsa_pubkey: Rsa3072PubKey = serde_json::from_str(&keystr).unwrap(); info!("[+] Got RSA public key of enclave"); @@ -54,15 +54,9 @@ impl Api { Ok(rsa_pubkey) } - pub fn get_stf_state( - &self, - getter: TrustedGetterSigned, - shard: &ShardIdentifier, - ) -> Result, ()> { - let getter_str = hex::encode(getter.encode()); - let shard_str = hex::encode(shard.encode()); - let request = format!("{}::{}::{}", MSG_GET_STF_STATE, getter_str, shard_str); - match Self::get(&self, &request) { + pub fn get_stf_state(&self, getter: Getter, shard: &ShardIdentifier) -> Result, ()> { + let req = ClientRequest::StfState(getter, shard.to_owned()); + match Self::get(&self, req) { Ok(res) => { let value_slice = hex::decode(&res).unwrap(); let value: Option> = Decode::decode(&mut &value_slice[..]).unwrap(); @@ -75,16 +69,15 @@ impl Api { } } - fn get(&self, request: &str) -> Result { + fn get(&self, request: ClientRequest) -> Result { let url = self.url.clone(); - let req = request.to_string(); let (port_in, port_out) = channel(); - info!("[Worker Api]: Sending request: {}", req); + info!("[Worker Api]: Sending request: {:?}", request); let client = thread::spawn(move || { match connect(url, |out| WsClient { out, - request: req.clone(), + request: request.clone(), result: port_in.clone(), }) { Ok(c) => c, diff --git a/worker/worker-api/src/requests.rs b/worker/worker-api/src/requests.rs index b9e6479bf2..437cbc91bc 100644 --- a/worker/worker-api/src/requests.rs +++ b/worker/worker-api/src/requests.rs @@ -15,6 +15,12 @@ */ -pub const MSG_GET_PUB_KEY_WORKER: &str = "get_pub_key_worker"; -pub const MSG_GET_MU_RA_PORT: &str = "get_mu_ra_port"; -pub const MSG_GET_STF_STATE: &str = "get_stf_state"; +use codec::{Decode, Encode}; +use substratee_stf::{Getter, ShardIdentifier}; + +#[derive(Encode, Decode, Clone, Debug)] +pub enum ClientRequest { + PubKeyWorker, + MuRaPortWorker, + StfState(Getter, ShardIdentifier), // (trusted_getter_encrypted, shard) +}