From 68ccafecfd504021188fc8d0e4fcc4b8f2eeb373 Mon Sep 17 00:00:00 2001 From: Steven Herbst Date: Mon, 20 May 2024 11:28:55 -0700 Subject: [PATCH] Single netlist mode (#223) --- .github/workflows/regression.yml | 3 +- examples/clean.sh | 12 ++ examples/network-fifo-chain/Makefile | 25 +++ examples/network-fifo-chain/test.py | 130 ++++++++++++ examples/network/Makefile | 8 + examples/network/test.py | 30 +-- examples/requirements.txt | 2 +- examples/test_examples.py | 2 + setup.py | 2 +- switchboard/autowrap.py | 287 ++++++++++++++++----------- switchboard/cmdline.py | 18 +- switchboard/network.py | 232 +++++++++++++++++----- switchboard/sbdut.py | 190 +++++++++++++----- switchboard/sc/__init__.py | 0 switchboard/sc/morty/__init__.py | 0 switchboard/sc/morty/morty.py | 14 ++ switchboard/sc/morty/uniquify.py | 56 ++++++ switchboard/sc/sed/__init__.py | 0 switchboard/sc/sed/remove.py | 41 ++++ switchboard/sc/sed/sed.py | 8 + switchboard/test_util.py | 20 +- 21 files changed, 841 insertions(+), 239 deletions(-) create mode 100755 examples/clean.sh create mode 100755 examples/network-fifo-chain/Makefile create mode 100755 examples/network-fifo-chain/test.py create mode 100644 switchboard/sc/__init__.py create mode 100644 switchboard/sc/morty/__init__.py create mode 100644 switchboard/sc/morty/morty.py create mode 100644 switchboard/sc/morty/uniquify.py create mode 100644 switchboard/sc/sed/__init__.py create mode 100644 switchboard/sc/sed/remove.py create mode 100644 switchboard/sc/sed/sed.py diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 72a8d92f..616a56a0 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -31,7 +31,8 @@ jobs: - name: Run pytest working-directory: examples - run: pytest --durations=0 -s + run: | + pytest --durations=0 -s - name: Run tests working-directory: tests diff --git a/examples/clean.sh b/examples/clean.sh new file mode 100755 index 00000000..90663745 --- /dev/null +++ b/examples/clean.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Loop through subdirectories, running 'make clean' + +for dir in */; do + # Run 'make clean' if a Makefile exists + + if [ -f "${dir}Makefile" ]; then + echo "Running 'make clean' in $dir" + (cd "$dir" && make clean) + fi +done diff --git a/examples/network-fifo-chain/Makefile b/examples/network-fifo-chain/Makefile new file mode 100755 index 00000000..4c4f3f39 --- /dev/null +++ b/examples/network-fifo-chain/Makefile @@ -0,0 +1,25 @@ +# Copyright (c) 2024 Zero ASIC Corporation +# This code is licensed under Apache License 2.0 (see LICENSE for details) + +.PHONY: verilator +verilator: + ./test.py --tool verilator --start-delay 5 --max-rate 1e3 + +.PHONY: verilator-single-netlist +verilator-single-netlist: + ./test.py --tool verilator --single-netlist + +.PHONY: icarus +icarus: + ./test.py --tool icarus --start-delay 2 --max-rate 1e3 --fifos 125 + +.PHONY: icarus-single-netlist +icarus-single-netlist: + ./test.py --tool icarus --single-netlist + +.PHONY: clean +clean: + rm -f queue-* *.q + rm -f *.vcd *.fst *.fst.hier + rm -rf obj_dir build + rm -f *.o *.vpi diff --git a/examples/network-fifo-chain/test.py b/examples/network-fifo-chain/test.py new file mode 100755 index 00000000..7db58914 --- /dev/null +++ b/examples/network-fifo-chain/test.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + +# Example showing how to wire up various modules using SbNetwork + +# Copyright (c) 2024 Zero ASIC Corporation +# This code is licensed under Apache License 2.0 (see LICENSE for details) + +import umi +from switchboard import SbNetwork, umi_loopback +from switchboard.cmdline import get_cmdline_args + +from pathlib import Path +THIS_DIR = Path(__file__).resolve().parent + + +def main(): + # create network + + extra_args = { + '--packets': dict(type=int, default=1000, help='Number of' + ' transactions to send into the FIFO during the test.'), + '--fifos': dict(type=int, default=500, help='Number of' + ' FIFOs to instantiate in series for this test.'), + '--fifos-per-sim': dict(type=int, default=1, help='Number of' + ' FIFOs to include in each simulation.') + } + + # workaround - need to see what type of simulation we're running + # (network of simulations, network of networks, single netlist) + + args = get_cmdline_args(extra_args=extra_args) + + assert args.fifos % args.fifos_per_sim == 0, \ + 'Number of FIFOs must be divisible by the number of FIFOs per simulation' + + if args.fifos_per_sim in [1, args.fifos]: + # single network + net = SbNetwork(cmdline=True, single_netlist=args.fifos_per_sim == args.fifos) + subnet = net + n = args.fifos + else: + # network of networks + net = SbNetwork(cmdline=True, single_netlist=False) + subnet = SbNetwork(name='subnet', cmdline=True, single_netlist=True) + n = args.fifos_per_sim + + subblock = make_umi_fifo(subnet) + + subblocks = [subnet.instantiate(subblock) for _ in range(n)] + + for i in range(len(subblocks) - 1): + subnet.connect(subblocks[i].umi_out, subblocks[i + 1].umi_in) + + if n < args.fifos: + subnet.external(subblocks[0].umi_in, name='umi_in') + subnet.external(subblocks[-1].umi_out, name='umi_out') + + blocks = [net.instantiate(subnet) for _ in range(args.fifos // args.fifos_per_sim)] + + for i in range(len(blocks) - 1): + net.connect(blocks[i].umi_out, blocks[i + 1].umi_in) + else: + blocks = subblocks + + net.external(blocks[0].umi_in, txrx='umi') + net.external(blocks[-1].umi_out, txrx='umi') + + # build simulator + + net.build() + + # launch the simulation + + net.simulate() + + # interact with the simulation + + umi_loopback(net.intfs['umi'], packets=args.packets) + + +def make_umi_fifo(net): + dw = 256 + aw = 64 + cw = 32 + + parameters = dict( + DW=dw, + AW=aw, + CW=cw + ) + + tieoffs = dict( + bypass="1'b0", + chaosmode="1'b0", + fifo_full=None, + fifo_empty=None, + vdd="1'b1", + vss="1'b0" + ) + + interfaces = { + 'umi_in': dict(type='umi', dw=dw, aw=aw, cw=cw, direction='input'), + 'umi_out': dict(type='umi', dw=dw, aw=aw, cw=cw, direction='output') + } + + clocks = [ + 'umi_in_clk', + 'umi_out_clk' + ] + + resets = [ + 'umi_in_nreset', + 'umi_out_nreset' + ] + + dut = net.make_dut('umi_fifo', parameters=parameters, interfaces=interfaces, + clocks=clocks, resets=resets, tieoffs=tieoffs) + + dut.use(umi) + dut.add('option', 'library', 'umi') + dut.add('option', 'library', 'lambdalib_stdlib') + dut.add('option', 'library', 'lambdalib_ramlib') + + dut.input('umi/rtl/umi_fifo.v', package='umi') + + return dut + + +if __name__ == '__main__': + main() diff --git a/examples/network/Makefile b/examples/network/Makefile index bd44d7f4..7ea6f7e2 100755 --- a/examples/network/Makefile +++ b/examples/network/Makefile @@ -5,10 +5,18 @@ verilator: ./test.py --tool verilator --start-delay 1 --max-rate 1e3 +.PHONY: verilator-single-netlist +verilator-single-netlist: + ./test.py --tool verilator --single-netlist + .PHONY: icarus icarus: ./test.py --tool icarus --start-delay 1 --max-rate 1e3 +.PHONY: icarus-single-netlist +icarus-single-netlist: + ./test.py --tool icarus --single-netlist + .PHONY: clean clean: rm -f queue-* *.q diff --git a/examples/network/test.py b/examples/network/test.py index da0c68ac..c57e3f9c 100755 --- a/examples/network/test.py +++ b/examples/network/test.py @@ -8,7 +8,7 @@ import numpy as np import umi -from switchboard import SbDut, SbNetwork +from switchboard import SbNetwork from pathlib import Path THIS_DIR = Path(__file__).resolve().parent @@ -21,9 +21,9 @@ def main(): # create the building blocks - umi_fifo = make_umi_fifo(args=net.args) - umi2axil = make_umi2axil(args=net.args) - axil_ram = make_axil_ram(args=net.args) + umi_fifo = make_umi_fifo(net) + umi2axil = make_umi2axil(net) + axil_ram = make_axil_ram(net) # connect them together @@ -68,7 +68,7 @@ def main(): assert wrdata == rddata -def make_umi_fifo(args): +def make_umi_fifo(net): dw = 256 aw = 64 cw = 32 @@ -103,18 +103,20 @@ def make_umi_fifo(args): 'umi_out_nreset' ] - dut = SbDut('umi_fifo', autowrap=True, parameters=parameters, interfaces=interfaces, - clocks=clocks, resets=resets, tieoffs=tieoffs, args=args) + dut = net.make_dut('umi_fifo', parameters=parameters, interfaces=interfaces, + clocks=clocks, resets=resets, tieoffs=tieoffs) dut.use(umi) dut.add('option', 'library', 'umi') dut.add('option', 'library', 'lambdalib_stdlib') dut.add('option', 'library', 'lambdalib_ramlib') + dut.input('umi/rtl/umi_fifo.v', package='umi') + return dut -def make_axil_ram(args): +def make_axil_ram(net): dw = 64 aw = 13 @@ -129,8 +131,8 @@ def make_axil_ram(args): resets = [dict(name='rst', delay=8)] - dut = SbDut('axil_ram', autowrap=True, parameters=parameters, interfaces=interfaces, - resets=resets, args=args) + dut = net.make_dut('axil_ram', parameters=parameters, + interfaces=interfaces, resets=resets) dut.register_package_source( 'verilog-axi', @@ -146,7 +148,7 @@ def make_axil_ram(args): return dut -def make_umi2axil(args): +def make_umi2axil(net): dw = 64 aw = 64 cw = 32 @@ -165,14 +167,16 @@ def make_umi2axil(args): resets = ['nreset'] - dut = SbDut('umi2axilite', autowrap=True, parameters=parameters, interfaces=interfaces, - resets=resets, args=args) + dut = net.make_dut('umi2axilite', parameters=parameters, + interfaces=interfaces, resets=resets) dut.use(umi) dut.add('option', 'library', 'umi') dut.add('option', 'library', 'lambdalib_stdlib') dut.add('option', 'library', 'lambdalib_ramlib') + dut.input('utils/rtl/umi2axilite.v', package='umi') + return dut diff --git a/examples/requirements.txt b/examples/requirements.txt index 2e95f4df..d61c4511 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,2 +1,2 @@ # Examples dependencies -umi @ git+https://github.com/zeroasiccorp/umi.git@f6d8fea9e6270b89a2c38860a4af512383cbc6ef +umi @ git+https://github.com/zeroasiccorp/umi.git@main diff --git a/examples/test_examples.py b/examples/test_examples.py index 1c14dcd0..445a843d 100755 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -21,6 +21,8 @@ ['minimal', 'PASS!', 'verilator'], ['network', None, 'verilator'], ['network', None, 'icarus'], + ['network', None, 'verilator-single-netlist'], + ['network', None, 'icarus-single-netlist'], ['python', 'PASS!', None], ['router', 'PASS!', None], ['stream', 'PASS!', None], diff --git a/setup.py b/setup.py index ffacab25..f75c7c14 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup, find_packages from pybind11.setup_helpers import Pybind11Extension, build_ext -__version__ = "0.2.1" +__version__ = "0.2.2" ################################################################################# # parse_reqs, long_desc from https://github.com/siliconcompiler/siliconcompiler # diff --git a/switchboard/autowrap.py b/switchboard/autowrap.py index 64d3ba9b..88161827 100644 --- a/switchboard/autowrap.py +++ b/switchboard/autowrap.py @@ -22,6 +22,12 @@ def normalize_interface(name, value): if 'type' not in value: value['type'] = 'sb' + if 'wire' not in value: + value['wire'] = name + + if 'external' not in value: + value['external'] = True + assert 'type' in value value['type'] = normalize_intf_type(value['type']) type = value['type'] @@ -184,7 +190,7 @@ def normalize_parameters(parameters): def autowrap( - design, + instances, toplevel='testbench', parameters=None, interfaces=None, @@ -197,11 +203,11 @@ def autowrap( ): # normalize inputs - parameters = normalize_parameters(parameters) - interfaces = normalize_interfaces(interfaces) - clocks = normalize_clocks(clocks) - resets = normalize_resets(resets) - tieoffs = normalize_tieoffs(tieoffs) + parameters = {k: normalize_parameters(v) for k, v in parameters.items()} + interfaces = {k: normalize_interfaces(v) for k, v in interfaces.items()} + clocks = {k: normalize_clocks(v) for k, v in clocks.items()} + resets = {k: normalize_resets(v) for k, v in resets.items()} + tieoffs = {k: normalize_tieoffs(v) for k, v in tieoffs.items()} # build up output lines @@ -223,68 +229,107 @@ def autowrap( '' ] + # wire declarations + + wires = {} + lines += [''] - for name, value in interfaces.items(): - type = value['type'] - direction = value['direction'] + for instance in instances: + for name, value in interfaces[instance].items(): + type = value['type'] - if type == 'sb': - dw = value['dw'] + if type not in wires: + wires[type] = set() - lines += [tab + f'`SB_WIRES({name}, {dw});'] + wire = value['wire'] - if direction_is_input(direction): - lines += [tab + f'`QUEUE_TO_SB_SIM({name}, {dw}, "");'] - elif direction_is_output(direction): - lines += [tab + f'`SB_TO_QUEUE_SIM({name}, {dw}, "");'] + if wire not in wires[type]: + decl_wire = True + wires[type].add(wire) else: - raise Exception(f'Unsupported SB direction: {direction}') - elif type == 'umi': - dw = value['dw'] - cw = value['cw'] - aw = value['aw'] - - lines += [tab + f'`SB_UMI_WIRES({name}, {dw}, {cw}, {aw});'] - - if direction_is_input(direction): - lines += [tab + f'`QUEUE_TO_UMI_SIM({name}, {dw}, {cw}, {aw}, "");'] - elif direction_is_output(direction): - lines += [tab + f'`UMI_TO_QUEUE_SIM({name}, {dw}, {cw}, {aw}, "");'] + decl_wire = False + + direction = value['direction'] + + external = value['external'] + + if type == 'sb': + dw = value['dw'] + + if decl_wire: + lines += [tab + f'`SB_WIRES({wire}, {dw});'] + + if external: + if direction_is_input(direction): + lines += [tab + f'`QUEUE_TO_SB_SIM({wire}, {dw}, "");'] + elif direction_is_output(direction): + lines += [tab + f'`SB_TO_QUEUE_SIM({wire}, {dw}, "");'] + else: + raise Exception(f'Unsupported SB direction: {direction}') + elif type == 'umi': + dw = value['dw'] + cw = value['cw'] + aw = value['aw'] + + if decl_wire: + lines += [tab + f'`SB_UMI_WIRES({wire}, {dw}, {cw}, {aw});'] + + if external: + if direction_is_input(direction): + lines += [tab + f'`QUEUE_TO_UMI_SIM({wire}, {dw}, {cw}, {aw}, "");'] + elif direction_is_output(direction): + lines += [tab + f'`UMI_TO_QUEUE_SIM({wire}, {dw}, {cw}, {aw}, "");'] + else: + raise Exception(f'Unsupported UMI direction: {direction}') + elif type == 'axi': + dw = value['dw'] + aw = value['aw'] + idw = value['idw'] + + if decl_wire: + lines += [tab + f'`SB_AXI_WIRES({wire}, {dw}, {aw}, {idw});'] + + if external: + if direction_is_subordinate(direction): + lines += [tab + f'`SB_AXI_M({wire}, {dw}, {aw}, {idw}, "");'] + elif direction_is_manager(direction): + lines += [tab + f'`SB_AXI_S({wire}, {dw}, {aw}, "");'] + else: + raise Exception(f'Unsupported AXI direction: {direction}') + elif type == 'axil': + dw = value['dw'] + aw = value['aw'] + + if decl_wire: + lines += [tab + f'`SB_AXIL_WIRES({wire}, {dw}, {aw});'] + + if external: + if direction_is_subordinate(direction): + lines += [tab + f'`SB_AXIL_M({wire}, {dw}, {aw}, "");'] + elif direction_is_manager(direction): + lines += [tab + f'`SB_AXIL_S({wire}, {dw}, {aw}, "");'] + else: + raise Exception(f'Unsupported AXI-Lite direction: {direction}') else: - raise Exception(f'Unsupported UMI direction: {direction}') - elif type == 'axi': - dw = value['dw'] - aw = value['aw'] - idw = value['idw'] - - lines += [tab + f'`SB_AXI_WIRES({name}, {dw}, {aw}, {idw});'] - - if direction_is_subordinate(direction): - lines += [tab + f'`SB_AXI_M({name}, {dw}, {aw}, {idw}, "");'] - elif direction_is_manager(direction): - lines += [tab + f'`SB_AXI_S({name}, {dw}, {aw}, "");'] - else: - raise Exception(f'Unsupported AXI direction: {direction}') - elif type == 'axil': - dw = value['dw'] - aw = value['aw'] + raise Exception(f'Unsupported interface type: "{type}"') - lines += [tab + f'`SB_AXIL_WIRES({name}, {dw}, {aw});'] + lines += [''] - if direction_is_subordinate(direction): - lines += [tab + f'`SB_AXIL_M({name}, {dw}, {aw}, "");'] - elif direction_is_manager(direction): - lines += [tab + f'`SB_AXIL_S({name}, {dw}, {aw}, "");'] - else: - raise Exception(f'Unsupported AXI-Lite direction: {direction}') - else: - raise Exception(f'Unsupported interface type: "{type}"') + max_rst_dly = None - lines += [''] + for inst_resets in resets.values(): + if len(inst_resets) > 0: + # find the max reset delay for this instance + inst_max_rst_dly = max(reset['delay'] for reset in inst_resets) - if len(resets) > 0: - max_rst_dly = max(reset['delay'] for reset in resets) + # update the overall max reset delay + if (max_rst_dly is None) or (inst_max_rst_dly > max_rst_dly): + max_rst_dly = inst_max_rst_dly + + if max_rst_dly is not None: + max_rst_dly = max(max(reset['delay'] for reset in inst_resets) + for inst_resets in resets.values()) lines += [ tab + f"reg [{max_rst_dly}:0] rstvec = '1;" @@ -302,74 +347,78 @@ def autowrap( '' ] - if len(parameters) > 0: - lines += [tab + f'{design} #('] - for n, (key, value) in enumerate(parameters.items()): - line = (2 * tab) + f'.{key}({value})' + for instance, module in instances.items(): + # start of the instantiation - if n != len(parameters) - 1: - line += ',' + if len(parameters[instance]) > 0: + lines += [tab + f'{module} #('] + for n, (key, value) in enumerate(parameters[instance].items()): + line = (2 * tab) + f'.{key}({value})' - lines += [line] - lines += [tab + f') {design}_i ('] - else: - lines += [tab + f'{design} {design}_i ('] + if n != len(parameters[instance]) - 1: + line += ',' - connections = [] + lines += [line] + lines += [tab + f') {instance} ('] + else: + lines += [tab + f'{module} {instance} ('] - # interfaces + connections = [] - for name, value in interfaces.items(): - type = value['type'] + # interfaces - if type_is_sb(type): - connections += [f'`SB_CONNECT({name}, {name})'] - elif type_is_umi(type): - connections += [f'`SB_UMI_CONNECT({name}, {name})'] - elif type_is_axi(type): - connections += [f'`SB_AXI_CONNECT({name}, {name})'] - elif type_is_axil(type): - connections += [f'`SB_AXIL_CONNECT({name}, {name})'] + for name, value in interfaces[instance].items(): + type = value['type'] + wire = value['wire'] - # clocks + if type_is_sb(type): + connections += [f'`SB_CONNECT({name}, {wire})'] + elif type_is_umi(type): + connections += [f'`SB_UMI_CONNECT({name}, {wire})'] + elif type_is_axi(type): + connections += [f'`SB_AXI_CONNECT({name}, {wire})'] + elif type_is_axil(type): + connections += [f'`SB_AXIL_CONNECT({name}, {wire})'] - for clock in clocks: - connections += [f'.{clock["name"]}(clk)'] + # clocks - # resets + for clock in clocks[instance]: + connections += [f'.{clock["name"]}(clk)'] - for reset in resets: - name = reset['name'] - polarity = reset['polarity'] - delay = reset['delay'] - - if polarity_is_positive(polarity): - value = f'rstvec[{delay}]' - elif polarity_is_negative(polarity): - value = f'~rstvec[{delay}]' - else: - raise ValueError(f'Unsupported reset polarity: "{polarity}"') + # resets - connections += [f'.{name}({value})'] + for reset in resets[instance]: + name = reset['name'] + polarity = reset['polarity'] + delay = reset['delay'] - # tieoffs + if polarity_is_positive(polarity): + value = f'rstvec[{delay}]' + elif polarity_is_negative(polarity): + value = f'~rstvec[{delay}]' + else: + raise ValueError(f'Unsupported reset polarity: "{polarity}"') - for key, value in tieoffs.items(): - if value is None: - value = '' - else: - value = str(value) - connections += [f'.{key}({value})'] + connections += [f'.{name}({value})'] - for n, connection in enumerate(connections): - if n != len(connections) - 1: - connection += ',' - lines += [(2 * tab) + connection] + # tieoffs - lines += [tab + ');'] - lines += [''] + for key, value in tieoffs[instance].items(): + if value is None: + value = '' + else: + value = str(value) + connections += [f'.{key}({value})'] + + for n, connection in enumerate(connections): + if n != len(connections) - 1: + connection += ',' + lines += [(2 * tab) + connection] - # initialize queue connections + lines += [tab + ');'] + lines += [''] + + # initialize queue connections for this instance lines += [ tab + 'string uri_sb_value;', @@ -378,12 +427,20 @@ def autowrap( (2 * tab) + '/* verilator lint_off IGNOREDRETURN */' ] - for name, value in interfaces.items(): - lines += [ - (2 * tab) + f'if($value$plusargs("{name}=%s", uri_sb_value)) begin', - (3 * tab) + f'{name}_sb_inst.init(uri_sb_value);', - (2 * tab) + 'end' - ] + for inst_interfaces in interfaces.values(): + for value in inst_interfaces.values(): + external = value['external'] + + if not external: + continue + + wire = value['wire'] + + lines += [ + (2 * tab) + f'if($value$plusargs("{wire}=%s", uri_sb_value)) begin', + (3 * tab) + f'{wire}_sb_inst.init(uri_sb_value);', + (2 * tab) + 'end' + ] lines += [ (2 * tab) + '/* verilator lint_on IGNOREDRETURN */', diff --git a/switchboard/cmdline.py b/switchboard/cmdline.py index 486b5a51..54f3f3b2 100644 --- a/switchboard/cmdline.py +++ b/switchboard/cmdline.py @@ -10,6 +10,8 @@ def get_cmdline_args( max_rate: float = -1, start_delay: float = None, fast: bool = False, + single_netlist: bool = False, + threads: int = None, extra_args: dict = None ): """ @@ -105,11 +107,21 @@ def get_cmdline_args( help='Delay before starting simulation, in seconds. Can be useful to prevent' ' simulations from stepping on each others toes when starting up.') + if not single_netlist: + parser.add_argument('--single-netlist', action='store_true', help='Run in single-netlist' + ' mode, where the network is constructed in Verilog and run in a single simulator.') + else: + parser.add_argument('--distributed', action='store_true', help='Run in distributed' + ' simulation mode, rather than single-netlist mode.') + + parser.add_argument('--threads', type=int, default=threads, + help='Number of threads to use when running a simulation.') + if extra_args is not None: for k, v in extra_args.items(): parser.add_argument(k, **v) - args = parser.parse_args() + args, _ = parser.parse_known_args() # standardize boolean flags @@ -121,6 +133,10 @@ def get_cmdline_args( args.fast = not args.rebuild del args.rebuild + if single_netlist: + args.single_netlist = not args.distributed + del args.distributed + # return arguments return args diff --git a/switchboard/network.py b/switchboard/network.py index 4cc0fb41..0cd68728 100644 --- a/switchboard/network.py +++ b/switchboard/network.py @@ -1,13 +1,16 @@ # Copyright (c) 2024 Zero ASIC Corporation # This code is licensed under Apache License 2.0 (see LICENSE for details) + +from pathlib import Path from copy import deepcopy from itertools import count from .sbdut import SbDut from .axi import axi_uris from .autowrap import (directions_are_compatible, normalize_intf_type, - type_is_umi, type_is_sb, create_intf_objs, type_is_axi, type_is_axil) + type_is_umi, type_is_sb, create_intf_objs, type_is_axi, type_is_axil, + autowrap) from .cmdline import get_cmdline_args from _switchboard import delete_queues @@ -24,17 +27,19 @@ def __init__(self, name, block): self.name = name self.block = block self.mapping = {} + self.external = set() for name, value in block.intf_defs.items(): - self.mapping[name] = None + self.mapping[name] = dict(uri=None, wire=None) self.__setattr__(name, SbIntf(inst=self, name=name)) class SbNetwork: def __init__(self, cmdline=False, tool: str = 'verilator', trace: bool = False, trace_type: str = 'vcd', frequency: float = 100e6, period: float = None, - max_rate: float = None, start_delay: float = None, fast: bool = False, - extra_args: dict = None, cleanup: bool = True, args=None): + max_rate: float = -1, start_delay: float = None, fast: bool = False, + extra_args: dict = None, cleanup: bool = True, args=None, + single_netlist: bool = False, threads: int = None, name: str = None): self.insts = {} @@ -44,12 +49,11 @@ def __init__(self, cmdline=False, tool: str = 'verilator', trace: bool = False, self.uri_set = set() self.uri_counters = {} - self.intf_defs = {} - if cmdline: self.args = get_cmdline_args(tool=tool, trace=trace, trace_type=trace_type, frequency=frequency, period=period, fast=fast, max_rate=max_rate, - start_delay=start_delay, extra_args=extra_args) + start_delay=start_delay, single_netlist=single_netlist, threads=threads, + extra_args=extra_args) elif args is not None: self.args = args else: @@ -64,6 +68,7 @@ def __init__(self, cmdline=False, tool: str = 'verilator', trace: bool = False, period = self.args.period max_rate = self.args.max_rate start_delay = self.args.start_delay + single_netlist = self.args.single_netlist # save settings @@ -78,6 +83,15 @@ def __init__(self, cmdline=False, tool: str = 'verilator', trace: bool = False, self.max_rate = max_rate self.start_delay = start_delay + self.single_netlist = single_netlist + + if single_netlist: + self.dut = SbDut(args=self.args) + else: + self._intf_defs = {} + + self.name = name + if cleanup: import atexit @@ -87,10 +101,26 @@ def cleanup_func(uri_set=self.uri_set): atexit.register(cleanup_func) - def instantiate(self, block: SbDut, name: str = None): + @property + def intf_defs(self): + if self.single_netlist: + return self.dut.intf_defs + else: + return self._intf_defs + + def instantiate(self, block, name: str = None): # generate a name if needed if name is None: - name = self.generate_inst_name(prefix=block.dut) + if isinstance(block, SbDut): + prefix = block.dut + else: + prefix = block.name + + assert prefix is not None, ('Cannot generate name for this instance.' + ' When block is an SbNetwork, make sure that its constructor set' + ' "name" if you want name generation to work here.') + + name = self.generate_inst_name(prefix=prefix) # make sure the name hasn't been used already assert name not in self.inst_name_set @@ -104,7 +134,7 @@ def instantiate(self, block: SbDut, name: str = None): # return the instance object return self.insts[name] - def connect(self, a, b, uri=None): + def connect(self, a, b, uri=None, wire=None): # retrieve the two interface definitions intf_def_a = a.inst.block.intf_defs[a.name] intf_def_b = b.inst.block.intf_defs[b.name] @@ -120,25 +150,80 @@ def connect(self, a, b, uri=None): # determine what the queue will be called that connects the two + if wire is None: + wire = f'{a.inst.name}_{a.name}_conn_{b.inst.name}_{b.name}' + if uri is None: - uri = f'{a.inst.name}_{a.name}_conn_{b.inst.name}_{b.name}' + uri = wire if type_is_sb(type_a) or type_is_umi(type_a): uri = uri + '.q' - self.register_uri(type=type_a, uri=uri) + if not self.single_netlist: + # internal connection, no need to register it for cleanup + self.register_uri(type=type_a, uri=uri) # tell both instances what they are connected to - a.inst.mapping[a.name] = uri - b.inst.mapping[b.name] = uri + + a.inst.mapping[a.name]['wire'] = wire + b.inst.mapping[b.name]['wire'] = wire + + a.inst.mapping[a.name]['uri'] = uri + b.inst.mapping[b.name]['uri'] = uri def build(self): unique_blocks = set(inst.block for inst in self.insts.values()) - for block in unique_blocks: - block.build() + if self.single_netlist: + passthroughs = [ + ('tool', 'verilator', 'task', 'compile', 'warningoff') + ] + + for block in unique_blocks: + self.dut.input(block.package()) + + for passthrough in passthroughs: + self.dut.add(*passthrough, block.get(*passthrough)) + + filename = Path(self.dut.get('option', 'builddir')).resolve() / 'testbench.sv' + + filename.parent.mkdir(exist_ok=True, parents=True) - def external(self, intf, name=None, txrx=None, uri=None): + # populate the interfaces dictionary + interfaces = {} + + for inst_name, inst in self.insts.items(): + # make a copy of the interface definitions for this block + intf_defs = deepcopy(inst.block.intf_defs) + + # wiring + for intf_name, props in inst.mapping.items(): + intf_defs[intf_name]['wire'] = props['wire'] + intf_defs[intf_name]['external'] = intf_name in inst.external + + interfaces[inst_name] = intf_defs + + # generate netlist that connects everything together, and input() it + self.dut.input( + autowrap( + instances={inst.name: inst.block.dut for inst in self.insts.values()}, + toplevel='testbench', + parameters={inst.name: inst.block.parameters for inst in self.insts.values()}, + interfaces=interfaces, + clocks={inst.name: inst.block.clocks for inst in self.insts.values()}, + resets={inst.name: inst.block.resets for inst in self.insts.values()}, + tieoffs={inst.name: inst.block.tieoffs for inst in self.insts.values()}, + filename=filename + ) + ) + + # build the single-netlist simulation + self.dut.build() + else: + for block in unique_blocks: + block.build() + + def external(self, intf, name=None, txrx=None, uri=None, wire=None): # make a copy of the interface definition since we will be modifying it intf_def = deepcopy(intf.inst.block.intf_defs[intf.name]) @@ -147,21 +232,27 @@ def external(self, intf, name=None, txrx=None, uri=None): type = intf_def['type'] + if wire is None: + wire = f'{intf.inst.name}_{intf.name}' + + intf_def['wire'] = wire + if uri is None: - uri = f'{intf.inst.name}_{intf.name}' + uri = wire if type_is_sb(type) or type_is_umi(type): uri = uri + '.q' + intf_def['uri'] = uri + # register the URI to make sure it doesn't collide with anything else self.register_uri(type=type, uri=uri) # propagate information about the URI mapping - intf_def['uri'] = uri - - intf.inst.mapping[intf.name] = uri + intf.inst.mapping[intf.name] = dict(uri=uri, wire=wire) + intf.inst.external.add(intf.name) # set txrx @@ -182,49 +273,65 @@ def external(self, intf, name=None, txrx=None, uri=None): self.intf_defs[name] = intf_def - def simulate(self): + return name + + def simulate(self, start_delay=None, run=None, intf_objs=True): + # set defaults + + if start_delay is None: + start_delay = self.start_delay + # create interface objects - self.intfs = create_intf_objs(self.intf_defs) + if self.single_netlist: + self.dut.simulate(start_delay=start_delay, run=run, intf_objs=intf_objs) + + if intf_objs: + self.intfs = self.dut.intfs + else: + if intf_objs: + self.intfs = create_intf_objs(self.intf_defs) + + if start_delay is not None: + import time + start = time.time() - if self.start_delay is not None: - import time - start = time.time() + insts = self.insts.values() - insts = self.insts.values() + try: + from tqdm import tqdm + insts = tqdm(insts) + except ModuleNotFoundError: + pass - try: - from tqdm import tqdm - insts = tqdm(insts) - except ModuleNotFoundError: - pass + for inst in insts: + block = inst.block - for inst in insts: - block = inst.block + for intf_name, props in inst.mapping.items(): + # check that the interface is wired up - for intf_name, uri in inst.mapping.items(): - # check that the interface is wired up + uri = props['uri'] - if uri is None: - raise Exception(f'{inst.name}.{intf_name} not connected') + if uri is None: + raise Exception(f'{inst.name}.{intf_name} not connected') - block.intf_defs[intf_name]['uri'] = uri + block.intf_defs[intf_name]['uri'] = uri - # calculate the start delay for this process by measuring the - # time left until the start delay for the whole network is over + # calculate the start delay for this process by measuring the + # time left until the start delay for the whole network is over - if self.start_delay is not None: - now = time.time() - dt = now - start - if dt < self.start_delay: - start_delay = self.start_delay - dt + if start_delay is not None: + now = time.time() + dt = now - start + if dt < start_delay: + start_delay = start_delay - dt + else: + start_delay = None else: start_delay = None - else: - start_delay = None - # launch an instance of simulation - block.simulate(run=inst.name, intf_objs=False, start_delay=start_delay) + # launch an instance of simulation + block.simulate(start_delay=start_delay, run=inst.name, intf_objs=False) def generate_inst_name(self, prefix): if prefix not in self.inst_name_counters: @@ -252,3 +359,28 @@ def register_uri(self, type, uri, fresh=True): if fresh: delete_queues(uris) + + def make_dut(self, *args, **kwargs): + # argument customizations + + cfg = {} + + cfg['args'] = self.args + + if self.single_netlist: + cfg['autowrap'] = False + cfg['subcomponent'] = True + else: + cfg['autowrap'] = True + cfg['subcomponent'] = False + + # add to keyword arguments without clobbering + # existing entries + + kwargs = deepcopy(kwargs) + + for k, v in cfg.items(): + if k not in kwargs: + kwargs[k] = v + + return SbDut(*args, **kwargs) diff --git a/switchboard/sbdut.py b/switchboard/sbdut.py index 16f56ee0..fa6aaabf 100644 --- a/switchboard/sbdut.py +++ b/switchboard/sbdut.py @@ -28,7 +28,6 @@ from .cmdline import get_cmdline_args import siliconcompiler -from siliconcompiler.flows import dvflow SB_DIR = sb_path() @@ -62,7 +61,10 @@ def __init__( tieoffs=None, buildroot=None, builddir=None, - args=None + args=None, + subcomponent=False, + suffix=None, + threads=None ): """ Parameters @@ -137,7 +139,7 @@ def __init__( # call the super constructor - if autowrap: + if autowrap and (not subcomponent): toplevel = 'testbench' else: toplevel = design @@ -149,7 +151,7 @@ def __init__( if cmdline: self.args = get_cmdline_args(tool=tool, trace=trace, trace_type=trace_type, frequency=frequency, period=period, fast=fast, max_rate=max_rate, - start_delay=start_delay, extra_args=extra_args) + start_delay=start_delay, threads=threads, extra_args=extra_args) elif args is not None: self.args = args else: @@ -164,6 +166,7 @@ def __init__( period = self.args.period max_rate = self.args.max_rate start_delay = self.args.start_delay + threads = self.args.threads # input validation @@ -186,11 +189,23 @@ def __init__( self.max_rate = max_rate self.start_delay = start_delay + self.threads = threads + self.timeunit = timeunit self.timeprecision = timeprecision self.autowrap = autowrap - self.dut = design + + if (suffix is None) and subcomponent: + suffix = f'_unq_{design}' + + self.suffix = suffix + + if suffix is not None: + self.dut = f'{design}{suffix}' + else: + self.dut = design + self.parameters = normalize_parameters(parameters) self.intf_defs = normalize_interfaces(interfaces) self.clocks = normalize_clocks(clocks) @@ -209,52 +224,80 @@ def __init__( buildroot = Path(buildroot).resolve() - builddir = buildroot / metadata_str(design=design, parameters=parameters, tool=tool, - trace=trace, trace_type=trace_type) + if subcomponent: + # the subcomponent build flow is tool-agnostic, producing a single Verilog + # file as output, as opposed to a simulator binary + builddir = buildroot / metadata_str(design=design, parameters=parameters) + else: + builddir = buildroot / metadata_str(design=design, parameters=parameters, + tool=tool, trace=trace, trace_type=trace_type, threads=threads) self.set('option', 'builddir', str(Path(builddir).resolve())) - if fpga: - # library dirs - self.set('option', 'ydir', sb_path() / 'verilog' / 'fpga') - self.add('option', 'ydir', sb_path() / 'deps' / 'verilog-axi' / 'rtl') + self.set('option', 'mode', 'sim') - # include dirs - self.set('option', 'idir', sb_path() / 'verilog' / 'fpga' / 'include') + if not subcomponent: + if fpga: + # library dirs + self.set('option', 'ydir', sb_path() / 'verilog' / 'fpga') + self.add('option', 'ydir', sb_path() / 'deps' / 'verilog-axi' / 'rtl') - for opt in ['ydir', 'idir']: - if not fpga: - self.set('option', opt, sb_path() / 'verilog' / 'sim') - self.add('option', opt, sb_path() / 'verilog' / 'common') + # include dirs + self.set('option', 'idir', sb_path() / 'verilog' / 'fpga' / 'include') - self.set('option', 'mode', 'sim') + for opt in ['ydir', 'idir']: + if not fpga: + self.set('option', opt, sb_path() / 'verilog' / 'sim') + self.add('option', opt, sb_path() / 'verilog' / 'common') - if trace: - self.set('option', 'trace', True) - self.set('option', 'define', 'SB_TRACE') + if trace: + self.set('option', 'trace', True) + self.set('option', 'define', 'SB_TRACE') - if self.trace_type == 'fst': - self.set('option', 'define', 'SB_TRACE_FST') + if self.trace_type == 'fst': + self.set('option', 'define', 'SB_TRACE_FST') - if tool == 'icarus': - self._configure_icarus() + if tool == 'icarus': + self._configure_icarus() + else: + if module is None: + if tool == 'verilator': + module = 'siliconcompiler' + else: + raise ValueError('Must specify the "module" argument,' + ' which is the name of the module containing the' + ' SiliconCompiler driver for this simulator.') + + self._configure_build( + module=module, + default_main=default_main, + fpga=fpga + ) + + if xyce: + self._configure_xyce() else: - if module is None: - if tool == 'verilator': - module = 'siliconcompiler' - else: - raise ValueError('Must specify the "module" argument,' - ' which is the name of the module containing the' - ' SiliconCompiler driver for this simulator.') - - self._configure_build( - module=module, - default_main=default_main, - fpga=fpga - ) + # special mode that produces a standalone Verilog netlist + # rather than building/running a simulation + + flowname = 'package' + + self.package_flow = siliconcompiler.Flow(self, flowname) - if xyce: - self._configure_xyce() + from siliconcompiler.tools.surelog import parse + self.package_flow.node(flowname, 'parse', parse) + + from .sc.sed import remove + self.package_flow.node(flowname, 'remove', remove) + + from .sc.morty import uniquify + self.package_flow.node(flowname, 'uniquify', uniquify) + + self.package_flow.edge(flowname, 'parse', 'remove') + self.package_flow.edge(flowname, 'remove', 'uniquify') + + self.use(self.package_flow) + self.set('option', 'flow', flowname) def _configure_build( self, @@ -306,6 +349,10 @@ def _configure_build( self.add('tool', 'verilator', 'task', 'compile', 'option', '--timescale') self.add('tool', 'verilator', 'task', 'compile', 'option', timescale) + if (self.threads is not None) and (self.tool == 'verilator'): + self.add('tool', 'verilator', 'task', 'compile', 'option', '--threads') + self.add('tool', 'verilator', 'task', 'compile', 'option', str(self.threads)) + self.set('option', 'libext', ['v', 'sv']) # Set up flow that compiles RTL @@ -320,7 +367,9 @@ def _configure_icarus(self): self.set('tool', 'icarus', 'task', 'compile', 'var', 'verilog_generation', '2012') # use dvflow to execute Icarus, but set steplist so we don't run sim + from siliconcompiler.flows import dvflow self.use(dvflow) + self.set('option', 'flow', 'dvflow') self.set('option', 'to', 'compile') @@ -389,9 +438,17 @@ def build(self, cwd: str = None, fast: bool = None): filename.parent.mkdir(exist_ok=True, parents=True) - autowrap(design=self.dut, parameters=self.parameters, - interfaces=self.intf_defs, clocks=self.clocks, resets=self.resets, - tieoffs=self.tieoffs, filename=filename) + instance = f'{self.dut}_i' + + autowrap( + instances={instance: self.dut}, + parameters={instance: self.parameters}, + interfaces={instance: self.intf_defs}, + clocks={instance: self.clocks}, + resets={instance: self.resets}, + tieoffs={instance: self.tieoffs}, + filename=filename + ) self.input(filename) @@ -497,7 +554,7 @@ def simulate( # add plusargs that define queue connections for name, value in self.intf_defs.items(): - plusargs += [(name, value['uri'])] + plusargs += [(value['wire'], value['uri'])] # run-specific configurations (if running the same simulator build multiple times # in parallel) @@ -671,9 +728,44 @@ def input_analog( self.input(verilog_wrapper) + def package(self, suffix=None, fast=None): + # set defaults + + if suffix is None: + suffix = self.suffix + + if fast is None: + fast = self.fast + + # see if we can exit early + + if fast: + package = self.find_package(suffix=suffix) + + if package is not None: + return package + + # if not, parse with surelog and postprocess with morty + + if suffix: + self.set('tool', 'morty', 'task', 'uniquify', 'var', 'suffix', suffix) + + self.set('tool', 'sed', 'task', 'remove', 'var', 'to_remove', '`resetall') + + self.run() -def metadata_str(design: str, tool: str, trace: bool, trace_type: str, - parameters: dict = None) -> Path: + # return the path to the output + return self.find_package(suffix=suffix) + + def find_package(self, suffix=None): + if suffix is None: + return self.find_result('v', step='parse') + else: + return self.find_result('v', step='uniquify') + + +def metadata_str(design: str, tool: str = None, trace: bool = False, + trace_type: str = None, threads: int = None, parameters: dict = None) -> Path: opts = [] @@ -683,12 +775,16 @@ def metadata_str(design: str, tool: str, trace: bool, trace_type: str, for k, v in parameters.items(): opts += [k, v] - opts += [tool] + if tool is not None: + opts += [tool] if trace: assert trace_type is not None opts += [trace_type] + if threads is not None: + opts += ['threads', threads] + return '-'.join(str(opt) for opt in opts) diff --git a/switchboard/sc/__init__.py b/switchboard/sc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/switchboard/sc/morty/__init__.py b/switchboard/sc/morty/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/switchboard/sc/morty/morty.py b/switchboard/sc/morty/morty.py new file mode 100644 index 00000000..97bdccdc --- /dev/null +++ b/switchboard/sc/morty/morty.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024 Zero ASIC Corporation +# This code is licensed under Apache License 2.0 (see LICENSE for details) + +def setup(chip): + tool = 'morty' + + chip.set('tool', tool, 'exe', 'morty') + chip.set('tool', tool, 'vendor', tool) + + chip.set('tool', tool, 'vswitch', '--version') + + +def parse_version(stdout): + return stdout.split()[-1] diff --git a/switchboard/sc/morty/uniquify.py b/switchboard/sc/morty/uniquify.py new file mode 100644 index 00000000..6512127b --- /dev/null +++ b/switchboard/sc/morty/uniquify.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024 Zero ASIC Corporation +# This code is licensed under Apache License 2.0 (see LICENSE for details) + +from .morty import setup as setup_tool + + +def setup(chip): + '''Task that uses morty to rewrite a Verilog file with a unique prefix and/or + suffix for all module definitions.''' + + setup_tool(chip) + + tool = 'morty' + step = chip.get('arg', 'step') + index = chip.get('arg', 'index') + task = chip._get_task(step, index) + + chip.set('tool', tool, 'task', task, 'var', 'suffix', + 'suffix to be added to the end of module names', + field='help') + + chip.set('tool', tool, 'task', task, 'var', 'prefix', + 'prefix to be added to the beginning of module names', + field='help') + + +def runtime_options(chip): + tool = 'morty' + step = chip.get('arg', 'step') + index = chip.get('arg', 'index') + task = chip._get_task(step, index) + design = chip.top() + + cmdlist = [] + + prefix = chip.get('tool', tool, 'task', task, 'var', 'prefix', step=step, index=index) + if prefix: + if isinstance(prefix, list) and (len(prefix) == 1) and isinstance(prefix[0], str): + cmdlist = ['--prefix', prefix[0]] + cmdlist + else: + raise ValueError('"prefix" does not have the expected format') + + suffix = chip.get('tool', tool, 'task', task, 'var', 'suffix', step=step, index=index) + if suffix: + if isinstance(suffix, list) and (len(suffix) == 1) and isinstance(suffix[0], str): + cmdlist = ['--suffix', suffix[0]] + cmdlist + else: + raise ValueError('"suffix" does not have the expected format') + + outfile = f'outputs/{design}.v' + cmdlist += ['-o', outfile] + + infile = f'inputs/{design}.v' + cmdlist += [infile] + + return cmdlist diff --git a/switchboard/sc/sed/__init__.py b/switchboard/sc/sed/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/switchboard/sc/sed/remove.py b/switchboard/sc/sed/remove.py new file mode 100644 index 00000000..006717e6 --- /dev/null +++ b/switchboard/sc/sed/remove.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024 Zero ASIC Corporation +# This code is licensed under Apache License 2.0 (see LICENSE for details) + +from .sed import setup as setup_tool + + +def setup(chip): + '''Task that removes specific strings from a Verilog source file.''' + + setup_tool(chip) + + tool = 'sed' + step = chip.get('arg', 'step') + index = chip.get('arg', 'index') + task = chip._get_task(step, index) + + chip.set('tool', tool, 'task', task, 'var', 'to_remove', + 'strings to remove from the Verilog source file', + field='help') + + +def runtime_options(chip): + tool = 'sed' + step = chip.get('arg', 'step') + index = chip.get('arg', 'index') + task = chip._get_task(step, index) + design = chip.top() + + infile = f'inputs/{design}.v' + outfile = f'outputs/{design}.v' + + to_remove = chip.get('tool', tool, 'task', task, 'var', 'to_remove', step=step, index=index) + + script = [f's/{elem}//g' for elem in to_remove] + script += [f'w {outfile}'] + + script = '; '.join(script) + + cmdlist = ['-n', f'"{script}"', infile] + + return cmdlist diff --git a/switchboard/sc/sed/sed.py b/switchboard/sc/sed/sed.py new file mode 100644 index 00000000..9a1d2950 --- /dev/null +++ b/switchboard/sc/sed/sed.py @@ -0,0 +1,8 @@ +# Copyright (c) 2024 Zero ASIC Corporation +# This code is licensed under Apache License 2.0 (see LICENSE for details) + +def setup(chip): + tool = 'sed' + + chip.set('tool', tool, 'exe', 'sed') + chip.set('tool', tool, 'vendor', tool) diff --git a/switchboard/test_util.py b/switchboard/test_util.py index 65d64922..648d5cf7 100644 --- a/switchboard/test_util.py +++ b/switchboard/test_util.py @@ -24,19 +24,19 @@ def test_cmd(args, expected=None, path=None): if isinstance(args, str): args = [args] args = [str(arg) for arg in args] - result = subprocess.run(args, check=True, capture_output=True, - text=True, cwd=cwd) - # print the output + process = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, bufsize=1, text=True, cwd=cwd) - stdout = result.stdout - if (stdout is not None) and (stdout != ''): - print(stdout, end='', flush=True) + # print output while saving it + stdout = '' + for line in process.stdout: + print(line, end='') + stdout += line - stderr = result.stderr - if (stderr is not None) and (stderr != ''): - print('### STDERR ###') - print(stderr, end='', flush=True) + # make sure that process exits cleanly + returncode = process.wait() + assert returncode == 0, f'Exited with non-zero code: {returncode}' # check the results if expected is not None: