From 8e27ed97e6a5f0eef3c36724d09bd17e2be338d1 Mon Sep 17 00:00:00 2001 From: Romain Dolbeau Date: Sat, 9 Jan 2021 08:17:46 -0500 Subject: [PATCH] pipeline AES DMAs --- NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.c | 6 +-- NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.h | 1 + sbus-to-ztex-gateware/aes_wrapper.vhd | 27 ++++++---- sbus-to-ztex-gateware/sbus_fsm.vhd | 68 +++++++++++++++++------- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.c b/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.c index 0c354a2..f6ff185 100644 --- a/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.c +++ b/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.c @@ -1084,11 +1084,11 @@ rdfpga_encdec_aes128cbc(struct rdfpga_softc *sw, const u_int8_t thesid, struct c memcpy(kvap, idat, tocopy); bus_dmamap_sync(sw->sc_dmatag, sw->sc_dmamap, 0, tocopy, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); - /* prepare write w/o start */ - ctrl = ((uint64_t)(RDFPGA_MASK_DMA_CTRL_AES | ((tocopy/16)-1))) | ((uint64_t)(uint32_t)(sw->sc_dmamap->dm_segs[0].ds_addr)) << 32; + /* start write */ + ctrl = ((uint64_t)(RDFPGA_MASK_DMA_CTRL_START | RDFPGA_MASK_DMA_CTRL_AES | ((tocopy/16)-1))) | ((uint64_t)(uint32_t)(sw->sc_dmamap->dm_segs[0].ds_addr)) << 32; bus_space_write_8(sw->sc_bustag, sw->sc_bhregs, (RDFPGA_REG_DMAW_ADDR), ctrl); /* start read */ - ctrl = ((uint64_t)(RDFPGA_MASK_DMA_CTRL_START | RDFPGA_MASK_DMA_CTRL_AES | ((tocopy/16)-1))) | ((uint64_t)(uint32_t)(sw->sc_dmamap->dm_segs[0].ds_addr)) << 32; + ctrl = ((uint64_t)(RDFPGA_MASK_DMA_CTRL_START | RDFPGA_MASK_DMA_CTRL_AES | RDFPGA_MASK_DMA_CTRL_CBC | ((tocopy/16)-1))) | ((uint64_t)(uint32_t)(sw->sc_dmamap->dm_segs[0].ds_addr)) << 32; bus_space_write_8(sw->sc_bustag, sw->sc_bhregs, (RDFPGA_REG_DMA_ADDR), ctrl); rdfpga_wait_dma_ready(sw, 50000); bus_dmamap_sync(sw->sc_dmatag, sw->sc_dmamap, 0, tocopy, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); diff --git a/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.h b/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.h index 3d986dc..4f44314 100644 --- a/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.h +++ b/NetBSD/9.0/usr/src/sys/dev/sbus/rdfpga.h @@ -79,6 +79,7 @@ struct rdfpga_softc { #define RDFPGA_MASK_DMA_CTRL_WRITE 0x10000000 /* for AES only */ #define RDFPGA_MASK_DMA_CTRL_GCM 0x08000000 #define RDFPGA_MASK_DMA_CTRL_AES 0x04000000 +#define RDFPGA_MASK_DMA_CTRL_CBC 0x02000000 #define RDFPGA_MASK_DMA_CTRL_BLKCNT 0x00000FFF #define RDFPGA_VAL_DMA_MAX_BLKCNT 4096 #define RDFPGA_VAL_DMA_MAX_SZ (RDFPGA_VAL_DMA_MAX_BLKCNT*16) diff --git a/sbus-to-ztex-gateware/aes_wrapper.vhd b/sbus-to-ztex-gateware/aes_wrapper.vhd index 21a368a..15822db 100644 --- a/sbus-to-ztex-gateware/aes_wrapper.vhd +++ b/sbus-to-ztex-gateware/aes_wrapper.vhd @@ -6,8 +6,8 @@ entity aes_wrapper is port ( aes_wrapper_rst : in std_logic; aes_wrapper_clk : in std_logic; --- iskey?, keylen, encdec, cbc, data (256 or 128 + 128) - input_fifo_out : in std_logic_vector(259 downto 0); +-- iskey?, keylen, encdec, cbc, internal cbc, data (256 or 128 + 128) + input_fifo_out : in std_logic_vector(260 downto 0); input_fifo_empty: in std_logic; input_fifo_rd_en : out std_logic; -- data (128) @@ -28,11 +28,14 @@ entity aes_wrapper is signal aes_block : std_logic_vector(127 downto 0); signal aes_result : std_logic_vector(127 downto 0); signal aes_result_valid : std_logic; + + signal aes_last : std_logic_vector(127 downto 0); - constant iskey_idx : integer := 259; - constant keylen_idx : integer := 258; - constant encdec_idx : integer := 257; - constant cbc_idx : integer := 256; + constant iskey_idx : integer := 260; + constant keylen_idx : integer := 259; + constant encdec_idx : integer := 258; + constant cbc_idx : integer := 257; + constant intcbc_idx : integer := 256; end aes_wrapper; @@ -97,12 +100,15 @@ begin ELSE aes_next <= '1'; aes_encdec <= input_fifo_out(encdec_idx); - IF (input_fifo_out(cbc_idx) = '0') THEN - -- normal mode - aes_block <= input_fifo_out(127 downto 0); - ELSE + IF (input_fifo_out(cbc_idx) = '1') THEN -- cbc mode aes_block <= input_fifo_out(127 downto 0) XOR input_fifo_out(255 downto 128); + ELSIF (input_fifo_out(intcbc_idx) = '1') THEN + -- internal cbc mode + aes_block <= input_fifo_out(127 downto 0) XOR aes_last; + ELSE + -- normal mode + aes_block <= input_fifo_out(127 downto 0); END IF; AES_State <= AES_CRYPT1; END IF; @@ -125,6 +131,7 @@ begin IF (aes_result_valid = '1') then output_fifo_wr_en <= '1'; output_fifo_in <= aes_result; + aes_last <= aes_result; AES_State <= AES_IDLE; END IF; diff --git a/sbus-to-ztex-gateware/sbus_fsm.vhd b/sbus-to-ztex-gateware/sbus_fsm.vhd index 0a813f1..0430245 100644 --- a/sbus-to-ztex-gateware/sbus_fsm.vhd +++ b/sbus-to-ztex-gateware/sbus_fsm.vhd @@ -119,6 +119,7 @@ ENTITY SBusFSM is constant DMA_CTRL_WRITE_IDX : integer := 28; -- unused constant DMA_CTRL_GCM_IDX : integer := 27; constant DMA_CTRL_AES_IDX : integer := 26; + constant DMA_CTRL_CBC_IDX : integer := 25; constant AES128_CTRL_START_IDX : integer := 31; constant AES128_CTRL_BUSY_IDX : integer := 30; @@ -266,10 +267,10 @@ ARCHITECTURE RTL OF SBusFSM IS signal r_TX_BYTE : std_logic_vector(7 downto 0) := (others => '0'); signal aes_wrapper_rst : std_logic := '0'; - signal fifo_toaes_din : STD_LOGIC_VECTOR ( 259 downto 0 ); + signal fifo_toaes_din : STD_LOGIC_VECTOR ( 260 downto 0 ); signal fifo_toaes_wr_en : STD_LOGIC; signal fifo_toaes_rd_en : STD_LOGIC; - signal fifo_toaes_dout : STD_LOGIC_VECTOR ( 259 downto 0 ); + signal fifo_toaes_dout : STD_LOGIC_VECTOR ( 260 downto 0 ); signal fifo_toaes_full : STD_LOGIC; signal fifo_toaes_empty : STD_LOGIC; signal fifo_fromaes_din : STD_LOGIC_VECTOR ( 127 downto 0 ); @@ -494,10 +495,10 @@ ARCHITECTURE RTL OF SBusFSM IS Port ( wr_clk : in STD_LOGIC; rd_clk : in STD_LOGIC; - din : in STD_LOGIC_VECTOR ( 259 downto 0 ); + din : in STD_LOGIC_VECTOR ( 260 downto 0 ); wr_en : in STD_LOGIC; rd_en : in STD_LOGIC; - dout : out STD_LOGIC_VECTOR ( 259 downto 0 ); + dout : out STD_LOGIC_VECTOR ( 260 downto 0 ); full : out STD_LOGIC; empty : out STD_LOGIC ); @@ -544,7 +545,7 @@ ARCHITECTURE RTL OF SBusFSM IS aes_wrapper_rst : in std_logic; aes_wrapper_clk : in std_logic; -- iskey?, keylen, encdec, cbc, data (256 or 128 + 128) - input_fifo_out : in std_logic_vector(259 downto 0); + input_fifo_out : in std_logic_vector(260 downto 0); input_fifo_empty: in std_logic; input_fifo_rd_en : out std_logic; -- data (128) @@ -815,9 +816,11 @@ BEGIN -- we have a DMA request pending and not been granted the bus IF ((REGISTERS(REG_INDEX_DMA_CTRL)(DMA_CTRL_GCM_IDX) = '1') OR ((REGISTERS(REG_INDEX_DMA_CTRL)(DMA_CTRL_AES_IDX) = '1') AND - (REGISTERS(REG_INDEX_AES128_CTRL) = 0)) OR + (REGISTERS(REG_INDEX_AES128_CTRL) = 0) AND + (fifo_toaes_full = '0')) OR ((REGISTERS(REG_INDEX_DMAW_CTRL)(DMA_CTRL_AES_IDX) = '1') AND - (REGISTERS(REG_INDEX_AES128_CTRL) = 0)) + (REGISTERS(REG_INDEX_AES128_CTRL) = 0) AND + (fifo_fromaes_empty = '0')) ) THEN fifo_wr_en <= '1'; fifo_din <= x"61"; -- "a" -- GCM is always available (1 cycle) @@ -835,12 +838,18 @@ BEGIN DATA_T <= '0'; -- set data buffer as output SM_T <= '0'; -- PPRD, SIZ becomes output (master mode) SMs_T <= '1'; - IF (REGISTERS(REG_INDEX_DMAW_CTRL)(DMA_CTRL_START_IDX) = '1') THEN + IF ((REGISTERS(REG_INDEX_DMAW_CTRL)(DMA_CTRL_START_IDX) = '1') AND + (fifo_fromaes_empty = '0')) THEN dma_write := true; dma_ctrl_idx := REG_INDEX_DMAW_CTRL; dma_addr_idx := REG_INDEX_DMAW_ADDR; BUF_DATA_O <= REGISTERS(REG_INDEX_DMAW_ADDR); -- virt address BUF_PPRD_O <= '0'; -- writing to slave + REGISTERS(REG_INDEX_AES128_OUT1) <= fifo_fromaes_dout(127 downto 96); + REGISTERS(REG_INDEX_AES128_OUT2) <= fifo_fromaes_dout( 95 downto 64); + REGISTERS(REG_INDEX_AES128_OUT3) <= fifo_fromaes_dout( 63 downto 32); + REGISTERS(REG_INDEX_AES128_OUT4) <= fifo_fromaes_dout( 31 downto 0); + fifo_fromaes_rd_en <= '1'; ELSE dma_write := false; dma_ctrl_idx := REG_INDEX_DMA_CTRL; @@ -1118,7 +1127,34 @@ BEGIN REGISTERS(REG_INDEX_AES128_DATA1 + (BURST_COUNTER mod 4)) <= BUF_DATA_I; BURST_COUNTER := BURST_COUNTER + 1; IF (BURST_COUNTER mod 4 = 0) THEN - REGISTERS(REG_INDEX_AES128_CTRL) <= x"88000000"; -- request to start a CBC block +-- REGISTERS(REG_INDEX_AES128_CTRL) <= x"88000000"; -- request to start a CBC block + -- enqueue the block in the AES FIFO + IF (REGISTERS(dma_ctrl_idx)(DMA_CTRL_CBC_IDX) = '0') THEN + fifo_toaes_din <= + '0' & -- !iskey + '0' & -- keylen, ignored + '1' & -- encdec + '0' & -- cbc + '1' & -- internal cbc + x"00000000000000000000000000000000" & + REGISTERS(REG_INDEX_AES128_DATA1) & REGISTERS(REG_INDEX_AES128_DATA2) & + REGISTERS(REG_INDEX_AES128_DATA3) & BUF_DATA_I; + fifo_toaes_wr_en <= '1'; + ELSE + fifo_toaes_din <= + '0' & -- !iskey + '0' & -- keylen, ignored + '1' & -- encdec + '0' & -- cbc + '0' & -- internal cbc + x"00000000000000000000000000000000" & + (REGISTERS(REG_INDEX_AES128_DATA1) XOR REGISTERS(REG_INDEX_AES128_OUT1)) & + (REGISTERS(REG_INDEX_AES128_DATA2) XOR REGISTERS(REG_INDEX_AES128_OUT2)) & + (REGISTERS(REG_INDEX_AES128_DATA3) XOR REGISTERS(REG_INDEX_AES128_OUT3)) & + (BUF_DATA_I XOR REGISTERS(REG_INDEX_AES128_OUT4)); + fifo_toaes_wr_en <= '1'; + REGISTERS(dma_ctrl_idx)(DMA_CTRL_CBC_IDX) <= '0'; + END IF; END IF; END IF; -- GCM | AES if (BURST_COUNTER = BURST_LIMIT) THEN @@ -1164,11 +1200,6 @@ BEGIN REGISTERS(dma_ctrl_idx)(11 downto 0) <= REGISTERS(dma_ctrl_idx)(11 downto 0) - (BURST_LIMIT/4); REGISTERS(dma_addr_idx) <= REGISTERS(dma_addr_idx) + (BURST_LIMIT*4); END IF; - -- for AES always write after read - IF (REGISTERS(dma_ctrl_idx)(DMA_CTRL_AES_IDX) = '1') THEN - REGISTERS(REG_INDEX_DMA_CTRL)(DMA_CTRL_START_IDX) <= '0'; - REGISTERS(REG_INDEX_DMAW_CTRL)(DMA_CTRL_START_IDX) <= '1'; - END IF; SBus_Set_Default(SBUS_3V3_INT1s, SBUS_3V3_INT7s, SBUS_DATA_OE_LED, SBUS_DATA_OE_LED_2, p_addr, DATA_T, SM_T, SMs_T, LED_RESET); @@ -1215,11 +1246,6 @@ BEGIN -- move to next block REGISTERS(dma_ctrl_idx)(11 downto 0) <= REGISTERS(dma_ctrl_idx)(11 downto 0) - (BURST_LIMIT/4); REGISTERS(dma_addr_idx) <= REGISTERS(dma_addr_idx) + (BURST_LIMIT*4); - -- only switch ro read if there's one more block - IF (REGISTERS(dma_ctrl_idx)(DMA_CTRL_AES_IDX) = '1') THEN -- should always be true ATM - REGISTERS(REG_INDEX_DMAW_CTRL)(DMA_CTRL_START_IDX) <= '0'; - REGISTERS(REG_INDEX_DMA_CTRL)(DMA_CTRL_START_IDX) <= '1'; - END IF; END IF; END IF; SBus_Set_Default(SBUS_3V3_INT1s, SBUS_3V3_INT7s, @@ -1252,10 +1278,10 @@ BEGIN CASE AES_State IS WHEN AES_IDLE => IF ((REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_START_IDX) = '1') AND - (REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_BUSY_IDX) = '0') AND (fifo_toaes_full = '0') ) THEN fifo_wr_en <= '1'; fifo_din <= x"30"; -- "0" + REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_BUSY_IDX) <= '1'; -- start & !busy & !aesbusy -> start processing if (REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_NEWKEY_IDX) = '1') THEN --newkey fifo_toaes_din <= @@ -1263,6 +1289,7 @@ BEGIN REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_AES256_IDX) & -- keylen '1' & -- encdec REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_CBCMOD_IDX) & -- cbc + '0' & -- internal cbc REGISTERS(REG_INDEX_AES128_KEY1) & REGISTERS(REG_INDEX_AES128_KEY2) & REGISTERS(REG_INDEX_AES128_KEY3) & REGISTERS(REG_INDEX_AES128_KEY4) & REGISTERS(REG_INDEX_AES128_KEY5) & REGISTERS(REG_INDEX_AES128_KEY6) & @@ -1275,6 +1302,7 @@ BEGIN REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_AES256_IDX) & -- keylen '1' & -- encdec REGISTERS(REG_INDEX_AES128_CTRL)(AES128_CTRL_CBCMOD_IDX) & -- cbc + '0' & -- internal cbc REGISTERS(REG_INDEX_AES128_OUT1) & REGISTERS(REG_INDEX_AES128_OUT2) & REGISTERS(REG_INDEX_AES128_OUT3) & REGISTERS(REG_INDEX_AES128_OUT4) & REGISTERS(REG_INDEX_AES128_DATA1) & REGISTERS(REG_INDEX_AES128_DATA2) &