LeCroy Exerciser Script - NVMe (2) IO Command ( + MCTP, Completion options )

%% LeCroy Exerciser Script - For NVMe Emulation
* Reference : PETrainerScriptManual_v*.pdf
http://teledynelecroy.com/protocolanalyzer/pci-express/summit-z3-16-exerciser-with-smbus-support

 

* LeCroy Exerciser Script
☞ http://ya-n-ds.tistory.com/3108 : PCIe Commands
☞ http://ya-n-ds.tistory.com/3116 : NVMe (1) Preparation for IO
☞ http://ya-n-ds.tistory.com/3131 : NVMe (2) IO Command ( + MCTP, Completion options )

 

 

< Step 0. Option Preparation for NVMe Host Emulation >
# Exerciser BAR Menu : Setup -> Generation Options -> Transactions
- Host Memory Regions : Mem_64, Address 0x4_2FAA_8000, Length 0x1_0000 ( 4KB Blocks )

< Step 1. Configuration for NVMe >
Config=Definitions
{
    HOST_REQUESTER_ID                  = (1:0:0)
    DEVICE_ID                          = (2:1:0)

 

    BAR0_ADDRESS = 0xEC03_0000  ;; Little Endian
    BAR0_ADDRESS_BIG_END = 0x0000_003EC

 

    BAR2_ADDRESS_BIG_END = 0x0000_002EC ;; Little Endian = 0xEC02_0000

 

    PXCAP_BASE_ADDR = 0x0000_0070 ;; PCIe Capability Base Address ( Device HW dependency )
    PXCAP_PXDC_OFFSET = 0x0000_0008 ;; PCIe Device Congrol register offset

 

    MSI_X_TABLE_OFFSET = 0x0000_4000  ;; Device HW dependency, This can be read from MSIXCP.MTAB register
    MSI_X_TABLE_BASE_ADDR = ( BAR0_ADDRESS + MSI_X_TABLE_OFFSET )

 

    MSI_X_CAP_BASE_ADDR = 0xB0  ;; Device HW dependency

 

    HOST_MSIX_INTR0_BASE_ADDR_BIG_END = 0x0000_E0FE  ;; MSI-X Interrupt0 BA in Host - for Admin CQ
    HOST_MSIX_INTR0_BASE_ADDR = 0xFEE0_0000  ;; Littel Endian
    HOST_MSIX_INTR1_BASE_ADDR_BIG_END = 0x0000_E2FE  ;; MSI-X Interrupt1 BA in Host - for IO CQ
    HOST_MSIX_INTR1_BASE_ADDR = 0xFEE2_0000  ;; Littel Endian

 

    HOST_MSIX_INTR0_MSG_DATA_BIG_END = 0xB040_0000  ;; MSI-X Interrupt0 Message Data, Littel Endian = 0x0000_40B0
    HOST_MSIX_INTR1_MSG_DATA_BIG_END = 0xA040_0000  ;; MSI-X Interrupt1 Message Data, Littel Endian = 0x0000_40A0

 

    CONTROLLER_REGISTERS_BASE          = BAR0_ADDRESS

 

    IO_QUEUE_SIZE = 0xFF  ;; Device dependency

 

    IO_SQ_NUM_PAYLOAD = 0x01000000  ;; 0x0000_0001
    ;IO_SQ_NUM_PAYLOAD = 0x02000000  ;; 0x0000_0002

 

    COUNT_REPEAT = 4 ;; Count is from 0 to 'N-1'

 

    TO_FOR_WAIT = 100000 ; SQ Tail Doorbell -> 33us -> [R<-] MRd -> 190us -> CplD
    MRd_TAG_NUM = 0

 

    IO_DB_NUM = 0x00000001  ;; IO Door Bell Number #1
    IO_DB_NUM_BIG_END = ( IO_DB_NUM << 24 )
}

 

template = TLP
{
    Name = "Temp_ConfigWrite0"
   
    TLPType=CfgWr0
    Length = 1
    RequesterId = HOST_REQUESTER_ID
    FirstDwBe = 0xF
    DeviceId = DEVICE_ID
}

 

template = TLP
{
    Name = "Temp_ConfigRead0"
   
    TLPType=CfgRd0
    Length = 1
    RequesterId = HOST_REQUESTER_ID
    FirstDwBe = 0xF
    DeviceId = DEVICE_ID
}

 

template = TLP
{
    Name = "Temp_MWr32_OneDword"
   
    TLPType = MWr32
    Length = 1
    RequesterId = HOST_REQUESTER_ID
    FirstDwBe = 0xF
}

 

template = TLP
{
    Name = "Temp_MRd32_OneDword"
   
    TLPType = MRd32
    Length = 1
    RequesterId = HOST_REQUESTER_ID
    FirstDwBe = 0xF
}

 

;; PCIe VDM Header
; Test for Payload size(Length), Packet number,  Message Tag
template = TLP
{
    Name = "Temp_PCIe_VDM_Header"
    PSN = Incr
   
    ;; ** PCIe Medium-Specifice Header
    ;; 1st Dword ( Byte0~3 )
    TLPType=MsgD   ; Fmt[6:5]=11b(4DW Header with Data) + Type[4:3]=10b(Message)
    ;MessageRoute = ByID  ; Type[2:0] ( ByID = 010b = 0x2 )
    MessageRoute = 0x2
    Length = 2    ; Payload size = 2 Dword for Set EID
   
    ;; 2nd Dword ( Byte4~7 )
    RequesterID = (1:0:0)
    Tag = 0x00     ; Rsvd[7:6] + PadLen[5:4] + MCTP VDM Code(0000b)
                   ; PadLen = 0 for Aligned Payload
    MessageCode = Vendor_Defined_Type1  ; Should be VDM type 1 for MCTP packets = 0x7F
   
    ;; 3rd DWord ( Byte8~11 )
    DeviceID = (2:0:0)   ; Byte8,9
    VendorId = 0x1AB4    ; Byte10,11 - DMTF Vendor ID
   
    ;; 4th Dword ( Byte12~15 : MCTP Transport Header )
    ;VendorData = 0x0108F0C8 ; Big Endian -  Packet Seq #[5:4] = 00b, TO[3]=0, Message Tag[2:0]=0x0
    VendorData = 0x0108F0EB ; Big Endian - Packet Seq # = 10b, TO=1, Message Tag=0x3   
    ;Byte12 bit[3:0]=Header Version=1  /  Byte13=Destination EID=0x08  /  Byte14=Source EID=0xF0
    ;   cf. Byte13 - Destination EID : 0=Null EID, 1~7=Reserved for Future definition, 0xFF=Broadcast EID
    ;Byte15 : bit[7]=SOM , bit[6]=EOM are set to 1, bit[5:4]=Pkt # =0
    ;Byte15 : bit[3]=TO(Tag Owner) =1(=Requester)  /  bit[2:0]=Msg Tag
   
    ;;** PCIe VDM Data = MCTP Packet Payload ( Message body )
    ;1st Dword
    ;   Byte1=Message Type('MCTP Control'=000_0000b),
    ;   Byte2 - bit[7]=Rq('1'=Request), bit[6]=D(Datagram=0), bit[4:0]=Instance ID
    ;   Byte3 - Command Code=0x01(=Set EID)
    ;   Byte4 - bit[1:0] MCTP_CMD_SETID_Operation
    ;               ( 00b=Set ID, 01b=Force ID, 10b=Reset EID, 11b=Set Discovered Flag )
    ;2nd Dword
    ;   Byte1 - MCTP_CMD_SEID_EPID= 0x09  /  Byte2~4 - PAD=0x0 ( determined by 'Pad Len' )
    ;Payload = (0x00800100 0x09000000)
}

 

 

< Step 4-1. NVMe IO Command - Write >
;; Pre-program command in the Host Memory Region. This is the Mem_64 Host region
;; defined in the generation options file "nvme_host_gen_options.gen"
Structure=NVMe
{
    Location = Mem64
    Offset = 0x10000
    NVMeStructType=NVMCommand
    OpcodeNvm = Write
    CID = 0
    NamespaceId = 1
    PRP1_Low = 0x2FB08000
    PRP1_High = 0x4
    ;PRP2_Low = 0x2FB18000
    ;PRP2_High = 0x4
    NumLBlocks = 0 ; 1 -> 0 ( Zero base value : 0=4KB ) - Only PRP1 is reqired
    ;NumLBlocks = 1 ; For 4KB block size, PRP2 is necessary
}

 

;; Write Queue 1 SQ Tail Doorbell
packet="Temp_MWr32_OneDword"
{
    Tag = 0
    Address = ( CONTROLLER_REGISTERS_BASE + 0x1008 )

    ;Payload = ( 0x01000000 )  ;; 0x0000_0001
    Payload = IO_SQ_NUM_PAYLOAD
}

 

;; Wait for the Controller to process the command.
;; The last thing would be the MSI-X interrupt at vector 1
;; Normally Host driver matches MSI-X vector # and Submission Queue #
wait=TLP
{
    TLPType = MWr32
    Address = HOST_MSIX_INTR1_BASE_ADDR
}

 

;; Write Queue 1 Completion Queue Head Doorbell
packet="Temp_MWr32_OneDword"
{
    Tag = 1
    Address = ( CONTROLLER_REGISTERS_BASE + 0x100C )

    ;Payload = ( 0x01000000 )  ;; 0x0000_0001
    Payload = IO_SQ_NUM_PAYLOAD
}

 

 

< Step 4-2. NVMe IO Command - Read >
;; Pre-program command in the Host Memory Region. This is the Mem_64 Host region
Structure=NVMe
{
    Location = Mem64
    Offset = 0x10000
    NVMeStructType=NVMCommand
    OpcodeNvm = Read
    CID = 0
    NamespaceId = 1
    PRP1_Low = 0x3F246000
    PRP1_High = 0x8
    NumLBlocks = 1
}

;; Write Queue 1 SQ Tail Doorbell
packet="Temp_MWr32_OneDword"
{
    Tag = 0
    Address = ( CONTROLLER_REGISTERS_BASE + 0x1008 )
    Payload = IO_SQ_NUM_PAYLOAD
}

 

;; Wait for the Controller to process the command.
;; The last thing would be the MSI-X interrupt at vector 1
;; Normally Host driver matches MSI-X vector # and Submission Queue #
wait=TLP
{
    TLPType = MWr32
    Address = CQ1_INT_VECTOR_ADDRESS
}

 

;; Write Queue 1 Completion Queue Head
packet="Temp_MWr32_OneDword"
{
    Tag = 1
    Address = ( CONTROLLER_REGISTERS_BASE + 0x100C )
    Payload = IO_SQ_NUM_PAYLOAD
}

 

< Step 4-3. NVMe IO Command - Write + MCTP >
Repeat = Begin { Count = (COUNT_REPEAT&IO_QUEUE_SIZE) Counter=qq }

 

;; Pre-program command in the Host Memory Region. This is the Mem_64 Host region
;; defined in the generation options file "nvme_host_gen_options.gen"
AddressSpace=Write
{
    Location = Mem64  ;; 0x4_2FAA_8000 ( defined in Exerciser Generation Options : Host Memory region of Transactions menu )
    Offset = 0x60000  ;; for 0x4_2FB0_8000 ( PRP1 for Write command )
    Size = 0x1000  ;; 4KB
    LoadFrom = Incr
}

 

AddressSpace=Write
{
    Location = Mem64  ;; 0x4_2FAA_8000
    Offset = 0x61000  ;; for 0x4_2FB1_8000 ( PRP1 for Write command )
    Size = 0x1000  ;; 4KB
    LoadFrom = Ones
}

Structure=NVMe
{
    Location = Mem64
    ;Offset = 0x10000
    Offset = ( 0x10000  + qq*0x40 )  ;; Stride : 16DW=64Byte

 

    NVMeStructType=NVMCommand
    OpcodeNvm = Write

 

    ;CID = 0
     CID = ( IO_QUEUE_SIZE & qq )

 

     NamespaceId = 1  ; Port0
    ;NamespaceId = 2  ; Port1

 

    ;PRP1_Low = 0x2FB08000
    PRP1_Low = ( 0x2FB08000 + ((0x1&qq)<<12) )  ;; Based on 'AddressSpace=Write'
    PRP1_High = 0x4

    NumLBlocks = 0 ; 1 -> 0 ( Zero base value : 0=4KB ) - Only PRP1 is reqired
    ;NumLBlocks = 1 ; For > 4KB block size, PRP2 is necessary
}

 

;; Write Queue 1 SQ Tail Doorbell
packet="Temp_MWr32_OneDword"
{
    ;Tag = 0
    Tag = (0 + qq*0x2)

    Address = ( CONTROLLER_REGISTERS_BASE + 0x1008 )

    ;Payload = IO_SQ_NUM_PAYLOAD
    Payload = ( [(1 + (qq&IO_QUEUE_SIZE))<<24 ] )  ;; Increase by 1 based on 'qq'
}

 

;; Wait for the Controller to process the command.
;; The last thing would be the MSI-X interrupt at vector 1
;; Normally Host driver matches MSI-X vector # and Submission Queue #
wait=TLP
{
    TLPType = MWr32
    Address = HOST_MSIX_INTR1_BASE_ADDR
}

;; Write Queue 1 Completion Queue Head
packet="Temp_MWr32_OneDword"
{
    ;Tag = 1
    Tag = (1 + qq*0x2)

    Address = ( CONTROLLER_REGISTERS_BASE + 0x100C )
    ;Payload = IO_SQ_NUM_PAYLOAD
    Payload = ( [(1 + (qq&IO_QUEUE_SIZE))<<24 ] )
}

 

;; MCTP over PCIe VDM Command
Loop = Begin { count = COUNT_LOOP }
Packet="Temp_PCIe_VDM_Header"
{
    ;2nd Dword
    ;   Byte1 - MCTP_CMD_SEID_EPID= 0x09  /  Byte2~4 - PAD=0x0 ( determined by 'Pad Len' )
    RequesterID = HOST_REQUESTER_ID  ;; Added for Previous NVMe setting
   
    ;; 3rd DWord ( Byte8~11 )
    DeviceID = DEVICE_ID ; Byte8,9

    ;; ** PCIe Medium-Specifice Header
    ;; 1st Dword ( Byte0~3 )
    Length = 16 ; 64B

    ;; 2nd Dword ( Byte4~7 )
    Tag = 0x00    ; Rsvd[7:6] + PadLen[5:4] + MCTP VDM Code(0000b)
        ; PadLen = 0 for Aligned Payload
    ;; ** Payload
    Payload = Incr
}

 

wait=DLLP
{    DLLPType = Ack   }

Loop = End

Repeat =End

 

< Step 4-4. NVMe IO Command for Completion with CA or UR >
;; CA(Completer Abort), UR(Unsupported Request)
Structure=NVMe
{
    Location = Mem64
   
    ;Offset = 0x10000
    Offset = ( 0x10000 + 0x40*(IO_DB_NUM_LITTLE_END - 1) ) ;; Co-relation to IO Doorbell Number
   
    NVMeStructType=NVMCommand
    OpcodeNvm = Write
    CID = 0
    NamespaceId = 1
    PRP1_Low = 0x2FB08000
    PRP1_High = 0x4
    ;PRP2_Low = 0x2FB18000
    ;PRP2_High = 0x4
    NumLBlocks = 0 ; 1 -> 0 ( Zero base value : 0=4KB ) - Only PRP1 is reqired
    ;NumLBlocks = 1 ; For > 4KB block size, PRP2 is necessary
}

 

;; Write Queue 1 SQ Tail Doorbell
packet="Temp_MWr32_OneDword"
{
    Tag = 0
    Address = ( CONTROLLER_REGISTERS_BASE + 0x1008 )
    Payload = IO_DB_NUM_BIG_END
}

 

;; For CA Completion in response to MRd from Device
; Wait for MRd TLP from Device or for Timeout for Next step
Wait=TLP
{
    TLPType=MRd64
    Tag = MRd_TAG_NUM  ;; Check the Tag Number
    Timeout = TO_FOR_WAIT  ;; [ns] If the TLP is not received, then the execution contienues after timeout expires
}

;; Cpl packet sent late

;; For Host to Send Completion with CA or UR to Device
Packet=TLP
{
    TLPType = CplD
    TD = 0   
    ;Length = 32  ;; Based on the Request from Device
    Length = 16
   
    Tag = MRd_TAG_NUM
    ComplStatus = CA  ;; Completer Abort
    ;ComplStatus = UR  ;; Unsuppoted Request
   
    ;ByteCount = 128
    ByteCount = 64
   
    Payload = Ones
    RequesterId = DEVICE_ID
}

 

;; Wait for the Controller to process the command.
wait=TLP
{
    TLPType = MWr32
    Address = HOST_MSIX_INTR1_BASE_ADDR
}

; Write Queue 1 Completion Queue Head Doorbell
packet="Temp_OneDwordWrite"
{
    Tag = 1
    Address = ( CONTROLLER_REGISTERS_BASE + 0x100C )
    Payload = IO_DB_NUM_BIG_END
}