gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* Added the original files of the driver as a basis for comparison

Browse Source 
Conflicts:
	kernel/include/hfi1/sdma.h
	kernel/sdma.c
	kernel/user_sdma.c
This commit is contained in:
Aram Santogidis
2017-08-01 15:17:39 +09:00
committed by Balazs Gerofi
parent 4a0e389953
commit 14b360e867
11 changed files with 5424 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1558
kernel/file_ops.c Normal file
View File

File diff suppressed because it is too large Load Diff

407
kernel/include/hfi1/common.h Normal file
View File

@@ -0,0 +1,407 @@
/*
* Copyright(c) 2015, 2016 Intel Corporation.
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef _COMMON_H
#define _COMMON_H
#include "update/hfi1_user.h"
/*
* This file contains defines, structures, etc. that are used
* to communicate between kernel and user code.
*/
/* version of protocol header (known to chip also). In the long run,
* we should be able to generate and accept a range of version numbers;
* for now we only accept one, and it's compiled in.
*/
#define IPS_PROTO_VERSION 2
/*
* These are compile time constants that you may want to enable or disable
* if you are trying to debug problems with code or performance.
* HFI1_VERBOSE_TRACING define as 1 if you want additional tracing in
* fast path code
* HFI1_TRACE_REGWRITES define as 1 if you want register writes to be
* traced in fast path code
* _HFI1_TRACING define as 0 if you want to remove all tracing in a
* compilation unit
*/
/*
* If a packet's QP[23:16] bits match this value, then it is
* a PSM packet and the hardware will expect a KDETH header
* following the BTH.
*/
#define DEFAULT_KDETH_QP 0x80
/* driver/hw feature set bitmask */
#define HFI1_CAP_USER_SHIFT 24
#define HFI1_CAP_MASK ((1UL << HFI1_CAP_USER_SHIFT) - 1)
/* locked flag - if set, only HFI1_CAP_WRITABLE_MASK bits can be set */
#define HFI1_CAP_LOCKED_SHIFT 63
#define HFI1_CAP_LOCKED_MASK 0x1ULL
#define HFI1_CAP_LOCKED_SMASK (HFI1_CAP_LOCKED_MASK << HFI1_CAP_LOCKED_SHIFT)
/* extra bits used between kernel and user processes */
#define HFI1_CAP_MISC_SHIFT (HFI1_CAP_USER_SHIFT * 2)
#define HFI1_CAP_MISC_MASK ((1ULL << (HFI1_CAP_LOCKED_SHIFT - \
HFI1_CAP_MISC_SHIFT)) - 1)
#define HFI1_CAP_KSET(cap) ({ hfi1_cap_mask |= HFI1_CAP_##cap; hfi1_cap_mask; })
#define HFI1_CAP_KCLEAR(cap) \
({ \
hfi1_cap_mask &= ~HFI1_CAP_##cap; \
hfi1_cap_mask; \
})
#define HFI1_CAP_USET(cap) \
({ \
hfi1_cap_mask |= (HFI1_CAP_##cap << HFI1_CAP_USER_SHIFT); \
hfi1_cap_mask; \
})
#define HFI1_CAP_UCLEAR(cap) \
({ \
hfi1_cap_mask &= ~(HFI1_CAP_##cap << HFI1_CAP_USER_SHIFT); \
hfi1_cap_mask; \
})
#define HFI1_CAP_SET(cap) \
({ \
hfi1_cap_mask |= (HFI1_CAP_##cap | (HFI1_CAP_##cap << \
HFI1_CAP_USER_SHIFT)); \
hfi1_cap_mask; \
})
#define HFI1_CAP_CLEAR(cap) \
({ \
hfi1_cap_mask &= ~(HFI1_CAP_##cap | \
(HFI1_CAP_##cap << HFI1_CAP_USER_SHIFT)); \
hfi1_cap_mask; \
})
#define HFI1_CAP_LOCK() \
({ hfi1_cap_mask |= HFI1_CAP_LOCKED_SMASK; hfi1_cap_mask; })
#define HFI1_CAP_LOCKED() (!!(hfi1_cap_mask & HFI1_CAP_LOCKED_SMASK))
/*
* The set of capability bits that can be changed after initial load
* This set is the same for kernel and user contexts. However, for
* user contexts, the set can be further filtered by using the
* HFI1_CAP_RESERVED_MASK bits.
*/
#define HFI1_CAP_WRITABLE_MASK (HFI1_CAP_SDMA_AHG | \
HFI1_CAP_HDRSUPP | \
HFI1_CAP_MULTI_PKT_EGR | \
HFI1_CAP_NODROP_RHQ_FULL | \
HFI1_CAP_NODROP_EGR_FULL | \
HFI1_CAP_ALLOW_PERM_JKEY | \
HFI1_CAP_STATIC_RATE_CTRL | \
HFI1_CAP_PRINT_UNIMPL | \
HFI1_CAP_TID_UNMAP | \
HFI1_CAP_OPFN | \
HFI1_CAP_TID_RDMA)
/*
* A set of capability bits that are "global" and are not allowed to be
* set in the user bitmask.
*/
#define HFI1_CAP_RESERVED_MASK ((HFI1_CAP_SDMA | \
HFI1_CAP_USE_SDMA_HEAD | \
HFI1_CAP_EXTENDED_PSN | \
HFI1_CAP_PRINT_UNIMPL | \
HFI1_CAP_NO_INTEGRITY | \
HFI1_CAP_PKEY_CHECK | \
HFI1_CAP_TID_RDMA | \
HFI1_CAP_OPFN) << \
HFI1_CAP_USER_SHIFT)
/*
* Set of capabilities that need to be enabled for kernel context in
* order to be allowed for user contexts, as well.
*/
#define HFI1_CAP_MUST_HAVE_KERN (HFI1_CAP_STATIC_RATE_CTRL)
/* Default enabled capabilities (both kernel and user) */
#define HFI1_CAP_MASK_DEFAULT (HFI1_CAP_HDRSUPP | \
HFI1_CAP_NODROP_RHQ_FULL | \
HFI1_CAP_NODROP_EGR_FULL | \
HFI1_CAP_SDMA | \
HFI1_CAP_PRINT_UNIMPL | \
HFI1_CAP_STATIC_RATE_CTRL | \
HFI1_CAP_PKEY_CHECK | \
HFI1_CAP_MULTI_PKT_EGR | \
HFI1_CAP_EXTENDED_PSN | \
((HFI1_CAP_HDRSUPP | \
HFI1_CAP_MULTI_PKT_EGR | \
HFI1_CAP_STATIC_RATE_CTRL | \
HFI1_CAP_PKEY_CHECK | \
HFI1_CAP_EARLY_CREDIT_RETURN) << \
HFI1_CAP_USER_SHIFT))
/*
* A bitmask of kernel/global capabilities that should be communicated
* to user level processes.
*/
#define HFI1_CAP_K2U (HFI1_CAP_SDMA | \
HFI1_CAP_EXTENDED_PSN | \
HFI1_CAP_PKEY_CHECK | \
HFI1_CAP_NO_INTEGRITY)
#define HFI1_USER_SWVERSION ((HFI1_USER_SWMAJOR << HFI1_SWMAJOR_SHIFT) | \
HFI1_USER_SWMINOR)
#ifndef HFI1_KERN_TYPE
#define HFI1_KERN_TYPE 0
#endif
/*
* Similarly, this is the kernel version going back to the user. It's
* slightly different, in that we want to tell if the driver was built as
* part of a Intel release, or from the driver from openfabrics.org,
* kernel.org, or a standard distribution, for support reasons.
* The high bit is 0 for non-Intel and 1 for Intel-built/supplied.
*
* It's returned by the driver to the user code during initialization in the
* spi_sw_version field of hfi1_base_info, so the user code can in turn
* check for compatibility with the kernel.
*/
#define HFI1_KERN_SWVERSION ((HFI1_KERN_TYPE << 31) | HFI1_USER_SWVERSION)
/*
* Define the driver version number. This is something that refers only
* to the driver itself, not the software interfaces it supports.
*/
#ifndef HFI1_DRIVER_VERSION_BASE
#define HFI1_DRIVER_VERSION_BASE "0.9-294"
#endif
/* create the final driver version string */
#ifdef HFI1_IDSTR
#define HFI1_DRIVER_VERSION HFI1_DRIVER_VERSION_BASE " " HFI1_IDSTR
#else
#define HFI1_DRIVER_VERSION HFI1_DRIVER_VERSION_BASE
#endif
/*
* Diagnostics can send a packet by writing the following
* struct to the diag packet special file.
*
* This allows a custom PBC qword, so that special modes and deliberate
* changes to CRCs can be used.
*/
#define _DIAG_PKT_VERS 1
struct diag_pkt {
__u16 version; /* structure version */
__u16 unit; /* which device */
__u16 sw_index; /* send sw index to use */
__u16 len; /* data length, in bytes */
__u16 port; /* port number */
__u16 unused;
__u32 flags; /* call flags */
__u64 data; /* user data pointer */
__u64 pbc; /* PBC for the packet */
};
/* diag_pkt flags */
#define F_DIAGPKT_WAIT 0x1 /* wait until packet is sent */
/*
* The next set of defines are for packet headers, and chip register
* and memory bits that are visible to and/or used by user-mode software.
*/
/*
* Receive Header Flags
*/
#define RHF_PKT_LEN_SHIFT 0
#define RHF_PKT_LEN_MASK 0xfffull
#define RHF_PKT_LEN_SMASK (RHF_PKT_LEN_MASK << RHF_PKT_LEN_SHIFT)
#define RHF_RCV_TYPE_SHIFT 12
#define RHF_RCV_TYPE_MASK 0x7ull
#define RHF_RCV_TYPE_SMASK (RHF_RCV_TYPE_MASK << RHF_RCV_TYPE_SHIFT)
#define RHF_USE_EGR_BFR_SHIFT 15
#define RHF_USE_EGR_BFR_MASK 0x1ull
#define RHF_USE_EGR_BFR_SMASK (RHF_USE_EGR_BFR_MASK << RHF_USE_EGR_BFR_SHIFT)
#define RHF_EGR_INDEX_SHIFT 16
#define RHF_EGR_INDEX_MASK 0x7ffull
#define RHF_EGR_INDEX_SMASK (RHF_EGR_INDEX_MASK << RHF_EGR_INDEX_SHIFT)
#define RHF_DC_INFO_SHIFT 27
#define RHF_DC_INFO_MASK 0x1ull
#define RHF_DC_INFO_SMASK (RHF_DC_INFO_MASK << RHF_DC_INFO_SHIFT)
#define RHF_RCV_SEQ_SHIFT 28
#define RHF_RCV_SEQ_MASK 0xfull
#define RHF_RCV_SEQ_SMASK (RHF_RCV_SEQ_MASK << RHF_RCV_SEQ_SHIFT)
#define RHF_EGR_OFFSET_SHIFT 32
#define RHF_EGR_OFFSET_MASK 0xfffull
#define RHF_EGR_OFFSET_SMASK (RHF_EGR_OFFSET_MASK << RHF_EGR_OFFSET_SHIFT)
#define RHF_HDRQ_OFFSET_SHIFT 44
#define RHF_HDRQ_OFFSET_MASK 0x1ffull
#define RHF_HDRQ_OFFSET_SMASK (RHF_HDRQ_OFFSET_MASK << RHF_HDRQ_OFFSET_SHIFT)
#define RHF_K_HDR_LEN_ERR (0x1ull << 53)
#define RHF_DC_UNC_ERR (0x1ull << 54)
#define RHF_DC_ERR (0x1ull << 55)
#define RHF_RCV_TYPE_ERR_SHIFT 56
#define RHF_RCV_TYPE_ERR_MASK 0x7ul
#define RHF_RCV_TYPE_ERR_SMASK (RHF_RCV_TYPE_ERR_MASK << RHF_RCV_TYPE_ERR_SHIFT)
#define RHF_TID_ERR (0x1ull << 59)
#define RHF_LEN_ERR (0x1ull << 60)
#define RHF_ECC_ERR (0x1ull << 61)
#define RHF_VCRC_ERR (0x1ull << 62)
#define RHF_ICRC_ERR (0x1ull << 63)
#define RHF_ERROR_SMASK 0xffe0000000000000ull /* bits 63:53 */
/* RHF receive types */
#define RHF_RCV_TYPE_EXPECTED 0
#define RHF_RCV_TYPE_EAGER 1
#define RHF_RCV_TYPE_IB 2 /* normal IB, IB Raw, or IPv6 */
#define RHF_RCV_TYPE_ERROR 3
#define RHF_RCV_TYPE_BYPASS 4
#define RHF_RCV_TYPE_INVALID5 5
#define RHF_RCV_TYPE_INVALID6 6
#define RHF_RCV_TYPE_INVALID7 7
/* RHF receive type error - expected packet errors */
#define RHF_RTE_EXPECTED_FLOW_SEQ_ERR 0x2
#define RHF_RTE_EXPECTED_FLOW_GEN_ERR 0x4
/* RHF receive type error - eager packet errors */
#define RHF_RTE_EAGER_NO_ERR 0x0
/* RHF receive type error - IB packet errors */
#define RHF_RTE_IB_NO_ERR 0x0
/* RHF receive type error - error packet errors */
#define RHF_RTE_ERROR_NO_ERR 0x0
#define RHF_RTE_ERROR_OP_CODE_ERR 0x1
#define RHF_RTE_ERROR_KHDR_MIN_LEN_ERR 0x2
#define RHF_RTE_ERROR_KHDR_HCRC_ERR 0x3
#define RHF_RTE_ERROR_KHDR_KVER_ERR 0x4
#define RHF_RTE_ERROR_CONTEXT_ERR 0x5
#define RHF_RTE_ERROR_KHDR_TID_ERR 0x6
/* RHF receive type error - bypass packet errors */
#define RHF_RTE_BYPASS_NO_ERR 0x0
/* IB - LRH header constants */
#define HFI1_LRH_GRH 0x0003 /* 1. word of IB LRH - next header: GRH */
#define HFI1_LRH_BTH 0x0002 /* 1. word of IB LRH - next header: BTH */
/* misc. */
#define SIZE_OF_CRC 1
#define LIM_MGMT_P_KEY 0x7FFF
#define FULL_MGMT_P_KEY 0xFFFF
#define DEFAULT_P_KEY LIM_MGMT_P_KEY
#define HFI1_FECN_SHIFT 31
#define HFI1_FECN_MASK 1
#define HFI1_FECN_SMASK BIT(HFI1_FECN_SHIFT)
#define HFI1_BECN_SHIFT 30
#define HFI1_BECN_MASK 1
#define HFI1_BECN_SMASK BIT(HFI1_BECN_SHIFT)
#define HFI1_PSM_IOC_BASE_SEQ 0x0
/* Number of BTH.PSN bits used for sequence number in expected rcvs */
#define HFI1_KDETH_BTH_SEQ_SHIFT 11
#define HFI1_KDETH_BTH_SEQ_MASK (BIT(HFI1_KDETH_BTH_SEQ_SHIFT) - 1)
static inline __u64 rhf_to_cpu(const __le32 *rbuf)
{
return __le64_to_cpu(*((__le64 *)rbuf));
}
static inline u64 rhf_err_flags(u64 rhf)
{
return rhf & RHF_ERROR_SMASK;
}
static inline u32 rhf_rcv_type(u64 rhf)
{
return (rhf >> RHF_RCV_TYPE_SHIFT) & RHF_RCV_TYPE_MASK;
}
static inline u32 rhf_rcv_type_err(u64 rhf)
{
return (rhf >> RHF_RCV_TYPE_ERR_SHIFT) & RHF_RCV_TYPE_ERR_MASK;
}
/* return size is in bytes, not DWORDs */
static inline u32 rhf_pkt_len(u64 rhf)
{
return ((rhf & RHF_PKT_LEN_SMASK) >> RHF_PKT_LEN_SHIFT) << 2;
}
static inline u32 rhf_egr_index(u64 rhf)
{
return (rhf >> RHF_EGR_INDEX_SHIFT) & RHF_EGR_INDEX_MASK;
}
static inline u32 rhf_rcv_seq(u64 rhf)
{
return (rhf >> RHF_RCV_SEQ_SHIFT) & RHF_RCV_SEQ_MASK;
}
/* returned offset is in DWORDS */
static inline u32 rhf_hdrq_offset(u64 rhf)
{
return (rhf >> RHF_HDRQ_OFFSET_SHIFT) & RHF_HDRQ_OFFSET_MASK;
}
static inline u64 rhf_use_egr_bfr(u64 rhf)
{
return rhf & RHF_USE_EGR_BFR_SMASK;
}
static inline u64 rhf_dc_info(u64 rhf)
{
return rhf & RHF_DC_INFO_SMASK;
}
static inline u32 rhf_egr_buf_offset(u64 rhf)
{
return (rhf >> RHF_EGR_OFFSET_SHIFT) & RHF_EGR_OFFSET_MASK;
}
#endif /* _COMMON_H */

2152
kernel/include/hfi1/hfi.h Normal file
View File

File diff suppressed because it is too large Load Diff

439
kernel/include/hfi1/hfi1_user.h Normal file
View File

@@ -0,0 +1,439 @@
/*
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright(c) 2015 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
/*
* This file contains defines, structures, etc. that are used
* to communicate between kernel and user code.
*/
#ifndef _LINUX__HFI1_USER_H
#define _LINUX__HFI1_USER_H
#include <linux/types.h>
/*
* This version number is given to the driver by the user code during
* initialization in the spu_userversion field of hfi1_user_info, so
* the driver can check for compatibility with user code.
*
* The major version changes when data structures change in an incompatible
* way. The driver must be the same for initialization to succeed.
*/
#define HFI1_USER_SWMAJOR 6
/*
* Minor version differences are always compatible
* a within a major version, however if user software is larger
* than driver software, some new features and/or structure fields
* may not be implemented; the user code must deal with this if it
* cares, or it must abort after initialization reports the difference.
*/
#define HFI1_USER_SWMINOR 3
/*
* We will encode the major/minor inside a single 32bit version number.
*/
#define HFI1_SWMAJOR_SHIFT 16
/*
* Set of HW and driver capability/feature bits.
* These bit values are used to configure enabled/disabled HW and
* driver features. The same set of bits are communicated to user
* space.
*/
#define HFI1_CAP_DMA_RTAIL (1UL << 0) /* Use DMA'ed RTail value */
#define HFI1_CAP_SDMA (1UL << 1) /* Enable SDMA support */
#define HFI1_CAP_SDMA_AHG (1UL << 2) /* Enable SDMA AHG support */
#define HFI1_CAP_EXTENDED_PSN (1UL << 3) /* Enable Extended PSN support */
#define HFI1_CAP_HDRSUPP (1UL << 4) /* Enable Header Suppression */
#define HFI1_CAP_TID_RDMA (1UL << 5) /* Enable TID RDMA operations */
#define HFI1_CAP_USE_SDMA_HEAD (1UL << 6) /* DMA Hdr Q tail vs. use CSR */
#define HFI1_CAP_MULTI_PKT_EGR (1UL << 7) /* Enable multi-packet Egr buffs*/
#define HFI1_CAP_NODROP_RHQ_FULL (1UL << 8) /* Don't drop on Hdr Q full */
#define HFI1_CAP_NODROP_EGR_FULL (1UL << 9) /* Don't drop on EGR buffs full */
#define HFI1_CAP_TID_UNMAP (1UL << 10) /* Disable Expected TID caching */
#define HFI1_CAP_PRINT_UNIMPL (1UL << 11) /* Show for unimplemented feats */
#define HFI1_CAP_ALLOW_PERM_JKEY (1UL << 12) /* Allow use of permissive JKEY */
#define HFI1_CAP_NO_INTEGRITY (1UL << 13) /* Enable ctxt integrity checks */
#define HFI1_CAP_PKEY_CHECK (1UL << 14) /* Enable ctxt PKey checking */
#define HFI1_CAP_STATIC_RATE_CTRL (1UL << 15) /* Allow PBC.StaticRateControl */
#define HFI1_CAP_OPFN (1UL << 16) /* Enable the OPFN protocol */
#define HFI1_CAP_SDMA_HEAD_CHECK (1UL << 17) /* SDMA head checking */
#define HFI1_CAP_EARLY_CREDIT_RETURN (1UL << 18) /* early credit return */
#define HFI1_RCVHDR_ENTSIZE_2 (1UL << 0)
#define HFI1_RCVHDR_ENTSIZE_16 (1UL << 1)
#define HFI1_RCVDHR_ENTSIZE_32 (1UL << 2)
/* User commands. */
#define HFI1_CMD_ASSIGN_CTXT 1 /* allocate HFI and context */
#define HFI1_CMD_CTXT_INFO 2 /* find out what resources we got */
#define HFI1_CMD_USER_INFO 3 /* set up userspace */
#define HFI1_CMD_TID_UPDATE 4 /* update expected TID entries */
#define HFI1_CMD_TID_FREE 5 /* free expected TID entries */
#define HFI1_CMD_CREDIT_UPD 6 /* force an update of PIO credit */
#define HFI1_CMD_RECV_CTRL 8 /* control receipt of packets */
#define HFI1_CMD_POLL_TYPE 9 /* set the kind of polling we want */
#define HFI1_CMD_ACK_EVENT 10 /* ack & clear user status bits */
#define HFI1_CMD_SET_PKEY 11 /* set context's pkey */
#define HFI1_CMD_CTXT_RESET 12 /* reset context's HW send context */
#define HFI1_CMD_TID_INVAL_READ 13 /* read TID cache invalidations */
#define HFI1_CMD_GET_VERS 14 /* get the version of the user cdev */
/*
* User IOCTLs can not go above 128 if they do then see common.h and change the
* base for the snoop ioctl
*/
#define IB_IOCTL_MAGIC 0x1b /* See Documentation/ioctl/ioctl-number.txt */
/*
* Make the ioctls occupy the last 0xf0-0xff portion of the IB range
*/
#define __NUM(cmd) (HFI1_CMD_##cmd + 0xe0)
struct hfi1_cmd;
#define HFI1_IOCTL_ASSIGN_CTXT \
_IOWR(IB_IOCTL_MAGIC, __NUM(ASSIGN_CTXT), struct hfi1_user_info)
#define HFI1_IOCTL_CTXT_INFO \
_IOW(IB_IOCTL_MAGIC, __NUM(CTXT_INFO), struct hfi1_ctxt_info)
#define HFI1_IOCTL_USER_INFO \
_IOW(IB_IOCTL_MAGIC, __NUM(USER_INFO), struct hfi1_base_info)
#define HFI1_IOCTL_TID_UPDATE \
_IOWR(IB_IOCTL_MAGIC, __NUM(TID_UPDATE), struct hfi1_tid_info)
#define HFI1_IOCTL_TID_FREE \
_IOWR(IB_IOCTL_MAGIC, __NUM(TID_FREE), struct hfi1_tid_info)
#define HFI1_IOCTL_CREDIT_UPD \
_IO(IB_IOCTL_MAGIC, __NUM(CREDIT_UPD))
#define HFI1_IOCTL_RECV_CTRL \
_IOW(IB_IOCTL_MAGIC, __NUM(RECV_CTRL), int)
#define HFI1_IOCTL_POLL_TYPE \
_IOW(IB_IOCTL_MAGIC, __NUM(POLL_TYPE), int)
#define HFI1_IOCTL_ACK_EVENT \
_IOW(IB_IOCTL_MAGIC, __NUM(ACK_EVENT), unsigned long)
#define HFI1_IOCTL_SET_PKEY \
_IOW(IB_IOCTL_MAGIC, __NUM(SET_PKEY), __u16)
#define HFI1_IOCTL_CTXT_RESET \
_IO(IB_IOCTL_MAGIC, __NUM(CTXT_RESET))
#define HFI1_IOCTL_TID_INVAL_READ \
_IOWR(IB_IOCTL_MAGIC, __NUM(TID_INVAL_READ), struct hfi1_tid_info)
#define HFI1_IOCTL_GET_VERS \
_IOR(IB_IOCTL_MAGIC, __NUM(GET_VERS), int)
#define _HFI1_EVENT_FROZEN_BIT 0
#define _HFI1_EVENT_LINKDOWN_BIT 1
#define _HFI1_EVENT_LID_CHANGE_BIT 2
#define _HFI1_EVENT_LMC_CHANGE_BIT 3
#define _HFI1_EVENT_SL2VL_CHANGE_BIT 4
#define _HFI1_EVENT_TID_MMU_NOTIFY_BIT 5
#define _HFI1_MAX_EVENT_BIT _HFI1_EVENT_TID_MMU_NOTIFY_BIT
#define HFI1_EVENT_FROZEN (1UL << _HFI1_EVENT_FROZEN_BIT)
#define HFI1_EVENT_LINKDOWN (1UL << _HFI1_EVENT_LINKDOWN_BIT)
#define HFI1_EVENT_LID_CHANGE (1UL << _HFI1_EVENT_LID_CHANGE_BIT)
#define HFI1_EVENT_LMC_CHANGE (1UL << _HFI1_EVENT_LMC_CHANGE_BIT)
#define HFI1_EVENT_SL2VL_CHANGE (1UL << _HFI1_EVENT_SL2VL_CHANGE_BIT)
#define HFI1_EVENT_TID_MMU_NOTIFY (1UL << _HFI1_EVENT_TID_MMU_NOTIFY_BIT)
/*
* These are the status bits readable (in ASCII form, 64bit value)
* from the "status" sysfs file. For binary compatibility, values
* must remain as is; removed states can be reused for different
* purposes.
*/
#define HFI1_STATUS_INITTED 0x1 /* basic initialization done */
/* Chip has been found and initialized */
#define HFI1_STATUS_CHIP_PRESENT 0x20
/* IB link is at ACTIVE, usable for data traffic */
#define HFI1_STATUS_IB_READY 0x40
/* link is configured, LID, MTU, etc. have been set */
#define HFI1_STATUS_IB_CONF 0x80
/* A Fatal hardware error has occurred. */
#define HFI1_STATUS_HWERROR 0x200
/*
* Number of supported shared contexts.
* This is the maximum number of software contexts that can share
* a hardware send/receive context.
*/
#define HFI1_MAX_SHARED_CTXTS 8
/*
* Poll types
*/
#define HFI1_POLL_TYPE_ANYRCV 0x0
#define HFI1_POLL_TYPE_URGENT 0x1
/*
* This structure is passed to the driver to tell it where
* user code buffers are, sizes, etc. The offsets and sizes of the
* fields must remain unchanged, for binary compatibility. It can
* be extended, if userversion is changed so user code can tell, if needed
*/
struct hfi1_user_info {
/*
* version of user software, to detect compatibility issues.
* Should be set to HFI1_USER_SWVERSION.
*/
__u32 userversion;
__u32 pad;
/*
* If two or more processes wish to share a context, each process
* must set the subcontext_cnt and subcontext_id to the same
* values. The only restriction on the subcontext_id is that
* it be unique for a given node.
*/
__u16 subctxt_cnt;
__u16 subctxt_id;
/* 128bit UUID passed in by PSM. */
__u8 uuid[16];
};
struct hfi1_ctxt_info {
__u64 runtime_flags; /* chip/drv runtime flags (HFI1_CAP_*) */
__u32 rcvegr_size; /* size of each eager buffer */
__u16 num_active; /* number of active units */
__u16 unit; /* unit (chip) assigned to caller */
__u16 ctxt; /* ctxt on unit assigned to caller */
__u16 subctxt; /* subctxt on unit assigned to caller */
__u16 rcvtids; /* number of Rcv TIDs for this context */
__u16 credits; /* number of PIO credits for this context */
__u16 numa_node; /* NUMA node of the assigned device */
__u16 rec_cpu; /* cpu # for affinity (0xffff if none) */
__u16 send_ctxt; /* send context in use by this user context */
__u16 egrtids; /* number of RcvArray entries for Eager Rcvs */
__u16 rcvhdrq_cnt; /* number of RcvHdrQ entries */
__u16 rcvhdrq_entsize; /* size (in bytes) for each RcvHdrQ entry */
__u16 sdma_ring_size; /* number of entries in SDMA request ring */
};
struct hfi1_tid_info {
/* virtual address of first page in transfer */
__u64 vaddr;
/* pointer to tid array. this array is big enough */
__u64 tidlist;
/* number of tids programmed by this request */
__u32 tidcnt;
/* length of transfer buffer programmed by this request */
__u32 length;
};
enum hfi1_sdma_comp_state {
FREE = 0,
QUEUED,
COMPLETE,
ERROR
};
/*
* SDMA completion ring entry
*/
struct hfi1_sdma_comp_entry {
__u32 status;
__u32 errcode;
};
/*
* Device status and notifications from driver to user-space.
*/
struct hfi1_status {
__u64 dev; /* device/hw status bits */
__u64 port; /* port state and status bits */
char freezemsg[0];
};
/*
* This structure is returned by the driver immediately after
* open to get implementation-specific info, and info specific to this
* instance.
*
* This struct must have explicit pad fields where type sizes
* may result in different alignments between 32 and 64 bit
* programs, since the 64 bit * bit kernel requires the user code
* to have matching offsets
*/
struct hfi1_base_info {
/* version of hardware, for feature checking. */
__u32 hw_version;
/* version of software, for feature checking. */
__u32 sw_version;
/* Job key */
__u16 jkey;
__u16 padding1;
/*
* The special QP (queue pair) value that identifies PSM
* protocol packet from standard IB packets.
*/
__u32 bthqp;
/* PIO credit return address, */
__u64 sc_credits_addr;
/*
* Base address of write-only pio buffers for this process.
* Each buffer has sendpio_credits*64 bytes.
*/
__u64 pio_bufbase_sop;
/*
* Base address of write-only pio buffers for this process.
* Each buffer has sendpio_credits*64 bytes.
*/
__u64 pio_bufbase;
/* address where receive buffer queue is mapped into */
__u64 rcvhdr_bufbase;
/* base address of Eager receive buffers. */
__u64 rcvegr_bufbase;
/* base address of SDMA completion ring */
__u64 sdma_comp_bufbase;
/*
* User register base for init code, not to be used directly by
* protocol or applications. Always maps real chip register space.
* the register addresses are:
* ur_rcvhdrhead, ur_rcvhdrtail, ur_rcvegrhead, ur_rcvegrtail,
* ur_rcvtidflow
*/
__u64 user_regbase;
/* notification events */
__u64 events_bufbase;
/* status page */
__u64 status_bufbase;
/* rcvhdrtail update */
__u64 rcvhdrtail_base;
/*
* shared memory pages for subctxts if ctxt is shared; these cover
* all the processes in the group sharing a single context.
* all have enough space for the num_subcontexts value on this job.
*/
__u64 subctxt_uregbase;
__u64 subctxt_rcvegrbuf;
__u64 subctxt_rcvhdrbuf;
};
enum sdma_req_opcode {
EXPECTED = 0,
EAGER
};
#define HFI1_SDMA_REQ_VERSION_MASK 0xF
#define HFI1_SDMA_REQ_VERSION_SHIFT 0x0
#define HFI1_SDMA_REQ_OPCODE_MASK 0xF
#define HFI1_SDMA_REQ_OPCODE_SHIFT 0x4
#define HFI1_SDMA_REQ_IOVCNT_MASK 0xFF
#define HFI1_SDMA_REQ_IOVCNT_SHIFT 0x8
struct sdma_req_info {
/*
* bits 0-3 - version (currently unused)
* bits 4-7 - opcode (enum sdma_req_opcode)
* bits 8-15 - io vector count
*/
__u16 ctrl;
/*
* Number of fragments contained in this request.
* User-space has already computed how many
* fragment-sized packet the user buffer will be
* split into.
*/
__u16 npkts;
/*
* Size of each fragment the user buffer will be
* split into.
*/
__u16 fragsize;
/*
* Index of the slot in the SDMA completion ring
* this request should be using. User-space is
* in charge of managing its own ring.
*/
__u16 comp_idx;
} __packed;
/*
* SW KDETH header.
* swdata is SW defined portion.
*/
struct hfi1_kdeth_header {
__le32 ver_tid_offset;
__le16 jkey;
__le16 hcrc;
__le32 swdata[7];
} __packed;
/*
* Structure describing the headers that User space uses. The
* structure above is a subset of this one.
*/
struct hfi1_pkt_header {
__le16 pbc[4];
__be16 lrh[4];
__be32 bth[3];
struct hfi1_kdeth_header kdeth;
} __packed;
/*
* The list of usermode accessible registers.
*/
enum hfi1_ureg {
/* (RO) DMA RcvHdr to be used next. */
ur_rcvhdrtail = 0,
/* (RW) RcvHdr entry to be processed next by host. */
ur_rcvhdrhead = 1,
/* (RO) Index of next Eager index to use. */
ur_rcvegrindextail = 2,
/* (RW) Eager TID to be processed next */
ur_rcvegrindexhead = 3,
/* (RO) Receive Eager Offset Tail */
ur_rcvegroffsettail = 4,
/* For internal use only; max register number. */
ur_maxreg,
/* (RW) Receive TID flow table */
ur_rcvtidflowtable = 256
};
#endif /* _LINIUX__HFI1_USER_H */

410
kernel/include/hfi1/iowait.h Normal file
View File

@@ -0,0 +1,410 @@
#ifndef _HFI1_IOWAIT_H
#define _HFI1_IOWAIT_H
/*
* Copyright(c) 2015 - 2017 Intel Corporation.
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/sched.h>
#include "sdma_txreq.h"
/*
* typedef (*restart_t)() - restart callback
* @work: pointer to work structure
*/
typedef void (*restart_t)(struct work_struct *work);
#define IOWAIT_PENDING_IB 0x0
#define IOWAIT_PENDING_TID 0x1
/*
* A QP can have multiple Send Engines (SEs).
*
* The current use case is for supporting a TID RDMA
* packet build/xmit mechanism independent from verbs.
*/
#define IOWAIT_SES 2
#define IOWAIT_IB_SE 0
#define IOWAIT_TID_SE 1
struct sdma_txreq;
struct sdma_engine;
/**
* @iowork: the work struct
* @tx_head: list of prebuilt packets
* @iow: the parent iowait structure
*
* This structure is the work item (process) specific
* details associated with the each of the two SEs of the
* QP.
*
* The workstruct and the queued TXs are unique to each
* SE.
*/
struct iowait;
struct iowait_work {
struct work_struct iowork;
struct list_head tx_head;
struct iowait *iow;
};
/**
* @list: used to add/insert into QP/PQ wait lists
* @tx_head: overflow list of sdma_txreq's
* @sleep: no space callback
* @wakeup: space callback wakeup
* @sdma_drained: sdma count drained
* @lock: lock protected head of wait queue
* @iowork: workqueue overhead
* @wait_dma: wait for sdma_busy == 0
* @wait_pio: wait for pio_busy == 0
* @sdma_busy: # of packets in flight
* @count: total number of descriptors in tx_head'ed list
* @tx_limit: limit for overflow queuing
* @tx_count: number of tx entry's in tx_head'ed list
* @flags: wait flags (one per QP)
* @wait: SE array
*
* This is to be embedded in user's state structure
* (QP or PQ).
*
* The sleep and wakeup members are a
* bit misnamed. They do not strictly
* speaking sleep or wake up, but they
* are callbacks for the ULP to implement
* what ever queuing/dequeuing of
* the embedded iowait and its containing struct
* when a resource shortage like SDMA ring space is seen.
*
* Both potentially have locks help
* so sleeping is not allowed.
*
* The wait_dma member along with the iow
*
* The lock field is used by waiters to record
* the seqlock_t that guards the list head.
* Waiters explicity know that, but the destroy
* code that unwaits QPs does not.
*/
struct iowait {
struct list_head list;
int (*sleep)(
struct sdma_engine *sde,
struct iowait_work *wait,
struct sdma_txreq *tx,
unsigned seq);
void (*wakeup)(struct iowait *wait, int reason);
void (*sdma_drained)(struct iowait *wait);
seqlock_t *lock;
wait_queue_head_t wait_dma;
wait_queue_head_t wait_pio;
atomic_t sdma_busy;
atomic_t pio_busy;
u32 count;
u32 tx_limit;
u32 tx_count;
unsigned long flags;
struct iowait_work wait[IOWAIT_SES];
};
#define SDMA_AVAIL_REASON 0
void iowait_set_flag(struct iowait *wait, u32 flag);
bool iowait_flag_set(struct iowait *wait, u32 flag);
void iowait_clear_flag(struct iowait *wait, u32 flag);
void iowait_init(
struct iowait *wait,
u32 tx_limit,
void (*func)(struct work_struct *work),
void (*tidfunc)(struct work_struct *work),
int (*sleep)(
struct sdma_engine *sde,
struct iowait_work *wait,
struct sdma_txreq *tx,
unsigned seq),
void (*wakeup)(struct iowait *wait, int reason),
void (*sdma_drained)(struct iowait *wait));
/**
* iowait_schedule() - schedule the default send engine work
* @wait: wait struct to schedule
* @wq: workqueue for schedule
* @cpu: cpu
*/
static inline bool iowait_schedule(
struct iowait *wait,
struct workqueue_struct *wq,
int cpu)
{
return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
}
/**
* iowait_tid_schedule - schedule the tid SE
* @wait: the iowait structure
* @wq: the work queue
* @cpu: the cpu
*/
static inline bool iowait_tid_schedule(
struct iowait *wait,
struct workqueue_struct *wq,
int cpu)
{
return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
}
/**
* iowait_sdma_drain() - wait for DMAs to drain
* @wait: iowait structure
*
* This will delay until the iowait sdmas have
* completed.
*/
static inline void iowait_sdma_drain(struct iowait *wait)
{
wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
}
/**
* iowait_sdma_pending() - return sdma pending count
*
* @wait: iowait structure
*
*/
static inline int iowait_sdma_pending(struct iowait *wait)
{
return atomic_read(&wait->sdma_busy);
}
/**
* iowait_sdma_inc - note sdma io pending
* @wait: iowait structure
*/
static inline void iowait_sdma_inc(struct iowait *wait)
{
atomic_inc(&wait->sdma_busy);
}
/**
* iowait_sdma_add - add count to pending
* @wait: iowait_work structure
*/
static inline void iowait_sdma_add(struct iowait *wait, int count)
{
atomic_add(count, &wait->sdma_busy);
}
/**
* iowait_pio_drain() - wait for pios to drain
*
* @wait: iowait structure
*
* This will delay until the iowait pios have
* completed.
*/
static inline void iowait_pio_drain(struct iowait *wait)
{
wait_event_timeout(wait->wait_pio,
!atomic_read(&wait->pio_busy),
HZ);
}
/**
* iowait_pio_pending() - return pio pending count
*
* @wait: iowait structure
*
*/
static inline int iowait_pio_pending(struct iowait *w)
{
return atomic_read(&w->pio_busy);
}
/**
* iowait_drain_wakeup() - trigger iowait_drain() waiter
* @wait: iowait structure
*
* This will trigger any waiters.
*/
static inline void iowait_drain_wakeup(struct iowait *w)
{
wake_up(&w->wait_dma);
wake_up(&w->wait_pio);
if (w->sdma_drained)
w->sdma_drained(w);
}
/**
* iowait_pio_inc - note pio pending
* @wait: iowait structure
*/
static inline void iowait_pio_inc(struct iowait *wait)
{
atomic_inc(&wait->pio_busy);
}
/**
* iowait_pio_dec - note pio complete
* @wait: iowait structure
*/
static inline int iowait_pio_dec(struct iowait *wait)
{
if (!wait)
return 0;
return atomic_dec_and_test(&wait->pio_busy);
}
/**
* iowait_sdma_dec - note pio complete
* @wait: iowait structure
*/
static inline int iowait_sdma_dec(struct iowait *wait)
{
if (!wait)
return 0;
return atomic_dec_and_test(&wait->sdma_busy);
}
/**
* iowait_get_txhead() - get packet off of iowait list
* @wait wait struture
*/
static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
{
struct sdma_txreq *tx = NULL;
if (!list_empty(&wait->tx_head)) {
tx = list_first_entry(
&wait->tx_head,
struct sdma_txreq,
list);
list_del_init(&tx->list);
}
return tx;
}
static inline u16 iowait_get_desc(struct iowait_work *w)
{
u16 num_desc = 0;
struct sdma_txreq *tx = NULL;
if (!list_empty(&w->tx_head)) {
tx = list_first_entry(
&w->tx_head,
struct sdma_txreq,
list);
num_desc = tx->num_desc;
}
return num_desc;
}
static inline u32 iowait_get_all_desc(struct iowait *w)
{
u32 num_desc = 0;
num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
return num_desc;
}
/**
* iowait_packet_queued() - determine if a packet it queued
* @wait: the wait structure
*/
static inline bool iowait_packet_queued(struct iowait_work *w)
{
return !list_empty(&w->tx_head);
}
/**
* inc_wait_count - increment wait counts
* @w: the log work struct
* @n: the count
*/
static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
{
if (!w)
return;
w->iow->tx_count++;
w->iow->count += n;
}
/**
* iowait_get_tid_work - return iowait_work for tid SE
* @w: the iowait struct
*/
static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
{
return &w->wait[IOWAIT_TID_SE];
}
/**
* iowait_get_ib_work - return iowait_work for ib SE
* @w: the iowait struct
*/
static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
{
return &w->wait[IOWAIT_IB_SE];
}
/**
* iowait_ioww_to_iow - return iowait given iowait_work
* @w: the iowait_work struct
*/
static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
{
if (likely(w))
return w->iow;
return NULL;
}
void iowait_cancel_work(struct iowait *w);
int iowait_set_work_flag(struct iowait_work *w);
#endif

59
kernel/include/hfi1/sdma.h
View File

@@ -47,11 +47,6 @@
*
*/
#include <hfi1/ihk_hfi1_common.h>
#include <hfi1/sdma_txreq.h>
#ifdef __HFI1_ORIG__
#include <linux/types.h>
#include <linux/list.h>
#include <asm/byteorder.h>
@@ -62,11 +57,6 @@
#include "verbs.h"
#include "sdma_txreq.h"
#define hfi1_cdbg(which, fmt, ...) \
__hfi1_trace_##which(__func__, fmt, ##__VA_ARGS__)
extern void __hfi1_trace_AIOWRITE(const char *func, char *fmt, ...);
#endif /* __HFI1_ORIG__ */
/* Hardware limit */
#define MAX_DESC 64
/* Hardware limit for SDMA packet size */
@@ -202,7 +192,6 @@ struct sdma_set_state_action {
unsigned go_s99_running_totrue:1;
};
#ifdef __HFI1_ORIG__
struct sdma_state {
struct kref kref;
struct completion comp;
@@ -214,11 +203,6 @@ struct sdma_state {
unsigned previous_op;
enum sdma_events last_event;
};
#else
struct sdma_state {
enum sdma_states current_state;
};
#endif /* __HFI1_ORIG__ */
/**
* DOC: sdma exported routines
@@ -410,7 +394,6 @@ struct sdma_engine {
/* private: */
struct list_head dmawait;
#ifdef __HFI1_ORIG__
/* CONFIG SDMA for now, just blindly duplicate */
/* private: */
struct tasklet_struct sdma_hw_clean_up_task
@@ -426,20 +409,14 @@ struct sdma_engine {
u32 progress_check_head;
/* private: */
struct work_struct flush_worker;
#endif /* __HFI1_ORIG__ */
/* protect flush list */
spinlock_t flushlist_lock;
/* private: */
struct list_head flushlist;
#ifdef __HFI1_ORIG__
struct cpumask cpu_mask;
struct kobject kobj;
#endif /* __HFI1_ORIG__ */
};
#ifdef __HFI1_ORIG__
int sdma_init(struct hfi1_devdata *dd, u8 port);
void sdma_start(struct hfi1_devdata *dd);
void sdma_exit(struct hfi1_devdata *dd);
@@ -464,7 +441,6 @@ static inline int sdma_empty(struct sdma_engine *sde)
return sde->descq_tail == sde->descq_head;
}
#endif /* __HFI1_ORIG__ */
static inline u16 sdma_descq_freecnt(struct sdma_engine *sde)
{
return sde->descq_cnt -
@@ -502,11 +478,9 @@ static inline int sdma_running(struct sdma_engine *engine)
unsigned long flags;
int ret;
hfi1_cdbg(AIOWRITE, "+");
spin_lock_irqsave(&engine->tail_lock, flags);
ret = __sdma_running(engine);
spin_unlock_irqrestore(&engine->tail_lock, flags);
hfi1_cdbg(AIOWRITE, "-");
return ret;
}
@@ -645,7 +619,6 @@ static inline int sdma_txinit(
{
return sdma_txinit_ahg(tx, flags, tlen, 0, 0, NULL, 0, cb);
}
#ifdef __HFI1_ORIG__
/* helpers - don't use */
static inline int sdma_mapping_type(struct sdma_desc *d)
@@ -666,7 +639,6 @@ static inline dma_addr_t sdma_mapping_addr(struct sdma_desc *d)
>> SDMA_DESC0_PHY_ADDR_SHIFT;
}
#endif /* __HFI1_ORIG__ */
static inline void make_tx_sdma_desc(
struct sdma_txreq *tx,
int type,
@@ -694,6 +666,7 @@ static inline void make_tx_sdma_desc(
int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
int type, void *kvaddr, struct page *page,
unsigned long offset, u16 len);
int _pad_sdma_tx_descs(struct hfi1_devdata *, struct sdma_txreq *);
void __sdma_txclean(struct hfi1_devdata *, struct sdma_txreq *);
static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx)
@@ -701,8 +674,6 @@ static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx)
if (tx->num_desc)
__sdma_txclean(dd, tx);
}
#ifdef __HFI1_ORIG__
int _pad_sdma_tx_descs(struct hfi1_devdata *, struct sdma_txreq *);
/* helpers used by public routines */
static inline void _sdma_close_tx(struct hfi1_devdata *dd,
@@ -718,7 +689,6 @@ static inline void _sdma_close_tx(struct hfi1_devdata *dd,
SDMA_DESC1_INT_REQ_FLAG);
}
#endif /* __HFI1_ORIG__ */
static inline int _sdma_txadd_daddr(
struct hfi1_devdata *dd,
int type,
@@ -737,13 +707,11 @@ static inline int _sdma_txadd_daddr(
/* special cases for last */
if (!tx->tlen) {
if (tx->packet_len & (sizeof(u32) - 1)) {
//TODO: _pad_sdma_tx_descs
//rval = _pad_sdma_tx_descs(dd, tx);
rval = _pad_sdma_tx_descs(dd, tx);
if (rval)
return rval;
} else {
//TODO: _sdma_close_tx
//_sdma_close_tx(dd, tx);
_sdma_close_tx(dd, tx);
}
}
tx->num_desc++;
@@ -775,7 +743,7 @@ static inline int sdma_txadd_page(
{
dma_addr_t addr;
int rval;
hfi1_cdbg(AIOWRITE, "+");
if ((unlikely(tx->num_desc == tx->desc_limit))) {
rval = ext_coal_sdma_tx_descs(dd, tx, SDMA_MAP_PAGE,
NULL, page, offset, len);
@@ -783,7 +751,6 @@ static inline int sdma_txadd_page(
return rval;
}
#ifdef __HFI1_ORIG__
addr = dma_map_page(
&dd->pcidev->dev,
page,
@@ -795,16 +762,7 @@ static inline int sdma_txadd_page(
__sdma_txclean(dd, tx);
return -ENOSPC;
}
#else
//TODO: dma_map_page
#endif /* __HFI1_ORIG__ */
hfi1_cdbg(AIOWRITE, "-");
/*
* XXX: It seems that this is the place where the reference to
* the payload is added, but addr is kernel virtual here.
* TODO: verify this by printing it out in Linux.
*/
return _sdma_txadd_daddr(
dd, SDMA_MAP_PAGE, tx, addr, len);
}
@@ -875,7 +833,6 @@ static inline int sdma_txadd_kvaddr(
return rval;
}
#ifdef __HFI1_ORIG__
addr = dma_map_single(
&dd->pcidev->dev,
kvaddr,
@@ -886,9 +843,6 @@ static inline int sdma_txadd_kvaddr(
__sdma_txclean(dd, tx);
return -ENOSPC;
}
#else
//TODO: dma_map_single
#endif /* __HFI1_ORIG__ */
return _sdma_txadd_daddr(
dd, SDMA_MAP_SINGLE, tx, addr, len);
@@ -931,7 +885,6 @@ static inline u32 sdma_build_ahg_descriptor(
((data & SDMA_AHG_VALUE_MASK) <<
SDMA_AHG_VALUE_SHIFT));
}
#ifdef __HFI1_ORIG__
/**
* sdma_progress - use seq number of detect head progress
@@ -1108,7 +1061,6 @@ struct sdma_engine *sdma_select_engine_sc(
u32 selector,
u8 sc5);
#endif /* __HFI1_ORIG__ */
struct sdma_engine *sdma_select_engine_vl(
struct hfi1_devdata *dd,
u32 selector,
@@ -1116,8 +1068,6 @@ struct sdma_engine *sdma_select_engine_vl(
struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
u32 selector, u8 vl);
#ifdef __HFI1_ORIG__
ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf);
ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,
size_t count);
@@ -1145,5 +1095,4 @@ extern uint mod_num_sdma;
void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid);
#endif /* __HFI1_ORIG__ */
#endif

135
kernel/include/hfi1/sdma_txreq.h Normal file
View File

@@ -0,0 +1,135 @@
/*
* Copyright(c) 2016 Intel Corporation.
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef HFI1_SDMA_TXREQ_H
#define HFI1_SDMA_TXREQ_H
/* increased for AHG */
#define NUM_DESC 6
/*
* struct sdma_desc - canonical fragment descriptor
*
* This is the descriptor carried in the tx request
* corresponding to each fragment.
*
*/
struct sdma_desc {
/* private: don't use directly */
u64 qw[2];
};
/**
* struct sdma_txreq - the sdma_txreq structure (one per packet)
* @list: for use by user and by queuing for wait
*
* This is the representation of a packet which consists of some
* number of fragments. Storage is provided to within the structure.
* for all fragments.
*
* The storage for the descriptors are automatically extended as needed
* when the currently allocation is exceeded.
*
* The user (Verbs or PSM) may overload this structure with fields
* specific to their use by putting this struct first in their struct.
* The method of allocation of the overloaded structure is user dependent
*
* The list is the only public field in the structure.
*
*/
#define SDMA_TXREQ_S_OK 0
#define SDMA_TXREQ_S_SENDERROR 1
#define SDMA_TXREQ_S_ABORTED 2
#define SDMA_TXREQ_S_SHUTDOWN 3
/* flags bits */
#define SDMA_TXREQ_F_URGENT 0x0001
#define SDMA_TXREQ_F_AHG_COPY 0x0002
#define SDMA_TXREQ_F_USE_AHG 0x0004
struct sdma_txreq;
typedef void (*callback_t)(struct sdma_txreq *, int);
struct iowait_wait;
struct sdma_txreq {
struct list_head list;
/* private: */
struct sdma_desc *descp;
/* private: */
void *coalesce_buf;
/* private: */
struct iowait *wait;
/* private: */
callback_t complete;
#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
u64 sn;
#endif
/* private: - used in coalesce/pad processing */
u16 packet_len;
/* private: - down-counted to trigger last */
u16 tlen;
/* private: */
u16 num_desc;
/* private: */
u16 desc_limit;
/* private: */
u16 next_descq_idx;
/* private: */
u16 coalesce_idx;
/* private: flags */
u16 flags;
/* private: */
struct sdma_desc descs[NUM_DESC];
};
static inline int sdma_txreq_built(struct sdma_txreq *tx)
{
return tx->num_desc;
}
#endif /* HFI1_SDMA_TXREQ_H */

157
kernel/include/hfi1/user_exp_rcv.h Normal file
View File

@@ -0,0 +1,157 @@
#ifndef _HFI1_USER_EXP_RCV_H
#define _HFI1_USER_EXP_RCV_H
/*
* Copyright(c) 2015, 2016 Intel Corporation.
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "hfi.h"
#define EXP_TID_TIDLEN_MASK 0x7FFULL
#define EXP_TID_TIDLEN_SHIFT 0
#define EXP_TID_TIDCTRL_MASK 0x3ULL
#define EXP_TID_TIDCTRL_SHIFT 20
#define EXP_TID_TIDIDX_MASK 0x3FFULL
#define EXP_TID_TIDIDX_SHIFT 22
#define EXP_TID_GET(tid, field) \
(((tid) >> EXP_TID_TID##field##_SHIFT) & EXP_TID_TID##field##_MASK)
#define EXP_TID_SET(field, value) \
(((value) & EXP_TID_TID##field##_MASK) << \
EXP_TID_TID##field##_SHIFT)
#define EXP_TID_CLEAR(tid, field) ({ \
(tid) &= ~(EXP_TID_TID##field##_MASK << \
EXP_TID_TID##field##_SHIFT); \
})
#define EXP_TID_RESET(tid, field, value) do { \
EXP_TID_CLEAR(tid, field); \
(tid) |= EXP_TID_SET(field, (value)); \
} while (0)
struct tid_group {
struct list_head list;
unsigned base;
u8 size;
u8 used;
u8 map;
};
struct tid_pageset {
u16 idx;
u16 count;
};
/*
* Write an "empty" RcvArray entry.
* This function exists so the TID registaration code can use it
* to write to unused/unneeded entries and still take advantage
* of the WC performance improvements. The HFI will ignore this
* write to the RcvArray entry.
*/
static inline void rcv_array_wc_fill(struct hfi1_devdata *dd, u32 index)
{
/*
* Doing the WC fill writes only makes sense if the device is
* present and the RcvArray has been mapped as WC memory.
*/
if ((dd->flags & HFI1_PRESENT) && dd->rcvarray_wc)
writeq(0, dd->rcvarray_wc + (index * 8));
}
static inline u32 rcventry2tidinfo(u32 rcventry)
{
u32 pair = rcventry & ~0x1;
return EXP_TID_SET(IDX, pair >> 1) |
EXP_TID_SET(CTRL, 1 << (rcventry - pair));
}
static inline void exp_tid_group_init(struct exp_tid_set *set)
{
INIT_LIST_HEAD(&set->list);
set->count = 0;
}
static inline void tid_group_remove(struct tid_group *grp,
struct exp_tid_set *set)
{
list_del_init(&grp->list);
set->count--;
}
static inline void tid_group_add_tail(struct tid_group *grp,
struct exp_tid_set *set)
{
list_add_tail(&grp->list, &set->list);
set->count++;
}
static inline struct tid_group *tid_group_pop(struct exp_tid_set *set)
{
struct tid_group *grp =
list_first_entry(&set->list, struct tid_group, list);
list_del_init(&grp->list);
set->count--;
return grp;
}
static inline void tid_group_move(struct tid_group *group,
struct exp_tid_set *s1,
struct exp_tid_set *s2)
{
tid_group_remove(group, s1);
tid_group_add_tail(group, s2);
}
u32 find_phys_blocks(struct page **, unsigned, struct tid_pageset *);
int alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd);
void free_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd);
int hfi1_user_exp_rcv_init(struct file *);
int hfi1_user_exp_rcv_free(struct hfi1_filedata *);
int hfi1_user_exp_rcv_setup(struct file *, struct hfi1_tid_info *);
int hfi1_user_exp_rcv_clear(struct file *, struct hfi1_tid_info *);
int hfi1_user_exp_rcv_invalid(struct file *, struct hfi1_tid_info *);
#endif /* _HFI1_USER_EXP_RCV_H */

128
kernel/include/hfi1/user_sdma.h Normal file
View File

@@ -0,0 +1,128 @@
/*
* Copyright(c) 2015, 2016 Intel Corporation.
*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* BSD LICENSE
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* - Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <linux/device.h>
#include <linux/wait.h>
#include "common.h"
#include "iowait.h"
#include "user_exp_rcv.h"
extern uint extended_psn;
/*
* Define fields in the KDETH header so we can update the header
* template.
*/
#define KDETH_OFFSET_SHIFT 0
#define KDETH_OFFSET_MASK 0x7fff
#define KDETH_OM_SHIFT 15
#define KDETH_OM_MASK 0x1
#define KDETH_TID_SHIFT 16
#define KDETH_TID_MASK 0x3ff
#define KDETH_TIDCTRL_SHIFT 26
#define KDETH_TIDCTRL_MASK 0x3
#define KDETH_INTR_SHIFT 28
#define KDETH_INTR_MASK 0x1
#define KDETH_SH_SHIFT 29
#define KDETH_SH_MASK 0x1
#define KDETH_KVER_SHIFT 30
#define KDETH_KVER_MASK 0x3
#define KDETH_JKEY_SHIFT 0x0
#define KDETH_JKEY_MASK 0xff
#define KDETH_HCRC_UPPER_SHIFT 16
#define KDETH_HCRC_UPPER_MASK 0xff
#define KDETH_HCRC_LOWER_SHIFT 24
#define KDETH_HCRC_LOWER_MASK 0xff
#define AHG_KDETH_INTR_SHIFT 12
#define AHG_KDETH_SH_SHIFT 13
#define KDETH_GET(val, field) \
(((le32_to_cpu((val))) >> KDETH_##field##_SHIFT) & KDETH_##field##_MASK)
#define KDETH_SET(dw, field, val) do { \
u32 dwval = le32_to_cpu(dw); \
dwval &= ~(KDETH_##field##_MASK << KDETH_##field##_SHIFT); \
dwval |= (((val) & KDETH_##field##_MASK) << \
KDETH_##field##_SHIFT); \
dw = cpu_to_le32(dwval); \
} while (0)
#define KDETH_RESET(dw, field, val) ({ dw = 0; KDETH_SET(dw, field, val); })
/* KDETH OM multipliers and switch over point */
#define KDETH_OM_SMALL 4
#define KDETH_OM_LARGE 64
#define KDETH_OM_MAX_SIZE (1 << ((KDETH_OM_LARGE / KDETH_OM_SMALL) + 1))
struct hfi1_user_sdma_pkt_q {
struct list_head list;
unsigned ctxt;
unsigned subctxt;
u16 n_max_reqs;
atomic_t n_reqs;
u16 reqidx;
struct hfi1_devdata *dd;
struct kmem_cache *txreq_cache;
struct user_sdma_request *reqs;
unsigned long *req_in_use;
struct iowait busy;
unsigned state;
wait_queue_head_t wait;
unsigned long unpinned;
struct mmu_rb_handler *handler;
atomic_t n_locked;
struct mm_struct *mm;
};
struct hfi1_user_sdma_comp_q {
u16 nentries;
struct hfi1_sdma_comp_entry *comps;
};
int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *, struct file *);
int hfi1_user_sdma_free_queues(struct hfi1_filedata *);
int hfi1_user_sdma_process_request(struct file *, struct iovec *, unsigned long,
unsigned long *);

84
kernel/sdma.c
View File

@@ -45,15 +45,6 @@
*
*/
#include <hfi1/ihk_hfi1_common.h>
#include <hfi1/user_sdma.h>
#include <hfi1/sdma.h>
#include <hfi1/common.h>
unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
#ifdef __HFI1_ORIG__
#include <linux/spinlock.h>
#include <linux/seqlock.h>
#include <linux/netdevice.h>
@@ -71,15 +62,11 @@ unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
#include "iowait.h"
#include "trace.h"
#endif /* __HFI1_ORIG__ */
/* must be a power of 2 >= 64 <= 32768 */
#define SDMA_DESCQ_CNT 2048
#define SDMA_DESC_INTR 64
#define INVALID_TAIL 0xffff
#ifdef __HFI1_ORIG__
static uint sdma_descq_cnt = SDMA_DESCQ_CNT;
module_param(sdma_descq_cnt, uint, S_IRUGO);
MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
@@ -239,9 +226,7 @@ static const struct sdma_set_state_action sdma_action_table[] = {
},
};
#endif /* __HFI1_ORIG__ */
#define SDMA_TAIL_UPDATE_THRESH 0x1F
#ifdef __HFI1_ORIG__
/* declare all statics here rather than keep sorting */
static void sdma_complete(struct kref *);
@@ -383,7 +368,7 @@ static inline void complete_tx(struct sdma_engine *sde,
/* protect against complete modifying */
struct iowait *wait = tx->wait;
callback_t complete = tx->complete;
hfi1_cdbg(AIOWRITE, "+");
#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
trace_hfi1_sdma_out_sn(sde, tx->sn);
if (WARN_ON_ONCE(sde->head_sn != tx->sn))
@@ -396,7 +381,6 @@ static inline void complete_tx(struct sdma_engine *sde,
(*complete)(tx, res);
if (iowait_sdma_dec(wait))
iowait_drain_wakeup(wait);
hfi1_cdbg(AIOWRITE, "-");
}
/*
@@ -789,16 +773,11 @@ struct sdma_engine *sdma_select_engine_vl(
struct sdma_map_elem *e;
struct sdma_engine *rval;
hfi1_cdbg(AIOWRITE, "+");
/* NOTE This should only happen if SC->VL changed after the initial
* checks on the QP/AH
* Default will return engine 0 below
*/
#ifdef __HFI1_ORIG__
if (vl >= num_vls) {
#else
if (vl >= HFI1_MAX_VLS_SUPPORTED) {
#endif /* __HFI1_ORIG__ */
rval = NULL;
goto done;
}
@@ -816,7 +795,6 @@ struct sdma_engine *sdma_select_engine_vl(
done:
rval = !rval ? &dd->per_sdma[0] : rval;
trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx);
hfi1_cdbg(AIOWRITE, "-");
return rval;
}
@@ -886,7 +864,6 @@ struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
const struct cpumask *current_mask = tsk_cpus_allowed(current);
unsigned long cpu_id;
hfi1_cdbg(AIOWRITE, "+");
/*
* To ensure that always the same sdma engine(s) will be
* selected make sure the process is pinned to this CPU only.
@@ -1681,7 +1658,6 @@ static inline void sdma_unmap_desc(
break;
}
}
#endif /* __HFI1_ORIG__ */
/*
* return the mode as indicated by the first
@@ -1713,15 +1689,13 @@ void __sdma_txclean(
if (tx->num_desc) {
u8 skip = 0, mode = ahg_mode(tx);
/* TODO: enable sdma_unmap_desc */
/* unmap first */
//sdma_unmap_desc(dd, &tx->descp[0]);
sdma_unmap_desc(dd, &tx->descp[0]);
/* determine number of AHG descriptors to skip */
if (mode > SDMA_AHG_APPLY_UPDATE1)
skip = mode >> 1;
/* TODO: enable sdma_unmap_desc */
// for (i = 1 + skip; i < tx->num_desc; i++)
// sdma_unmap_desc(dd, &tx->descp[i]);
for (i = 1 + skip; i < tx->num_desc; i++)
sdma_unmap_desc(dd, &tx->descp[i]);
tx->num_desc = 0;
}
kfree(tx->coalesce_buf);
@@ -1732,7 +1706,6 @@ void __sdma_txclean(
kfree(tx->descp);
}
}
#ifdef __HFI1_ORIG__
static inline u16 sdma_gethead(struct sdma_engine *sde)
{
@@ -1851,7 +1824,6 @@ static void sdma_make_progress(struct sdma_engine *sde, u64 status)
u16 hwhead, swhead;
int idle_check_done = 0;
hfi1_cdbg(AIOWRITE, "+");
hwhead = sdma_gethead(sde);
/* The reason for some of the complexity of this code is that
@@ -1903,7 +1875,6 @@ retry:
sde->last_status = status;
if (progress)
sdma_desc_avail(sde, sdma_descq_freecnt(sde));
hfi1_cdbg(AIOWRITE, "-");
}
/*
@@ -1917,7 +1888,6 @@ retry:
*/
void sdma_engine_interrupt(struct sdma_engine *sde, u64 status)
{
hfi1_cdbg(AIOWRITE, "+");
trace_hfi1_sdma_engine_interrupt(sde, status);
write_seqlock(&sde->head_lock);
sdma_set_desc_cnt(sde, sdma_desct_intr);
@@ -1929,7 +1899,6 @@ void sdma_engine_interrupt(struct sdma_engine *sde, u64 status)
sde->sdma_int_cnt++;
sdma_make_progress(sde, status);
write_sequnlock(&sde->head_lock);
hfi1_cdbg(AIOWRITE, "-");
}
/**
@@ -2031,15 +2000,12 @@ static void sdma_setlengen(struct sdma_engine *sde)
(4ULL << SD(LEN_GEN_GENERATION_SHIFT)));
}
#endif /* __HFI1_ORIG__ */
static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail)
{
hfi1_cdbg(AIOWRITE, ".");
/* Commit writes to memory and advance the tail on the chip */
smp_wmb(); /* see get_txhead() */
writeq(tail, sde->tail_csr);
}
#ifdef __HFI1_ORIG__
/*
* This is called when changing to state s10_hw_start_up_halt_wait as
@@ -2304,7 +2270,6 @@ void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde)
}
}
#endif /* __HFI1_ORIG__ */
/*
* add the generation number into
* the qw1 and return
@@ -2341,12 +2306,12 @@ static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx)
u16 tail;
struct sdma_desc *descp = tx->descp;
u8 skip = 0, mode = ahg_mode(tx);
hfi1_cdbg(AIOWRITE, "+");
tail = sde->descq_tail & sde->sdma_mask;
sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1]));
// trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1],
// tail, &sde->descq[tail]);
trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1],
tail, &sde->descq[tail]);
tail = ++sde->descq_tail & sde->sdma_mask;
descp++;
if (mode > SDMA_AHG_APPLY_UPDATE1)
@@ -2364,19 +2329,18 @@ static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx)
qw1 = add_gen(sde, descp->qw[1]);
}
sde->descq[tail].qw[1] = cpu_to_le64(qw1);
// trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1,
// tail, &sde->descq[tail]);
trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1,
tail, &sde->descq[tail]);
tail = ++sde->descq_tail & sde->sdma_mask;
}
tx->next_descq_idx = tail;
#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
tx->sn = sde->tail_sn++;
// trace_hfi1_sdma_in_sn(sde, tx->sn);
trace_hfi1_sdma_in_sn(sde, tx->sn);
WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]);
#endif
sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx;
sde->desc_avail -= tx->num_desc;
hfi1_cdbg(AIOWRITE, "-");
return tail;
}
@@ -2390,7 +2354,6 @@ static int sdma_check_progress(
{
int ret;
hfi1_cdbg(AIOWRITE, "+");
sde->desc_avail = sdma_descq_freecnt(sde);
if (tx->num_desc <= sde->desc_avail)
return -EAGAIN;
@@ -2406,10 +2369,8 @@ static int sdma_check_progress(
} else {
ret = -EBUSY;
}
hfi1_cdbg(AIOWRITE, "-");
return ret;
}
#ifdef __HFI1_ORIG__
/**
* sdma_send_txreq() - submit a tx req to ring
@@ -2433,7 +2394,6 @@ int sdma_send_txreq(struct sdma_engine *sde,
u16 tail;
unsigned long flags;
hfi1_cdbg(AIOWRITE, "+");
/* user should have supplied entire packet */
if (unlikely(tx->tlen))
return -EINVAL;
@@ -2450,7 +2410,6 @@ retry:
sdma_update_tail(sde, tail);
unlock:
spin_unlock_irqrestore(&sde->tail_lock, flags);
hfi1_cdbg(AIOWRITE, "-");
return ret;
unlock_noconn:
if (wait)
@@ -2477,7 +2436,6 @@ nodesc:
goto unlock;
}
#endif /* __HFI1_ORIG__ */
/**
* sdma_send_txlist() - submit a list of tx req to ring
* @sde: sdma engine to use
@@ -2515,13 +2473,8 @@ int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
u16 tail = INVALID_TAIL;
u32 submit_count = 0, flush_count = 0, total_count;
hfi1_cdbg(AIOWRITE, "+");
spin_lock_irqsave(&sde->tail_lock, flags);
retry:
/*
* XXX: do we really need to build tx_list??
* We could just submit the requests straight to the HW..
*/
list_for_each_entry_safe(tx, tx_next, tx_list, list) {
tx->wait = iowait_ioww_to_iow(wait);
if (unlikely(!__sdma_running(sde)))
@@ -2549,7 +2502,6 @@ update_tail:
sdma_update_tail(sde, tail);
spin_unlock_irqrestore(&sde->tail_lock, flags);
*count_out = total_count;
hfi1_cdbg(AIOWRITE, "-");
return ret;
unlock_noconn:
spin_lock(&sde->flushlist_lock);
@@ -2559,15 +2511,14 @@ unlock_noconn:
tx->next_descq_idx = 0;
#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
tx->sn = sde->tail_sn++;
// trace_hfi1_sdma_in_sn(sde, tx->sn);
trace_hfi1_sdma_in_sn(sde, tx->sn);
#endif
list_add_tail(&tx->list, &sde->flushlist);
flush_count++;
iowait_inc_wait_count(wait, tx->num_desc);
}
spin_unlock(&sde->flushlist_lock);
// TODO: schedule_work
//schedule_work(&sde->flush_worker);
schedule_work(&sde->flush_worker);
ret = -ECOMM;
goto update_tail;
nodesc:
@@ -2579,7 +2530,6 @@ nodesc:
sde->descq_full_count++;
goto update_tail;
}
#ifdef __HFI1_ORIG__
static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event)
{
@@ -3133,7 +3083,6 @@ enomem:
return -ENOMEM;
}
#endif /* __HFI1_ORIG__ */
/*
* ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors
*
@@ -3154,8 +3103,6 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
int type, void *kvaddr, struct page *page,
unsigned long offset, u16 len)
{
//TODO: ext_coal_sdma_tx_descs
#ifdef __HFI1_ORIG__
int pad_len, rval;
dma_addr_t addr;
@@ -3215,10 +3162,9 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,
addr, tx->tlen);
}
#endif /* __HFI1_ORIG__ */
return 1;
}
#ifdef __HFI1_ORIG__
/* Update sdes when the lmc changes */
void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid)
@@ -3263,7 +3209,6 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
return rval;
}
#endif /* __HFI1_ORIG__ */
/*
* Add ahg to the sdma_txreq
*
@@ -3371,7 +3316,6 @@ void sdma_ahg_free(struct sdma_engine *sde, int ahg_index)
return;
clear_bit(ahg_index, &sde->ahg_bits);
}
#ifdef __HFI1_ORIG__
/*
* SPC freeze handling for SDMA engines. Called when the driver knows
@@ -3466,5 +3410,3 @@ void _sdma_engine_progress_schedule(
CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)),
sde->progress_mask);
}
#endif /* __HFI1_ORIG__ */

135
kernel/user_sdma.c
View File

@@ -44,13 +44,6 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <hfi1/user_sdma.h>
#include <hfi1/user_exp_rcv.h>
#include <hfi1/common.h>
#ifdef __HFI1_ORIG__
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/device.h>
@@ -76,11 +69,9 @@
#include "trace.h"
#include "mmu_rb.h"
static uint hfi1_sdma_comp_ring_size = 128;
module_param_named(sdma_comp_size, hfi1_sdma_comp_ring_size, uint, S_IRUGO);
MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 128");
#endif /* __HFI1_ORIG__ */
static uint hfi1_sdma_comp_ring_size = 128;
/* The maximum number of Data io vectors per message/request */
#define MAX_VECTORS_PER_REQ 8
@@ -142,11 +133,8 @@ static unsigned initial_pkt_count = 8;
#define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */
#ifdef __HFI1_ORIG__
struct sdma_mmu_node;
#endif /* __HFI1_ORIG__ */
struct user_sdma_iovec {
struct list_head list;
struct iovec iov;
@@ -161,7 +149,6 @@ struct user_sdma_iovec {
u64 offset;
struct sdma_mmu_node *node;
};
#ifdef __HFI1_ORIG__
struct sdma_mmu_node {
struct mmu_rb_node rb;
@@ -177,7 +164,6 @@ struct evict_data {
u32 target; /* target count to evict */
};
#endif /* __HFI1_ORIG__ */
struct user_sdma_request {
struct sdma_req_info info;
struct hfi1_user_sdma_pkt_q *pq;
@@ -251,7 +237,6 @@ struct user_sdma_txreq {
unsigned busycount;
u64 seqnum;
};
#ifdef __HFI1_ORIG__
#define SDMA_DBG(req, fmt, ...) \
hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \
@@ -261,28 +246,24 @@ struct user_sdma_txreq {
hfi1_cdbg(SDMA, "[%u:%u:%u] " fmt, (pq)->dd->unit, (pq)->ctxt, \
(pq)->subctxt, ##__VA_ARGS__)
#endif /* __HFI1_ORIG__ */
static int user_sdma_send_pkts(struct user_sdma_request *, unsigned);
static int num_user_pages(const struct iovec *);
static void user_sdma_txreq_cb(struct sdma_txreq *, int);
static inline void pq_update(struct hfi1_user_sdma_pkt_q *);
static void user_sdma_free_request(struct user_sdma_request *, bool);
static int pin_vector_pages(struct user_sdma_request *,
struct user_sdma_iovec *);
static void unpin_vector_pages(struct mm_struct *, struct page **, unsigned,
unsigned);
static int check_header_template(struct user_sdma_request *,
struct hfi1_pkt_header *, u32, u32);
static int set_txreq_header(struct user_sdma_request *,
struct user_sdma_txreq *, u32);
static int set_txreq_header_ahg(struct user_sdma_request *,
struct user_sdma_txreq *, u32);
static void user_sdma_free_request(struct user_sdma_request *, bool);
static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *,
struct hfi1_user_sdma_comp_q *,
u16, enum hfi1_sdma_comp_state, int);
static void user_sdma_txreq_cb(struct sdma_txreq *, int);
#ifdef __HFI1_ORIG__
static int num_user_pages(const struct iovec *);
static int pin_vector_pages(struct user_sdma_request *,
struct user_sdma_iovec *);
static void unpin_vector_pages(struct mm_struct *, struct page **, unsigned,
unsigned);
static inline u32 set_pkt_bth_psn(__be32, u8, u32);
static inline u32 get_lrh_len(struct hfi1_pkt_header, u32 len);
@@ -495,8 +476,6 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd)
return 0;
}
#endif /* __HFI1_ORIG__ */
static u8 dlid_to_selector(u16 dlid)
{
static u8 mapping[256];
@@ -518,19 +497,11 @@ static u8 dlid_to_selector(u16 dlid)
return mapping[hash];
}
#ifdef __HFI1_ORIG__
int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec,
unsigned long dim, unsigned long *count)
{
int ret = 0, i;
struct hfi1_filedata *fd = fp->private_data;
#else
int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
unsigned long dim, unsigned long *count)
{
int ret = 0, i;
struct hfi1_filedata *fd = private_data;
#endif /* __HFI1_ORIG__ */
struct hfi1_ctxtdata *uctxt = fd->uctxt;
struct hfi1_user_sdma_pkt_q *pq = fd->pq;
struct hfi1_user_sdma_comp_q *cq = fd->cq;
@@ -544,7 +515,6 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
u16 dlid;
u32 selector;
hfi1_cdbg(AIOWRITE, "+");
if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {
hfi1_cdbg(
SDMA,
@@ -560,8 +530,8 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
return -EFAULT;
}
// trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt,
// (u16 *)&info);
trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt,
(u16 *)&info);
if (info.comp_idx >= hfi1_sdma_comp_ring_size) {
hfi1_cdbg(SDMA,
@@ -611,12 +581,6 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
memcpy(&req->info, &info, sizeof(info));
/*
* Seems like iovec[0] is always the header and if EXPECTED,
* TID is present is in iovec[dim - 1]
* The rest in between are data.
*/
if (req_opcode(info.ctrl) == EXPECTED) {
/* expected must have a TID info and at least one data vector */
if (req->data_iovs < 2) {
@@ -655,9 +619,6 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
ret = -EINVAL;
goto free_req;
}
// TODO: Enable this validation and checking
#ifdef __HFI1_ORIG__
/*
* Validate the vl. Do not trust packets from user space blindly.
* VL comes from PBC, SC comes from LRH, and the VL needs to
@@ -679,7 +640,6 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
ret = -EINVAL;
goto free_req;
}
#endif /* __HFI1_ORIG__ */
/*
* Also should check the BTH.lnh. If it says the next header is GRH then
@@ -706,22 +666,12 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
/* Save all the IO vector structures */
for (i = 0; i < req->data_iovs; i++) {
INIT_LIST_HEAD(&req->iovs[i].list);
/*
* XXX: iovec is still user-space in McKernel here!!
*
* req->iovs[] contain only the data.
*/
memcpy(&req->iovs[i].iov, iovec + idx++, sizeof(struct iovec));
#ifdef __HFI1_ORIG__
hfi1_cdbg(AIOWRITE, "+pin_vector_pages");
// TODO: pin_vector_pages
ret = pin_vector_pages(req, &req->iovs[i]);
hfi1_cdbg(AIOWRITE, "-pin_vector_pages");
if (ret) {
req->status = ret;
goto free_req;
}
#endif /* __HFI1_ORIG__ */
req->data_len += req->iovs[i].iov.iov_len;
}
SDMA_DBG(req, "total data length %u", req->data_len);
@@ -769,13 +719,7 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
dlid = be16_to_cpu(req->hdr.lrh[1]);
selector = dlid_to_selector(dlid);
selector += uctxt->ctxt + fd->subctxt;
/* TODO: check the rcu stuff */
/*
* XXX: didn't we conclude that we don't need to worry about RCU here?
* the mapping is created at driver initialization, the rest of the
* accesses are read-only
*/
//req->sde = sdma_select_user_engine(dd, selector, vl);
req->sde = sdma_select_user_engine(dd, selector, vl);
if (!req->sde || !sdma_running(req->sde)) {
ret = -ECOMM;
@@ -828,28 +772,20 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
goto free_req;
return ret;
}
#ifdef __HFI1_ORIG__
hfi1_cdbg(AIOWRITE, "+wait_event_interruptible_timeout");
wait_event_interruptible_timeout(
pq->busy.wait_dma,
(pq->state == SDMA_PKT_Q_ACTIVE),
msecs_to_jiffies(
SDMA_IOWAIT_TIMEOUT));
hfi1_cdbg(AIOWRITE, "-wait_event_interruptible_timeout");
#else
while (pq->state != SDMA_PKT_Q_ACTIVE) cpu_pause();
#endif /* __HFI1_ORIG__ */
}
}
*count += idx;
hfi1_cdbg(AIOWRITE, "-");
return 0;
free_req:
user_sdma_free_request(req, true);
if (req_queued)
pq_update(pq);
set_comp_state(pq, cq, info.comp_idx, ERROR, req->status);
hfi1_cdbg(AIOWRITE, "-");
return ret;
}
@@ -926,7 +862,6 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
struct hfi1_user_sdma_pkt_q *pq = NULL;
struct user_sdma_iovec *iovec = NULL;
hfi1_cdbg(AIOWRITE, "+");
if (!req->pq)
return -EINVAL;
@@ -964,11 +899,7 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
return -EFAULT;
}
#ifdef __HFI1_ORIG__
tx = kmem_cache_alloc(pq->txreq_cache, GFP_KERNEL);
#else
tx = kmalloc(sizeof(struct user_sdma_txreq), GFP_KERNEL | __GFP_ZERO);
#endif /* __HFI1_ORIG__ */
if (!tx)
return -ENOMEM;
@@ -1094,11 +1025,6 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
unsigned long base, offset;
unsigned pageidx, len;
/*
* XXX: this seems to operate with the assumption
* that all user buffers need to be processed in
* PAGE_SIZE steps, how about large pages?
*/
base = (unsigned long)iovec->iov.iov_base;
offset = offset_in_page(base + iovec->offset +
iov_offset);
@@ -1161,21 +1087,14 @@ dosend:
if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags))
sdma_ahg_free(req->sde, req->ahg_idx);
}
hfi1_cdbg(AIOWRITE, "-");
return ret;
free_txreq:
sdma_txclean(pq->dd, &tx->txreq);
free_tx:
#ifdef __HFI1_ORIG__
kmem_cache_free(pq->txreq_cache, tx);
hfi1_cdbg(AIOWRITE, "-");
#else
kfree(tx);
#endif /* __HFI1_ORIG__ */
return ret;
}
#ifdef __HFI1_ORIG__
/*
* How many pages in this iovec element?
@@ -1293,7 +1212,6 @@ static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,
kfree(pages);
}
#endif /* __HFI1_ORIG__ */
static int check_header_template(struct user_sdma_request *req,
struct hfi1_pkt_header *hdr, u32 lrhlen,
u32 datalen)
@@ -1470,8 +1388,8 @@ static int set_txreq_header(struct user_sdma_request *req,
req->omfactor != KDETH_OM_SMALL);
}
done:
// trace_hfi1_sdma_user_header(pq->dd, pq->ctxt, pq->subctxt,
// req->info.comp_idx, hdr, tidval);
trace_hfi1_sdma_user_header(pq->dd, pq->ctxt, pq->subctxt,
req->info.comp_idx, hdr, tidval);
return sdma_txadd_kvaddr(pq->dd, &tx->txreq, hdr, sizeof(*hdr));
}
@@ -1557,9 +1475,9 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
AHG_HEADER_SET(req->ahg, diff, 7, 16, 14, val);
}
// trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt,
// req->info.comp_idx, req->sde->this_idx,
// req->ahg_idx, req->ahg, diff, tidval);
trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt,
req->info.comp_idx, req->sde->this_idx,
req->ahg_idx, req->ahg, diff, tidval);
return diff;
}
@@ -1592,8 +1510,7 @@ static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status)
}
req->seqcomp = tx->seqnum;
//TODO: kmem_cache_free
//kmem_cache_free(pq->txreq_cache, tx);
kmem_cache_free(pq->txreq_cache, tx);
tx = NULL;
idx = req->info.comp_idx;
@@ -1621,14 +1538,12 @@ static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq)
{
if (atomic_dec_and_test(&pq->n_reqs)) {
xchg(&pq->state, SDMA_PKT_Q_INACTIVE);
//TODO: wake_up
//wake_up(&pq->wait);
wake_up(&pq->wait);
}
}
static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)
{
hfi1_cdbg(AIOWRITE, "+");
if (!list_empty(&req->txps)) {
struct sdma_txreq *t, *p;
@@ -1637,9 +1552,7 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)
container_of(t, struct user_sdma_txreq, txreq);
list_del_init(&t->list);
sdma_txclean(req->pq->dd, t);
#ifdef __HFI1_ORIG__
kmem_cache_free(req->pq->txreq_cache, tx);
#endif /* __HFI1_ORIG__ */
}
}
if (req->data_iovs) {
@@ -1651,20 +1564,17 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin)
if (!node)
continue;
//TODO:
#ifdef __HFI1_ORIG__
if (unpin)
hfi1_mmu_rb_remove(req->pq->handler,
&node->rb);
else
atomic_dec(&node->refcount);
#endif /* __HFI1_ORIG__ */
}
}
kfree(req->tids);
clear_bit(req->info.comp_idx, req->pq->req_in_use);
hfi1_cdbg(AIOWRITE, "-");
}
static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,
struct hfi1_user_sdma_comp_q *cq,
u16 idx, enum hfi1_sdma_comp_state state,
@@ -1675,10 +1585,9 @@ static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,
cq->comps[idx].status = state;
if (state == ERROR)
cq->comps[idx].errcode = -ret;
// trace_hfi1_sdma_user_completion(pq->dd, pq->ctxt, pq->subctxt,
// idx, state, ret);
trace_hfi1_sdma_user_completion(pq->dd, pq->ctxt, pq->subctxt,
idx, state, ret);
}
#ifdef __HFI1_ORIG__
static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
unsigned long len)
@@ -1742,5 +1651,3 @@ static int sdma_rb_invalidate(void *arg, struct mmu_rb_node *mnode)
return 1;
return 0;
}
#endif /* __HFI1_ORIG__ */